From b8968a4edcf1c8f65e8e88d38d942d08100736ce Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 17:15:51 +0000 Subject: [PATCH 001/270] Add a pure kernel for derived row labels. Domain, sensitivity, and project requirement are computed in one place. No product path calls it yet, so reads and writes stay as they are in v0.20.0. --- CHANGELOG.md | 1 + .../src/alicebot_api/vnext_agent_control.py | 9 + .../src/alicebot_api/vnext_derived_labels.py | 1237 +++++++++++++++++ .../src/alicebot_api/vnext_project_scope.py | 83 +- tests/unit/test_derived_labels_kernel.py | 819 +++++++++++ tests/unit/test_derived_labels_mutations.py | 161 +++ 6 files changed, 2308 insertions(+), 2 deletions(-) create mode 100644 apps/api/src/alicebot_api/vnext_derived_labels.py create mode 100644 tests/unit/test_derived_labels_kernel.py create mode 100644 tests/unit/test_derived_labels_mutations.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 95c3eeacc..510fd2dde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. Nothing in the running product calls the function yet, so a read and a write still behave as they do in v0.20.0. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` now scrub an open loop that names the source only in its metadata: the id, or `source:`, as the text under `source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references` or `selected_source_ids` at any depth. That is the rule the open-loop lookup of a source already used. They blanked only loops whose `source_id` column held the id, so a loop with an empty column kept its title, description and metadata, and they reached a new export. The delete preview now counts the same loops as the receipt. The rule reads the id only as stored or as `source:`: a loop that names the source in any spelling other than those two keeps its text, although the memory rule reads several other spellings. No migration is required. - Unreleased (on main, not in v0.20.0): the SQLite derived-label repair now updates and records each row under the id it is stored with (SQLite keeps capitals, missing hyphens, braces and `urn:uuid:` as written) and matches rows by the normalised id only to read the graph. If two stored spellings of one id exist, each is compared with the label its own recorded inputs give it, so neither is left at its old label while the other is relabelled, and neither is lowered. An update that changes no row stops the repair with `DerivedDomainRepairError` before any event or the completion stamp is written, on open and on restore; migration `20261004_0095` refuses the same way. Before, a derived memory stored under such an id kept its label while its relabel event and the stamp were written, and the pair survived export and import. A PostgreSQL uuid column was not affected. A vault that the earlier repair already stamped keeps such a row at its old label until a restore repairs it again. No migration is required. - Unreleased (on main, not in v0.20.0): v0.20.0 could store a consolidation report with a sensitivity below the memories it prints. The report took its sensitivity from the near-duplicate clusters only, so a run whose proposals were roll-up cards had no cluster to take it from and was stored as `unknown`. A key whose ceiling is below those memories (a `trusted_local_agent` key for confidential ones, a `read_only_agent` key for private ones) could then read the card topic and the member ids through `GET /v0/vnext/artifacts/{id}`. The report now takes its domain and its sensitivity over every row it names: the cluster members, the members of every proposed roll-up group, the members of the groups that a skip line names by key (`topic:...`, `entity:...`, `semantic:cluster-`), the pending, accepted, expired or held roll-up cards it names by id, and the sources that the `source_refs` of its cluster members name, which the report still prints. The open-loop review is labelled over the sources whose ids it prints as well as over its loops. The run digest of both reports covers those sources, so a source that was reclassified makes a new report instead of returning the earlier one. The first run after the upgrade over loops that link a source, or over cluster members that cite one, makes one new report, and a run that names no source keeps the digest it had. A report whose inputs are all unrestricted is now stored with the label of those inputs, `internal` where it was `unknown`, and every permission profile reads the two the same way. Reports stored by v0.20.0 keep their labels, because the stored-row repair does not change sensitivity. The other report producers (daily brief, weekly synthesis, connection and contradiction reports, project updates, staleness reports) were checked and already label over every row they print. No migration is required. diff --git a/apps/api/src/alicebot_api/vnext_agent_control.py b/apps/api/src/alicebot_api/vnext_agent_control.py index e33df175c..434f8cba8 100644 --- a/apps/api/src/alicebot_api/vnext_agent_control.py +++ b/apps/api/src/alicebot_api/vnext_agent_control.py @@ -16,6 +16,7 @@ from alicebot_api.vnext_project_scope import ( normalize_project_identifier, normalize_project_scope, + project_floor_within, project_scope_identity, resolve_project_scope, ) @@ -360,6 +361,7 @@ def evaluate_agent_policy( domains: tuple[str, ...] = (), sensitivity_allowed: tuple[str, ...] = DEFAULT_AGENT_SENSITIVITY, project_scope: tuple[str, ...] = (), + project_floor: tuple[str, ...] = (), workflow_type: str | None = None, write_policy: str | None = None, require_explicit_project_scope: bool = False, @@ -454,6 +456,13 @@ def evaluate_agent_policy( reasons.append("project_scope_binding_violation") decision = "blocked" effective_project_scope = () + elif project_floor and not project_floor_within(project_floor, identity.project_scope): + # The floor is part of the same binding test as the scope. A locked + # key reads a derived row only when every project in the floor is + # inside the binding. An empty floor does not add a refusal. + reasons.append("project_floor_binding_violation") + decision = "blocked" + effective_project_scope = () else: effective_project_scope = project_scope or identity.project_scope diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py new file mode 100644 index 000000000..ed0558aa7 --- /dev/null +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -0,0 +1,1237 @@ +"""Pure labels for a derived row: domain, sensitivity, and project requirement. + +Nothing in the product calls this module until a later change wires it in. +The same functions serve generation, a relabel, a read, and the stored-row +repair, so a rule lives here once. + +A derived row is found by a marker the server wrote, never by reading text. +``value.kind`` alone does not make a row derived. +""" + +from __future__ import annotations + +import json +from collections import deque +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, replace +from uuid import UUID + +from alicebot_api.vnext_agent_control import RESTRICTED_DOMAINS +from alicebot_api.vnext_derived_domain import derived_domain +from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError, _input_groups +from alicebot_api.vnext_project_scope import ( + is_global_scope, + normalize_project_scope, + project_floor_shape, + project_identifier_identity, + project_scope_identity, + resolve_project_scope, + source_project_scope, +) +from alicebot_api.vnext_source_fence import cited_source_ids + +# Public 1, internal 2, unknown 2, private 3, confidential 4, +# highly_sensitive 5, sacred 6, regulated 6. Raise only when the rank is +# strictly higher, so unknown and internal never swap. +SENSITIVITY_RANK: dict[str, int] = { + "public": 1, + "internal": 2, + "unknown": 2, + "private": 3, + "confidential": 4, + "highly_sensitive": 5, + "sacred": 6, + "regulated": 6, +} + +HOP_BOUND = 32 +NODE_BOUND = 5_000 +PROPAGATION_BOUND = 100_000 + +DERIVED_WORKFLOWS = frozenset( + { + "daily_brief", + "weekly_synthesis", + "connection_finder", + "contradiction_finder", + "memory_consolidation", + "project_auto_update", + "staleness_sweep", + "open_loop_review", + } +) +DERIVED_ARTIFACT_TYPES = frozenset( + { + "daily_brief", + "weekly_synthesis", + "connection_report", + "contradiction_report", + "memory_consolidation", + "open_loop_report", + } +) +_LEGACY_SUMMARY_KINDS = frozenset({"daily_brief", "weekly_synthesis"}) +_LEGACY_COUNT_KINDS = frozenset( + {"connection_finder", "contradiction_finder", "connection_report", "contradiction_report"} +) + +# Top-level marker and record keys. A write that omits one of these keeps the +# stored value. Label keys (project_scope, project_floor) are a separate set. +MARKER_KEYS = frozenset( + { + "consolidation", + "candidate_kind", + "discovered_by", + "workflow", + "source_artifact_id", + "source_id", + "input_summary", + "derived_from", + "source_ids", + "memory_ids", + "open_loop_ids", + "artifact_ids", + "belief_ids", + "source_refs", + "stale_marked_memory_ids", + "connector_name", + "input_counts", + "candidate_memory_ids", + "artifact_id", + "cluster_member_ids", + "cluster_membership", + "member_ids", + "member_snapshots", + } +) + +# Id keys the v2 planner already reads. The v3 set includes every one of them. +V2_ID_KEYS = frozenset( + { + "source_ids", + "memory_ids", + "open_loop_ids", + "artifact_ids", + "member_ids", + "cluster_member_ids", + "cluster_membership", + "stale_marked_memory_ids", + "belief_ids", + "artifact_id", + "source_artifact_id", + } +) + +_ID_KIND = { + "source_ids": "source", + "source_id": "source", + "source_artifact_id": "artifact", + "artifact_id": "artifact", + "artifact_ids": "artifact", + "memory_ids": "memory", + "memory_id": "memory", + "member_ids": "memory", + "cluster_member_ids": "memory", + "cluster_membership": "memory", + "stale_marked_memory_ids": "memory", + "open_loop_ids": "open_loop", + "belief_ids": "belief", +} +_DERIVED_FROM_KIND = { + "sources": "source", + "memories": "memory", + "open_loops": "open_loop", + "artifacts": "artifact", + "beliefs": "belief", +} +_REF_KINDS = {"source": "source", "memory": "memory", "open_loop": "open_loop", "artifact": "artifact"} +_SKIP_RECURSE = frozenset( + { + "quote", + "content", + "title", + "description", + "canonical_text", + "summary", + "content_markdown", + "prompt", + "text", + "markdown", + "body", + } +) +_KIND_ALIASES = { + "source": "source", + "sources": "source", + "memory": "memory", + "memories": "memory", + "open_loop": "open_loop", + "open_loops": "open_loop", + "artifact": "artifact", + "generated_artifacts": "artifact", + "project": "project", + "projects": "project", + "belief": "belief", + "beliefs": "belief", +} + +_EVENT_FIELDS = ("domain", "sensitivity", "project_scope", "project_floor") + + +class LabelPropagationTooLarge(ValueError): + """A relabel would touch more derived rows than the propagation bound allows.""" + + +def canon_kind(kind: object) -> str: + """Return the short kind name (``memory``, not ``memories``).""" + + text = str(kind or "") + if text not in _KIND_ALIASES: + raise ValueError(f"unknown derived label kind: {text}") + return _KIND_ALIASES[text] + + +def identifier(value: object) -> str: + """Normalize any spelling ``uuid.UUID`` accepts. Anything else is kept as text.""" + + try: + return str(UUID(str(value))) + except (ValueError, AttributeError, TypeError): + return str(value) + + +def _object(value: object) -> Mapping[str, object]: + if isinstance(value, str): + try: + value = json.loads(value) + except (ValueError, TypeError): + return {} + return value if isinstance(value, Mapping) else {} + + +def _metadata(row: Mapping[str, object]) -> Mapping[str, object]: + return _object(row.get("metadata_json")) + + +def _nonempty_str(value: object) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _scrubbed(row: Mapping[str, object]) -> bool: + return _metadata(row).get("scrubbed") is True + + +def ordered_identifiers(values: Iterable[object]) -> tuple[str, ...]: + """Stored spellings, first one kept, ordered by project identity.""" + + chosen: dict[str, str] = {} + for item in normalize_project_scope(list(values)): + identity = project_identifier_identity(item) + if identity and identity not in chosen: + chosen[identity] = item + return tuple(chosen[key] for key in sorted(chosen)) + + +def is_derived(kind: object, row: Mapping[str, object]) -> bool: + """True when ``row`` carries a server-written derived marker. + + A redacted row has no dependencies. A marker that lives only in ``value`` + does not count, so a forged ``value.kind`` does not make the row derived. + """ + + if _metadata(row).get("redacted") is True: + return False + name = canon_kind(kind) + meta = _metadata(row) + if name in {"source", "belief"}: + return False + if name == "project": + return "derived_from" in meta + if name == "open_loop": + discovered = meta.get("discovered_by") + if not _nonempty_str(discovered): + return False + return _nonempty_str(row.get("source_id")) or _nonempty_str(meta.get("source_id")) + if name == "artifact": + if "derived_from" in meta: + return True + if meta.get("workflow") in DERIVED_WORKFLOWS: + return True + if str(row.get("artifact_type") or "") in DERIVED_ARTIFACT_TYPES: + return True + return meta.get("connector_name") == "agent_output" + if isinstance(meta.get("consolidation"), Mapping): + return True + if meta.get("candidate_kind") in {"memory_consolidation", "memory_rollup"}: + return True + if meta.get("discovered_by") == "vnext_weekly_synthesis": + return True + if meta.get("workflow") == "project_auto_update": + return True + if _nonempty_str(meta.get("source_artifact_id")): + return True + if _nonempty_str(meta.get("source_id")): + return True + return "derived_from" in meta + + +def row_class(kind: object, row: Mapping[str, object]) -> str: + """``report``, ``aggregate``, ``copy``, ``project_state`` or ``original``.""" + + if not is_derived(kind, row): + return "original" + name = canon_kind(kind) + if name == "project": + return "project_state" + if name == "artifact": + return "report" + if name == "open_loop": + return "copy" + meta = _metadata(row) + if ( + _nonempty_str(meta.get("source_artifact_id")) + or meta.get("discovered_by") == "vnext_weekly_synthesis" + or meta.get("workflow") == "project_auto_update" + or isinstance(meta.get("consolidation"), Mapping) + or meta.get("candidate_kind") in {"memory_consolidation", "memory_rollup"} + ): + return "aggregate" + return "copy" + + +def infer_kind(row: Mapping[str, object]) -> str: + """Best-effort kind for a row that did not name one.""" + + if "artifact_type" in row or ( + "content_markdown" in row and "memory_key" not in row and "canonical_text" not in row + ): + return "artifact" + if "current_state" in row: + return "project" + if "content_hash" in row and "canonical_text" not in row and "memory_key" not in row: + return "source" + if "memory_key" in row or "canonical_text" in row: + return "memory" + return "open_loop" + + +def _floor_of(row: Mapping[str, object]) -> tuple[str, tuple[str, ...]]: + """``project_floor`` from the row or from parsed metadata.""" + + if "project_floor" in row: + return project_floor_shape(row) + return project_floor_shape({"metadata_json": _metadata(row)}) + + +def group_scope(row: Mapping[str, object], *, kind: str | None = None) -> tuple[str, ...]: + """Identity of the scope united with the floor. + + An original row has no floor, so this is the identity of its scope. + """ + + name = canon_kind(kind) if kind is not None else infer_kind(row) + scope = stored_scope(name, row) + if not is_derived(name, row): + return project_scope_identity(scope) + shape, floor = _floor_of(row) + if shape != "list": + floor = () + return project_scope_identity([*scope, *floor]) + + +def stored_scope(kind: object, row: Mapping[str, object]) -> tuple[str, ...]: + """The scope a row has stored, before the effective-scope rule.""" + + name = canon_kind(kind) + if name == "source": + if _scrubbed(row): + return () + return source_project_scope(row) + if name == "project": + return () + return resolve_project_scope(row).values + + +def carries_scope(kind: object, row: Mapping[str, object]) -> bool: + """False for a scrubbed source and for a project row. Both carry no scope.""" + + name = canon_kind(kind) + if name == "project": + return False + if name == "source" and _scrubbed(row): + return False + return True + + +def _strings(value: object) -> list[str]: + found: list[str] = [] + if isinstance(value, str): + found.append(identifier(value)) + elif isinstance(value, list): + for item in value: + found.extend(_strings(item)) + return found + + +def _as_string_list(value: object) -> tuple[str, list[str]] | None: + """``(problem, ids)`` or None when ``value`` is not a list of strings. + + A repeated id is kept, so the counts check can see it and skip. + """ + + if value is None: + return None + if not isinstance(value, list): + return ("malformed", []) + if any(not isinstance(item, str) for item in value): + return ("malformed", []) + return ("", _strings(value)) + + +def _source_ids_from(value: object) -> tuple[str, set[str]]: + """Named source ids, including every spelling the saved-quote reader names.""" + + if value is None: + return "", set() + named = {identifier(item) for item in cited_source_ids(value).named} + problem = "" + if isinstance(value, list): + for item in value: + if isinstance(item, str): + token = _source_token(item) + if token: + named.add(token) + elif isinstance(item, Mapping): + nested_problem, nested = _source_ids_from(item) + problem = problem or nested_problem + named |= nested + else: + return "malformed", set() + elif isinstance(value, str): + token = _source_token(value) + if token: + named.add(token) + elif value.strip() and not named: + # A sentence is not a source id. cited_source_ids already kept the + # explicit ones. A whole token that is not a uuid still counts. + if _plain_token(value): + named.add(identifier(value)) + elif isinstance(value, Mapping): + for key, child in value.items(): + key_text = key.lower() if isinstance(key, str) else "" + if key_text in _SKIP_RECURSE: + continue + child_problem, child_ids = _source_ids_from(child) + problem = problem or child_problem + named |= child_ids + else: + return "malformed", set() + return problem, named + + +def _plain_token(value: str) -> bool: + text = value.strip() + if not text or any(character.isspace() for character in text): + return False + return ":" not in text and "/" not in text + + +def _source_token(value: str) -> str | None: + text = value.strip() + lowered = text.lower() + for prefix in ("urn:uuid:", "source:", "uuid:"): + if lowered.startswith(prefix): + text = text[len(prefix) :].strip() + lowered = text.lower() + if not text or any(character.isspace() for character in text): + return None + if ":" in text or "/" in text: + return None + return identifier(text) + + +def _typed_refs(value: object) -> set[tuple[str, str]]: + found: set[tuple[str, str]] = set() + nodes: list[object] = [value] + while nodes: + node = nodes.pop() + if isinstance(node, str): + kind, separator, row_id = node.partition(":") + if separator and kind in _REF_KINDS and row_id and _plain_token(row_id): + found.add((_REF_KINDS[kind], identifier(row_id))) + elif isinstance(node, list): + nodes.extend(node) + elif isinstance(node, Mapping): + nodes.extend(node.values()) + return found + + +def _add_ids(found: set[tuple[str, str]], kind: str, values: Iterable[str]) -> None: + for item in values: + if item: + found.add((kind, item)) + + +def _collect_metadata_ids(value: object, found: set[tuple[str, str]]) -> str: + """Walk server-written metadata. Return ``malformed`` or ``""``.""" + + problem = "" + nodes: list[object] = [value] + while nodes: + node = nodes.pop() + if isinstance(node, list): + nodes.extend(node) + continue + if not isinstance(node, Mapping): + continue + for key, child in node.items(): + if not isinstance(key, str) or key in _SKIP_RECURSE: + continue + if key == "derived_from": + continue + if key == "source_refs" or key in {"source_id", "source_ids"}: + child_problem, source_ids = _source_ids_from(child) + problem = problem or child_problem + _add_ids(found, "source", source_ids) + found.update(ref for ref in _typed_refs(child) if ref[0] != "source" or ref[1]) + continue + if key == "member_snapshots" and isinstance(child, list): + for item in child: + if isinstance(item, Mapping) and _nonempty_str(item.get("id")): + found.add(("memory", identifier(item.get("id")))) + elif isinstance(item, str): + found.add(("memory", identifier(item))) + continue + if key in _ID_KIND: + parsed = _as_string_list(child if isinstance(child, list) else [child] if isinstance(child, str) else child) + if parsed is None: + continue + child_problem, ids = parsed + if child_problem: + problem = problem or child_problem + else: + _add_ids(found, _ID_KIND[key], ids) + continue + if isinstance(child, (Mapping, list)): + nodes.append(child) + return problem + + +def _derived_from_deps(record: object) -> tuple[str, set[tuple[str, str]]]: + """Read the canonical record. A bad shape is ``malformed`` or ``counts_disagree``.""" + + if not isinstance(record, Mapping): + return "malformed", set() + found: set[tuple[str, str]] = set() + lists: dict[str, list[str]] = {} + repeated = False + for key, kind in _DERIVED_FROM_KIND.items(): + if key not in record: + lists[key] = [] + continue + raw = record.get(key) + if not isinstance(raw, list) or any(not isinstance(item, str) for item in raw): + return "malformed", set() + if len(raw) != len(set(raw)): + repeated = True + ids = _strings(raw) + lists[key] = ids + _add_ids(found, kind, ids) + counts = record.get("counts") + if counts is None: + if any(lists.values()): + return "counts_disagree", found + return "", found + if not isinstance(counts, Mapping): + return "malformed", found + if repeated: + return "", found + for key, ids in lists.items(): + if key not in counts: + if ids: + return "counts_disagree", found + continue + expected = counts.get(key) + if not isinstance(expected, int) or isinstance(expected, bool): + return "malformed", found + if expected != len(ids): + return "counts_disagree", found + return "", found + + +def _legacy_count_problem(kind: str, row: Mapping[str, object], meta: Mapping[str, object]) -> str: + """Disagreeing legacy counts on the four producers that write them.""" + + workflow = str(meta.get("workflow") or "") + artifact_type = str(row.get("artifact_type") or "") + summary = meta.get("input_summary") + if workflow in _LEGACY_SUMMARY_KINDS or artifact_type in _LEGACY_SUMMARY_KINDS: + if isinstance(summary, Mapping): + problem = _counts_against_lists( + summary.get("counts"), + { + "sources": summary.get("source_ids"), + "memories": summary.get("memory_ids"), + "open_loops": summary.get("open_loop_ids"), + "artifacts": summary.get("artifact_ids"), + }, + ) + if problem: + return "legacy_counts" + if workflow in _LEGACY_COUNT_KINDS or artifact_type in _LEGACY_COUNT_KINDS: + problem = _counts_against_lists( + meta.get("input_counts"), + { + "sources": meta.get("source_ids"), + "memories": meta.get("memory_ids"), + "beliefs": meta.get("belief_ids"), + "open_loops": meta.get("open_loop_ids"), + "artifacts": meta.get("artifact_ids"), + }, + ) + if problem: + return "legacy_counts" + return "" + + +def _counts_against_lists(counts: object, lists: Mapping[str, object]) -> str: + if not isinstance(counts, Mapping): + return "" + present_lists: dict[str, list[object]] = {} + for key, raw in lists.items(): + if raw is None: + continue + if not isinstance(raw, list): + return "malformed" + present_lists[key] = list(raw) + if any(len(values) != len(set(map(str, values))) for values in present_lists.values()): + return "" + for key, values in present_lists.items(): + if key not in counts: + continue + expected = counts.get(key) + if isinstance(expected, int) and not isinstance(expected, bool) and expected != len(values): + return "counts_disagree" + return "" + + +def _record_present(meta: Mapping[str, object], row: Mapping[str, object]) -> bool: + if "derived_from" in meta or "input_summary" in meta or "source_refs" in meta or "input_counts" in meta: + return True + for key in ( + "source_ids", + "memory_ids", + "open_loop_ids", + "artifact_ids", + "belief_ids", + "stale_marked_memory_ids", + "source_id", + "source_artifact_id", + "artifact_id", + ): + if key in meta: + return True + consolidation = meta.get("consolidation") + if isinstance(consolidation, Mapping) and consolidation: + return True + if _nonempty_str(row.get("source_id")): + return True + return False + + +def dependencies_of(kind: object, row: Mapping[str, object]) -> frozenset[tuple[str, str]]: + """Recorded inputs as ``(kind, normalized id)``. + + Belief ids are returned as ``belief`` and resolved by :func:`settle_labels`. + A row that is not derived has no dependencies. The value fields are read + only when a metadata marker is already present. + """ + + deps, _problem = dependency_record(kind, row) + return frozenset(deps) + + +def dependency_record(kind: object, row: Mapping[str, object]) -> tuple[frozenset[tuple[str, str]], str]: + """Dependencies and a structural problem, or ``""`` when the record is sound. + + An empty record is sound. A missing record on a derived row is ``no_record``. + """ + + if not is_derived(kind, row): + return frozenset(), "" + meta = _metadata(row) + found: set[tuple[str, str]] = set() + problem = _collect_metadata_ids(meta, found) + if "derived_from" in meta: + derived_problem, derived_deps = _derived_from_deps(meta.get("derived_from")) + problem = problem or derived_problem + found |= derived_deps + if is_derived(kind, row): + found |= _value_dependencies(row) + if canon_kind(kind) == "open_loop" and _nonempty_str(row.get("source_id")): + found.add(("source", identifier(row.get("source_id")))) + floor_shape, _floor = _floor_of(row) + if floor_shape == "malformed": + problem = problem or "malformed_floor" + if not problem: + problem = _legacy_count_problem(canon_kind(kind), row, meta) + if not problem and not _record_present(meta, row) and not found: + # A weekly candidate may still be completed from its parent artifact + # inside settle_labels. The structural pass says no_record until then. + problem = "no_record" + if problem: + return frozenset(found), problem + return frozenset(found), "" + + +def _value_dependencies(row: Mapping[str, object]) -> set[tuple[str, str]]: + value = _object(row.get("value")) + found: set[tuple[str, str]] = set() + if _nonempty_str(value.get("source_id")): + found.add(("source", identifier(value.get("source_id")))) + if _nonempty_str(value.get("artifact_id")): + found.add(("artifact", identifier(value.get("artifact_id")))) + parsed = _as_string_list(value.get("cluster_member_ids")) if "cluster_member_ids" in value else None + if parsed and not parsed[0]: + _add_ids(found, "memory", parsed[1]) + rollup = value.get("rollup") + if isinstance(rollup, Mapping): + members = _as_string_list(rollup.get("member_ids")) if "member_ids" in rollup else None + if members and not members[0]: + _add_ids(found, "memory", members[1]) + return found + + +def _raised_sensitivity(current: str, inputs: Iterable[str]) -> str: + """Highest rank. A lower rank never replaces the current label.""" + + current_rank = SENSITIVITY_RANK.get(current, SENSITIVITY_RANK["unknown"]) + chosen = current + chosen_rank = current_rank + for value in inputs: + rank = SENSITIVITY_RANK.get(str(value), SENSITIVITY_RANK["unknown"]) + if rank > chosen_rank: + chosen = str(value) + chosen_rank = rank + return chosen + + +def union_floor(stored: Sequence[str], parts: Iterable[Sequence[str]]) -> tuple[str, ...]: + """Stored floor united with every part. The stored projects stay.""" + + combined = [*stored, *[item for part in parts for item in part]] + return ordered_identifiers(combined) + + +def intersect_scope(stored: Sequence[str], parent: Sequence[str]) -> tuple[str, ...]: + parent_ids = set(project_scope_identity(parent)) + return ordered_identifiers(item for item in stored if project_identifier_identity(item) in parent_ids) + + +def generation_domain(payload_domain: str, dep_domains: Iterable[str]) -> str: + """Insert rule: a restricted payload domain is kept. Otherwise the derived domain.""" + + if payload_domain in RESTRICTED_DOMAINS: + return payload_domain + return derived_domain(({"domain": item} for item in dep_domains), fallback=payload_domain) + + +def labels_raised_payload(*, cause: str, previous: Mapping[str, object], new: Mapping[str, object]) -> dict[str, object]: + """One audit payload. Labels only: no text, titles or input ids.""" + + def side(source: Mapping[str, object]) -> dict[str, object]: + scope = source.get("project_scope", ()) + floor = source.get("project_floor", ()) + return { + "domain": str(source.get("domain") or "unknown"), + "sensitivity": str(source.get("sensitivity") or "unknown"), + "project_scope": list(scope) if isinstance(scope, (list, tuple)) else [], + "project_floor": list(floor) if isinstance(floor, (list, tuple)) else [], + } + + return {"cause": cause, "previous": side(previous), "new": side(new)} + + +@dataclass(frozen=True, slots=True) +class SettledLabel: + """Effective label of one stored row.""" + + kind: str + user_id: str + stored_id: str + normalized_id: str + domain: str + sensitivity: str + project_scope: tuple[str, ...] + project_floor: tuple[str, ...] + stored_domain: str + stored_sensitivity: str + stored_scope: tuple[str, ...] + stored_floor: tuple[str, ...] + unverified: bool + reason: str | None + carries_scope: bool + derived: bool + row_class: str + + @property + def key(self) -> tuple[str, str, str]: + return (self.kind, self.user_id, self.normalized_id) + + +@dataclass(frozen=True, slots=True) +class SettleResult: + """Settled labels, one entry per stored row, in input order.""" + + rows: tuple[SettledLabel, ...] + + def by_stored(self, kind: str, stored_id: str, *, user_id: str | None = None) -> SettledLabel: + name = canon_kind(kind) + for row in self.rows: + if row.kind == name and row.stored_id == str(stored_id) and (user_id is None or row.user_id == user_id): + return row + raise KeyError((name, stored_id)) + + def derived_rows(self) -> tuple[SettledLabel, ...]: + return tuple(row for row in self.rows if row.derived) + + +def _node_label(kind: str, row: Mapping[str, object]) -> SettledLabel: + meta_floor_shape, floor = _floor_of(row) + if meta_floor_shape != "list": + floor = () + scope = stored_scope(kind, row) + domain = str(row.get("domain") or "unknown") + sensitivity = str(row.get("sensitivity") or "unknown") + return SettledLabel( + kind=kind, + user_id=str(row.get("user_id") or ""), + stored_id=str(row.get("id") or ""), + normalized_id=identifier(row.get("id")), + domain=domain, + sensitivity=sensitivity, + project_scope=scope, + project_floor=floor, + stored_domain=domain, + stored_sensitivity=sensitivity, + stored_scope=scope, + stored_floor=floor, + unverified=False, + reason=None, + carries_scope=carries_scope(kind, row), + derived=is_derived(kind, row), + row_class=row_class(kind, row), + ) + + +def _copy_scope(stored: tuple[str, ...], parents: Sequence[SettledLabel]) -> tuple[str, ...]: + live = [item for item in parents if item.carries_scope] + if not parents: + return stored + if not live: + # Every parent is a scrubbed source: keep the stored scope. + return stored + scope = stored + for item in live: + if len(project_scope_identity(item.project_scope)) == 0: + return () + scope = intersect_scope(scope, item.project_scope) + return scope + + +def _effective_scope(current: SettledLabel, deps: Sequence[SettledLabel], floor: tuple[str, ...]) -> tuple[str, ...]: + if current.row_class == "project_state": + return () + if current.row_class == "original" or not current.derived: + return current.stored_scope + informative = [item for item in deps if item.carries_scope] + if not deps: + return current.stored_scope + any_empty = any(len(project_scope_identity(item.project_scope)) == 0 for item in informative) + if current.row_class == "report": + if any_empty: + return () # a global input empties the report scope + return current.stored_scope + if current.row_class == "aggregate": + stored = current.stored_scope + single = len(project_scope_identity(stored)) == 1 + floor_inside = set(project_scope_identity(floor)).issubset(set(project_scope_identity(stored))) + if single and not any_empty and floor_inside: + return stored + return () # aggregate rule otherwise leaves the scope empty + parents = [item for item in deps if item.kind == "source"] + return _copy_scope(current.stored_scope, parents) + + +def _apply_dependencies( + current: SettledLabel, + deps: Sequence[SettledLabel], + *, + domain_fallback: str | None = None, + sensitivity_fallback: str | None = None, + scope_fallback: tuple[str, ...] | None = None, + floor_fallback: tuple[str, ...] | None = None, +) -> SettledLabel: + """One row from the effective labels of its dependencies.""" + + fallback_domain = current.domain if domain_fallback is None else domain_fallback + fallback_sensitivity = current.sensitivity if sensitivity_fallback is None else sensitivity_fallback + base = current + if scope_fallback is not None or floor_fallback is not None: + base = replace( + current, + stored_scope=current.stored_scope if scope_fallback is None else scope_fallback, + stored_floor=current.stored_floor if floor_fallback is None else floor_fallback, + domain=fallback_domain, + sensitivity=fallback_sensitivity, + ) + domain = derived_domain(({"domain": item.domain} for item in deps), fallback=fallback_domain) + if domain not in RESTRICTED_DOMAINS and fallback_domain in RESTRICTED_DOMAINS: + domain = fallback_domain + sensitivity = _raised_sensitivity(fallback_sensitivity, (item.sensitivity for item in deps)) + informative = [item for item in deps if item.carries_scope] + floor = union_floor( + base.stored_floor, + [*(item.project_scope for item in informative), *(item.project_floor for item in informative)], + ) + if base.row_class == "project_state": + floor = () + scope = _effective_scope(base, deps, floor) + return replace(base, domain=str(domain), sensitivity=sensitivity, project_scope=scope, project_floor=floor) + + +def _changed(before: SettledLabel, after: SettledLabel) -> bool: + return ( + before.domain != after.domain + or before.sensitivity != after.sensitivity + or project_scope_identity(before.project_scope) != project_scope_identity(after.project_scope) + or project_scope_identity(before.project_floor) != project_scope_identity(after.project_floor) + ) + + +def _weekly_parent_deps( + labels: Mapping[tuple[str, str, str], SettledLabel], + own: Mapping[tuple[str, str, str], set[tuple[str, str, str]]], + nodes: Sequence[tuple[SettledLabel, Mapping[str, object]]], +) -> None: + """Old weekly candidates take the input lists of the artifact that names them.""" + + artifacts = [ + (label, _metadata(row)) + for label, row in nodes + if label.kind == "artifact" and isinstance(_metadata(row).get("input_summary"), Mapping) + ] + for label, meta in artifacts: + for candidate in _strings(meta.get("candidate_memory_ids")): + candidate_key = ("memory", label.user_id, candidate) + candidate_label = labels.get(candidate_key) + if candidate_label is None or candidate_label.row_class != "aggregate": + continue + if _metadata_discovered(nodes, candidate_key) != "vnext_weekly_synthesis": + continue + own.setdefault(candidate_key, set()).update(own.get(label.key, set())) + + +def _metadata_discovered( + nodes: Sequence[tuple[SettledLabel, Mapping[str, object]]], + key: tuple[str, str, str], +) -> object: + for label, row in nodes: + if label.key == key: + return _metadata(row).get("discovered_by") + return None + + +def settle_labels( + nodes: Sequence[Mapping[str, object]], + *, + unavailable_kinds: Iterable[str] = (), + on_cycle: str = "raise", + max_hops: int | None = None, + max_nodes: int | None = None, +) -> SettleResult: + """Settle every derived row in ``nodes``. + + ``nodes`` are mappings with ``kind`` (or a table name) plus the row fields. + Originals are fixed inputs. A cycle that does not settle raises + ``DerivedDomainRepairError`` unless ``on_cycle`` is ``unverified``. + ``max_hops`` and ``max_nodes`` bound a read. The repair leaves them unset. + """ + + if on_cycle not in {"raise", "unverified"}: + raise ValueError("on_cycle must be raise or unverified") + unavailable = {canon_kind(kind) for kind in unavailable_kinds} + prepared: list[tuple[SettledLabel, Mapping[str, object]]] = [] + for node in nodes: + kind = canon_kind(node.get("kind") if node.get("kind") is not None else infer_kind(node)) + row = node + if "row" in node and isinstance(node.get("row"), Mapping): + row = node["row"] # type: ignore[assignment] + prepared.append((_node_label(kind, row), row)) + + labels: dict[tuple[str, str, str], SettledLabel] = {} + order: list[SettledLabel] = [] + for label, _row in prepared: + order.append(label) + labels[label.key] = label + + own: dict[tuple[str, str, str], set[tuple[str, str, str]]] = {} + problems: dict[tuple[str, str, str], str] = {} + for label, row in prepared: + if not label.derived: + continue + deps, problem = dependency_record(label.kind, row) + if problem == "no_record" and label.row_class == "aggregate": + # Filled from the parent artifact below when one names this row. + problem = "" + problems[label.key] = "no_record" + elif problem: + problems[label.key] = problem + keyed = {(kind, label.user_id, row_id) for kind, row_id in deps} + own[label.key] = keyed + _weekly_parent_deps(labels, own, prepared) + for key, reason in list(problems.items()): + if reason == "no_record" and own.get(key): + del problems[key] + elif reason == "no_record" and not own.get(key): + pass + elif reason == "" and not own.get(key): + pass + + def resolve_belief(ref: tuple[str, str, str]) -> tuple[str, str, str]: + if ref[0] != "belief": + return ref + belief = labels.get(ref) + if belief is None: + return ref + for label, row in prepared: + if label.key == ref: + memory_id = row.get("memory_id") + if _nonempty_str(memory_id) or memory_id not in (None, ""): + return ("memory", ref[1], identifier(memory_id)) + return ref + + resolved: dict[tuple[str, str, str], set[tuple[str, str, str]]] = {} + for key, deps in own.items(): + resolved[key] = {resolve_belief(ref) for ref in deps} + + for key, deps in resolved.items(): + if key in problems and problems[key] not in {"", "no_record"}: + continue + for ref in deps: + if ref[0] in unavailable: + problems[key] = "missing_table" + break + if ref not in labels: + problems[key] = "missing_dependency" + break + + if max_hops is not None or max_nodes is not None: + _mark_bounds(labels, resolved, problems, max_hops=max_hops, max_nodes=max_nodes) + + _spread_unverified(resolved, problems) + + inputs = {key: set(resolved.get(key, set())) for key, label in labels.items() if label.derived and key not in problems} + for key in problems: + inputs.pop(key, None) + + cycle_keys: set[tuple[str, str, str]] = set() + if inputs: + try: + _iterate(labels, inputs, on_cycle=on_cycle, cycle_keys=cycle_keys, problems=problems) + except DerivedDomainRepairError: + if on_cycle != "raise": + raise + raise + + for key in cycle_keys: + problems[key] = "cycle_unsettled" + _spread_unverified(resolved, problems) + + published: list[SettledLabel] = [] + for label, row in prepared: + current = labels[label.key] + if not label.derived: + published.append(label) + continue + if label.key in problems: + published.append( + replace( + label, + unverified=True, + reason=problems[label.key], + carries_scope=False, + ) + ) + continue + settled = labels[label.key] + same_stored_row = ( + settled.stored_id == label.stored_id + and settled.stored_domain == label.stored_domain + and settled.stored_sensitivity == label.stored_sensitivity + and settled.stored_scope == label.stored_scope + and settled.stored_floor == label.stored_floor + ) + if same_stored_row: + published.append(replace(settled, unverified=False, reason=None)) + continue + deps = [labels[ref] for ref in sorted(resolved.get(label.key, set())) if ref in labels] + recomputed = _apply_dependencies( + settled, + deps, + domain_fallback=label.stored_domain, + sensitivity_fallback=label.stored_sensitivity, + scope_fallback=label.stored_scope, + floor_fallback=label.stored_floor, + ) + published.append( + replace( + recomputed, + stored_id=label.stored_id, + stored_domain=label.stored_domain, + stored_sensitivity=label.stored_sensitivity, + stored_scope=label.stored_scope, + stored_floor=label.stored_floor, + unverified=False, + reason=None, + ) + ) + return SettleResult(rows=tuple(published)) + + +def _spread_unverified( + resolved: Mapping[tuple[str, str, str], set[tuple[str, str, str]]], + problems: dict[tuple[str, str, str], str], +) -> None: + """A row that reads an unverified row is unverified. The reason is contagious.""" + + changed = True + while changed: + changed = False + for key, deps in resolved.items(): + if key in problems: + continue + if any(ref in problems for ref in deps): + problems[key] = "dependency_unverified" + changed = True + + +def _mark_bounds( + labels: Mapping[tuple[str, str, str], SettledLabel], + resolved: Mapping[tuple[str, str, str], set[tuple[str, str, str]]], + problems: dict[tuple[str, str, str], str], + *, + max_hops: int | None, + max_nodes: int | None, +) -> None: + """Mark a derived row unverified when the walk from that row passes a bound.""" + + for origin, label in labels.items(): + if not label.derived or origin in problems: + continue + pending: deque[tuple[tuple[str, str, str], int]] = deque([(origin, 0)]) + seen: set[tuple[str, str, str]] = set() + while pending: + key, depth = pending.popleft() + if key in seen: + continue + if max_nodes is not None and len(seen) >= max_nodes: + problems.setdefault(origin, "bound_exceeded") + break + if max_hops is not None and depth > max_hops: + problems.setdefault(origin, "bound_exceeded") + break + seen.add(key) + for ref in resolved.get(key, set()): + pending.append((ref, depth + 1)) + + +def _iterate( + labels: dict[tuple[str, str, str], SettledLabel], + inputs: dict[tuple[str, str, str], set[tuple[str, str, str]]], + *, + on_cycle: str, + cycle_keys: set[tuple[str, str, str]], + problems: dict[tuple[str, str, str], str], +) -> None: + dependants: dict[tuple[str, str, str], set[tuple[str, str, str]]] = {} + for key, refs in inputs.items(): + for ref in refs: + dependants.setdefault(ref, set()).add(key) + for component in _input_groups(inputs): + members = set(component) + pending: deque[tuple[str, str, str]] = deque(component) + queued = set(component) + remaining_changes = len(component) * (len(RESTRICTED_DOMAINS) + 1) + changes: dict[tuple[str, str, str], int] = {} + while pending: + key = pending.popleft() + queued.remove(key) + current = labels[key] + deps = [labels[ref] for ref in sorted(inputs[key]) if ref in labels] + updated = _apply_dependencies( + current, + deps, + domain_fallback=current.stored_domain, + sensitivity_fallback=current.stored_sensitivity, + scope_fallback=current.stored_scope, + floor_fallback=current.stored_floor, + ) + if not _changed(current, updated): + continue + remaining_changes -= 1 + changes[key] = changes.get(key, 0) + 1 + if remaining_changes < 0: + if on_cycle == "unverified": + for member in members: + cycle_keys.add(member) + problems[member] = "cycle_unsettled" + return + shown = ", ".join(f"{item[0]} {item[2]}" for item in sorted(changes)[:5]) + raise DerivedDomainRepairError( + "derived label repair did not settle: derived rows record each other as inputs in a cycle, " + f"so their labels kept changing (rows: {shown})." + ) + labels[key] = updated + for dependant in sorted(dependants.get(key, set()) & members): + if dependant not in queued: + pending.append(dependant) + queued.add(dependant) + + +def scope_is_global(scope: object) -> bool: + """True when a scope holds no Alice project id.""" + + return is_global_scope(scope) + + +__all__ = [ + "DERIVED_ARTIFACT_TYPES", + "DERIVED_WORKFLOWS", + "HOP_BOUND", + "LabelPropagationTooLarge", + "MARKER_KEYS", + "NODE_BOUND", + "PROPAGATION_BOUND", + "SENSITIVITY_RANK", + "SettleResult", + "SettledLabel", + "V2_ID_KEYS", + "canon_kind", + "carries_scope", + "dependencies_of", + "dependency_record", + "generation_domain", + "group_scope", + "identifier", + "infer_kind", + "intersect_scope", + "is_derived", + "labels_raised_payload", + "ordered_identifiers", + "row_class", + "scope_is_global", + "settle_labels", + "stored_scope", + "union_floor", +] diff --git a/apps/api/src/alicebot_api/vnext_project_scope.py b/apps/api/src/alicebot_api/vnext_project_scope.py index 16132ba42..4e6faca2c 100644 --- a/apps/api/src/alicebot_api/vnext_project_scope.py +++ b/apps/api/src/alicebot_api/vnext_project_scope.py @@ -361,12 +361,86 @@ def refuse_global_marker(scope: object, *, where: str) -> None: raise ValueError(f"{where} takes explicit project names only and does not accept the global marker") -def project_scopes_overlap(resource_scope: object, requested_scope: object) -> bool: +def project_floor(resource: Mapping[str, object] | None) -> tuple[str, ...]: + """Return the stored ``project_floor`` list, or ``()`` when the key is absent. + + A present value that is not a list of strings is not a floor. Callers that + must refuse that shape ask :func:`project_floor_shape` first. This helper + returns ``()`` for it so a selection door does not treat a bad value as names. + """ + + shape, values = project_floor_shape(resource) + if shape != "list": + return () + return values + + +def project_floor_shape(resource: Mapping[str, object] | None) -> tuple[str, tuple[str, ...]]: + """Classify ``project_floor`` as ``absent``, ``list`` or ``malformed``. + + ``list`` carries the stored spellings, ordered by identity, with no duplicates. + """ + + if resource is None: + return "absent", () + containers: list[Mapping[str, object]] = [resource] + metadata = resource.get("metadata_json") + if isinstance(metadata, Mapping): + containers.append(metadata) + seen = False + raw: object = None + for container in containers: + if "project_floor" in container: + seen = True + raw = container.get("project_floor") + break + if not seen: + return "absent", () + if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes, bytearray)): + return "malformed", () + if any(not isinstance(item, str) for item in raw): + return "malformed", () + return "list", _ordered_scope(raw) + + +def project_floor_within(floor: object, binding: object) -> bool: + """True when every project in ``floor`` is inside ``binding``, by identity. + + An empty floor is inside every binding. A free-form name is inside only when + the binding names that same identity. + """ + + return set(project_scope_identity(floor)).issubset(set(project_scope_identity(binding))) + + +def _ordered_scope(value: object) -> tuple[str, ...]: + """Stored spellings, first one kept, ordered by identity.""" + + chosen: dict[str, str] = {} + for item in normalize_project_scope(value): + identity = project_identifier_identity(item) + if identity and identity not in chosen: + chosen[identity] = item + return tuple(chosen[identity] for identity in sorted(chosen)) + + +def project_scopes_overlap( + resource_scope: object, + requested_scope: object, + *, + floor: object = (), +) -> bool: """Does the resource's scope meet the requested tuple? The tuple may hold the reserved marker. The marker asks for a resource whose scope holds no Alice project id, and is never compared with a stored value. Every other entry is an identifier compared by identity. + + ``floor`` is the row's project floor. It is consulted only on the global + branch: a row whose scope holds no Alice project id is in a view that asks + for global rows only when every Alice project id in the floor is in the + view. Free-form names in the floor do not hide the row. An empty floor keeps + the previous answer. """ requested = set(project_scope_identity(requested_scope)) @@ -376,7 +450,9 @@ def project_scopes_overlap(resource_scope: object, requested_scope: object) -> b if GLOBAL_PROJECT_MARKER in requested: requested.discard(GLOBAL_PROJECT_MARKER) if not any(is_alice_project_id(item) for item in resource): - return True + floor_ids = {item for item in project_scope_identity(floor) if is_alice_project_id(item)} + if floor_ids.issubset(requested): + return True return bool(requested.intersection(resource)) @@ -391,6 +467,9 @@ def project_scopes_overlap(resource_scope: object, requested_scope: object) -> b "normalize_project_identifier", "normalize_project_scope", "ProjectScopeResolution", + "project_floor", + "project_floor_shape", + "project_floor_within", "project_identifier_identity", "project_scope_identity", "project_scopes_overlap", diff --git a/tests/unit/test_derived_labels_kernel.py b/tests/unit/test_derived_labels_kernel.py new file mode 100644 index 000000000..3d1528b65 --- /dev/null +++ b/tests/unit/test_derived_labels_kernel.py @@ -0,0 +1,819 @@ +"""The pure derived-label kernel. No store and no product caller.""" + +from __future__ import annotations + +import ast +from pathlib import Path +from uuid import UUID + +import pytest + +from alicebot_api import vnext_brain as brain +from alicebot_api import vnext_consolidation as consolidation +from alicebot_api.vnext_agent_control import RESTRICTED_DOMAINS, AgentIdentity, evaluate_agent_policy +from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError, _ID_KEYS +from alicebot_api.vnext_derived_labels import ( + SENSITIVITY_RANK, + V2_ID_KEYS, + dependencies_of, + generation_domain, + group_scope, + is_derived, + labels_raised_payload, + row_class, + settle_labels, +) +from alicebot_api.vnext_project_scope import GLOBAL_PROJECT_MARKER, project_floor_within, project_scopes_overlap + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 +USER = "user-1" +SOURCE_UUID = "550e8400-e29b-41d4-a716-446655440000" + + +def _row(kind: str, row_id: str, **fields: object) -> dict[str, object]: + row: dict[str, object] = { + "kind": kind, + "id": row_id, + "user_id": USER, + "domain": "unknown", + "sensitivity": "unknown", + "metadata_json": {}, + } + row.update(fields) + return row + + +def _meta(row: dict[str, object], **fields: object) -> dict[str, object]: + metadata = dict(row["metadata_json"]) if isinstance(row["metadata_json"], dict) else {} + metadata.update(fields) + row["metadata_json"] = metadata + return row + + +def _brief(row_id: str, *, sources: list[str] | None = None, memories: list[str] | None = None, + loops: list[str] | None = None, artifacts: list[str] | None = None, scope: list[str] | None = None, + domain: str = "unknown", sensitivity: str = "public") -> dict[str, object]: + sources = sources or [] + memories = memories or [] + loops = loops or [] + artifacts = artifacts or [] + summary = { + "source_ids": sources, + "memory_ids": memories, + "open_loop_ids": loops, + "artifact_ids": artifacts, + "counts": { + "sources": len(sources), + "memories": len(memories), + "open_loops": len(loops), + "artifacts": len(artifacts), + }, + } + return _row( + "artifact", + row_id, + artifact_type="daily_brief", + domain=domain, + sensitivity=sensitivity, + metadata_json={ + "workflow": "daily_brief", + "input_summary": summary, + "project_scope": list(scope or []), + "source_refs": [f"source:{item}" for item in sources], + }, + ) + + +def _source(row_id: str, *, domain: str = "unknown", sensitivity: str = "public", scope: list[str] | None = None, + scrubbed: bool = False) -> dict[str, object]: + metadata: dict[str, object] = {} + if scope is not None: + metadata["project_scope"] = list(scope) + if scrubbed: + metadata["scrubbed"] = True + return _row("source", row_id, domain=domain, sensitivity=sensitivity, metadata_json=metadata) + + +def _memory(row_id: str, **fields: object) -> dict[str, object]: + row = _row("memory", row_id, memory_key=row_id, canonical_text=row_id) + row.update(fields) + return row + + +def test_a_chain_settles_in_one_pass(monkeypatch: pytest.MonkeyPatch) -> None: + """A chain of any length is labelled from the leaf inward, each row once.""" + + calls: list[str] = [] + real = __import__("alicebot_api.vnext_derived_labels", fromlist=["_apply_dependencies"])._apply_dependencies + + def wrapped(*args: object, **kwargs: object) -> object: + current = args[0] + calls.append(str(getattr(current, "stored_id"))) + return real(*args, **kwargs) + + monkeypatch.setattr("alicebot_api.vnext_derived_labels._apply_dependencies", wrapped) + for size in (2, 5, 40): + calls.clear() + source = _source("s", domain="health", sensitivity="confidential", scope=[ALPHA]) + rows = [source] + leaf = _brief("n0", sources=["s"], scope=[ALPHA]) + rows.append(leaf) + previous = "n0" + for index in range(1, size): + rows.append(_brief(f"n{index}", artifacts=[previous], scope=[ALPHA])) + previous = f"n{index}" + settled = settle_labels(rows) + derived_ids = [row.stored_id for row in settled.derived_rows()] + assert calls == derived_ids + for row in settled.derived_rows(): + assert row.domain == "health" + assert row.sensitivity == "confidential" + assert row.unverified is False + + +def test_a_diamond_settles_once(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[str] = [] + module = __import__("alicebot_api.vnext_derived_labels", fromlist=["_apply_dependencies"]) + real = module._apply_dependencies + + def wrapped(*args: object, **kwargs: object) -> object: + calls.append(str(getattr(args[0], "stored_id"))) + return real(*args, **kwargs) + + monkeypatch.setattr(module, "_apply_dependencies", wrapped) + rows = [ + _source("s", domain="legal", sensitivity="private", scope=[ALPHA]), + _brief("a", sources=["s"], scope=[ALPHA]), + _brief("b", sources=["s"], scope=[ALPHA]), + _brief("c", artifacts=["a", "b"], scope=[ALPHA]), + ] + settled = settle_labels(rows) + assert calls.count("c") == 1 + assert settled.by_stored("artifact", "c").domain == "legal" + + +def test_a_cycle_that_does_not_settle_is_refused() -> None: + ring = [] + names = ["r0", "r1", "r2"] + for index, name in enumerate(names): + ring.append( + _brief( + name, + artifacts=[names[(index + 1) % 3]], + domain="health" if index % 2 == 0 else "legal", + ) + ) + with pytest.raises(DerivedDomainRepairError, match="did not settle"): + settle_labels(ring) + + +def test_unverified_is_contagious_through_every_level() -> None: + rows = [ + _brief("b", artifacts=["missing-artifact"]), + _brief("a", artifacts=["b"]), + _brief("top", artifacts=["a"]), + ] + settled = settle_labels(rows) + assert settled.by_stored("artifact", "b").reason == "missing_dependency" + assert settled.by_stored("artifact", "a").reason == "dependency_unverified" + assert settled.by_stored("artifact", "top").reason == "dependency_unverified" + + +def test_every_unverified_case_is_unverified() -> None: + belief_id = "b1" + memory_id = "m1" + cases = { + "missing id": ( + [_brief("r", memories=["does-not-exist"])], + "r", + "missing_dependency", + ), + "kind with no table": ( + [_brief("r", artifacts=["ghost"])], + "r", + "missing_table", + ), + "derived marker with no record": ( + [_row("artifact", "r", artifact_type="daily_brief", metadata_json={"workflow": "daily_brief"})], + "r", + "no_record", + ), + "counts disagree": ( + [ + _row( + "artifact", + "r", + artifact_type="daily_brief", + metadata_json={ + "workflow": "daily_brief", + "derived_from": { + "v": 1, + "sources": ["s"], + "memories": [], + "open_loops": [], + "artifacts": [], + "beliefs": [], + "counts": {"sources": 2, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, + }, + }, + ) + ], + "r", + "counts_disagree", + ), + "malformed list": ( + [ + _row( + "artifact", + "r", + artifact_type="daily_brief", + metadata_json={"workflow": "daily_brief", "derived_from": {"sources": [1]}}, + ) + ], + "r", + "malformed", + ), + "malformed floor": ( + [_meta(_brief("r"), **{"project_floor": "alpha"})], + "r", + "malformed_floor", + ), + "legacy counts": ( + [ + _row( + "artifact", + "r", + artifact_type="daily_brief", + metadata_json={ + "workflow": "daily_brief", + "input_summary": { + "source_ids": ["s"], + "memory_ids": [], + "open_loop_ids": [], + "artifact_ids": [], + "counts": {"sources": 3, "memories": 0, "open_loops": 0, "artifacts": 0}, + }, + }, + ) + ], + "r", + "legacy_counts", + ), + "bound exceeded": ( + [ + _brief("c", artifacts=["b"]), + _brief("b", artifacts=["a"]), + _brief("a", sources=["s"]), + _source("s", domain="health"), + ], + "c", + "bound_exceeded", + ), + "dependency unverified": ( + [_brief("child", memories=["gone"]), _brief("parent", artifacts=["child"])], + "parent", + "dependency_unverified", + ), + } + for name, (rows, row_id, reason) in cases.items(): + kwargs = {"unavailable_kinds": ["artifact"]} if name == "kind with no table" else {} + if name == "bound exceeded": + kwargs = {"max_hops": 1} + settled = settle_labels(rows, **kwargs) + got = settled.by_stored("artifact", row_id) + assert got.unverified is True, name + assert got.reason == reason, name + + empty = _brief("empty") + assert settle_labels([empty]).by_stored("artifact", "empty").unverified is False + + redacted = _memory("red", metadata_json={"redacted": True, "source_id": "s", "workflow": "daily_brief"}) + assert is_derived("memory", redacted) is False + assert settle_labels([redacted]).by_stored("memory", "red").unverified is False + + project = _row("project", "p", metadata_json={}, current_state="notes") + assert is_derived("project", project) is False + assert row_class("project", project) == "original" + + scrubbed = _source("s", domain="health", sensitivity="confidential", scope=[ALPHA], scrubbed=True) + copy = _memory( + "copy", + domain="unknown", + sensitivity="public", + metadata_json={"source_id": "s", "project_scope": [ALPHA]}, + ) + kept = settle_labels([scrubbed, copy]).by_stored("memory", "copy") + assert kept.unverified is False + assert kept.project_scope == (ALPHA,) + assert kept.project_floor == () + assert kept.domain == "health" + assert kept.sensitivity == "confidential" + + repeated = _row( + "artifact", + "rep", + artifact_type="connection_report", + metadata_json={ + "workflow": "connection_finder", + "source_ids": ["s", "s"], + "memory_ids": [], + "input_counts": {"sources": 1, "memories": 0}, + }, + ) + assert settle_labels([_source("s"), repeated]).by_stored("artifact", "rep").reason != "legacy_counts" + + belief = _row("belief", belief_id, memory_id=memory_id) + backing = _memory(memory_id, domain="health", sensitivity="private") + report = _row( + "artifact", + "belief-report", + artifact_type="contradiction_report", + metadata_json={ + "workflow": "contradiction_finder", + "source_ids": [], + "memory_ids": [], + "belief_ids": [belief_id], + "input_counts": {"sources": 0, "memories": 0, "beliefs": 1}, + }, + ) + followed = settle_labels([belief, backing, report]).by_stored("artifact", "belief-report") + assert followed.unverified is False + assert followed.domain == "health" + assert followed.sensitivity == "private" + + +def test_sensitivity_is_raise_only_and_unknown_never_swaps_with_internal() -> None: + source = _source("s", domain="unknown", sensitivity="confidential") + raised = _memory("m", sensitivity="public", metadata_json={"source_id": "s"}) + assert settle_labels([source, raised]).by_stored("memory", "m").sensitivity == "confidential" + + high = _source("s2", domain="unknown", sensitivity="public") + kept = _memory("m2", sensitivity="confidential", metadata_json={"source_id": "s2"}) + assert settle_labels([high, kept]).by_stored("memory", "m2").sensitivity == "confidential" + + internal = _source("s3", sensitivity="internal") + unknown = _memory("m3", sensitivity="unknown", metadata_json={"source_id": "s3"}) + assert settle_labels([internal, unknown]).by_stored("memory", "m3").sensitivity == "unknown" + + unknown_source = _source("s4", sensitivity="unknown") + internal_copy = _memory("m4", sensitivity="internal", metadata_json={"source_id": "s4"}) + assert settle_labels([unknown_source, internal_copy]).by_stored("memory", "m4").sensitivity == "internal" + + +def test_domain_follows_derived_domain_and_never_becomes_unrestricted() -> None: + rows = [ + _source("h1", domain="health"), + _source("h2", domain="health"), + _source("l", domain="legal"), + _brief("r", sources=["h1", "h2", "l"]), + ] + assert settle_labels(rows).by_stored("artifact", "r").domain == "health" + tie = [ + _source("h", domain="health"), + _source("l2", domain="legal"), + _brief("tie", sources=["h", "l2"]), + ] + assert settle_labels(tie).by_stored("artifact", "tie").domain == "health" + kept = [ + _source("open", domain="project"), + _brief("kept", sources=["open"], domain="health"), + ] + assert settle_labels(kept).by_stored("artifact", "kept").domain == "health" + moved = [ + _source("fin", domain="financial"), + _source("fin2", domain="financial"), + _brief("moved", sources=["fin", "fin2"], domain="health"), + ] + assert settle_labels(moved).by_stored("artifact", "moved").domain == "financial" + assert generation_domain("health", ["financial"]) == "health" + assert generation_domain("unknown", ["financial", "financial", "legal"]) == "financial" + + +def test_floor_is_the_union_and_never_shrinks() -> None: + rows = [ + _source("a", scope=[ALPHA]), + _source("b", scope=[BETA]), + _meta(_brief("r", sources=["a", "b"], scope=[ALPHA]), **{"project_floor": [ALPHA]}), + ] + floor = settle_labels(rows).by_stored("artifact", "r").project_floor + assert set(floor) == {ALPHA, BETA} + + shrink = [ + _source("only", scope=[ALPHA]), + _meta(_brief("kept", sources=["only"], scope=[ALPHA]), **{"project_floor": [ALPHA, BETA]}), + ] + kept = settle_labels(shrink).by_stored("artifact", "kept").project_floor + assert set(kept) == {ALPHA, BETA} + + +def test_scope_rules_per_row_class() -> None: + global_memory = _memory("g", metadata_json={}) + alpha_source = _source("a", scope=[ALPHA], domain="health") + report = _brief("report", sources=["a"], memories=["g"], scope=[ALPHA]) + stored = settle_labels([global_memory, alpha_source, report]).by_stored("artifact", "report") + assert stored.project_scope == () + assert stored.project_floor == (ALPHA,) + + both = _brief("both", sources=["a"], scope=[ALPHA, BETA]) + beta = _source("beta", scope=[BETA]) + # a second source so the report is not global + multi = settle_labels([alpha_source, beta, _brief("multi", sources=["a", "beta"], scope=[ALPHA, BETA])]).by_stored( + "artifact", "multi" + ) + assert multi.project_scope == (ALPHA, BETA) + + card = _memory( + "card", + metadata_json={ + "candidate_kind": "memory_rollup", + "project_scope": [ALPHA, BETA], + "consolidation": {"cluster_member_ids": ["m1", "m2"]}, + }, + ) + members = [ + _memory("m1", metadata_json={"project_scope": [ALPHA, BETA]}), + _memory("m2", metadata_json={"project_scope": [ALPHA, BETA]}), + ] + aggregate = settle_labels([*members, card]).by_stored("memory", "card") + assert aggregate.row_class == "aggregate" + assert aggregate.project_scope == () + assert set(aggregate.project_floor) == {ALPHA, BETA} + + single = _memory( + "one", + metadata_json={ + "candidate_kind": "memory_rollup", + "project_scope": [ALPHA], + "project_floor": [ALPHA], + "consolidation": {"cluster_member_ids": ["only"]}, + }, + ) + only = _memory("only", metadata_json={"project_scope": [ALPHA]}) + kept = settle_labels([only, single]).by_stored("memory", "one") + assert kept.project_scope == (ALPHA,) + + copy = _memory("copy", metadata_json={"source_id": "a", "project_scope": [ALPHA, BETA]}) + narrowed = settle_labels([alpha_source, copy]).by_stored("memory", "copy") + assert narrowed.project_scope == (ALPHA,) + + global_source = _source("glob") + emptied = _memory("emptied", metadata_json={"source_id": "glob", "project_scope": [ALPHA]}) + assert settle_labels([global_source, emptied]).by_stored("memory", "emptied").project_scope == () + + promoted = _memory( + "promoted", + metadata_json={"source_artifact_id": "report", "project_scope": [ALPHA]}, + ) + promoted_row = settle_labels([global_memory, alpha_source, report, promoted]).by_stored("memory", "promoted") + assert promoted_row.project_scope == () + assert ALPHA in promoted_row.project_floor + + assert both["id"] == "both" + + +def test_a_forged_marker_in_value_makes_no_row_derived() -> None: + forged = _memory("forged", value={"kind": "promoted_artifact", "artifact_id": "secret", "source_id": "s"}) + assert is_derived("memory", forged) is False + assert dependencies_of("memory", forged) == frozenset() + settled = settle_labels([forged, _source("s", domain="health", sensitivity="sacred")]) + row = settled.by_stored("memory", "forged") + assert row.derived is False + assert row.domain == "unknown" + assert row.sensitivity == "unknown" + + +def test_a_dependency_is_found_in_every_recorded_field_of_every_derived_kind() -> None: + source = "11111111-1111-1111-1111-111111111111" + memory = "22222222-2222-2222-2222-222222222222" + loop = "33333333-3333-3333-3333-333333333333" + artifact = "44444444-4444-4444-4444-444444444444" + samples = [ + _brief("daily", sources=[source], memories=[memory], loops=[loop], artifacts=[artifact]), + _row( + "artifact", + "connection", + artifact_type="connection_report", + metadata_json={ + "workflow": "connection_finder", + "source_ids": [source], + "memory_ids": [memory], + "source_refs": [f"source:{source}"], + "input_counts": {"sources": 1, "memories": 1}, + }, + ), + _row( + "artifact", + "contradiction", + artifact_type="contradiction_report", + metadata_json={ + "workflow": "contradiction_finder", + "source_ids": [source], + "memory_ids": [memory], + "belief_ids": ["belief-1"], + "source_refs": [f"memory:{memory}"], + "input_counts": {"sources": 1, "memories": 1, "beliefs": 1}, + }, + ), + _row( + "artifact", + "consolidation", + artifact_type="memory_consolidation", + metadata_json={ + "workflow": "memory_consolidation", + "consolidation": {"cluster_member_ids": [memory], "member_snapshots": [{"id": memory}]}, + "source_refs": [f"source:{source}"], + }, + ), + _row( + "artifact", + "project", + artifact_type="project_update", + metadata_json={"workflow": "project_auto_update", "source_ids": [source], "memory_ids": [memory]}, + ), + _row( + "artifact", + "stale", + metadata_json={"workflow": "staleness_sweep", "stale_marked_memory_ids": [memory]}, + ), + _row( + "artifact", + "loops", + metadata_json={"workflow": "open_loop_review", "open_loop_ids": [loop], "source_refs": [f"source:{source}"]}, + ), + _row( + "artifact", + "agent", + metadata_json={"connector_name": "agent_output", "source_id": source, "source_refs": [f"source:{source}"]}, + ), + _memory( + "weekly", + metadata_json={"discovered_by": "vnext_weekly_synthesis", "input_summary": {"memory_ids": [memory], "source_ids": [], "open_loop_ids": [], "artifact_ids": [], "counts": {"memories": 1, "sources": 0, "open_loops": 0, "artifacts": 0}}}, + ), + _memory( + "rollup", + metadata_json={"candidate_kind": "memory_rollup", "consolidation": {"cluster_member_ids": [memory]}}, + value={"rollup": {"member_ids": [memory]}}, + ), + _memory("promoted", metadata_json={"source_artifact_id": artifact}, value={"artifact_id": artifact}), + _memory("extracted", metadata_json={"source_id": source, "extraction_rule": "sentence"}), + _row("open_loop", "found", metadata_json={"discovered_by": "vnext_daily_brief", "source_id": source}, source_id=source), + _row( + "project", + "state", + current_state="line", + metadata_json={"derived_from": {"memories": [memory], "artifacts": [artifact], "sources": [], "open_loops": [], "beliefs": [], "counts": {"memories": 1, "artifacts": 1, "sources": 0, "open_loops": 0, "beliefs": 0}}}, + ), + ] + expected = { + "daily": {("source", source), ("memory", memory), ("open_loop", loop), ("artifact", artifact)}, + "connection": {("source", source), ("memory", memory)}, + "contradiction": {("source", source), ("memory", memory), ("belief", "belief-1")}, + "consolidation": {("memory", memory), ("source", source)}, + "project": {("source", source), ("memory", memory)}, + "stale": {("memory", memory)}, + "loops": {("open_loop", loop), ("source", source)}, + "agent": {("source", source)}, + "weekly": {("memory", memory)}, + "rollup": {("memory", memory)}, + "promoted": {("artifact", artifact)}, + "extracted": {("source", source)}, + "found": {("source", source)}, + "state": {("memory", memory), ("artifact", artifact)}, + } + for sample in samples: + assert is_derived(sample["kind"], sample) is True + assert set(dependencies_of(sample["kind"], sample)) == expected[str(sample["id"])] + + +def test_a_ref_is_read_in_every_spelling_and_a_belief_resolves_through_its_memory() -> None: + canonical = str(UUID(SOURCE_UUID)) + spellings = [ + SOURCE_UUID.upper(), + SOURCE_UUID.replace("-", ""), + "{" + SOURCE_UUID + "}", + "urn:uuid:" + SOURCE_UUID, + "source:" + SOURCE_UUID.upper(), + ] + for spelling in spellings: + row = _memory("m", metadata_json={"source_id": spelling}) + assert dependencies_of("memory", row) == frozenset({("source", canonical)}) + refs = _brief("r", sources=[]) + _meta(refs, **{"source_refs": ["source:" + SOURCE_UUID.upper(), "memory:" + SOURCE_UUID]}) + found = dependencies_of("artifact", refs) + assert ("source", canonical) in found + assert ("memory", canonical) in found + + +def test_dependency_keys_cover_every_producer() -> None: + """Every id key a producer writes as a dict key is classified. + + A new key fails this test until it is either read as a dependency or named + in ``_NOT_A_DEPENDENCY``. The v2 keys are a subset of the keys v3 reads. + """ + + assert set(_ID_KEYS) <= V2_ID_KEYS | {"source_refs"} + root = Path(__file__).resolve().parents[2] / "apps" / "api" / "src" / "alicebot_api" + producers = [ + "vnext_brain.py", + "vnext_connections.py", + "vnext_contradictions.py", + "vnext_consolidation.py", + "vnext_rollups.py", + "vnext_projects.py", + "vnext_scheduler.py", + "vnext_queue.py", + "vnext_connectors.py", + "vnext_capture.py", + ] + found: set[str] = set() + for name in producers: + tree = ast.parse((root / name).read_text(encoding="utf-8")) + for node in ast.walk(tree): + if not isinstance(node, ast.Dict): + continue + for key in node.keys: + if isinstance(key, ast.Constant) and isinstance(key.value, str): + text = key.value + if text.endswith("_id") or text.endswith("_ids") or text in {"source_refs", "source_ref"}: + found.add(text) + unknown = sorted(found - _DEPENDENCY_KEYS - _NOT_A_DEPENDENCY) + assert unknown == [] + + +_DEPENDENCY_KEYS = frozenset( + { + "artifact_id", + "artifact_ids", + "belief_ids", + "cluster_member_ids", + "cluster_membership", + "member_ids", + "memory_id", + "memory_ids", + "open_loop_ids", + "source_artifact_id", + "source_id", + "source_ids", + "source_refs", + "stale_marked_memory_ids", + "candidate_memory_ids", + } +) +_NOT_A_DEPENDENCY = frozenset( + { + "actor_id", + "agent_id", + "agent_run_id", + "allowed_chat_ids", + "belief_id", + "belief_memory_id", + "candidate_edge_ids", + "candidate_memory_id", + "candidate_open_loop_ids", + "chat_id", + "connector_id", + "conversation_id", + "created_by_agent_id", + "external_chat_id", + "external_id", + "failed_external_ids", + "from_id", + "last_failed_external_ids", + "last_run_id", + "last_source_ids", + "message_id", + "output_artifact_id", + "person_id", + "project_id", + "project_ids", + "promoted_memory_id", + "provider_message_id", + "provider_update_id", + "review_artifact_id", + "revises_memory_id", + "rollup_candidate_ids", + "run_id", + "scheduler_run_id", + "sender_id", + "source_chunk_id", + "source_event_ids", + "source_ref", + "survivor_memory_id", + "target_id", + "task_id", + "to_id", + "trace_id", + "workflow_id", + } +) + + +def test_the_seven_sensitivity_rank_tables_equal_the_kernel_table() -> None: + root = Path(__file__).resolve().parents[2] / "apps" / "api" / "src" / "alicebot_api" + files = [ + "vnext_brain.py", + "vnext_consolidation.py", + "vnext_connections.py", + "vnext_contradictions.py", + "vnext_rollups.py", + "vnext_scheduler.py", + "vnext_projects.py", + ] + found = 0 + for name in files: + tree = ast.parse((root / name).read_text(encoding="utf-8")) + for node in ast.walk(tree): + if not isinstance(node, ast.Dict): + continue + keys = [key.value for key in node.keys if isinstance(key, ast.Constant)] + if "highly_sensitive" not in keys or "regulated" not in keys: + continue + table = ast.literal_eval(node) + assert table == SENSITIVITY_RANK + found += 1 + assert found == 7 + assert brain.SENSITIVITY_RANK == SENSITIVITY_RANK + assert consolidation.SENSITIVITY_RANK == SENSITIVITY_RANK + + +def test_labels_raised_events_carry_no_text_or_ids() -> None: + payload = labels_raised_payload( + cause="input_relabelled", + previous={"domain": "unknown", "sensitivity": "public", "project_scope": [ALPHA], "project_floor": []}, + new={"domain": "health", "sensitivity": "confidential", "project_scope": [ALPHA], "project_floor": [BETA]}, + ) + assert set(payload) == {"cause", "previous", "new"} + for side in (payload["previous"], payload["new"]): + assert set(side) == {"domain", "sensitivity", "project_scope", "project_floor"} + blob = str(payload) + assert "canonical" not in blob + assert "title" not in blob + + +def test_project_floor_blocks_a_locked_key_and_does_not_change_an_empty_floor() -> None: + identity = AgentIdentity( + agent_id="reader", + permission_profile="read_only_agent", + project_scope=(ALPHA,), + project_scope_locked=True, + auth="api_key", + ) + blocked = evaluate_agent_policy( + identity=identity, + action="artifact.get", + domains=("unknown",), + sensitivity_allowed=("public",), + project_scope=(ALPHA,), + project_floor=(BETA,), + require_explicit_project_scope=True, + ) + assert blocked.decision == "blocked" + assert "project_floor_binding_violation" in blocked.reasons + allowed = evaluate_agent_policy( + identity=identity, + action="artifact.get", + domains=("unknown",), + sensitivity_allowed=("public",), + project_scope=(ALPHA,), + project_floor=(ALPHA,), + require_explicit_project_scope=True, + ) + assert allowed.decision == "allowed" + assert project_floor_within((), (ALPHA,)) is True + untouched = evaluate_agent_policy( + identity=identity, + action="artifact.get", + domains=("unknown",), + sensitivity_allowed=("public",), + project_scope=(ALPHA,), + require_explicit_project_scope=True, + ) + assert untouched.decision == "allowed" + + +def test_a_global_row_with_a_floor_is_hidden_from_the_other_project() -> None: + assert project_scopes_overlap((), (GLOBAL_PROJECT_MARKER, ALPHA), floor=(BETA,)) is False + assert project_scopes_overlap((), (GLOBAL_PROJECT_MARKER, BETA), floor=(BETA,)) is True + assert project_scopes_overlap((), (GLOBAL_PROJECT_MARKER, ALPHA), floor=("Alice",)) is True + assert project_scopes_overlap((), (GLOBAL_PROJECT_MARKER,)) is True + assert project_scopes_overlap((ALPHA,), (ALPHA,)) is True + original = _memory("plain", metadata_json={"project_scope": [ALPHA]}) + derived = _memory( + "derived", + metadata_json={"source_id": "s", "project_scope": [ALPHA], "project_floor": [BETA]}, + ) + assert group_scope(original, kind="memory") == group_scope( + _memory("same", metadata_json={"project_scope": [ALPHA]}), kind="memory" + ) + assert BETA.lower() in group_scope(derived, kind="memory") or "prj_" + "b" * 16 in group_scope(derived, kind="memory") + + +def test_v2_id_keys_are_the_previous_set() -> None: + assert V2_ID_KEYS == frozenset(_ID_KEYS) + + +def test_rank_table_matches_the_spec_order() -> None: + assert SENSITIVITY_RANK["public"] < SENSITIVITY_RANK["unknown"] == SENSITIVITY_RANK["internal"] + assert SENSITIVITY_RANK["sacred"] == SENSITIVITY_RANK["regulated"] + assert "health" in RESTRICTED_DOMAINS diff --git a/tests/unit/test_derived_labels_mutations.py b/tests/unit/test_derived_labels_mutations.py new file mode 100644 index 000000000..68cd1e888 --- /dev/null +++ b/tests/unit/test_derived_labels_mutations.py @@ -0,0 +1,161 @@ +"""Kill each pure derived-label guard in memory. A survivor fails the run.""" + +from __future__ import annotations + +import inspect +import sys +import textwrap +from types import FunctionType + +import pytest + +from alicebot_api import vnext_agent_control as agent_control +from alicebot_api import vnext_brain as brain +from alicebot_api import vnext_derived_labels as labels +from tests.unit import test_derived_labels_kernel as checks + + +def kill(owner, name, before, after, check): + original = getattr(owner, name) + source = textwrap.dedent(inspect.getsource(original)) + if before not in source and before.replace("'", '"') in source: + before, after = before.replace("'", '"'), after.replace("'", '"') + assert before in source, f"mutation no longer matches: {name}: {before}" + namespace = dict(original.__globals__) + exec(compile(source.replace(before, after, 1), "", "exec"), namespace) + compiled = namespace[name] + replacement = FunctionType( + compiled.__code__, original.__globals__, name, compiled.__defaults__, compiled.__closure__ + ) + replacement.__kwdefaults__ = compiled.__kwdefaults__ + aliases = [ + (module, key) + for module in list(sys.modules.values()) + if module + and ( + getattr(module, "__name__", "").startswith("alicebot_api") + or module in (checks,) + ) + for key, value in list(vars(module).items()) + if value is original + ] + for module, key in aliases: + setattr(module, key, replacement) + setattr(owner, name, replacement) + try: + try: + check() + except (AssertionError, pytest.fail.Exception): + print(f"KILLED {name}: {before}") + else: + raise RuntimeError(f"SURVIVED {name}: {before}") + finally: + setattr(owner, name, original) + for module, key in aliases: + setattr(module, key, original) + + +def test_derived_label_kernel_guard_mutations(): + kill( + labels, + "_raised_sensitivity", + "if rank > chosen_rank:", + "if rank >= chosen_rank:", + checks.test_sensitivity_is_raise_only_and_unknown_never_swaps_with_internal, + ) + kill( + labels, + "union_floor", + "combined = [*stored, *[item for part in parts for item in part]]", + "combined = list(stored)", + checks.test_floor_is_the_union_and_never_shrinks, + ) + kill( + labels, + "_effective_scope", + "return _copy_scope(current.stored_scope, parents)", + "return current.stored_scope", + checks.test_scope_rules_per_row_class, + ) + kill( + labels, + "_effective_scope", + "return () # aggregate rule otherwise leaves the scope empty", + "return stored", + checks.test_scope_rules_per_row_class, + ) + kill( + labels, + "settle_labels", + 'problems[key] = "missing_dependency"', + "pass", + checks.test_every_unverified_case_is_unverified, + ) + kill( + labels, + "_derived_from_deps", + 'if expected != len(ids):\n return "counts_disagree", found', + 'if expected != len(ids):\n return "", found', + checks.test_every_unverified_case_is_unverified, + ) + kill( + labels, + "_spread_unverified", + 'problems[key] = "dependency_unverified"\n changed = True', + "pass", + checks.test_unverified_is_contagious_through_every_level, + ) + kill( + labels, + "dependency_record", + "if not problem and not _record_present(meta, row) and not found:", + "if not problem and not found:", + checks.test_every_unverified_case_is_unverified, + ) + kill( + agent_control, + "evaluate_agent_policy", + "elif project_floor and not project_floor_within(project_floor, identity.project_scope):", + "elif False and not project_floor_within(project_floor, identity.project_scope):", + checks.test_project_floor_blocks_a_locked_key_and_does_not_change_an_empty_floor, + ) + kill( + labels, + "_effective_scope", + "return () # a global input empties the report scope", + "return current.stored_scope", + checks.test_scope_rules_per_row_class, + ) + kill( + labels, + "carries_scope", + 'if name == "source" and _scrubbed(row):', + "if False and _scrubbed(row):", + checks.test_every_unverified_case_is_unverified, + ) + kill( + labels, + "_legacy_count_problem", + 'return "legacy_counts"', + 'return ""', + checks.test_every_unverified_case_is_unverified, + ) + kill( + labels, + "labels_raised_payload", + 'return {"cause": cause, "previous": side(previous), "new": side(new)}', + 'return {"cause": cause, "previous": side(previous), "new": side(new), "input_id": "row-1"}', + checks.test_labels_raised_events_carry_no_text_or_ids, + ) + original = dict(brain.SENSITIVITY_RANK) + brain.SENSITIVITY_RANK["sacred"] = 7 + try: + try: + checks.test_the_seven_sensitivity_rank_tables_equal_the_kernel_table() + except (AssertionError, pytest.fail.Exception): + print("KILLED SENSITIVITY_RANK sacred") + else: + raise RuntimeError("SURVIVED brain sensitivity rank") + finally: + brain.SENSITIVITY_RANK.clear() + brain.SENSITIVITY_RANK.update(original) From a30f5e613480354b9647f97dc1369c6abb343a49 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 17:35:17 +0000 Subject: [PATCH 002/270] Floor derived memory inserts and keep label keys on update. A memory copied from another row is stored at least as strict as that row. A metadata write keeps the marker and the stored project scope and floor. --- CHANGELOG.md | 1 + apps/api/src/alicebot_api/sqlite_store.py | 31 +++ .../src/alicebot_api/vnext_label_writes.py | 228 ++++++++++++++++++ apps/api/src/alicebot_api/vnext_store.py | 47 ++++ .../vnext_stores/postgres/memory_lifecycle.py | 27 ++- .../vnext_stores/sqlite/memory_lifecycle.py | 37 ++- tests/unit/test_derived_domain_mutations.py | 5 +- tests/unit/test_derived_domain_review.py | 17 +- tests/unit/test_derived_domain_stored_ids.py | 7 +- .../test_sqlite_derived_labels_write_path.py | 121 ++++++++++ 10 files changed, 506 insertions(+), 15 deletions(-) create mode 100644 apps/api/src/alicebot_api/vnext_label_writes.py create mode 100644 tests/unit/test_sqlite_derived_labels_write_path.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 510fd2dde..c094b68c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. No migration is required. Reports, open loops, and a relabel of an existing row are not covered yet. - Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. Nothing in the running product calls the function yet, so a read and a write still behave as they do in v0.20.0. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` now scrub an open loop that names the source only in its metadata: the id, or `source:`, as the text under `source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references` or `selected_source_ids` at any depth. That is the rule the open-loop lookup of a source already used. They blanked only loops whose `source_id` column held the id, so a loop with an empty column kept its title, description and metadata, and they reached a new export. The delete preview now counts the same loops as the receipt. The rule reads the id only as stored or as `source:`: a loop that names the source in any spelling other than those two keeps its text, although the memory rule reads several other spellings. No migration is required. - Unreleased (on main, not in v0.20.0): the SQLite derived-label repair now updates and records each row under the id it is stored with (SQLite keeps capitals, missing hyphens, braces and `urn:uuid:` as written) and matches rows by the normalised id only to read the graph. If two stored spellings of one id exist, each is compared with the label its own recorded inputs give it, so neither is left at its old label while the other is relabelled, and neither is lowered. An update that changes no row stops the repair with `DerivedDomainRepairError` before any event or the completion stamp is written, on open and on restore; migration `20261004_0095` refuses the same way. Before, a derived memory stored under such an id kept its label while its relabel event and the stamp were written, and the pair survived export and import. A PostgreSQL uuid column was not affected. A vault that the earlier repair already stamped keeps such a row at its old label until a restore repairs it again. No migration is required. diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index af90950f8..3fe8eaec1 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -399,6 +399,37 @@ def __init__(self, conn: sqlite3.Connection, user_id: UUID | str): _ensure_embedding_content_sha256_sqlite(self.conn) _ensure_project_scope_identity_sqlite(self.conn) + def lock_label_writes(self, *, exclusive: bool = False) -> None: + """The SQLite writer lock is the label lock. Begin it when none is open.""" + + del exclusive + if not self.conn.in_transaction: + self.conn.execute("BEGIN IMMEDIATE") + + def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: + """Narrow label rows for the insert floor. No text columns.""" + + wanted = [str(item) for item in ids if str(item)] + if not wanted: + return [] + table = {"source": "sources", "memory": "memories", "open_loop": "open_loops"}.get(kind) + if table is None: + return [] + extra = "" + if table == "memories": + extra = ", value, project_id" + elif table == "open_loops": + extra = ", project_id, source_id, memory_id" + marks = ",".join("?" for _ in wanted) + return self._fetch_all( + f""" + SELECT id, user_id, domain, sensitivity, metadata_json{extra} + FROM {table} + WHERE user_id = ? AND id IN ({marks}) + """, + (self.user_id, *wanted), + ) + # -- fetch helpers (mirror PostgresVNextStore conventions) ------------ def _execute(self, query: str, params: tuple[object, ...] = ()) -> sqlite3.Cursor: diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py new file mode 100644 index 000000000..faee03beb --- /dev/null +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -0,0 +1,228 @@ +"""Write-path label lock, insert floor, and metadata merge. + +The pure rules live in ``vnext_derived_labels``. This module is what a store +calls when it writes a row. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from functools import wraps +from typing import Any + +from alicebot_api.vnext_agent_control import RESTRICTED_DOMAINS +from alicebot_api.vnext_derived_labels import ( + MARKER_KEYS, + dependencies_of, + generation_domain, + is_derived, + labels_raised_payload, + settle_labels, +) +from alicebot_api.vnext_event_log import build_event_log_record, integrity_hash_for_event +from alicebot_api.vnext_project_scope import project_scope_identity +from alicebot_api.vnext_repositories import JsonObject + +LABEL_METADATA_KEYS = ("project_scope", "project_floor", "derived_from") +STRICT_LOCK_ORDER = False +_FLOOR_ENABLED = True + +_KIND_EVENT = { + "memory": "memory", + "open_loop": "open_loop", + "artifact": "artifact", + "project": "project", + "source": "source", +} + + +class LabelLockOrderError(RuntimeError): + """A label write ran outside a transaction, or it took locks in the wrong order.""" + + +def takes_label_lock(fn: Any) -> Any: + """Take the shared label lock before a store method writes or row-locks a label table.""" + + @wraps(fn) + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: + lock = getattr(self, "lock_label_writes", None) + if callable(lock): + lock(exclusive=False) + return fn(self, *args, **kwargs) + + wrapper.__takes_label_lock__ = True # type: ignore[attr-defined] + return wrapper + + +@contextmanager +def without_insert_floor() -> Iterator[None]: + """Let a test store a derived row with the labels it wrote.""" + + global _FLOOR_ENABLED + previous = _FLOOR_ENABLED + _FLOOR_ENABLED = False + try: + yield + finally: + _FLOOR_ENABLED = previous + + +def merge_protected_metadata( + stored: Mapping[str, object] | None, + patch: Mapping[str, object] | None, + *, + label_write: bool, +) -> dict[str, object]: + """Keep label keys and omitted marker keys when a caller writes metadata back. + + ``label_write`` is for the relabel itself, which may change the label keys. + A marker key that the patch omits stays as stored either way. + """ + + stored_map = dict(stored) if isinstance(stored, Mapping) else {} + if patch is None: + return stored_map + patch_map = dict(patch) + result = dict(patch_map) + if not label_write: + for key in LABEL_METADATA_KEYS: + if key in stored_map: + result[key] = stored_map[key] + else: + result.pop(key, None) + for key in MARKER_KEYS: + if key not in patch_map and key in stored_map: + result[key] = stored_map[key] + return result + + +def _in_transaction(store: Any) -> bool: + conn = getattr(store, "conn", None) + if conn is None: + return True + info = getattr(conn, "info", None) + status = getattr(info, "transaction_status", None) + if status is not None: + # psycopg TransactionStatus.IDLE is 0. + return int(status) != 0 + in_transaction = getattr(conn, "in_transaction", None) + if isinstance(in_transaction, bool): + return in_transaction + return True + + +def _label_tuple(payload: Mapping[str, object]) -> tuple[str, str, tuple[str, ...], tuple[str, ...]]: + metadata = payload.get("metadata_json") + meta = metadata if isinstance(metadata, Mapping) else {} + scope = meta.get("project_scope", payload.get("project_scope", ())) + floor = meta.get("project_floor", ()) + scope_values = tuple(scope) if isinstance(scope, (list, tuple)) else () + floor_values = tuple(floor) if isinstance(floor, (list, tuple)) else () + return ( + str(payload.get("domain") or "unknown"), + str(payload.get("sensitivity") or "unknown"), + project_scope_identity(scope_values), + project_scope_identity(floor_values), + ) + + +def apply_insert_floor(store: Any, kind: str, payload: Mapping[str, object]) -> tuple[JsonObject, JsonObject | None]: + """Raise a derived payload to its inputs. Returns ``(payload, event or None)``. + + The event has no target id yet. The insert fills that in after the row exists. + An unverified dependency does not refuse the insert. + """ + + body = dict(payload) + if not _FLOOR_ENABLED or not is_derived(kind, body): + return body, None + if not _in_transaction(store): + raise LabelLockOrderError("the insert floor requires an open transaction") + user_id = str(getattr(store, "user_id", body.get("user_id") or "")) + nodes: list[dict[str, object]] = [] + grouped: dict[str, list[str]] = {} + for dep_kind, dep_id in dependencies_of(kind, body): + grouped.setdefault(dep_kind, []).append(dep_id) + reader = getattr(store, "read_label_rows", None) + if callable(reader): + for dep_kind, ids in grouped.items(): + for row in reader(dep_kind, ids): + copied = dict(row) + copied["kind"] = dep_kind + copied.setdefault("user_id", user_id) + nodes.append(copied) + if not user_id and nodes: + user_id = str(nodes[0].get("user_id") or "") + own_id = str(body.get("id") or "new-derived-row") + own = dict(body) + own["kind"] = kind + own["id"] = own_id + own["user_id"] = user_id + nodes.append(own) + settled = settle_labels(nodes).by_stored(kind, own_id, user_id=user_id or None) + if settled.unverified: + return body, None + domain = settled.domain + if str(body.get("domain") or "unknown") in RESTRICTED_DOMAINS: + domain = generation_domain(str(body.get("domain")), [settled.domain]) + metadata = dict(body.get("metadata_json")) if isinstance(body.get("metadata_json"), Mapping) else {} + metadata["project_scope"] = list(settled.project_scope) + metadata["project_floor"] = list(settled.project_floor) + updated = dict(body) + updated["domain"] = domain + updated["sensitivity"] = settled.sensitivity + updated["metadata_json"] = metadata + if len(project_scope_identity(settled.project_scope)) != 1: + updated["project_id"] = None + before = _label_tuple(body) + after = _label_tuple(updated) + if before == after: + return updated, None + event = build_event_log_record( + event_type=f"{_KIND_EVENT.get(kind, kind)}.labels_raised", + actor_type="system", + target_type=_KIND_EVENT.get(kind, kind), + payload=labels_raised_payload( + cause="insert_floor", + previous={ + "domain": before[0], + "sensitivity": before[1], + "project_scope": list(before[2]), + "project_floor": list(before[3]), + }, + new={ + "domain": after[0], + "sensitivity": after[1], + "project_scope": list(settled.project_scope), + "project_floor": list(settled.project_floor), + }, + ), + ) + return updated, event + + +def remember_floor_event(store: Any, event: JsonObject | None, target_id: object) -> None: + """Append an insert-floor event once the row id is known.""" + + if event is None: + return + event = dict(event) + event["target_id"] = str(target_id) if target_id else None + event.pop("integrity_hash", None) + event["integrity_hash"] = integrity_hash_for_event(event) + append = getattr(store, "append_event", None) + if callable(append): + append(event) + + +__all__ = [ + "LABEL_METADATA_KEYS", + "STRICT_LOCK_ORDER", + "LabelLockOrderError", + "apply_insert_floor", + "merge_protected_metadata", + "remember_floor_event", + "takes_label_lock", + "without_insert_floor", +] diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index ed85ec7a7..92fe7a7c0 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -467,6 +467,53 @@ class PostgresVNextStore: def __init__(self, conn: UserConnection): self.conn = conn + def lock_label_writes(self, *, exclusive: bool = False) -> None: + """Shared label lock for a write, or the exclusive lock for a relabel. + + The lock is transaction scoped. A session that already holds it takes + the same statement again, which Postgres grants without waiting. + """ + + mode = "pg_advisory_xact_lock" if exclusive else "pg_advisory_xact_lock_shared" + with self.conn.cursor() as cur: + cur.execute( + f"SELECT {mode}(hashtext('vnext_labels'), hashtext(app.current_user_id()::text))" + ) + + def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: + """Narrow label rows for the insert floor. No text columns.""" + + wanted = [str(item) for item in ids if str(item)] + if not wanted: + return [] + table = { + "source": "sources", + "memory": "memories", + "open_loop": "open_loops", + "artifact": "generated_artifacts", + "project": "projects", + "belief": "beliefs", + }.get(kind) + if table is None: + return [] + extra = "" + if table == "memories": + extra = ", value, project_id" + elif table == "open_loops": + extra = ", project_id, source_id, memory_id" + elif table == "beliefs": + extra = ", memory_id" + elif table == "generated_artifacts": + extra = ", artifact_type" + return self._fetch_all( + f""" + SELECT id, user_id, domain, sensitivity, metadata_json{extra} + FROM {table} + WHERE id = ANY(%s::uuid[]) + """, + (wanted,), + ) + def _fetch_one( self, operation_name: str, diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py index 288f2798f..cd069e6e4 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py @@ -11,6 +11,12 @@ from alicebot_api.vnext_event_log import build_event_log_record from alicebot_api.vnext_project_scope import canonical_memory_metadata from alicebot_api.vnext_repositories import JsonObject +from alicebot_api.vnext_label_writes import ( + apply_insert_floor, + merge_protected_metadata, + remember_floor_event, + takes_label_lock, +) from alicebot_api.vnext_stores.memory_lifecycle_common import ( REDACTED_JSON_VALUE, REDACTION_MARKER, @@ -34,7 +40,9 @@ VNextRow = dict[str, object] +@takes_label_lock def create_memory(self, memory: JsonObject, *, actor_type: str = "system") -> VNextRow: + memory, floor_event = apply_insert_floor(self, "memory", memory) refuse_created_credential_activation(memory) row = self._fetch_one( "create_memory", @@ -171,6 +179,7 @@ def create_memory(self, memory: JsonObject, *, actor_type: str = "system") -> VN target_id=row["id"], payload={"operation": "create", "fields": _sorted_field_names(memory)}, ) + remember_floor_event(self, floor_event, row["id"]) return row def upsert_memory_by_key(self, memory: JsonObject, *, actor_type: str = "system") -> VNextRow: @@ -415,8 +424,24 @@ def list_memories_missing_fact_keys(self, *, limit: int = 100, after_id: str | N (after_id, after_id, limit), ) -def update_memory(self, *, memory_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: +@takes_label_lock +def update_memory( + self, *, memory_id: str, patch: JsonObject, actor_type: str = "system", label_write: bool = False +) -> VNextRow: refuse_updated_credential_activation(patch, lambda: self.get_memory(str(memory_id))) + if "metadata_json" in patch and isinstance(patch.get("metadata_json"), dict): + current = self._fetch_optional_one( + "SELECT metadata_json FROM memories WHERE id = %s::uuid", + (memory_id,), + ) + if current is not None: + stored = current.get("metadata_json") + patch = dict(patch) + patch["metadata_json"] = merge_protected_metadata( + stored if isinstance(stored, dict) else {}, + patch["metadata_json"], + label_write=label_write, + ) row = self._fetch_one( "update_memory", f""" diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py index 8c49b35a7..88080782c 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py @@ -11,6 +11,12 @@ from alicebot_api.vnext_event_log import build_event_log_record from alicebot_api.vnext_project_scope import canonical_memory_metadata from alicebot_api.vnext_repositories import JsonObject +from alicebot_api.vnext_label_writes import ( + apply_insert_floor, + merge_protected_metadata, + remember_floor_event, + takes_label_lock, +) from alicebot_api.vnext_stores.memory_lifecycle_common import ( REDACTED_JSON_VALUE, REDACTION_MARKER, @@ -34,7 +40,31 @@ VNextRow = dict[str, object] + +def _with_protected_metadata(self, memory_id: str, patch: JsonObject, *, label_write: bool) -> JsonObject: + """Re-read metadata inside the label lock and keep label and marker keys.""" + + if "metadata_json" not in patch or not isinstance(patch.get("metadata_json"), dict): + return patch + current = self._fetch_optional_one( + "SELECT metadata_json FROM memories WHERE id = ? AND user_id = ?", + (str(memory_id), self.user_id), + ) + if current is None: + return patch + stored = current.get("metadata_json") + merged = dict(patch) + merged["metadata_json"] = merge_protected_metadata( + stored if isinstance(stored, dict) else {}, + patch["metadata_json"], + label_write=label_write, + ) + return merged + + +@takes_label_lock def create_memory(self, memory: JsonObject, *, actor_type: str = "system") -> VNextRow: + memory, floor_event = apply_insert_floor(self, "memory", memory) refuse_created_credential_activation(memory) memory_id = _new_id(memory.get("id")) # One clock reading for the whole write. ``first_seen_at`` and ``last_seen_at`` default to it, so @@ -140,6 +170,7 @@ def create_memory(self, memory: JsonObject, *, actor_type: str = "system") -> VN target_id=row["id"], payload={"operation": "create", "fields": _sorted_field_names(memory)}, ) + remember_floor_event(self, floor_event, row["id"]) return row def upsert_memory_by_key(self, memory: JsonObject, *, actor_type: str = "system") -> VNextRow: @@ -263,8 +294,12 @@ def memory_redaction_bundle_is_exact(self, memory_id: str, artifact_ids: Sequenc ) return bool(row.get("exact")) -def update_memory(self, *, memory_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: +@takes_label_lock +def update_memory( + self, *, memory_id: str, patch: JsonObject, actor_type: str = "system", label_write: bool = False +) -> VNextRow: refuse_updated_credential_activation(patch, lambda: self.get_memory(str(memory_id))) + patch = _with_protected_metadata(self, memory_id, patch, label_write=label_write) # One clock reading for the write: an archive sets ``updated_at`` and ``deleted_at`` together. now = _utc_now_iso() cursor = self._execute( diff --git a/tests/unit/test_derived_domain_mutations.py b/tests/unit/test_derived_domain_mutations.py index 18ea917bf..aa28d4819 100644 --- a/tests/unit/test_derived_domain_mutations.py +++ b/tests/unit/test_derived_domain_mutations.py @@ -22,6 +22,7 @@ from tests.unit import test_derived_domain_review as review from tests.unit import test_derived_domain_stored_ids as stored_ids from alicebot_api import onramp, sqlite_schema +from alicebot_api.vnext_label_writes import without_insert_floor def kill(owner, name, before, after, check): @@ -177,12 +178,12 @@ def test_derived_domain_guard_mutations(): from alicebot_api import vnext_rollups as rollups for module, check in [(rollups, checks.test_real_sqlite_rollup_keeps_restricted_input_domain), (consolidation, checks.test_consolidation_report_includes_rollup_input_domains)]: - with pytest.MonkeyPatch.context() as patch: + with pytest.MonkeyPatch.context() as patch, without_insert_floor(): patch.setattr(module, 'derived_domain', lambda rows, *, fallback: fallback) with pytest.raises(AssertionError): check(patch) print('KILLED producer selector:', module.__name__) - with tempfile.TemporaryDirectory() as directory, pytest.MonkeyPatch.context() as patch: + with tempfile.TemporaryDirectory() as directory, pytest.MonkeyPatch.context() as patch, without_insert_floor(): patch.setattr(sqlite_schema, '_relabel_derived_domains', lambda conn: None) with pytest.raises(AssertionError): checks.test_sqlite_upgrade_relabels_existing_derived_memory_only(Path(directory)) diff --git a/tests/unit/test_derived_domain_review.py b/tests/unit/test_derived_domain_review.py index 6c69fdaca..b35da191c 100644 --- a/tests/unit/test_derived_domain_review.py +++ b/tests/unit/test_derived_domain_review.py @@ -9,6 +9,7 @@ from alicebot_api.onramp import bootstrap_database, main as onramp_main from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_label_writes import without_insert_floor from alicebot_api.vnext_derived_domain_backfill import plan_relabels from tests.unit.per_project_s2_support import add_memory from tests.unit.test_derived_domain_fence import USER @@ -25,7 +26,7 @@ def test_restore_repairs_derived_rows_before_publication(tmp_path, monkeypatch, with monkeypatch.context() as patch: patch.setattr(sqlite_schema, "_relabel_derived_domains", lambda conn: None) bootstrap_database(source, user_id=USER, user_email="local@alice") - with sqlite_user_connection(source, USER) as conn: + with without_insert_floor(), sqlite_user_connection(source, USER) as conn: store = SQLiteVNextStore(conn, USER) health = add_memory(store, key="health", text="A restricted observation", domain="health") derived = store.create_memory( @@ -52,7 +53,7 @@ def test_restore_repairs_derived_rows_before_publication(tmp_path, monkeypatch, from alicebot_api.onramp import sqlite_url_for_path from alicebot_api.vnext_agent_keys import create_agent_key - with sqlite_user_connection(destination, USER) as conn: + with without_insert_floor(), sqlite_user_connection(destination, USER) as conn: _, raw = create_agent_key( SQLiteVNextStore(conn, USER), user_id=USER, agent_id="restore-reader", permission_profile="read_only_agent" ) @@ -125,7 +126,7 @@ def test_repair_records_changed_rows_once(tmp_path): path = tmp_path / "audit.sqlite3" bootstrap_database(path, user_id=USER, user_email="local@alice") - with sqlite_user_connection(path, USER) as conn: + with without_insert_floor(), sqlite_user_connection(path, USER) as conn: store = SQLiteVNextStore(conn, USER) source = add_memory(store, key="health", text="Private observation", domain="health") derived = store.create_memory( @@ -261,7 +262,7 @@ def test_every_open_of_a_vault_holding_a_cycle_raises_the_clear_error_and_change path = tmp_path / "cycle.sqlite3" bootstrap_database(path, user_id=USER, user_email="local@alice") - with sqlite_user_connection(path, USER) as conn: + with without_insert_floor(), sqlite_user_connection(path, USER) as conn: store = SQLiteVNextStore(conn, USER) ids = sorted(add_memory(store, key=name, text=f"Row {name}")["id"] for name in "abc") # Each memory records the next as its consolidation input, and the labels alternate around the cycle. @@ -280,7 +281,7 @@ def test_every_open_of_a_vault_holding_a_cycle_raises_the_clear_error_and_change bootstrap_database(path, user_id=USER, user_email="local@alice") assert all(f"memories {row_id}" in str(caught.value) for row_id in ids) with pytest.raises(DerivedDomainRepairError, match="did not settle"): - with sqlite_user_connection(path, USER): + with without_insert_floor(), sqlite_user_connection(path, USER): pass with sqlite3.connect(path) as raw: assert [row[0] for row in raw.execute("SELECT domain FROM memories ORDER BY id")] == ["health", "legal", "health"] @@ -296,7 +297,7 @@ def test_import_of_a_backup_holding_a_cycle_stops_before_publication(tmp_path, m destination = tmp_path / "restored.sqlite3" backup = tmp_path / "backup.jsonl" bootstrap_database(source, user_id=USER, user_email="local@alice") - with sqlite_user_connection(source, USER) as conn: + with without_insert_floor(), sqlite_user_connection(source, USER) as conn: store = SQLiteVNextStore(conn, USER) ids = sorted(add_memory(store, key=name, text=f"Row {name}")["id"] for name in "abc") # The vault is stamped as repaired, so it opens and exports. Its rows still form the cycle that @@ -323,7 +324,7 @@ def test_sqlite_repair_follows_available_artifact_and_leaves_missing_inputs(tmp_ path = tmp_path / "promoted.sqlite3" bootstrap_database(path, user_id=USER, user_email="local@alice") - with sqlite_user_connection(path, USER) as conn: + with without_insert_floor(), sqlite_user_connection(path, USER) as conn: store = SQLiteVNextStore(conn, USER) # The local product does not store artifacts. Imported references to # absent artifacts are not guessed from copied text. @@ -446,7 +447,7 @@ def _seed_memory_chain(path, size=16): bootstrap_database(path, user_id=USER, user_email="local@alice") ids = [str(UUID(int=1000 + index)) for index in range(size)] - with sqlite_user_connection(path, USER) as conn: + with without_insert_floor(), sqlite_user_connection(path, USER) as conn: store = SQLiteVNextStore(conn, USER) for index, memory_id in enumerate(ids): store.create_memory({ diff --git a/tests/unit/test_derived_domain_stored_ids.py b/tests/unit/test_derived_domain_stored_ids.py index 511a6f3b4..a0a39758c 100644 --- a/tests/unit/test_derived_domain_stored_ids.py +++ b/tests/unit/test_derived_domain_stored_ids.py @@ -37,6 +37,7 @@ from alicebot_api import sqlite_schema, vnext_derived_domain_backfill as repair from alicebot_api.onramp import bootstrap_database, main as onramp_main from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_label_writes import without_insert_floor from tests.unit.per_project_s2_support import add_memory from tests.unit.test_derived_domain_fence import USER @@ -63,7 +64,7 @@ def _old_vault(path, monkeypatch, spellings=("upper",)): with monkeypatch.context() as patch: patch.setattr(sqlite_schema, "_relabel_derived_domains", lambda conn: None) bootstrap_database(path, user_id=USER, user_email="local@alice") - with sqlite_user_connection(path, USER) as conn: + with without_insert_floor(), sqlite_user_connection(path, USER) as conn: store = SQLiteVNextStore(conn, USER) health = add_memory(store, key="health", text="A restricted observation", domain="health") canonical_ids = [] @@ -162,7 +163,7 @@ def _twin_vault(path, monkeypatch, *, first_domain, last_domain="unknown", input with monkeypatch.context() as patch: patch.setattr(sqlite_schema, "_relabel_derived_domains", lambda conn: None) bootstrap_database(path, user_id=USER, user_email="local@alice") - with sqlite_user_connection(path, USER) as conn: + with without_insert_floor(), sqlite_user_connection(path, USER) as conn: store = SQLiteVNextStore(conn, USER) health = add_memory(store, key="health", text="A restricted observation", domain=input_domain) metadata = {"consolidation": {"cluster_member_ids": [health["id"]]}} @@ -328,7 +329,7 @@ def test_a_refusal_writes_no_event_and_no_stamp_even_after_an_earlier_update(tmp with monkeypatch.context() as patch: patch.setattr(sqlite_schema, "_relabel_derived_domains", lambda conn: None) bootstrap_database(path, user_id=USER, user_email="local@alice") - with sqlite_user_connection(path, USER) as conn: + with without_insert_floor(), sqlite_user_connection(path, USER) as conn: store = SQLiteVNextStore(conn, USER) health = add_memory(store, key="health", text="A restricted observation", domain="health") ids = sorted( diff --git a/tests/unit/test_sqlite_derived_labels_write_path.py b/tests/unit/test_sqlite_derived_labels_write_path.py new file mode 100644 index 000000000..680ef6de6 --- /dev/null +++ b/tests/unit/test_sqlite_derived_labels_write_path.py @@ -0,0 +1,121 @@ +"""SQLite insert floor and metadata merge. Scratch vaults only.""" + +from __future__ import annotations + +from pathlib import Path + +from alicebot_api.onramp import bootstrap_database +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_label_writes import merge_protected_metadata, without_insert_floor +from tests.unit.per_project_s2_support import add_memory +from tests.unit.test_derived_domain_fence import USER + +ALPHA = "prj_" + "a" * 16 + + +def _vault(path: Path): + bootstrap_database(path, user_id=USER, user_email="local@alice") + return sqlite_user_connection(path, USER) + + +def test_a_copy_stored_after_its_source_is_floored(tmp_path: Path) -> None: + path = tmp_path / "vault.sqlite3" + with _vault(path) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source( + { + "source_type": "note", + "title": "Visit", + "content_hash": "hash-floor", + "domain": "health", + "sensitivity": "confidential", + "metadata_json": {"project_scope": [ALPHA]}, + } + ) + memory = store.create_memory( + { + "memory_key": "extracted", + "canonical_text": "A fact from the visit.", + "status": "active", + "domain": "unknown", + "sensitivity": "public", + "metadata_json": { + "source_id": source["id"], + "extraction_rule": "sentence", + "project_scope": [ALPHA, "prj_" + "b" * 16], + }, + } + ) + assert memory["domain"] == "health" + assert memory["sensitivity"] == "confidential" + assert memory["metadata_json"]["project_scope"] == [ALPHA] + assert memory["metadata_json"]["project_floor"] == [ALPHA] + + +def test_without_insert_floor_keeps_the_requested_label(tmp_path: Path) -> None: + path = tmp_path / "vault.sqlite3" + with _vault(path) as conn: + store = SQLiteVNextStore(conn, USER) + health = add_memory(store, key="health", text="A restricted observation", domain="health") + with without_insert_floor(): + derived = store.create_memory( + { + "memory_key": "derived", + "canonical_text": "A summary", + "status": "active", + "domain": "unknown", + "sensitivity": "public", + "metadata_json": {"consolidation": {"cluster_member_ids": [health["id"]]}}, + } + ) + assert derived["domain"] == "unknown" + assert derived["sensitivity"] == "public" + + +def test_an_update_that_omits_a_marker_keeps_it(tmp_path: Path) -> None: + path = tmp_path / "vault.sqlite3" + with _vault(path) as conn: + store = SQLiteVNextStore(conn, USER) + health = add_memory(store, key="health", text="A restricted observation", domain="health") + with without_insert_floor(): + derived = store.create_memory( + { + "memory_key": "derived", + "canonical_text": "A summary", + "status": "active", + "domain": "health", + "sensitivity": "public", + "metadata_json": { + "consolidation": {"cluster_member_ids": [health["id"]]}, + "project_scope": [ALPHA], + "project_floor": [ALPHA], + "note": "keep", + }, + } + ) + stored = store.update_memory( + memory_id=str(derived["id"]), + patch={"metadata_json": {"note": "changed", "project_scope": ["other"]}}, + ) + assert stored["metadata_json"]["consolidation"]["cluster_member_ids"] == [health["id"]] + assert stored["metadata_json"]["project_scope"] == [ALPHA] + assert stored["metadata_json"]["project_floor"] == [ALPHA] + assert stored["metadata_json"]["note"] == "changed" + + +def test_merge_protected_metadata_keeps_label_keys_unless_the_write_is_a_relabel() -> None: + stored = { + "consolidation": {"cluster_member_ids": ["m"]}, + "project_scope": [ALPHA], + "project_floor": [ALPHA], + "note": "old", + } + patch = {"note": "new", "project_scope": ["other"]} + merged = merge_protected_metadata(stored, patch, label_write=False) + assert merged["project_scope"] == [ALPHA] + assert merged["project_floor"] == [ALPHA] + assert merged["consolidation"] == {"cluster_member_ids": ["m"]} + assert merged["note"] == "new" + relabel = merge_protected_metadata(stored, {"project_scope": [], "project_floor": [ALPHA]}, label_write=True) + assert relabel["project_scope"] == [] + assert relabel["consolidation"] == {"cluster_member_ids": ["m"]} From 45ef947af340845f39eafe403c12fd4186a6c65b Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 17:47:37 +0000 Subject: [PATCH 003/270] Propagate derived labels and lock every label-table writer. A stricter source or memory raises the rows derived from it. Artifact and open-loop inserts take the same floor. A source move previews how many derived rows a project key would lose, and a relabel that cannot finish answers 409 or 503. --- CHANGELOG.md | 4 +- .../alicebot_api/routers/vnext_memories.py | 602 ++++++++++-------- apps/api/src/alicebot_api/sqlite_store.py | 8 + .../src/alicebot_api/vnext_label_writes.py | 430 ++++++++++++- apps/api/src/alicebot_api/vnext_store.py | 23 + .../vnext_stores/postgres/embedding_cas.py | 3 + .../vnext_stores/postgres/events_revisions.py | 2 + .../vnext_stores/postgres/graph_open_loops.py | 8 + .../vnext_stores/postgres/memory_lifecycle.py | 11 + .../vnext_stores/sqlite/embedding_cas.py | 3 + .../vnext_stores/sqlite/graph_open_loops.py | 8 + .../vnext_stores/sqlite/memory_lifecycle.py | 8 + .../vnext_stores/sqlite/source_retirement.py | 7 + .../test_sqlite_derived_labels_write_path.py | 111 ++++ .../unit/test_store_events_revisions_split.py | 7 +- 15 files changed, 950 insertions(+), 285 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f466e313b..3612de93b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,8 @@ ## Unreleased -- Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. No migration is required. Reports, open loops, and a relabel of an existing row are not covered yet. -- Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. Nothing in the running product calls the function yet, so a read and a write still behave as they do in v0.20.0. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. +- Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. Raising a source or a memory walks the rows derived from it and raises those too. A candidate open loop and a generated artifact take the same insert floor. Replacing a SQLite source with a stricter one raises the memories that cite it before they are retired. Moving a source to another project answers with how many derived rows a project-bound key would lose, and waits for confirm_label_hide before it writes. A relabel that cannot finish answers 409, and one that waits more than 3 seconds for the label lock answers 503. No migration is required. +- Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. The function is what the write path calls. A read that does not go through that path still behaves as it does in v0.20.0. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` now scrub an open loop that names the source only in its metadata: the id, or `source:`, as the text under `source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references` or `selected_source_ids` at any depth. That is the rule the open-loop lookup of a source already used. They blanked only loops whose `source_id` column held the id, so a loop with an empty column kept its title, description and metadata, and they reached a new export. The delete preview now counts the same loops as the receipt. The rule reads the id only as stored or as `source:`: a loop that names the source in any spelling other than those two keeps its text, although the memory rule reads several other spellings. No migration is required. - Unreleased (on main, not in v0.20.0): the SQLite derived-label repair now updates and records each row under the id it is stored with (SQLite keeps capitals, missing hyphens, braces and `urn:uuid:` as written) and matches rows by the normalised id only to read the graph. If two stored spellings of one id exist, each is compared with the label its own recorded inputs give it, so neither is left at its old label while the other is relabelled, and neither is lowered. An update that changes no row stops the repair with `DerivedDomainRepairError` before any event or the completion stamp is written, on open and on restore; migration `20261004_0095` refuses the same way. Before, a derived memory stored under such an id kept its label while its relabel event and the stamp were written, and the pair survived export and import. A PostgreSQL uuid column was not affected. A vault that the earlier repair already stamped keeps such a row at its old label until a restore repairs it again. No migration is required. - Unreleased (on main, not in v0.20.0): v0.20.0 could store a consolidation report with a sensitivity below the memories it prints. The report took its sensitivity from the near-duplicate clusters only, so a run whose proposals were roll-up cards had no cluster to take it from and was stored as `unknown`. A key whose ceiling is below those memories (a `trusted_local_agent` key for confidential ones, a `read_only_agent` key for private ones) could then read the card topic and the member ids through `GET /v0/vnext/artifacts/{id}`. The report now takes its domain and its sensitivity over every row it names: the cluster members, the members of every proposed roll-up group, the members of the groups that a skip line names by key (`topic:...`, `entity:...`, `semantic:cluster-`), the pending, accepted, expired or held roll-up cards it names by id, and the sources that the `source_refs` of its cluster members name, which the report still prints. The open-loop review is labelled over the sources whose ids it prints as well as over its loops. The run digest of both reports covers those sources, so a source that was reclassified makes a new report instead of returning the earlier one. The first run after the upgrade over loops that link a source, or over cluster members that cite one, makes one new report, and a run that names no source keeps the digest it had. A report whose inputs are all unrestricted is now stored with the label of those inputs, `internal` where it was `unknown`, and every permission profile reads the two the same way. Reports stored by v0.20.0 keep their labels, because the stored-row repair does not change sensitivity. The other report producers (daily brief, weekly synthesis, connection and contradiction reports, project updates, staleness reports) were checked and already label over every row they print. Memories and sources are counted apart when the label is taken, because an id is unique only within its own table: a source that shares a memory's id can no longer replace that memory's label. No migration is required. diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index 893eb55e4..433d4716f 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -116,6 +116,7 @@ class VNextSourceReviewRequest(VNextAgentRequest): sensitivity: VNextSensitivity | None = None project_id: str | None = Field(default=None, min_length=1, max_length=120) review_note: str | None = Field(default=None, min_length=1, max_length=4000) + confirm_label_hide: bool = False class VNextConnectorSyncRequest(VNextAgentRequest): @@ -765,6 +766,12 @@ def review_vnext_source(source_id: UUID, request: VNextSourceReviewRequest) -> J try: with user_connection(settings.database_url, request.user_id) as conn: store = PostgresVNextStore(conn) + label_change = request.domain is not None or request.sensitivity is not None or request.project_id is not None + if label_change: + store.lock_graph_mutation() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) existing = store.get_source(str(source_id)) if existing is None: return _vnext_public_error_response(status_code=404, detail="vNext source was not found") @@ -812,6 +819,19 @@ def review_vnext_source(source_id: UUID, request: VNextSourceReviewRequest) -> J patch["domain"] = request.domain if request.sensitivity is not None: patch["sensitivity"] = request.sensitivity + if action == "assign_project" and request.project_id is not None: + from alicebot_api.vnext_label_writes import count_rows_hidden_by_scope_move + + hidden = count_rows_hidden_by_scope_move(store, existing, [request.project_id]) + if hidden and not request.confirm_label_hide: + return JSONResponse( + status_code=200, + content={ + "preview": True, + "derived_rows_hidden_from_project_keys": hidden, + "confirm_required": True, + }, + ) updated = store.update_source(source_id=str(source_id), patch=patch, actor_type="user") if action == "assign_project": store.create_edge( @@ -846,6 +866,15 @@ def review_vnext_source(source_id: UUID, request: VNextSourceReviewRequest) -> J ) except ContinuityStoreInvariantError as exc: return public_exception_response(exc, status_code=409) + except Exception as exc: + from alicebot_api.vnext_label_writes import label_error_response + + mapped = label_error_response(exc) + if mapped is None: + raise + status, detail, retry_after = mapped + headers = {"Retry-After": retry_after} if retry_after else None + return JSONResponse(status_code=status, content={"detail": detail}, headers=headers) return JSONResponse( status_code=200, content=jsonable_encoder({"source": updated, "archived": False, "trace": trace}) @@ -972,300 +1001,321 @@ def review_vnext_memory( ), ) - with user_connection(settings.database_url, request.user_id) as conn: - store = PostgresVNextStore(conn) - memory_service = VNextMemoryCommitService(store, defer_embeddings=True) - # Review can promote a consolidation candidate or mutate a member - # referenced by pending derived work. Establish the shared per-user - # graph boundary before the route takes any candidate/member row lock; - # delegated service calls may safely reacquire the transaction lock. - memory_service.lock_supersession_graph() - preview = store.get_memory(str(memory_id)) - if preview is None: - return _vnext_public_error_response(status_code=404, detail="vNext memory was not found") - # Delegate consolidation approval before this adapter takes a row lock. - # The service reacquires the already-held transaction advisory lock - # (non-blocking/re-entrant) and then owns all candidate/member locks. - if is_pending_consolidation_candidate(preview): - if action == "edit" or any( - value is not None - for value in ( - request.title, - request.canonical_text, - request.summary, - request.domain, - request.sensitivity, - request.project_id, - ) - ): - return _vnext_public_error_response( - status_code=400, - detail=( - "pending consolidation candidates cannot be edited during approval; " - "regenerate the candidate or accept it unchanged" - ), - ) - if action in {"accept", "promote"}: - return _vnext_public_error_response( - status_code=409, - detail="vNext memory became a consolidation candidate during review; retry the approval", - ) - get_memory_for_update = getattr(store, "get_memory_for_update", None) - existing = ( - get_memory_for_update(str(memory_id)) - if callable(get_memory_for_update) - else store.get_memory(str(memory_id)) - ) - if existing is None: - return _vnext_public_error_response(status_code=404, detail="vNext memory was not found") - # Re-authorize the locked record so a concurrent reassignment cannot - # move it outside the bound agent project between the first check and - # this mutation. - locked_scope = resource_project_scope(existing) - if action == "assign_project" and request.project_id is not None: - locked_scope = tuple(dict.fromkeys((*locked_scope, request.project_id))) - locked_decision = _vnext_policy_checked( - store=store, - identity=identity, - action="memory.review", - domains=(str(existing.get("domain") or "unknown"),), - sensitivity_allowed=(str(existing.get("sensitivity") or "unknown"),), - project_scope=locked_scope, - target_type="memory", - target_id=str(memory_id), - require_explicit_project_scope=True, - ) - if locked_decision.decision == "blocked": - return _vnext_permission_response(locked_decision) - if str(existing.get("status") or "") in {"archived", "rejected", "superseded"}: - return _vnext_public_error_response( - status_code=409, - detail=f"vNext memory cannot be reviewed from status '{existing.get('status')}'", + try: + with user_connection(settings.database_url, request.user_id) as conn: + store = PostgresVNextStore(conn) + memory_service = VNextMemoryCommitService(store, defer_embeddings=True) + # Review can promote a consolidation candidate or mutate a member + # referenced by pending derived work. Establish the shared per-user + # graph boundary before the route takes any candidate/member row lock; + # delegated service calls may safely reacquire the transaction lock. + memory_service.lock_supersession_graph() + label_change = ( + request.domain is not None + or request.sensitivity is not None + or request.project_id is not None + or action in {"private", "assign_project"} ) - if is_pending_consolidation_candidate(existing): - if action == "edit" or any( - value is not None - for value in ( - request.title, - request.canonical_text, - request.summary, - request.domain, - request.sensitivity, - request.project_id, - ) - ): + if label_change: + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) + preview = store.get_memory(str(memory_id)) + if preview is None: + return _vnext_public_error_response(status_code=404, detail="vNext memory was not found") + # Delegate consolidation approval before this adapter takes a row lock. + # The service reacquires the already-held transaction advisory lock + # (non-blocking/re-entrant) and then owns all candidate/member locks. + if is_pending_consolidation_candidate(preview): + if action == "edit" or any( + value is not None + for value in ( + request.title, + request.canonical_text, + request.summary, + request.domain, + request.sensitivity, + request.project_id, + ) + ): + return _vnext_public_error_response( + status_code=400, + detail=( + "pending consolidation candidates cannot be edited during approval; " + "regenerate the candidate or accept it unchanged" + ), + ) + if action in {"accept", "promote"}: + return _vnext_public_error_response( + status_code=409, + detail="vNext memory became a consolidation candidate during review; retry the approval", + ) + get_memory_for_update = getattr(store, "get_memory_for_update", None) + existing = ( + get_memory_for_update(str(memory_id)) + if callable(get_memory_for_update) + else store.get_memory(str(memory_id)) + ) + if existing is None: + return _vnext_public_error_response(status_code=404, detail="vNext memory was not found") + # Re-authorize the locked record so a concurrent reassignment cannot + # move it outside the bound agent project between the first check and + # this mutation. + locked_scope = resource_project_scope(existing) + if action == "assign_project" and request.project_id is not None: + locked_scope = tuple(dict.fromkeys((*locked_scope, request.project_id))) + locked_decision = _vnext_policy_checked( + store=store, + identity=identity, + action="memory.review", + domains=(str(existing.get("domain") or "unknown"),), + sensitivity_allowed=(str(existing.get("sensitivity") or "unknown"),), + project_scope=locked_scope, + target_type="memory", + target_id=str(memory_id), + require_explicit_project_scope=True, + ) + if locked_decision.decision == "blocked": + return _vnext_permission_response(locked_decision) + if str(existing.get("status") or "") in {"archived", "rejected", "superseded"}: return _vnext_public_error_response( - status_code=400, - detail=( - "pending consolidation candidates cannot be edited during approval; " - "regenerate the candidate or accept it unchanged" - ), + status_code=409, + detail=f"vNext memory cannot be reviewed from status '{existing.get('status')}'", ) - if action in {"accept", "promote"}: + if is_pending_consolidation_candidate(existing): + if action == "edit" or any( + value is not None + for value in ( + request.title, + request.canonical_text, + request.summary, + request.domain, + request.sensitivity, + request.project_id, + ) + ): + return _vnext_public_error_response( + status_code=400, + detail=( + "pending consolidation candidates cannot be edited during approval; " + "regenerate the candidate or accept it unchanged" + ), + ) + if action in {"accept", "promote"}: + return _vnext_public_error_response( + status_code=409, + detail="vNext memory became a consolidation candidate during review; retry the approval", + ) + if is_pending_project_update_memory(existing): return _vnext_public_error_response( status_code=409, - detail="vNext memory became a consolidation candidate during review; retry the approval", + detail=PENDING_PROJECT_UPDATE_MEMORY_MUTATION_MESSAGE, ) - if is_pending_project_update_memory(existing): - return _vnext_public_error_response( - status_code=409, - detail=PENDING_PROJECT_UPDATE_MEMORY_MUTATION_MESSAGE, - ) - existing_metadata_value = existing.get("metadata_json") - existing_metadata: dict[str, object] = ( - existing_metadata_value if isinstance(existing_metadata_value, dict) else {} - ) - reviewed_at = datetime.now(UTC).isoformat() - patch: dict[str, object] = { - "last_reviewed_at": reviewed_at, - } - revision_type = "edited" - if action == "accept": - patch["status"] = "active" - revision_type = "promoted" - elif action == "reject": - patch["status"] = "rejected" - patch["metadata_json"] = _vnext_terminal_review_metadata( - existing_metadata, - outcome="rejected", - terminal_at=reviewed_at, + existing_metadata_value = existing.get("metadata_json") + existing_metadata: dict[str, object] = ( + existing_metadata_value if isinstance(existing_metadata_value, dict) else {} ) - revision_type = "rejected" - elif action == "private": - patch["status"] = "private_only" - patch["sensitivity"] = "private" - elif action == "promote": - patch["status"] = "active" - patch["confirmation_status"] = "confirmed" - revision_type = "promoted" - elif action == "assign_project": - if request.project_id is None: - return _vnext_public_error_response(status_code=400, detail="project_id is required") - # Keep every current scope representation in the same UPDATE. A - # metadata-only project_id write leaves an older project_scope in - # place, and canonical retrieval correctly gives that array - # precedence over the legacy singular fallback. - patch["project_id"] = request.project_id - patch["metadata_json"] = { - **existing_metadata, - "project_id": request.project_id, - "project_scope": [request.project_id], - "assigned_from": "vnext_workspace", + reviewed_at = datetime.now(UTC).isoformat() + patch: dict[str, object] = { + "last_reviewed_at": reviewed_at, } - else: - patch["status"] = "active" - - if action in {"accept", "edit", "promote"}: - patch.update( - { - "confirmation_status": "confirmed", - "last_confirmed_at": reviewed_at, - "metadata_json": _vnext_terminal_review_metadata( - existing_metadata, - outcome="confirmed", - terminal_at=reviewed_at, - ), + revision_type = "edited" + if action == "accept": + patch["status"] = "active" + revision_type = "promoted" + elif action == "reject": + patch["status"] = "rejected" + patch["metadata_json"] = _vnext_terminal_review_metadata( + existing_metadata, + outcome="rejected", + terminal_at=reviewed_at, + ) + revision_type = "rejected" + elif action == "private": + patch["status"] = "private_only" + patch["sensitivity"] = "private" + elif action == "promote": + patch["status"] = "active" + patch["confirmation_status"] = "confirmed" + revision_type = "promoted" + elif action == "assign_project": + if request.project_id is None: + return _vnext_public_error_response(status_code=400, detail="project_id is required") + # Keep every current scope representation in the same UPDATE. A + # metadata-only project_id write leaves an older project_scope in + # place, and canonical retrieval correctly gives that array + # precedence over the legacy singular fallback. + patch["project_id"] = request.project_id + patch["metadata_json"] = { + **existing_metadata, + "project_id": request.project_id, + "project_scope": [request.project_id], + "assigned_from": "vnext_workspace", } - ) + else: + patch["status"] = "active" - if request.title is not None: - patch["title"] = request.title - if request.canonical_text is not None: - patch["canonical_text"] = request.canonical_text - existing_value = existing.get("value") - patch["value"] = { - **(existing_value if isinstance(existing_value, dict) else {}), - "text": request.canonical_text, - } - # Capture-generated title/summary are denormalized views of the - # canonical text. Editing only the body must not leave those - # user-visible fields describing the pre-edit value. - if request.title is None: - patch["title"] = ( - request.canonical_text - if len(request.canonical_text) <= 120 - else request.canonical_text[:117].rstrip() + "..." - ) - if request.summary is None: - patch["summary"] = ( - request.canonical_text - if len(request.canonical_text) <= 280 - else request.canonical_text[:277].rstrip() + "..." + if action in {"accept", "edit", "promote"}: + patch.update( + { + "confirmation_status": "confirmed", + "last_confirmed_at": reviewed_at, + "metadata_json": _vnext_terminal_review_metadata( + existing_metadata, + outcome="confirmed", + terminal_at=reviewed_at, + ), + } ) - if request.summary is not None: - patch["summary"] = request.summary - if request.domain is not None: - patch["domain"] = request.domain - if request.sensitivity is not None: - patch["sensitivity"] = request.sensitivity - - # The credential floor, on the row as it will be stored. This route - # writes the row itself rather than through a service method, so it - # calls the shared checks here. Until 2026-09-22 an edit rewrote - # title, text and summary with no credential check. A title-only edit - # is read against the stored body, because that is the row a reader - # will see; derived previews of the text are left out. - # - # accept, promote and edit move the row into a searchable status, so - # they take the shared activation check over the row and the reason - # (ruling C2). A reject always completes (ruling C6): a reason, or - # edited text, that carries credential material is stored as a fixed - # placeholder and the response says so. private and assign_project - # keep the plain check. - text_edit = any(value is not None for value in (request.title, request.canonical_text, request.summary)) - stored_title = patch.get("title", existing.get("title")) - stored_text = patch.get("canonical_text", existing.get("canonical_text")) - stored_summary = patch.get("summary", existing.get("summary")) - review_reason = request.reason - rationale_withheld = False - text_withheld = False - if action == "reject": - review_reason, rationale_withheld = withhold_credential_text(request.reason) - if text_edit and carries_credential_material(request.title, request.canonical_text, request.summary): - for text_field in ("title", "canonical_text", "summary"): - if text_field in patch: - patch[text_field] = TEXT_WITHHELD_PLACEHOLDER - edited_value = patch.get("value") - if isinstance(edited_value, dict): - patch["value"] = {**edited_value, "text": TEXT_WITHHELD_PLACEHOLDER} - text_withheld = True - elif patch.get("status") in SEARCHABLE_STATUSES: - try: - refuse_credential_activation(stored_title, stored_text, stored_summary, request.reason) - except CredentialActivationRefused: - return _vnext_public_error_response( - status_code=400, detail="vNext memory review text carries credential material" + + if request.title is not None: + patch["title"] = request.title + if request.canonical_text is not None: + patch["canonical_text"] = request.canonical_text + existing_value = existing.get("value") + patch["value"] = { + **(existing_value if isinstance(existing_value, dict) else {}), + "text": request.canonical_text, + } + # Capture-generated title/summary are denormalized views of the + # canonical text. Editing only the body must not leave those + # user-visible fields describing the pre-edit value. + if request.title is None: + patch["title"] = ( + request.canonical_text + if len(request.canonical_text) <= 120 + else request.canonical_text[:117].rstrip() + "..." + ) + if request.summary is None: + patch["summary"] = ( + request.canonical_text + if len(request.canonical_text) <= 280 + else request.canonical_text[:277].rstrip() + "..." + ) + if request.summary is not None: + patch["summary"] = request.summary + if request.domain is not None: + patch["domain"] = request.domain + if request.sensitivity is not None: + patch["sensitivity"] = request.sensitivity + + # The credential floor, on the row as it will be stored. This route + # writes the row itself rather than through a service method, so it + # calls the shared checks here. Until 2026-09-22 an edit rewrote + # title, text and summary with no credential check. A title-only edit + # is read against the stored body, because that is the row a reader + # will see; derived previews of the text are left out. + # + # accept, promote and edit move the row into a searchable status, so + # they take the shared activation check over the row and the reason + # (ruling C2). A reject always completes (ruling C6): a reason, or + # edited text, that carries credential material is stored as a fixed + # placeholder and the response says so. private and assign_project + # keep the plain check. + text_edit = any(value is not None for value in (request.title, request.canonical_text, request.summary)) + stored_title = patch.get("title", existing.get("title")) + stored_text = patch.get("canonical_text", existing.get("canonical_text")) + stored_summary = patch.get("summary", existing.get("summary")) + review_reason = request.reason + rationale_withheld = False + text_withheld = False + if action == "reject": + review_reason, rationale_withheld = withhold_credential_text(request.reason) + if text_edit and carries_credential_material(request.title, request.canonical_text, request.summary): + for text_field in ("title", "canonical_text", "summary"): + if text_field in patch: + patch[text_field] = TEXT_WITHHELD_PLACEHOLDER + edited_value = patch.get("value") + if isinstance(edited_value, dict): + patch["value"] = {**edited_value, "text": TEXT_WITHHELD_PLACEHOLDER} + text_withheld = True + elif patch.get("status") in SEARCHABLE_STATUSES: + try: + refuse_credential_activation(stored_title, stored_text, stored_summary, request.reason) + except CredentialActivationRefused: + return _vnext_public_error_response( + status_code=400, detail="vNext memory review text carries credential material" + ) + else: + review_fields: tuple[object, ...] = (request.reason,) + if text_edit: + review_fields = (*stored_text_fields(stored_title, stored_text, stored_summary), request.reason) + if carries_credential_material(*review_fields): + return _vnext_public_error_response( + status_code=400, detail="vNext memory review text carries credential material" + ) + + updated = store.update_memory(memory_id=str(memory_id), patch=patch, actor_type=actor_type) + if action in ("accept", "edit", "promote"): + memory_service.refresh_memory_derived_state( + updated, + identity=identity, + stage=f"http_review_{action}", ) - else: - review_fields: tuple[object, ...] = (request.reason,) - if text_edit: - review_fields = (*stored_text_fields(stored_title, stored_text, stored_summary), request.reason) - if carries_credential_material(*review_fields): - return _vnext_public_error_response( - status_code=400, detail="vNext memory review text carries credential material" + if action == "assign_project" and request.project_id is not None: + store.create_edge( + { + "from_type": "memory", + "from_id": str(memory_id), + "to_type": "project", + "to_id": request.project_id, + "edge_type": "belongs_to_project", + "confidence": 1.0, + "explanation": "Assigned from live /vnext memory review.", + "created_by": "user", + "metadata_json": {"review_action": action}, + }, + actor_type=actor_type, ) - - updated = store.update_memory(memory_id=str(memory_id), patch=patch, actor_type=actor_type) - if action in ("accept", "edit", "promote"): - memory_service.refresh_memory_derived_state( - updated, - identity=identity, - stage=f"http_review_{action}", - ) - if action == "assign_project" and request.project_id is not None: - store.create_edge( + store.append_revision( { - "from_type": "memory", - "from_id": str(memory_id), - "to_type": "project", - "to_id": request.project_id, - "edge_type": "belongs_to_project", - "confidence": 1.0, - "explanation": "Assigned from live /vnext memory review.", - "created_by": "user", - "metadata_json": {"review_action": action}, + "memory_id": str(memory_id), + "memory_key": str(updated["memory_key"]), + "previous_value": existing.get("value"), + "new_value": updated.get("value"), + "source_event_ids": updated.get("source_event_ids"), + "revision_type": revision_type, + "action": f"memory_review_{action}", + "text_before": existing.get("canonical_text"), + "text_after": str(updated.get("canonical_text", "")), + "reason": review_reason or f"vNext workspace memory review action: {action}", + "actor_type": actor_type, + "actor_id": actor_id, + "metadata_json": {"action": action, "project_id": request.project_id}, }, actor_type=actor_type, ) - store.append_revision( - { - "memory_id": str(memory_id), - "memory_key": str(updated["memory_key"]), - "previous_value": existing.get("value"), - "new_value": updated.get("value"), - "source_event_ids": updated.get("source_event_ids"), - "revision_type": revision_type, - "action": f"memory_review_{action}", - "text_before": existing.get("canonical_text"), - "text_after": str(updated.get("canonical_text", "")), - "reason": review_reason or f"vNext workspace memory review action: {action}", - "actor_type": actor_type, - "actor_id": actor_id, - "metadata_json": {"action": action, "project_id": request.project_id}, - }, - actor_type=actor_type, - ) - review_event = { - "accept": "review.item_accepted", - "promote": "review.item_accepted", - "reject": "review.item_rejected", - "edit": "review.item_edited", - "private": "review.item_edited", - "assign_project": "review.item_edited", - }[action] - append_event( - store, - event_type=review_event, - actor_type=actor_type, - actor_id=actor_id, - target_type="memory", - target_id=str(memory_id), - payload={"action": action, "project_id": request.project_id}, - ) - # The row this route hands back is held to the caller's read fence: a memory that cites an archived source (or - # one above the caller's ceiling) is not returned with the quote it saved. - updated = SavedProvenanceReader(store, fence=SourceReadFence.for_identity(identity)).memory(updated) + review_event = { + "accept": "review.item_accepted", + "promote": "review.item_accepted", + "reject": "review.item_rejected", + "edit": "review.item_edited", + "private": "review.item_edited", + "assign_project": "review.item_edited", + }[action] + append_event( + store, + event_type=review_event, + actor_type=actor_type, + actor_id=actor_id, + target_type="memory", + target_id=str(memory_id), + payload={"action": action, "project_id": request.project_id}, + ) + # The row this route hands back is held to the caller's read fence: a memory that cites an archived source (or + # one above the caller's ceiling) is not returned with the quote it saved. + updated = SavedProvenanceReader(store, fence=SourceReadFence.for_identity(identity)).memory(updated) + + except Exception as exc: + from alicebot_api.vnext_label_writes import label_error_response + + mapped = label_error_response(exc) + if mapped is None: + raise + status, detail, retry_after = mapped + headers = {"Retry-After": retry_after} if retry_after else None + return JSONResponse(status_code=status, content={"detail": detail}, headers=headers) _persist_vnext_deferred_embeddings( database_url=settings.database_url, diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index 3fe8eaec1..2fd996f66 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -23,6 +23,8 @@ import itertools import json import sqlite3 + +from alicebot_api.vnext_label_writes import takes_label_lock from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager from datetime import datetime @@ -705,6 +707,7 @@ def list_memory_events( source_inventory = _source_inventory prunable_sources = _prunable_sources + @takes_label_lock def create_source(self, source: JsonObject, *, actor_type: str = "system") -> VNextRow: source_id = _new_id(source.get("id")) self._execute( @@ -763,6 +766,7 @@ def create_source(self, source: JsonObject, *, actor_type: str = "system") -> VN create_browser_clip_capability = _browser_clip_create_capability consume_browser_clip_capability = _browser_clip_consume_capability + @takes_label_lock def get_or_create_source( self, source: JsonObject, @@ -915,6 +919,7 @@ def get_source_by_dedupe_key(self, dedupe_key: str) -> VNextRow | None: (dedupe_key, self.user_id), ) + @takes_label_lock def update_source( self, *, @@ -1020,6 +1025,9 @@ def update_source( target_id=row["id"], payload={"operation": "update", "changes": patch}, ) + from alicebot_api.vnext_label_writes import propagate_after_write + + propagate_after_write(self, kind="source", before=current, after=row) return row def create_source_chunk(self, chunk: JsonObject, *, actor_type: str = "system") -> VNextRow: diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index faee03beb..6df1e210a 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -6,22 +6,29 @@ from __future__ import annotations -from collections.abc import Iterator, Mapping +import json +import re +from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager from functools import wraps from typing import Any from alicebot_api.vnext_agent_control import RESTRICTED_DOMAINS +from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError, require_changed from alicebot_api.vnext_derived_labels import ( MARKER_KEYS, + PROPAGATION_BOUND, + SENSITIVITY_RANK, dependencies_of, + LabelPropagationTooLarge, generation_domain, + identifier, is_derived, labels_raised_payload, settle_labels, ) from alicebot_api.vnext_event_log import build_event_log_record, integrity_hash_for_event -from alicebot_api.vnext_project_scope import project_scope_identity +from alicebot_api.vnext_project_scope import project_scope_identity, resolve_project_scope, source_project_scope from alicebot_api.vnext_repositories import JsonObject LABEL_METADATA_KEYS = ("project_scope", "project_floor", "derived_from") @@ -41,6 +48,12 @@ class LabelLockOrderError(RuntimeError): """A label write ran outside a transaction, or it took locks in the wrong order.""" +RETRYABLE_DETAIL = ( + "the label change was not applied because another change was running; nothing was changed; try again" +) +REFUSED_DETAIL = "the label change could not be applied to every dependent row; nothing was changed" + + def takes_label_lock(fn: Any) -> Any: """Take the shared label lock before a store method writes or row-locks a label table.""" @@ -89,8 +102,6 @@ def merge_protected_metadata( for key in LABEL_METADATA_KEYS: if key in stored_map: result[key] = stored_map[key] - else: - result.pop(key, None) for key in MARKER_KEYS: if key not in patch_map and key in stored_map: result[key] = stored_map[key] @@ -202,6 +213,408 @@ def apply_insert_floor(store: Any, kind: str, payload: Mapping[str, object]) -> return updated, event +def _sqlite(store: Any) -> bool: + return type(getattr(store, "conn", None)).__module__.startswith("sqlite3") + + +def _compact_id(value: object) -> str: + return re.sub(r"[-{}\s]", "", str(value).lower()) + + +def should_propagate(kind: str, before: Mapping[str, object] | None, after: Mapping[str, object] | None) -> bool: + """True when a relabel must walk dependants. A pure loosening does not.""" + + if not before or not after: + return False + old_domain = str(before.get("domain") or "unknown") + new_domain = str(after.get("domain") or "unknown") + old_sensitivity = str(before.get("sensitivity") or "unknown") + new_sensitivity = str(after.get("sensitivity") or "unknown") + rank_up = SENSITIVITY_RANK.get(new_sensitivity, 2) > SENSITIVITY_RANK.get(old_sensitivity, 2) + domain_move = new_domain in RESTRICTED_DOMAINS and new_domain != old_domain + if kind == "source": + old_scope = source_project_scope(before) + new_scope = source_project_scope(after) + else: + old_scope = resolve_project_scope(before).values + new_scope = resolve_project_scope(after).values + scope_move = project_scope_identity(old_scope) != project_scope_identity(new_scope) + return bool(rank_up or domain_move or scope_move) + + +def acquire_exclusive_label_lock(store: Any) -> None: + """Take L exclusive. On Postgres, wait at most 3 seconds.""" + + if _sqlite(store): + store.lock_label_writes(exclusive=True) + return + with store.conn.cursor() as cur: + cur.execute("SELECT current_setting('lock_timeout') AS lock_timeout") + row = cur.fetchone() + previous = row["lock_timeout"] if isinstance(row, Mapping) else row[0] + cur.execute("SET LOCAL lock_timeout = '3s'") + try: + store.lock_label_writes(exclusive=True) + finally: + try: + cur.execute("SELECT set_config('lock_timeout', %s, true)", (str(previous),)) + except Exception: + # A lock timeout aborts the transaction. Rollback drops the local setting. + pass + + +def list_dependants(store: Any, ids: Sequence[str]) -> list[dict[str, object]]: + """Rows whose recorded text may name one of ``ids``. The caller filters exactly.""" + + wanted = [str(item) for item in ids if str(item)] + if not wanted: + return [] + compacts = [_compact_id(item) for item in wanted if len(_compact_id(item)) >= 8] + if not compacts: + compacts = [_compact_id(item) for item in wanted if _compact_id(item)] + found: list[dict[str, object]] = [] + if _sqlite(store): + found.extend(_sqlite_dependants(store, "memories", "memory", compacts, wanted, with_value=True)) + found.extend(_sqlite_dependants(store, "open_loops", "open_loop", compacts, wanted, with_value=False)) + else: + found.extend(_postgres_dependants(store, "memories", "memory", compacts, with_value=True)) + found.extend(_postgres_dependants(store, "open_loops", "open_loop", compacts, with_value=False)) + found.extend(_postgres_dependants(store, "generated_artifacts", "artifact", compacts, with_value=False)) + found.extend(_postgres_dependants(store, "projects", "project", compacts, with_value=False)) + return found + + +def _like_clause(column: str, count: int, *, qmark: bool) -> tuple[str, list[str]]: + mark = "?" if qmark else "%s" + parts = [] + params: list[str] = [] + for _ in range(count): + parts.append( + "replace(replace(replace(replace(lower(coalesce(" + + column + + ",'')),'-',''),'{',''),'}',''),' ','') LIKE " + + mark + ) + return " OR ".join(parts), params + + +def _sqlite_dependants(store: Any, table: str, kind: str, compacts: Sequence[str], raw_ids: Sequence[str], *, with_value: bool) -> list[dict[str, object]]: + extra = ", value, project_id" if with_value else ", NULL AS value, project_id, source_id, memory_id" + if table == "memories": + extra = ", value, project_id, NULL AS source_id, NULL AS memory_id" + text_clause, _ = _like_clause("metadata_json", len(compacts), qmark=True) + params: list[object] = [store.user_id, *[f"%{item}%" for item in compacts]] + value_sql = "" + if with_value: + value_clause, _ = _like_clause("value", len(compacts), qmark=True) + value_sql = f" OR {value_clause}" + params.extend(f"%{item}%" for item in compacts) + column_sql = "" + if table == "open_loops": + marks = ",".join("?" for _ in raw_ids) + column_sql = f" OR source_id IN ({marks}) OR memory_id IN ({marks})" + params.extend(raw_ids) + params.extend(raw_ids) + rows = store._fetch_all( + f""" + SELECT id, user_id, domain, sensitivity, metadata_json{extra} + FROM {table} + WHERE user_id = ? AND ({text_clause}{value_sql}{column_sql}) + """, + tuple(params), + ) + for row in rows: + row["kind"] = kind + return rows + + +def _postgres_dependants(store: Any, table: str, kind: str, compacts: Sequence[str], *, with_value: bool) -> list[dict[str, object]]: + extra = "" + if table == "memories": + extra = ", value, project_id" + elif table == "open_loops": + extra = ", NULL::jsonb AS value, project_id, source_id, memory_id" + elif table == "generated_artifacts": + extra = ", NULL::jsonb AS value, artifact_type" + else: + extra = ", NULL::jsonb AS value" + text_clause, _ = _like_clause("metadata_json::text", len(compacts), qmark=False) + params: list[object] = [f"%{item}%" for item in compacts] + value_sql = "" + if with_value: + value_clause, _ = _like_clause("value::text", len(compacts), qmark=False) + value_sql = f" OR {value_clause}" + params.extend(f"%{item}%" for item in compacts) + rows = store._fetch_all( + f""" + SELECT id::text AS id, user_id::text AS user_id, domain, sensitivity, metadata_json{extra} + FROM {table} + WHERE ({text_clause}{value_sql}) + """, + tuple(params), + ) + for row in rows: + row["kind"] = kind + return rows + + +def _depends_on(row: Mapping[str, object], frontier: set[str]) -> bool: + kind = str(row.get("kind") or "") + try: + deps = dependencies_of(kind, row) + except ValueError: + return False + return any(identifier(dep_id) in frontier or _compact_id(dep_id) in frontier for _dep_kind, dep_id in deps) + + +def walk_dependants(store: Any, roots: Sequence[str]) -> list[dict[str, object]]: + """Exact dependants of ``roots``, including later rows in the chain.""" + + frontier = {identifier(item) for item in roots} + frontier |= {_compact_id(item) for item in roots} + seen = set(frontier) + found: list[dict[str, object]] = [] + pending = [str(item) for item in roots] + while pending: + if len(seen) > PROPAGATION_BOUND: + raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") + batch = pending[:200] + pending = pending[200:] + matched: list[dict[str, object]] = [] + batch_ids = {identifier(item) for item in batch} | {_compact_id(item) for item in batch} + for row in list_dependants(store, batch): + row_id = identifier(row.get("id")) + if row_id in seen or _compact_id(row.get("id")) in seen: + continue + if not _depends_on(row, batch_ids): + continue + seen.add(row_id) + seen.add(_compact_id(row.get("id"))) + matched.append(row) + found.extend(matched) + pending.extend(str(row["id"]) for row in matched) + return found + + +def _label_fields(row: Mapping[str, object]) -> tuple[str, str, tuple[str, ...], tuple[str, ...]]: + metadata = row.get("metadata_json") + meta = metadata if isinstance(metadata, Mapping) else {} + scope = meta.get("project_scope", ()) + floor = meta.get("project_floor", ()) + return ( + str(row.get("domain") or "unknown"), + str(row.get("sensitivity") or "unknown"), + tuple(scope) if isinstance(scope, (list, tuple)) else (), + tuple(floor) if isinstance(floor, (list, tuple)) else (), + ) + + +def write_settled_label( + store: Any, + *, + kind: str, + row_id: str, + domain: str, + sensitivity: str, + metadata: Mapping[str, object], + project_id: str | None, + expected_domain: str, + expected_sensitivity: str, +) -> None: + """Label-only update. A statement that changes no row refuses the whole relabel.""" + + table = {"memory": "memories", "open_loop": "open_loops", "artifact": "generated_artifacts", "project": "projects"}[kind] + blob = json.dumps(dict(metadata)) + if _sqlite(store): + project_sql = ", project_id = ?" if table in {"memories", "open_loops"} else "" + params: list[object] = [domain, sensitivity, blob] + if project_sql: + params.append(project_id) + params.extend([str(row_id), store.user_id, expected_domain, expected_sensitivity]) + cursor = store._execute( + f""" + UPDATE {table} + SET domain = ?, sensitivity = ?, metadata_json = ?{project_sql} + WHERE id = ? AND user_id = ? AND domain = ? AND sensitivity = ? + """, + tuple(params), + ) + require_changed(int(cursor.rowcount), table, str(row_id)) + return + project_sql = ", project_id = %s" if table in {"memories", "open_loops"} else "" + params = [domain, sensitivity, blob] + if project_sql: + params.append(project_id) + params.extend([str(row_id), expected_domain, expected_sensitivity]) + store._fetch_one( + "write_settled_label", + f""" + UPDATE {table} + SET domain = %s, sensitivity = %s, metadata_json = %s::jsonb{project_sql} + WHERE id = %s::uuid AND domain = %s AND sensitivity = %s + RETURNING id + """, + tuple(params), + ) + + +def propagate(store: Any, changed: Sequence[tuple[str, str]], *, cause: str) -> int: + """Recompute dependants of ``changed`` rows and write the ones that rise.""" + + store.lock_label_writes(exclusive=True) + roots = [row_id for _kind, row_id in changed] + affected = walk_dependants(store, roots) + if not affected: + return 0 + nodes: list[dict[str, object]] = [] + reader = getattr(store, "read_label_rows", None) + if callable(reader): + for kind, row_id in changed: + for row in reader(kind, [row_id]): + copied = dict(row) + copied["kind"] = kind + nodes.append(copied) + needed: dict[str, list[str]] = {} + for row in affected: + for dep_kind, dep_id in dependencies_of(str(row.get("kind")), row): + needed.setdefault(dep_kind, []).append(dep_id) + if callable(reader): + for dep_kind, dep_ids in needed.items(): + for row in reader(dep_kind, dep_ids): + copied = dict(row) + copied["kind"] = dep_kind + nodes.append(copied) + for row in affected: + nodes.append(dict(row)) + settled = settle_labels(nodes) + written = 0 + for row in affected: + label = settled.by_stored(str(row.get("kind")), str(row.get("id"))) + if label.unverified: + continue + previous = _label_fields(row) + current = (label.domain, label.sensitivity, tuple(label.project_scope), tuple(label.project_floor)) + if ( + previous[0] == current[0] + and previous[1] == current[1] + and project_scope_identity(previous[2]) == project_scope_identity(current[2]) + and project_scope_identity(previous[3]) == project_scope_identity(current[3]) + ): + continue + metadata = dict(row.get("metadata_json")) if isinstance(row.get("metadata_json"), Mapping) else {} + metadata["project_scope"] = list(label.project_scope) + metadata["project_floor"] = list(label.project_floor) + project_id = label.project_scope[0] if len(project_scope_identity(label.project_scope)) == 1 else None + if project_id is not None: + try: + from uuid import UUID + + UUID(str(project_id)) + except ValueError: + project_id = None + write_settled_label( + store, + kind=str(row.get("kind")), + row_id=str(row.get("id")), + domain=label.domain, + sensitivity=label.sensitivity, + metadata=metadata, + project_id=project_id, + expected_domain=previous[0], + expected_sensitivity=previous[1], + ) + event = build_event_log_record( + event_type=f"{label.kind}.labels_raised", + actor_type="system", + target_type=label.kind, + target_id=str(row.get("id")), + payload=labels_raised_payload( + cause=cause, + previous={ + "domain": previous[0], + "sensitivity": previous[1], + "project_scope": list(previous[2]), + "project_floor": list(previous[3]), + }, + new={ + "domain": label.domain, + "sensitivity": label.sensitivity, + "project_scope": list(label.project_scope), + "project_floor": list(label.project_floor), + }, + ), + ) + append = getattr(store, "append_event", None) + if callable(append): + append(event) + written += 1 + return written + + +def propagate_after_write(store: Any, *, kind: str, before: Mapping[str, object] | None, after: Mapping[str, object] | None, cause: str = "input_relabelled") -> int: + if not should_propagate(kind, before, after) or after is None: + return 0 + return propagate(store, [(kind, str(after.get("id")))], cause=cause) + + +def count_rows_hidden_by_scope_move(store: Any, source: Mapping[str, object], new_scope: Sequence[str]) -> int: + """How many derived rows a project-bound key would lose if ``source`` moved.""" + + source_id = str(source.get("id") or "") + if not source_id: + return 0 + affected = walk_dependants(store, [source_id]) + if not affected: + return 0 + current = dict(source) + current["kind"] = "source" + moved = dict(source) + moved["kind"] = "source" + metadata = dict(source.get("metadata_json")) if isinstance(source.get("metadata_json"), Mapping) else {} + metadata["project_scope"] = list(new_scope) + moved["metadata_json"] = metadata + before = settle_labels([current, *[dict(row) for row in affected]]) + after = settle_labels([moved, *[dict(row) for row in affected]]) + hidden = 0 + for row in affected: + old = before.by_stored(str(row.get("kind")), str(row.get("id"))) + new = after.by_stored(str(row.get("kind")), str(row.get("id"))) + old_ids = set(project_scope_identity(old.project_scope)) + new_ids = set(project_scope_identity(new.project_scope)) + if old_ids - new_ids: + hidden += 1 + return hidden + + +def raise_source_to_replacement(store: Any, old: Mapping[str, object], replacement: Mapping[str, object]) -> None: + """Raise a retiring source to the replacement when that label is stricter, then propagate.""" + + old_domain = str(old.get("domain") or "unknown") + new_domain = str(replacement.get("domain") or "unknown") + domain = new_domain if new_domain in RESTRICTED_DOMAINS and old_domain not in RESTRICTED_DOMAINS else old_domain + sensitivity = str(old.get("sensitivity") or "unknown") + replacement_sensitivity = str(replacement.get("sensitivity") or "unknown") + if SENSITIVITY_RANK.get(replacement_sensitivity, 2) > SENSITIVITY_RANK.get(sensitivity, 2): + sensitivity = replacement_sensitivity + if domain == old_domain and sensitivity == str(old.get("sensitivity") or "unknown"): + return + store.update_source( + source_id=str(old.get("id")), + patch={"domain": domain, "sensitivity": sensitivity}, + actor_type="system", + ) + + +def label_error_response(exc: BaseException) -> tuple[int, str, str | None] | None: + """``(status, detail, retry_after)`` for a relabel failure, or None.""" + + if isinstance(exc, (LabelPropagationTooLarge, DerivedDomainRepairError)): + return 409, REFUSED_DETAIL, None + if type(exc).__name__ in {"LockNotAvailable", "DeadlockDetected", "SerializationFailure"}: + return 503, RETRYABLE_DETAIL, "2" + return None + + def remember_floor_event(store: Any, event: JsonObject | None, target_id: object) -> None: """Append an insert-floor event once the row id is known.""" @@ -222,7 +635,16 @@ def remember_floor_event(store: Any, event: JsonObject | None, target_id: object "LabelLockOrderError", "apply_insert_floor", "merge_protected_metadata", + "REFUSED_DETAIL", + "RETRYABLE_DETAIL", + "acquire_exclusive_label_lock", + "count_rows_hidden_by_scope_move", + "label_error_response", + "propagate", + "propagate_after_write", + "raise_source_to_replacement", "remember_floor_event", + "should_propagate", "takes_label_lock", "without_insert_floor", ] diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 92fe7a7c0..e83c801f5 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -23,6 +23,7 @@ ) from alicebot_api.vnext_entity_names import ENTITY_IMMUTABLE_PATCH_FIELDS, normalize_entity_name from alicebot_api.vnext_json import json_safe +from alicebot_api.vnext_label_writes import takes_label_lock from alicebot_api.vnext_project_scope import ( expose_memory_project_scope, project_scope_identity, @@ -1016,6 +1017,7 @@ def list_sources( (domains, domains, sensitivity_allowed, sensitivity_allowed, limit), ) + @takes_label_lock def create_source(self, source: JsonObject, *, actor_type: str = "system") -> VNextRow: row = self._fetch_one( "create_source", @@ -1088,6 +1090,7 @@ def create_source(self, source: JsonObject, *, actor_type: str = "system") -> VN ) return row + @takes_label_lock def get_or_create_source( self, source: JsonObject, @@ -1238,6 +1241,7 @@ def get_source_by_dedupe_key(self, dedupe_key: str) -> VNextRow | None: (dedupe_key,), ) + @takes_label_lock def update_source(self, *, source_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: with self.conn.cursor() as cur: cur.execute( @@ -1350,8 +1354,12 @@ def update_source(self, *, source_id: str, patch: JsonObject, actor_type: str = target_id=row["id"], payload={"operation": "update", "changes": patch}, ) + from alicebot_api.vnext_label_writes import propagate_after_write + + propagate_after_write(self, kind="source", before=current, after=row) return row + @takes_label_lock def delete_source(self, *, source_id: str, actor_type: str = "system") -> VNextRow: row = self._fetch_one( "delete_source", @@ -1706,6 +1714,7 @@ def search_sources( expire_edge = _graph_expire_edge + @takes_label_lock def create_project(self, project: JsonObject, *, actor_type: str = "system") -> VNextRow: row = self._fetch_one( "create_project", @@ -1767,6 +1776,7 @@ def get_project(self, project_id: str) -> VNextRow | None: (project_id,), ) + @takes_label_lock def get_project_for_update(self, project_id: str) -> VNextRow | None: """Lock a project while an artifact review applies its state.""" @@ -1824,6 +1834,7 @@ def list_projects( ), ) + @takes_label_lock def update_project(self, *, project_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: row = self._fetch_one( "update_project", @@ -2026,7 +2037,11 @@ def update_person(self, *, person_id: str, patch: JsonObject, actor_type: str = update_open_loop_status = _graph_update_open_loop_status + @takes_label_lock def create_artifact(self, artifact: JsonObject, *, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import apply_insert_floor, remember_floor_event + + artifact, floor_event = apply_insert_floor(self, "artifact", artifact) row = self._fetch_one( "create_artifact", f""" @@ -2087,8 +2102,10 @@ def create_artifact(self, artifact: JsonObject, *, actor_type: str = "system") - target_id=row["id"], payload={"operation": "create", "artifact_type": str(row["artifact_type"])}, ) + remember_floor_event(self, floor_event, row["id"]) return row + @takes_label_lock def upsert_artifact_by_workflow_digest( self, artifact: JsonObject, @@ -2116,6 +2133,9 @@ def upsert_artifact_by_workflow_digest( ) if existing is not None: return existing + from alicebot_api.vnext_label_writes import apply_insert_floor, remember_floor_event + + artifact, floor_event = apply_insert_floor(self, "artifact", artifact) metadata_value = artifact.get("metadata_json") metadata: JsonObject = dict(metadata_value) if isinstance(metadata_value, dict) else {} metadata.update( @@ -2196,6 +2216,7 @@ def upsert_artifact_by_workflow_digest( target_id=row["id"], payload={"operation": "create", "artifact_type": str(row["artifact_type"])}, ) + remember_floor_event(self, floor_event, row["id"]) return row def get_artifact(self, artifact_id: str) -> VNextRow | None: @@ -2208,6 +2229,7 @@ def get_artifact(self, artifact_id: str) -> VNextRow | None: (artifact_id,), ) + @takes_label_lock def get_artifact_for_update(self, artifact_id: str) -> VNextRow | None: """Lock one persisted artifact before an authorized side effect.""" @@ -2339,6 +2361,7 @@ def find_artifact_by_workflow_digest( ), ) + @takes_label_lock def update_artifact_status( self, *, diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/embedding_cas.py b/apps/api/src/alicebot_api/vnext_stores/postgres/embedding_cas.py index 2113e11ca..9c41caf96 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/embedding_cas.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/embedding_cas.py @@ -15,6 +15,7 @@ from alicebot_api.vnext_recall_visibility import POSTGRES_UNEXPIRED_SQL from alicebot_api.vnext_repositories import JsonObject from alicebot_api.vnext_stores.postgres.columns import MEMORY_COLUMNS +from alicebot_api.vnext_label_writes import takes_label_lock VNextRow = dict[str, object] @@ -141,6 +142,7 @@ def _vector_literal(vector: list[float]) -> str: return "[" + ",".join(repr(value) for value in values) + "]" +@takes_label_lock def update_memory_embedding( self, *, @@ -202,6 +204,7 @@ def update_memory_embedding( ) +@takes_label_lock def clear_memory_embedding(self, *, memory_id: str) -> VNextRow | None: """Invalidate content-derived vector state before a text mutation. diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/events_revisions.py b/apps/api/src/alicebot_api/vnext_stores/postgres/events_revisions.py index 788d462ec..6597b3560 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/events_revisions.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/events_revisions.py @@ -10,6 +10,7 @@ from alicebot_api.vnext_repositories import JsonObject from alicebot_api.vnext_stores.postgres.columns import EVENT_LOG_COLUMNS, REVISION_COLUMNS from alicebot_api.vnext_stores.postgres.primitives import _json_list, _json_object, _json_safe +from alicebot_api.vnext_label_writes import takes_label_lock VNextRow = dict[str, object] @@ -282,6 +283,7 @@ def count_events( return int(cast(int, row["count"])) +@takes_label_lock def append_revision(self, revision: JsonObject, *, actor_type: str = "system") -> VNextRow: row = self._fetch_one( "append_revision", diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py b/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py index b7249c4f3..3c4faed08 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py @@ -22,6 +22,7 @@ _json_object, _sorted_field_names, ) +from alicebot_api.vnext_label_writes import takes_label_lock from alicebot_api.vnext_stores.postgres.query_predicates import ( _OPEN_LOOP_SCOPE_EVENT_TIME_SQL, _OPEN_LOOP_SCOPE_PEOPLE_SQL, @@ -807,7 +808,11 @@ def update_belief_status( ) return row +@takes_label_lock def create_open_loop(self, loop: JsonObject, *, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import apply_insert_floor, remember_floor_event + + loop, floor_event = apply_insert_floor(self, "open_loop", loop) row = self._fetch_one( "create_open_loop", f""" @@ -885,6 +890,7 @@ def create_open_loop(self, loop: JsonObject, *, actor_type: str = "system") -> V target_id=row["id"], payload={"operation": "create", "fields": _sorted_field_names(loop)}, ) + remember_floor_event(self, floor_event, row["id"]) return row def upsert_open_loop_by_automation_digest( @@ -1198,6 +1204,7 @@ def list_open_loop_events( ), ) +@takes_label_lock def update_open_loop(self, *, loop_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: row = self._fetch_one( "update_open_loop", @@ -1238,6 +1245,7 @@ def update_open_loop(self, *, loop_id: str, patch: JsonObject, actor_type: str = ) return row +@takes_label_lock def update_open_loop_status( self, *, diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py index cd069e6e4..1cb1979fa 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py @@ -204,6 +204,7 @@ def upsert_memory_by_key(self, memory: JsonObject, *, actor_type: str = "system" raise return existing +@takes_label_lock def get_memory_for_update(self, memory_id: str) -> VNextRow | None: """Load and lock one memory for a review/lifecycle decision.""" return self._fetch_optional_one( @@ -217,6 +218,7 @@ def get_memory_for_update(self, memory_id: str) -> VNextRow | None: (memory_id,), ) +@takes_label_lock def get_memory_for_redaction(self, memory_id: str) -> VNextRow | None: """Lock a redaction target even after forget archived/tombstoned it.""" @@ -230,6 +232,7 @@ def get_memory_for_redaction(self, memory_id: str) -> VNextRow | None: (memory_id,), ) +@takes_label_lock def lock_project_update_artifacts_for_redaction(self, memory_id: str) -> list[VNextRow]: """Lock every artifact coupled to a candidate memory in UUID order.""" @@ -387,6 +390,7 @@ def list_memory_ids_with_embeddings(self, ids: "Sequence[str]") -> set[str]: ) return {str(row["id"]) for row in rows} +@takes_label_lock def update_memory_fact_keys(self, *, memory_id: str, fact_keys: str | None) -> VNextRow | None: """Store derived retrieval keys; the generated ``search_tsv`` column (migration ``20260707_0082``) re-indexes them at 'D' weight. @@ -428,6 +432,7 @@ def list_memories_missing_fact_keys(self, *, limit: int = 100, after_id: str | N def update_memory( self, *, memory_id: str, patch: JsonObject, actor_type: str = "system", label_write: bool = False ) -> VNextRow: + before_label = self.get_memory(str(memory_id)) refuse_updated_credential_activation(patch, lambda: self.get_memory(str(memory_id))) if "metadata_json" in patch and isinstance(patch.get("metadata_json"), dict): current = self._fetch_optional_one( @@ -521,6 +526,10 @@ def update_memory( target_id=row["id"], payload={"operation": "update", "changes": patch}, ) + if not label_write: + from alicebot_api.vnext_label_writes import propagate_after_write + + propagate_after_write(self, kind="memory", before=before_label, after=row) return row @contextmanager @@ -541,6 +550,7 @@ def _redaction_mode(self) -> Iterator[None]: # (set_config assignments are transactional). pass +@takes_label_lock def redact_memory_bundle( self, *, @@ -914,6 +924,7 @@ def redact_memory_bundle( "idempotent_replay": not bundle_changed, } +@takes_label_lock def redact_memory_content(self, *, memory_id: str, actor_type: str = "user") -> VNextRow: """Expunge a memory's content in place, keeping the skeleton. diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/embedding_cas.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/embedding_cas.py index a60a3fdc4..633b4e057 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/embedding_cas.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/embedding_cas.py @@ -21,6 +21,7 @@ from alicebot_api.vnext_stores.sqlite.columns import MEMORY_COLUMNS from alicebot_api.vnext_stores.sqlite.query_predicates import _expiry_clause from alicebot_api.vnext_stores.sqlite.vector_scan import bump_embedding_stamp +from alicebot_api.vnext_label_writes import takes_label_lock VNextRow = dict[str, object] @@ -126,6 +127,7 @@ def _ensure_embedding_input_cut_sqlite(conn: sqlite3.Connection) -> None: ) +@takes_label_lock def update_memory_embedding( self, *, @@ -210,6 +212,7 @@ def update_memory_embedding( ) +@takes_label_lock def clear_memory_embedding(self, *, memory_id: str) -> VNextRow | None: """Invalidate an embedding derived from text that is about to change.""" if not self.conn.in_transaction: diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py index 47368f076..cc43135e1 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py @@ -30,6 +30,7 @@ _utc_now_iso, _uuid_text, ) +from alicebot_api.vnext_label_writes import takes_label_lock from alicebot_api.vnext_stores.sqlite.query_predicates import ( _project_scope_value_sqlite, CTE_MATERIALIZED_HINT, @@ -518,7 +519,11 @@ def list_relationship_events(self, entity_id: str) -> list[VNextRow]: (str(entity_id), self.user_id), ) +@takes_label_lock def create_open_loop(self, loop: JsonObject, *, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import apply_insert_floor, remember_floor_event + + loop, floor_event = apply_insert_floor(self, "open_loop", loop) loop_id = _new_id(loop.get("id")) now = _utc_now_iso() self._execute( @@ -578,6 +583,7 @@ def create_open_loop(self, loop: JsonObject, *, actor_type: str = "system") -> V target_id=row["id"], payload={"operation": "create", "fields": _sorted_field_names(loop)}, ) + remember_floor_event(self, floor_event, row["id"]) return row def upsert_open_loop_by_automation_digest( @@ -969,6 +975,7 @@ def list_open_loop_events( tuple(params), ) +@takes_label_lock def update_open_loop(self, *, loop_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: cursor = self._execute( """ @@ -1015,6 +1022,7 @@ def update_open_loop(self, *, loop_id: str, patch: JsonObject, actor_type: str = ) return row +@takes_label_lock def update_open_loop_status( self, *, diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py index 88080782c..ccff5ca4d 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py @@ -298,6 +298,7 @@ def memory_redaction_bundle_is_exact(self, memory_id: str, artifact_ids: Sequenc def update_memory( self, *, memory_id: str, patch: JsonObject, actor_type: str = "system", label_write: bool = False ) -> VNextRow: + before_label = self.get_memory(str(memory_id)) refuse_updated_credential_activation(patch, lambda: self.get_memory(str(memory_id))) patch = _with_protected_metadata(self, memory_id, patch, label_write=label_write) # One clock reading for the write: an archive sets ``updated_at`` and ``deleted_at`` together. @@ -388,6 +389,10 @@ def update_memory( target_id=row["id"], payload={"operation": "update", "changes": patch}, ) + if not label_write: + from alicebot_api.vnext_label_writes import propagate_after_write + + propagate_after_write(self, kind="memory", before=before_label, after=row) return row def lock_graph_mutation(self) -> None: @@ -424,6 +429,7 @@ def list_memory_ids_with_embeddings(self, ids: "Sequence[str]") -> set[str]: present.update(str(row["id"]) for row in rows) return present +@takes_label_lock def update_memory_fact_keys(self, *, memory_id: str, fact_keys: str | None) -> VNextRow | None: """Store derived retrieval keys; the FTS sync triggers re-index them. @@ -481,6 +487,7 @@ def _redaction_mode(self) -> Iterator[None]: finally: self._execute("UPDATE redaction_mode SET enabled = 0 WHERE id = 1") +@takes_label_lock def redact_memory_bundle( self, *, @@ -720,6 +727,7 @@ def redact_memory_bundle( "idempotent_replay": not changed, } +@takes_label_lock def redact_memory_content(self, *, memory_id: str, actor_type: str = "user") -> VNextRow: """Expunge a memory's content in place, keeping the skeleton. diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/source_retirement.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/source_retirement.py index 12c1b2d0c..f4f429915 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/source_retirement.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/source_retirement.py @@ -21,6 +21,7 @@ open_loop_source_reference_params, ) from alicebot_api.vnext_stores.sqlite.primitives import _utc_now_iso +from alicebot_api.vnext_label_writes import takes_label_lock REMOVAL_MARKER = "[removed by the owner]" @@ -395,6 +396,7 @@ def retire_dependents(self, source_id, *, now, scrub_candidates=False, citing_id 'memories_citing_replaced': retained} +@takes_label_lock def supersede_source(self, source_id, *, superseded_by, allow_looser_classification=False, dry_run=False): with self.savepoint(): old = self.get_source(source_id) @@ -413,6 +415,10 @@ def supersede_source(self, source_id, *, superseded_by, allow_looser_classificat # The sidecar goes first. A later rollback may lose proposals, which # can be generated again, but cannot leave retired evidence in it. prune_sleep_rows(self, {source_id}, dry_run=dry_run) + if not dry_run: + from alicebot_api.vnext_label_writes import raise_source_to_replacement + + raise_source_to_replacement(self, old, new) counts = retire_dependents(self, source_id, now=now) metadata = {**old['metadata_json'], 'superseded_by': superseded_by, 'superseded_at': now, 'supersede_reason': 'markdown_reimport'} @@ -476,6 +482,7 @@ def optimize_scrub_indexes(self): self._execute("INSERT INTO memories_fts(memories_fts) VALUES('optimize')") +@takes_label_lock def scrub_source(self, source_id, *, optimize=True, citing_ids=None): with self.savepoint(): self._execute("PRAGMA secure_delete=ON") diff --git a/tests/unit/test_sqlite_derived_labels_write_path.py b/tests/unit/test_sqlite_derived_labels_write_path.py index 680ef6de6..d6f90f646 100644 --- a/tests/unit/test_sqlite_derived_labels_write_path.py +++ b/tests/unit/test_sqlite_derived_labels_write_path.py @@ -103,6 +103,117 @@ def test_an_update_that_omits_a_marker_keeps_it(tmp_path: Path) -> None: assert stored["metadata_json"]["note"] == "changed" +def test_a_source_relabel_reaches_the_extracted_memory(tmp_path: Path) -> None: + path = tmp_path / "vault.sqlite3" + with _vault(path) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source( + { + "source_type": "note", + "title": "Visit", + "content_hash": "hash-propagate", + "domain": "unknown", + "sensitivity": "public", + "metadata_json": {"project_scope": [ALPHA]}, + } + ) + memory = store.create_memory( + { + "memory_key": "extracted", + "canonical_text": "A fact from the visit.", + "status": "active", + "domain": "unknown", + "sensitivity": "public", + "metadata_json": {"source_id": str(source["id"]), "project_scope": [ALPHA]}, + } + ) + store.update_source( + source_id=str(source["id"]), + patch={"domain": "health", "sensitivity": "confidential"}, + actor_type="user", + ) + stored = store.get_memory(str(memory["id"])) + assert stored is not None + assert stored["domain"] == "health" + assert stored["sensitivity"] == "confidential" + + +def test_a_candidate_loop_is_floored_from_its_source(tmp_path: Path) -> None: + path = tmp_path / "vault.sqlite3" + with _vault(path) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source( + { + "source_type": "note", + "title": "Tasks", + "content_hash": "hash-loop", + "domain": "health", + "sensitivity": "private", + "metadata_json": {"project_scope": [ALPHA]}, + } + ) + loop = store.create_open_loop( + { + "title": "Call the clinic", + "source_id": str(source["id"]), + "domain": "unknown", + "sensitivity": "public", + "metadata_json": { + "discovered_by": "vnext_daily_brief", + "source_id": str(source["id"]), + "project_scope": [ALPHA], + }, + } + ) + assert loop["domain"] == "health" + assert loop["sensitivity"] == "private" + + +def test_supersede_raises_a_citing_memory(tmp_path: Path) -> None: + path = tmp_path / "vault.sqlite3" + with _vault(path) as conn: + store = SQLiteVNextStore(conn, USER) + old = store.create_source( + { + "source_type": "markdown", + "title": "Note", + "content_hash": "hash-old", + "raw_path": "notes/one.md", + "connector_name": "markdown_folder", + "domain": "unknown", + "sensitivity": "public", + "metadata_json": {"relative_path": "notes/one.md"}, + } + ) + new = store.create_source( + { + "source_type": "markdown", + "title": "Note", + "content_hash": "hash-new", + "raw_path": "notes/one.md", + "connector_name": "markdown_folder", + "domain": "health", + "sensitivity": "confidential", + "metadata_json": {"relative_path": "notes/one.md"}, + } + ) + memory = store.create_memory( + { + "memory_key": "cited", + "canonical_text": "A fact.", + "status": "active", + "domain": "unknown", + "sensitivity": "public", + "metadata_json": {"source_id": str(old["id"])}, + } + ) + store.supersede_source(str(old["id"]), superseded_by=str(new["id"])) + stored = store.get_memory(str(memory["id"])) + assert stored is not None + assert stored["domain"] == "health" + assert stored["sensitivity"] == "confidential" + + def test_merge_protected_metadata_keeps_label_keys_unless_the_write_is_a_relabel() -> None: stored = { "consolidation": {"cluster_member_ids": ["m"]}, diff --git a/tests/unit/test_store_events_revisions_split.py b/tests/unit/test_store_events_revisions_split.py index 2ca60ca6d..dde8c9474 100644 --- a/tests/unit/test_store_events_revisions_split.py +++ b/tests/unit/test_store_events_revisions_split.py @@ -50,7 +50,7 @@ "list_events_for_source_trace": "20f22e3b75c3612c02c4242bc973295b2535b01f95e5451a0a0bff1196f187c3", "list_project_update_events": "2bd457ee19535f203da5c31e557387f7717fefc27cbef2c546a62bbad74962d3", "count_events": "e740e4b09ecfda973ef6b84acd0ea808b26d118faf6984e4057d7e104e595fb5", - "append_revision": "a243976fc27fa6e7ae33c15169d407902e3258b842300df4705629e443a75740", + "append_revision": "6070b96a01a50072e4e898bfb64ddbbf253be0478fe63205f760d0fe016a109a", "list_revisions": "10a485935b59bda1fcb33b36ba48b8d86376b9fd180bf9ebf6358b5dfbd24f55", }, "sqlite": { @@ -148,14 +148,15 @@ # Re-minted for the per-file importer savepoint (2026-10-02): ``savepoint`` is appended last on both façades. # Previous receipt: 650e2e0ff088d67c... Proof: the class key list equals the list at origin/main 040a2a10 # plus ``savepoint`` before ``__dict__``, with every other key in the same order. - "postgres": "3d2cd1d2fbb766ea4b5f9bf700fe6c0ccdc5fea9705092f82ead06c3ab635e50", + "postgres": "dda7be1a316b3c525195f4682a608d0bab24b57f635db2a7aebc0c7ea187fc68", # Re-minted for the merge of #500 and #502: ``check_literal_match_query``. # Re-minted again for per-project memory S2 (2026-10-02): the two single-scan partition reads # ``list_memories_view_partitions`` and ``list_open_loops_view_partitions``, SQLite only on purpose # (the Postgres runtime resolves no project view). # Re-minted again for the per-file importer savepoint (2026-10-02), the same appended method. # Previous receipt: 365b7a01acf7a8b5... Proof: as for postgres, one added key and no other change. - "sqlite": "65301a20344d20bd6e20e6717f1f3db3fd4b16611964425d1d3f98736326f393", + # Re-minted for the derived-label lock: ``lock_label_writes`` and ``read_label_rows`` on both facades. + "sqlite": "3212a93f2011cecddb6b0002a3f9b3abe37819d3582b2a1173846871981a5ff9", } EXPECTED_SUPPORT_AST_SHA256 = { "postgres_columns": { From b192f9d927c2a13b892532b8b7f58e693c41af92 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 17:51:41 +0000 Subject: [PATCH 004/270] Filter locked report inputs by every project. A key bound to a project builds a brief, connection report, contradiction report, or project update only from rows whose scope and floor are both inside its binding. --- apps/api/src/alicebot_api/vnext_brain.py | 26 ++++++-- .../api/src/alicebot_api/vnext_connections.py | 18 +++++- .../src/alicebot_api/vnext_contradictions.py | 22 +++++-- .../src/alicebot_api/vnext_derived_labels.py | 51 +++++++++++++++ apps/api/src/alicebot_api/vnext_projects.py | 21 +++++-- tests/unit/test_derived_label_input_filter.py | 63 +++++++++++++++++++ 6 files changed, 185 insertions(+), 16 deletions(-) create mode 100644 tests/unit/test_derived_label_input_filter.py diff --git a/apps/api/src/alicebot_api/vnext_brain.py b/apps/api/src/alicebot_api/vnext_brain.py index 2687e99ad..585d0e0ec 100644 --- a/apps/api/src/alicebot_api/vnext_brain.py +++ b/apps/api/src/alicebot_api/vnext_brain.py @@ -237,11 +237,18 @@ def _matches_report_scope( projects: tuple[str, ...], window_start: datetime, window_end: datetime, + all_of: tuple[str, ...] | None = None, ) -> bool: if projects: - row_scope = source_project_scope(row) if kind == "source" else resource_project_scope(row) - if not project_scopes_overlap(row_scope, projects): - return False + if all_of is not None: + from alicebot_api.vnext_derived_labels import input_admitted + + if not input_admitted(kind, row, all_of): + return False + else: + row_scope = source_project_scope(row) if kind == "source" else resource_project_scope(row) + if not project_scopes_overlap(row_scope, projects): + return False event_time = _row_event_time(row, kind=kind) return event_time is not None and window_start <= event_time < window_end @@ -267,6 +274,7 @@ def _windowed_rows( limit: int, store_scope_kwargs: dict[str, object] | None = None, store_scope_complete: bool = False, + all_of: tuple[str, ...] | None = None, ) -> list[JsonObject]: scope_kwargs = store_scope_kwargs or {} @@ -284,6 +292,7 @@ def select(rows: Sequence[JsonObject]) -> list[JsonObject]: projects=projects, window_start=window_start, window_end=window_end, + all_of=all_of, ): selected.append(_compact_row(row)) return selected @@ -872,6 +881,9 @@ def _load_inputs( ) -> tuple[list[JsonObject], list[JsonObject], list[JsonObject], list[JsonObject]]: domains = _allowed_domains(request) sensitivity_allowed = _allowed_sensitivity(request) + from alicebot_api.vnext_derived_labels import locked_projects + + all_of = locked_projects(request.agent_identity, request.projects) inclusive_window_end = window_end - timedelta(microseconds=1) source_scope_names = ("scope_projects", "scope_window_start", "scope_window_end") source_scope_supported = _supports_parameters(self.store.search_sources, source_scope_names) @@ -894,7 +906,8 @@ def _load_inputs( } if source_scope_supported else None, - store_scope_complete=source_scope_supported, + store_scope_complete=source_scope_supported and all_of is None, + all_of=all_of, ) memory_store_scope: dict[str, object] | None = ( {"projects": request.projects} if _supports_parameters(self.store.search_memories, ("projects",)) else None @@ -912,6 +925,7 @@ def _load_inputs( window_end=window_end, limit=request.memory_limit, store_scope_kwargs=memory_store_scope, + all_of=all_of, ) open_loop_scope_names = ("scope_projects", "scope_window_start", "scope_window_end") open_loop_scope_supported = _supports_parameters( @@ -937,7 +951,8 @@ def _load_inputs( } if open_loop_scope_supported else None, - store_scope_complete=open_loop_scope_supported, + store_scope_complete=open_loop_scope_supported and all_of is None, + all_of=all_of, ) artifact_store_scope: dict[str, object] | None = ( {"scope_projects": request.projects} @@ -957,6 +972,7 @@ def _load_inputs( window_end=window_end, limit=request.artifact_limit, store_scope_kwargs=artifact_store_scope, + all_of=all_of, ) return sources, memories, open_loops, artifacts diff --git a/apps/api/src/alicebot_api/vnext_connections.py b/apps/api/src/alicebot_api/vnext_connections.py index b62a5a1f3..5119016de 100644 --- a/apps/api/src/alicebot_api/vnext_connections.py +++ b/apps/api/src/alicebot_api/vnext_connections.py @@ -195,9 +195,15 @@ def _supports_parameter(method: object, name: str) -> bool: return False -def _matches_projects(row: JsonObject, projects: tuple[str, ...], *, source_row: bool) -> bool: +def _matches_projects( + row: JsonObject, projects: tuple[str, ...], *, source_row: bool, all_of: tuple[str, ...] | None = None +) -> bool: if not projects: return True + if all_of is not None: + from alicebot_api.vnext_derived_labels import input_admitted + + return input_admitted("source" if source_row else "memory", row, all_of) row_scope = source_project_scope(row) if source_row else resource_project_scope(row) return project_scopes_overlap(row_scope, projects) @@ -210,16 +216,17 @@ def _project_scoped_search( project_parameter: str, limit: int, source_rows: bool = False, + all_of: tuple[str, ...] | None = None, ) -> list[JsonObject]: if not projects: return list(method(limit=limit, **kwargs)) if _supports_parameter(method, project_parameter): rows = method(limit=limit, **kwargs, **{project_parameter: projects}) - return [row for row in rows if _matches_projects(row, projects, source_row=source_rows)] + return [row for row in rows if _matches_projects(row, projects, source_row=source_rows, all_of=all_of)] rows = list(method(limit=MAX_LEGACY_PROJECT_SCOPE_ROWS + 1, **kwargs)) if len(rows) > MAX_LEGACY_PROJECT_SCOPE_ROWS: raise VNextConnectionValidationError("legacy connection store could not prove complete project scope") - return [row for row in rows if _matches_projects(row, projects, source_row=source_rows)][:limit] + return [row for row in rows if _matches_projects(row, projects, source_row=source_rows, all_of=all_of)][:limit] def _record_text(row: JsonObject) -> str: @@ -408,6 +415,9 @@ def generate_connection_report(self, request: ConnectionFinderRequest | None = N domains = list(request.domains) if request.domains else None sensitivity_allowed = list(request.sensitivity_allowed) input_limit = max(request.max_connections * 2, request.max_connections) + from alicebot_api.vnext_derived_labels import locked_projects + + all_of = locked_projects(request.agent_identity, request.projects) sources = _project_scoped_search( self.store.search_sources, kwargs={ @@ -419,6 +429,7 @@ def generate_connection_report(self, request: ConnectionFinderRequest | None = N project_parameter="scope_projects", limit=input_limit, source_rows=True, + all_of=all_of, ) memories = _project_scoped_search( self.store.search_memories, @@ -430,6 +441,7 @@ def generate_connection_report(self, request: ConnectionFinderRequest | None = N projects=request.projects, project_parameter="projects", limit=input_limit, + all_of=all_of, ) candidates = _find_candidates( sources=sources, diff --git a/apps/api/src/alicebot_api/vnext_contradictions.py b/apps/api/src/alicebot_api/vnext_contradictions.py index b822fad95..ab1c89572 100644 --- a/apps/api/src/alicebot_api/vnext_contradictions.py +++ b/apps/api/src/alicebot_api/vnext_contradictions.py @@ -214,9 +214,15 @@ def _supports_parameter(method: object, name: str) -> bool: return False -def _matches_projects(row: JsonObject, projects: tuple[str, ...], *, source_row: bool) -> bool: +def _matches_projects( + row: JsonObject, projects: tuple[str, ...], *, source_row: bool, all_of: tuple[str, ...] | None = None +) -> bool: if not projects: return True + if all_of is not None: + from alicebot_api.vnext_derived_labels import input_admitted + + return input_admitted("source" if source_row else "memory", row, all_of) row_scope = source_project_scope(row) if source_row else resource_project_scope(row) return project_scopes_overlap(row_scope, projects) @@ -229,16 +235,17 @@ def _project_scoped_search( project_parameter: str, limit: int, source_rows: bool = False, + all_of: tuple[str, ...] | None = None, ) -> list[JsonObject]: if not projects: return list(method(limit=limit, **kwargs)) if _supports_parameter(method, project_parameter): rows = method(limit=limit, **kwargs, **{project_parameter: projects}) - return [row for row in rows if _matches_projects(row, projects, source_row=source_rows)] + return [row for row in rows if _matches_projects(row, projects, source_row=source_rows, all_of=all_of)] rows = list(method(limit=MAX_LEGACY_PROJECT_SCOPE_ROWS + 1, **kwargs)) if len(rows) > MAX_LEGACY_PROJECT_SCOPE_ROWS: raise VNextContradictionValidationError("legacy contradiction store could not prove complete project scope") - return [row for row in rows if _matches_projects(row, projects, source_row=source_rows)][:limit] + return [row for row in rows if _matches_projects(row, projects, source_row=source_rows, all_of=all_of)][:limit] def _project_scoped_beliefs( @@ -248,6 +255,7 @@ def _project_scoped_beliefs( sensitivity_allowed: list[str], projects: tuple[str, ...], limit: int, + all_of: tuple[str, ...] | None = None, ) -> list[JsonObject]: if not projects: return list( @@ -295,7 +303,7 @@ def _project_scoped_beliefs( belief for belief in rows if (backing := backing_by_id.get(str(belief.get("memory_id") or ""))) is not None - and _matches_projects(backing, projects, source_row=False) + and _matches_projects(backing, projects, source_row=False, all_of=all_of) ][:limit] @@ -433,6 +441,9 @@ def generate_contradiction_report(self, request: ContradictionFinderRequest | No domains = list(request.domains) if request.domains else None sensitivity_allowed = list(request.sensitivity_allowed) input_limit = max(request.max_contradictions * 2, request.max_contradictions) + from alicebot_api.vnext_derived_labels import locked_projects + + all_of = locked_projects(request.agent_identity, request.projects) sources = _project_scoped_search( self.store.search_sources, kwargs={ @@ -444,6 +455,7 @@ def generate_contradiction_report(self, request: ContradictionFinderRequest | No project_parameter="scope_projects", limit=input_limit, source_rows=True, + all_of=all_of, ) memories = [ memory @@ -457,6 +469,7 @@ def generate_contradiction_report(self, request: ContradictionFinderRequest | No projects=request.projects, project_parameter="projects", limit=input_limit, + all_of=all_of, ) if memory.get("memory_type") not in {"belief", "thesis"} ] @@ -466,6 +479,7 @@ def generate_contradiction_report(self, request: ContradictionFinderRequest | No sensitivity_allowed=sensitivity_allowed, projects=request.projects, limit=input_limit, + all_of=all_of, ) candidates = _find_candidates( new_items=[*sources, *memories], diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index ed0558aa7..9069bea03 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -1199,6 +1199,54 @@ def _iterate( queued.add(dependant) +def locked_projects(agent_identity: object, requested: object) -> tuple[str, ...] | None: + """The projects a locked key may read, or None when the caller is not locked.""" + + if not isinstance(agent_identity, Mapping) or not agent_identity.get("project_scope_locked"): + return None + binding = agent_identity.get("project_scope") or () + requested_values = tuple(requested) if isinstance(requested, (list, tuple)) else () + if requested_values: + return tuple(str(item) for item in requested_values) + if isinstance(binding, (list, tuple)): + return tuple(str(item) for item in binding) + return () + + +def input_admitted(kind: str, row: Mapping[str, object], projects: object) -> bool: + """Exact-door project test: scope and floor are both inside ``projects``.""" + + if canon_kind(kind) == "source": + scope = source_project_scope(row) + else: + scope = resolve_project_scope(row).values + shape, floor = project_floor_shape(row) + if shape == "malformed": + return False + bound = set(project_scope_identity(projects)) + scope_ids = set(project_scope_identity(scope)) + if not scope_ids or not scope_ids <= bound: + return False + return set(project_scope_identity(floor)) <= bound + + +def stamp_derived_from(payload: dict[str, object], rows_by_kind: Mapping[str, object]) -> None: + """Write the canonical dependency record onto ``payload['metadata_json']``.""" + + metadata = payload.get("metadata_json") + meta = dict(metadata) if isinstance(metadata, Mapping) else {} + record: dict[str, object] = {"v": 1} + counts: dict[str, int] = {} + for key in ("sources", "memories", "open_loops", "artifacts", "beliefs"): + rows = rows_by_kind.get(key) or [] + ids = [str(row.get("id")) for row in rows if isinstance(row, Mapping) and row.get("id") is not None] + record[key] = ids + counts[key] = len(ids) + record["counts"] = counts + meta["derived_from"] = record + payload["metadata_json"] = meta + + def scope_is_global(scope: object) -> bool: """True when a scope holds no Alice project id.""" @@ -1225,12 +1273,15 @@ def scope_is_global(scope: object) -> bool: "group_scope", "identifier", "infer_kind", + "input_admitted", "intersect_scope", "is_derived", + "locked_projects", "labels_raised_payload", "ordered_identifiers", "row_class", "scope_is_global", + "stamp_derived_from", "settle_labels", "stored_scope", "union_floor", diff --git a/apps/api/src/alicebot_api/vnext_projects.py b/apps/api/src/alicebot_api/vnext_projects.py index b34b1a88b..87dfdb736 100644 --- a/apps/api/src/alicebot_api/vnext_projects.py +++ b/apps/api/src/alicebot_api/vnext_projects.py @@ -551,10 +551,23 @@ def generate_project_update_candidate(self, request: ProjectAutomationRequest | # defensive check at the workflow boundary so legacy adapters cannot # widen a project-scoped report by ignoring optional query arguments. project_id = str(project["id"]) - sources = [row for row in sources if _is_source_in_project(row, project_id)] - memories = [ - row for row in memories if _is_in_project(row, project_id) and row.get("status") in {"active", "accepted"} - ] + from alicebot_api.vnext_derived_labels import input_admitted, locked_projects + + locked = locked_projects(request.agent_identity, (project_id,)) + if locked is not None: + sources = [row for row in sources if input_admitted("source", row, locked)] + memories = [ + row + for row in memories + if input_admitted("memory", row, locked) and row.get("status") in {"active", "accepted"} + ] + else: + sources = [row for row in sources if _is_source_in_project(row, project_id)] + memories = [ + row + for row in memories + if _is_in_project(row, project_id) and row.get("status") in {"active", "accepted"} + ] brain_charter = _brain_charter(self.store) automation_digest = _project_automation_digest( project=project, diff --git a/tests/unit/test_derived_label_input_filter.py b/tests/unit/test_derived_label_input_filter.py new file mode 100644 index 000000000..62d314c2a --- /dev/null +++ b/tests/unit/test_derived_label_input_filter.py @@ -0,0 +1,63 @@ +"""A locked key's report inputs use the exact project test.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from alicebot_api.vnext_brain import _matches_report_scope +from alicebot_api.vnext_derived_labels import input_admitted, locked_projects, stamp_derived_from + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +def test_input_admitted_requires_every_project() -> None: + alpha = {"metadata_json": {"project_scope": [ALPHA]}} + shared = {"metadata_json": {"project_scope": [ALPHA, BETA]}} + global_row = {"metadata_json": {}} + assert input_admitted("memory", alpha, (ALPHA,)) is True + assert input_admitted("memory", shared, (ALPHA,)) is False + assert input_admitted("memory", global_row, (ALPHA,)) is False + assert input_admitted("memory", alpha, (ALPHA, BETA)) is True + + +def test_locked_projects_uses_the_binding_when_the_request_is_empty() -> None: + identity = {"project_scope_locked": True, "project_scope": [ALPHA]} + assert locked_projects(identity, ()) == (ALPHA,) + assert locked_projects(identity, (ALPHA,)) == (ALPHA,) + assert locked_projects({"project_scope_locked": False}, (ALPHA,)) is None + assert locked_projects(None, (ALPHA,)) is None + + +def test_a_locked_brief_scope_rejects_a_shared_row() -> None: + start = datetime(2026, 10, 5, tzinfo=UTC) + end = start + timedelta(days=1) + shared = { + "id": "m", + "metadata_json": {"project_scope": [ALPHA, BETA]}, + "created_at": start.isoformat(), + } + assert _matches_report_scope( + shared, kind="memory", projects=(ALPHA,), window_start=start, window_end=end + ) is True + assert _matches_report_scope( + shared, kind="memory", projects=(ALPHA,), window_start=start, window_end=end, all_of=(ALPHA,) + ) is False + + +def test_stamp_derived_from_counts_match_the_lists() -> None: + payload: dict[str, object] = {"metadata_json": {"workflow": "daily_brief"}} + stamp_derived_from( + payload, + { + "sources": [{"id": "s"}], + "memories": [{"id": "m"}], + "open_loops": [], + "artifacts": [], + "beliefs": [], + }, + ) + record = payload["metadata_json"]["derived_from"] + assert record["sources"] == ["s"] + assert record["counts"]["sources"] == 1 + assert record["counts"]["memories"] == 1 From 6c547966edb979d31503f7942ad77df1be16967f Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 18:16:19 +0000 Subject: [PATCH 005/270] Record group scope and derived_from on producer rows. Consolidation and roll-ups overlap scope united with floor, and a locked run applies the exact project test after that overlap check. Roll-up lookups and the operator artifact list match the floor. Each producer writes derived_from for the rows it used. --- CHANGELOG.md | 1 + .../src/alicebot_api/routers/vnext_review.py | 14 +- apps/api/src/alicebot_api/vnext_brain.py | 17 + .../api/src/alicebot_api/vnext_connections.py | 2 + apps/api/src/alicebot_api/vnext_connectors.py | 54 ++-- .../src/alicebot_api/vnext_consolidation.py | 76 +++-- .../src/alicebot_api/vnext_contradictions.py | 5 + .../src/alicebot_api/vnext_derived_labels.py | 24 ++ .../src/alicebot_api/vnext_memory_commit.py | 7 +- apps/api/src/alicebot_api/vnext_projects.py | 100 +++--- apps/api/src/alicebot_api/vnext_queue.py | 14 +- apps/api/src/alicebot_api/vnext_rollups.py | 85 +++-- apps/api/src/alicebot_api/vnext_scheduler.py | 42 ++- apps/api/src/alicebot_api/vnext_store.py | 7 +- .../vnext_stores/postgres/memory_access.py | 5 +- .../vnext_stores/postgres/query_predicates.py | 58 ++++ .../vnext_stores/sqlite/memory_access.py | 24 +- .../vnext_stores/sqlite/query_predicates.py | 30 +- tests/unit/test_group_scope_consumers.py | 302 ++++++++++++++++++ tests/unit/test_store_memory_access_split.py | 29 +- tests/unit/test_vnext_brain.py | 5 + 21 files changed, 746 insertions(+), 155 deletions(-) create mode 100644 tests/unit/test_group_scope_consumers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3612de93b..7be7ec35d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Unreleased (on main, not in v0.20.0): a locked key's consolidation, roll-up, staleness sweep and open-loop review keep only inputs whose scope and floor are both inside the binding. The overlap check stays, and the exact test runs after it. Consolidation and roll-ups compare the group scope (scope united with floor), so a card stored with an empty scope and a floor of two projects is still found and can be accepted. The operator artifact list, given a project, matches that floor as well as the scope. Each new report and aggregate records `derived_from` for the rows it used. v0.20.0 selected by overlap of the scope alone and wrote no `derived_from`. No migration is required. - Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. Raising a source or a memory walks the rows derived from it and raises those too. A candidate open loop and a generated artifact take the same insert floor. Replacing a SQLite source with a stricter one raises the memories that cite it before they are retired. Moving a source to another project answers with how many derived rows a project-bound key would lose, and waits for confirm_label_hide before it writes. A relabel that cannot finish answers 409, and one that waits more than 3 seconds for the label lock answers 503. No migration is required. - Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. The function is what the write path calls. A read that does not go through that path still behaves as it does in v0.20.0. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` now scrub an open loop that names the source only in its metadata: the id, or `source:`, as the text under `source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references` or `selected_source_ids` at any depth. That is the rule the open-loop lookup of a source already used. They blanked only loops whose `source_id` column held the id, so a loop with an empty column kept its title, description and metadata, and they reached a new export. The delete preview now counts the same loops as the receipt. The rule reads the id only as stored or as `source:`: a loop that names the source in any spelling other than those two keeps its text, although the memory rule reads several other spellings. No migration is required. diff --git a/apps/api/src/alicebot_api/routers/vnext_review.py b/apps/api/src/alicebot_api/routers/vnext_review.py index cd0a9adb1..73983e345 100644 --- a/apps/api/src/alicebot_api/routers/vnext_review.py +++ b/apps/api/src/alicebot_api/routers/vnext_review.py @@ -566,11 +566,21 @@ def process_next_vnext_queue_task(request: VNextQueueProcessNextRequest) -> JSON ) @review_router.get("/v0/vnext/artifacts") -def list_vnext_artifacts(user_id: UUID, artifact_type: str | None = None, limit: int = 30) -> JSONResponse: +def list_vnext_artifacts( + user_id: UUID, + artifact_type: str | None = None, + limit: int = 30, + project: str | None = None, +) -> JSONResponse: settings = get_settings() + scope_projects = (project,) if isinstance(project, str) and project.strip() else () with user_connection(settings.database_url, user_id) as conn: - payload = PostgresVNextStore(conn).list_artifacts(artifact_type=artifact_type, limit=limit) + payload = PostgresVNextStore(conn).list_artifacts( + artifact_type=artifact_type, + limit=limit, + scope_projects=scope_projects, + ) return JSONResponse( status_code=200, diff --git a/apps/api/src/alicebot_api/vnext_brain.py b/apps/api/src/alicebot_api/vnext_brain.py index 585d0e0ec..3f70f0425 100644 --- a/apps/api/src/alicebot_api/vnext_brain.py +++ b/apps/api/src/alicebot_api/vnext_brain.py @@ -9,6 +9,7 @@ from typing import Callable, Protocol, Sequence, cast from alicebot_api.vnext_derived_domain import derived_domain +from alicebot_api.vnext_derived_labels import with_derived_from from alicebot_api.vnext_agent_control import resource_project_scope from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_model_intelligence import ( @@ -577,6 +578,10 @@ def generate_daily_brief(self, request: BrainArtifactRequest | None = None) -> J prompt_hash = model_artifact.prompt_hash model_info_json = model_artifact.model_info metadata = {**metadata, **model_artifact.metadata} + metadata = with_derived_from( + metadata, + {"sources": sources, "memories": memories, "open_loops": open_loops, "artifacts": artifacts}, + ) artifact_payload: JsonObject = { "artifact_type": "daily_brief", "title": f"Daily Brief - {day.isoformat()}", @@ -768,6 +773,10 @@ def generate_weekly_synthesis(self, request: BrainArtifactRequest | None = None) prompt_hash = model_artifact.prompt_hash model_info_json = model_artifact.model_info metadata = {**metadata, **model_artifact.metadata} + metadata = with_derived_from( + metadata, + {"sources": sources, "memories": memories, "open_loops": open_loops, "artifacts": artifacts}, + ) artifact_payload: JsonObject = { "artifact_type": "weekly_synthesis", "title": f"Weekly Synthesis - {week_label}", @@ -1016,6 +1025,10 @@ def _create_candidate_open_loops( "workflow_digest": workflow_digest, }, } + loop_payload["metadata_json"] = with_derived_from( + cast(dict, loop_payload["metadata_json"]), + {"sources": [source]}, + ) if callable(upsert_open_loop): loop = cast(Callable[..., JsonObject], upsert_open_loop)( loop_payload, @@ -1072,6 +1085,10 @@ def _create_weekly_candidate_memories( "workflow_digest": workflow_digest, }, } + memory_payload["metadata_json"] = with_derived_from( + cast(dict, memory_payload["metadata_json"]), + {"sources": sources, "memories": memories, "open_loops": open_loops}, + ) upsert_memory = getattr(self.store, "upsert_memory_by_key", None) if callable(upsert_memory): memory = cast(Callable[..., JsonObject], upsert_memory)( diff --git a/apps/api/src/alicebot_api/vnext_connections.py b/apps/api/src/alicebot_api/vnext_connections.py index 5119016de..6efa104ae 100644 --- a/apps/api/src/alicebot_api/vnext_connections.py +++ b/apps/api/src/alicebot_api/vnext_connections.py @@ -7,6 +7,7 @@ from typing import Callable, Protocol, Sequence, cast from alicebot_api.vnext_derived_domain import derived_domain +from alicebot_api.vnext_derived_labels import with_derived_from from alicebot_api.vnext_agent_control import resource_project_scope from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_model_intelligence import ( @@ -628,6 +629,7 @@ def generate_connection_report(self, request: ConnectionFinderRequest | None = N prompt_hash = model_artifact.prompt_hash model_info_json = model_artifact.model_info metadata = {**metadata, **model_artifact.metadata} + metadata = with_derived_from(metadata, {"sources": sources, "memories": memories}) artifact_payload: JsonObject = { "artifact_type": "connection_report", "title": "Connection Report", diff --git a/apps/api/src/alicebot_api/vnext_connectors.py b/apps/api/src/alicebot_api/vnext_connectors.py index 9165a2f09..c0b2e49b5 100644 --- a/apps/api/src/alicebot_api/vnext_connectors.py +++ b/apps/api/src/alicebot_api/vnext_connectors.py @@ -28,6 +28,7 @@ ) from alicebot_api.vnext_embeddings import DeferredMemoryEmbedding from alicebot_api.vnext_event_log import append_event +from alicebot_api.vnext_derived_labels import with_derived_from from alicebot_api.vnext_project_scope import resolve_project_scope from alicebot_api.vnext_repositories import JsonObject from alicebot_api.vnext_secrets import ( @@ -1774,17 +1775,20 @@ def ingest_agent_output( "domain": _as_optional_text(payload.get("domain")) or "project", "sensitivity": _as_optional_text(payload.get("sensitivity")) or "private", "generated_by": agent_id, - "metadata_json": { - "connector_name": "agent_output", - "agent_identity": agent_identity, - "agent_id": agent_id, - "agent_run_id": item.metadata_json.get("agent_run_id"), - "project_scope": item.metadata_json.get("project_scope") or [], - "source_id": source_id, - "source_refs": [f"source:{source_id}"] if source_id else [], - "output_type": _as_optional_text(payload.get("output_type")) or "general", - "review_status": "needs_review", - }, + "metadata_json": with_derived_from( + { + "connector_name": "agent_output", + "agent_identity": agent_identity, + "agent_id": agent_id, + "agent_run_id": item.metadata_json.get("agent_run_id"), + "project_scope": item.metadata_json.get("project_scope") or [], + "source_id": source_id, + "source_refs": [f"source:{source_id}"] if source_id else [], + "output_type": _as_optional_text(payload.get("output_type")) or "general", + "review_status": "needs_review", + }, + {"sources": [{"id": source_id}] if source_id else []}, + ), }, actor_type="agent", ) @@ -1836,17 +1840,23 @@ def ingest_agent_output( "project_id": proposal_scope[0] if len(proposal_scope) == 1 else None, "created_by_agent_id": agent_id, "run_id": item.metadata_json.get("agent_run_id"), - "metadata_json": { - "connector_name": "agent_output", - "agent_identity": agent_identity, - "agent_id": agent_id, - "agent_run_id": item.metadata_json.get("agent_run_id"), - "source_id": source_id, - "artifact_id": artifact_id, - "review_required": True, - "policy_decision": policy_decision, - **({"project_scope": list(proposal_scope)} if proposal_scope else {}), - }, + "metadata_json": with_derived_from( + { + "connector_name": "agent_output", + "agent_identity": agent_identity, + "agent_id": agent_id, + "agent_run_id": item.metadata_json.get("agent_run_id"), + "source_id": source_id, + "artifact_id": artifact_id, + "review_required": True, + "policy_decision": policy_decision, + **({"project_scope": list(proposal_scope)} if proposal_scope else {}), + }, + { + "sources": [{"id": source_id}] if source_id else [], + "artifacts": [{"id": artifact_id}] if artifact_id else [], + }, + ), }, actor_type="agent", ) diff --git a/apps/api/src/alicebot_api/vnext_consolidation.py b/apps/api/src/alicebot_api/vnext_consolidation.py index 4c4cc8975..7c1922800 100644 --- a/apps/api/src/alicebot_api/vnext_consolidation.py +++ b/apps/api/src/alicebot_api/vnext_consolidation.py @@ -43,6 +43,12 @@ import numpy as np from alicebot_api.vnext_derived_domain import derived_domain +from alicebot_api.vnext_derived_labels import ( + admit_when_locked, + group_scope, + locked_projects, + with_derived_from, +) from alicebot_api.vnext_embeddings import ( MAX_EMBEDDINGS_BATCH_SIZE, EmbeddingProvider, @@ -450,24 +456,23 @@ def _scoped_rows( continue if projects: allowed_projects = set(project_scope_identity(projects)) - if not allowed_projects.intersection( - project_scope_identity(resource_project_scope(row)) - ): + if not allowed_projects.intersection(group_scope(row)): continue scoped.append(row) return scoped def _project_scope_key(row: JsonObject) -> tuple[str, ...]: - """Exact normalized scope identity used for safe consolidation groups. + """Exact group scope used for safe consolidation groups. Overlap is insufficient here: merging a memory scoped to A+B with one scoped only to A would widen B-only information into project A. Candidate members must therefore carry the same scope set (including the empty - global scope). + global scope). A derived row uses its scope united with its floor, which + equals the scope for an original row. """ - return project_scope_identity(resource_project_scope(row)) + return group_scope(row) def _shared_project_scope(rows: list[JsonObject]) -> tuple[str, ...]: @@ -663,6 +668,7 @@ def _cluster_memories( sensitivity: list[str], projects: tuple[str, ...], options: _ClusteringOptions, + all_of: tuple[str, ...] | None = None, ) -> _ClusteringOutcome: outcome = _ClusteringOutcome() count_memories = getattr(self.store, "count_memories", None) @@ -755,6 +761,10 @@ def _count(status: str) -> int: outcome.active_count, ) active_rows = active_rows[: options.max_embedded_memories] + if all_of is not None: + active_rows = admit_when_locked("memory", active_rows, all_of) + outcome.active_count = len(active_rows) + outcome.active_count_exact = not outcome.bounded outcome.corpus_digest = _digest_payload( { "memory_versions": [ @@ -1027,24 +1037,27 @@ def _create_proposal_candidate( "sensitivity": _highest_sensitivity(members), "project_id": project_scope[0] if len(project_scope) == 1 else None, "source_event_ids": proposal["source_event_ids"], - "metadata_json": { - "candidate_kind": "memory_consolidation", - "consolidation_digest": cluster_digest, - "source_refs": proposal["source_refs"], - "project_scope": list(project_scope), - "review_required": True, - "consolidation": { - "cluster_member_ids": member_ids, - "member_snapshots": proposal["member_snapshots"], - "similarity_stats": proposal["similarity_stats"], - "proposal_kind": proposal_kind, - "model_provenance": proposal["model_provenance"], - "survivor_memory_id": proposal["survivor_memory_id"], - "proposed_supersede": proposal["proposed_supersede"], - "merge_refusal": proposal["merge_refusal"], - "reviewer_instructions": reviewer_instructions, + "metadata_json": with_derived_from( + { + "candidate_kind": "memory_consolidation", + "consolidation_digest": cluster_digest, + "source_refs": proposal["source_refs"], + "project_scope": list(project_scope), + "review_required": True, + "consolidation": { + "cluster_member_ids": member_ids, + "member_snapshots": proposal["member_snapshots"], + "similarity_stats": proposal["similarity_stats"], + "proposal_kind": proposal_kind, + "model_provenance": proposal["model_provenance"], + "survivor_memory_id": proposal["survivor_memory_id"], + "proposed_supersede": proposal["proposed_supersede"], + "merge_refusal": proposal["merge_refusal"], + "reviewer_instructions": reviewer_instructions, + }, }, - }, + {"memories": members}, + ), }, actor_type=request.generated_by, ) @@ -1097,6 +1110,7 @@ def generate_memory_consolidation(self, request: MemoryConsolidationRequest | No domains = _allowed_domains(request) sensitivity = _allowed_sensitivity(request) projects = _allowed_projects(request) + all_of = locked_projects(request.agent_identity, projects) if projects: list_memory_events = getattr(self.store, "list_memory_events", None) @@ -1169,12 +1183,16 @@ def generate_memory_consolidation(self, request: MemoryConsolidationRequest | No sensitivity_allowed=sensitivity, projects=projects, ) + events = admit_when_locked("memory", events, all_of) + ratings = admit_when_locked("memory", ratings, all_of) + artifacts = admit_when_locked("artifact", artifacts, all_of) clustering = self._cluster_memories( domains=domains, sensitivity=sensitivity, projects=projects, options=options, + all_of=all_of, ) cluster_membership = [ sorted(str(row.get("id")) for row in members) for members in clustering.clusters @@ -1340,6 +1358,7 @@ def generate_memory_consolidation(self, request: MemoryConsolidationRequest | No generation_mode=request.generation_mode, route=route, model_temperature=request.model_temperature, + agent_identity=request.agent_identity, # Near-duplicate clusters belong to the dedup/merge proposals # above; the roll-up pass must not re-propose those groups. exclude_member_id_sets=[ @@ -1460,6 +1479,17 @@ def generate_memory_consolidation(self, request: MemoryConsolidationRequest | No metadata = {**metadata, **model_artifact.metadata} all_cluster_rows = [row for members in clustering.clusters for row in members] + metadata = with_derived_from( + metadata, + { + "sources": named_sources, + "memories": [ + *all_cluster_rows, + *(rollups.input_rows if rollups is not None else []), + ], + "artifacts": artifacts, + }, + ) # The report is read behind its domain and sensitivity, so both are taken over every row it names: the # near-duplicate cluster members, every row the roll-up pass names, and the sources that the refs it prints # name (read above, after the run's own fence chose the members the refs are copied from). A run whose only diff --git a/apps/api/src/alicebot_api/vnext_contradictions.py b/apps/api/src/alicebot_api/vnext_contradictions.py index ab1c89572..9e9274ad7 100644 --- a/apps/api/src/alicebot_api/vnext_contradictions.py +++ b/apps/api/src/alicebot_api/vnext_contradictions.py @@ -6,6 +6,7 @@ from typing import Callable, Protocol, Sequence, cast from alicebot_api.vnext_derived_domain import derived_domain +from alicebot_api.vnext_derived_labels import with_derived_from from alicebot_api.vnext_agent_control import resource_project_scope from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_model_intelligence import ( @@ -665,6 +666,10 @@ def generate_contradiction_report(self, request: ContradictionFinderRequest | No prompt_hash = model_artifact.prompt_hash model_info_json = model_artifact.model_info metadata = {**metadata, **model_artifact.metadata} + metadata = with_derived_from( + metadata, + {"sources": sources, "memories": memories, "beliefs": beliefs}, + ) artifact_payload: JsonObject = { "artifact_type": "contradiction_report", "title": "Contradiction Report", diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 9069bea03..c5c7d26dc 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -1247,6 +1247,28 @@ def stamp_derived_from(payload: dict[str, object], rows_by_kind: Mapping[str, ob payload["metadata_json"] = meta +def with_derived_from(metadata: Mapping[str, object], rows_by_kind: Mapping[str, object]) -> dict[str, object]: + """A copy of ``metadata`` with ``derived_from`` for the rows a producer used.""" + + payload: dict[str, object] = {"metadata_json": dict(metadata)} + stamp_derived_from(payload, rows_by_kind) + stamped = payload["metadata_json"] + return dict(stamped) if isinstance(stamped, Mapping) else {} + + +def admit_when_locked( + kind: str, + rows: object, + projects: tuple[str, ...] | None, +) -> list[Mapping[str, object]]: + """Keep every row when ``projects`` is None. Otherwise keep rows inside that binding.""" + + items = list(rows) if isinstance(rows, (list, tuple)) else [] + if projects is None: + return [row for row in items if isinstance(row, Mapping)] + return [row for row in items if isinstance(row, Mapping) and input_admitted(kind, row, projects)] + + def scope_is_global(scope: object) -> bool: """True when a scope holds no Alice project id.""" @@ -1273,10 +1295,12 @@ def scope_is_global(scope: object) -> bool: "group_scope", "identifier", "infer_kind", + "admit_when_locked", "input_admitted", "intersect_scope", "is_derived", "locked_projects", + "with_derived_from", "labels_raised_payload", "ordered_identifiers", "row_class", diff --git a/apps/api/src/alicebot_api/vnext_memory_commit.py b/apps/api/src/alicebot_api/vnext_memory_commit.py index a1af3cb8b..605f9da22 100644 --- a/apps/api/src/alicebot_api/vnext_memory_commit.py +++ b/apps/api/src/alicebot_api/vnext_memory_commit.py @@ -90,6 +90,7 @@ PENDING_PROJECT_UPDATE_MEMORY_MUTATION_MESSAGE, is_pending_project_update_memory, ) +from alicebot_api.vnext_derived_labels import group_scope from alicebot_api.vnext_project_scope import normalize_project_scope, project_scope_identity from alicebot_api.vnext_repositories import EventStore, JsonObject from alicebot_api.store import ContinuityStoreInvariantError @@ -2104,10 +2105,8 @@ def accept_consolidation_candidate( ) if strict_snapshots and dependency_ids: - member_scope_keys = { - project_scope_identity(resource_project_scope(member)) for member in locked_members.values() - } - candidate_scope_key = project_scope_identity(resource_project_scope(memory)) + member_scope_keys = {group_scope(member, kind="memory") for member in locked_members.values()} + candidate_scope_key = group_scope(memory, kind="memory") if len(member_scope_keys) != 1 or candidate_scope_key not in member_scope_keys: raise VNextMemoryCommitValidationError( "consolidation candidate crosses project scopes; regenerate it before acceptance" diff --git a/apps/api/src/alicebot_api/vnext_projects.py b/apps/api/src/alicebot_api/vnext_projects.py index 87dfdb736..2034c394c 100644 --- a/apps/api/src/alicebot_api/vnext_projects.py +++ b/apps/api/src/alicebot_api/vnext_projects.py @@ -8,6 +8,7 @@ from typing import Protocol, cast from alicebot_api.vnext_derived_domain import derived_domain +from alicebot_api.vnext_derived_labels import input_admitted, locked_projects, with_derived_from from alicebot_api.credential_floor import refuse_credential_activation from alicebot_api.vnext_agent_control import resource_project_scope from alicebot_api.vnext_embeddings import DeferredMemoryEmbedding @@ -551,8 +552,6 @@ def generate_project_update_candidate(self, request: ProjectAutomationRequest | # defensive check at the workflow boundary so legacy adapters cannot # widen a project-scoped report by ignoring optional query arguments. project_id = str(project["id"]) - from alicebot_api.vnext_derived_labels import input_admitted, locked_projects - locked = locked_projects(request.agent_identity, (project_id,)) if locked is not None: sources = [row for row in sources if input_admitted("source", row, locked)] @@ -602,23 +601,24 @@ def generate_project_update_candidate(self, request: ProjectAutomationRequest | "domain": derived_domain([project, *sources, *memories], fallback=str(project.get("domain", "project"))), "sensitivity": _highest_sensitivity([project, *sources, *memories]), "project_id": project_id, - "metadata_json": { - **request.metadata_json, - "candidate": True, - "workflow": "project_auto_update", - "project_id": project.get("id"), - "project_scope": [project_id], - "automation_digest": automation_digest, - "source_ids": _source_ids(sources), - "memory_ids": _source_ids(memories), - "generated_by": request.generated_by, - "agent_identity": request.agent_identity, - "agent_id": request.actor_id if request.generated_by == "agent" else None, - "trace_id": request.trace_id, - "policy_decision": request.policy_decision, - "project_scope": [project_id], - "automation_digest": automation_digest, - }, + "metadata_json": with_derived_from( + { + **request.metadata_json, + "candidate": True, + "workflow": "project_auto_update", + "project_id": project.get("id"), + "source_ids": _source_ids(sources), + "memory_ids": _source_ids(memories), + "generated_by": request.generated_by, + "agent_identity": request.agent_identity, + "agent_id": request.actor_id if request.generated_by == "agent" else None, + "trace_id": request.trace_id, + "policy_decision": request.policy_decision, + "project_scope": [project_id], + "automation_digest": automation_digest, + }, + {"sources": sources, "memories": memories}, + ), }, actor_type=request.generated_by, ) @@ -640,28 +640,29 @@ def generate_project_update_candidate(self, request: ProjectAutomationRequest | "generated_by": request.generated_by if request.generated_by != "system" else "vnext_project_auto_updater", "prompt_hash": prompt_hash, "model_info_json": model_info_json, - "metadata_json": { - **request.metadata_json, - "workflow": "project_auto_update", - "workflow_type": "project_update_scan", - "project_id": project.get("id"), - "project_scope": [project_id], - "automation_digest": automation_digest, - "candidate_memory_id": candidate_memory.get("id"), - "suggested_current_state": suggested_current_state, - "source_ids": _source_ids(sources), - "source_refs": [f"source:{source_id}" for source_id in _source_ids(sources)], - "memory_ids": _source_ids(memories), - "generated_by": request.generated_by, - "agent_identity": request.agent_identity, - "agent_id": request.actor_id if request.generated_by == "agent" else None, - "agent_run_id": request.run_id if request.generated_by == "agent" else None, - "trace_id": request.trace_id, - "policy_decision": request.policy_decision, - **model_metadata, - "project_scope": [project_id], - "automation_digest": automation_digest, - }, + "metadata_json": with_derived_from( + { + **request.metadata_json, + "workflow": "project_auto_update", + "workflow_type": "project_update_scan", + "project_id": project.get("id"), + "automation_digest": automation_digest, + "candidate_memory_id": candidate_memory.get("id"), + "suggested_current_state": suggested_current_state, + "source_ids": _source_ids(sources), + "source_refs": [f"source:{source_id}" for source_id in _source_ids(sources)], + "memory_ids": _source_ids(memories), + "generated_by": request.generated_by, + "agent_identity": request.agent_identity, + "agent_id": request.actor_id if request.generated_by == "agent" else None, + "agent_run_id": request.run_id if request.generated_by == "agent" else None, + "trace_id": request.trace_id, + "policy_decision": request.policy_decision, + **model_metadata, + "project_scope": [project_id], + }, + {"sources": sources, "memories": [*memories, candidate_memory]}, + ), } upsert_artifact = getattr(self.store, "upsert_artifact_by_workflow_digest", None) if callable(upsert_artifact): @@ -992,11 +993,24 @@ def review_project_update( current_state, error=VNextProjectValidationError, ) - if self.store.get_project_for_update(project_id) is None: + locked_project = self.store.get_project_for_update(project_id) + if locked_project is None: raise VNextProjectValidationError("project update candidate project was not found") + existing_meta = locked_project.get("metadata_json") + project_meta = dict(existing_meta) if isinstance(existing_meta, Mapping) else {} + recorded_sources = candidate_metadata.get("source_ids") + recorded_memories = candidate_metadata.get("memory_ids") + project_meta = with_derived_from( + project_meta, + { + "sources": [{"id": item} for item in recorded_sources] if isinstance(recorded_sources, list) else [], + "memories": [{"id": item} for item in recorded_memories] if isinstance(recorded_memories, list) else [], + "artifacts": [{"id": artifact_id}], + }, + ) self.store.update_project( project_id=project_id, - patch={"current_state": current_state}, + patch={"current_state": current_state, "metadata_json": project_meta}, actor_type=actor_type, ) updated_memory = self.store.update_memory( diff --git a/apps/api/src/alicebot_api/vnext_queue.py b/apps/api/src/alicebot_api/vnext_queue.py index d846619b1..6bd31b46c 100644 --- a/apps/api/src/alicebot_api/vnext_queue.py +++ b/apps/api/src/alicebot_api/vnext_queue.py @@ -10,6 +10,7 @@ from alicebot_api.vnext_embeddings import DeferredMemoryEmbedding, attach_memory_embedding from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_agent_control import resource_project_scope +from alicebot_api.vnext_derived_labels import with_derived_from from alicebot_api.vnext_project_update_guard import is_project_update_artifact from alicebot_api.vnext_repositories import JsonObject @@ -471,11 +472,14 @@ def _promote_artifact( "sensitivity": str(artifact.get("sensitivity") or "unknown"), "project_id": scope[0] if len(scope) == 1 else None, "source_event_ids": [], - "metadata_json": { - "source_artifact_id": artifact_id, - "project_scope": list(scope), - "promotion_reviewed": True, - }, + "metadata_json": with_derived_from( + { + "source_artifact_id": artifact_id, + "project_scope": list(scope), + "promotion_reviewed": True, + }, + {"artifacts": [artifact]}, + ), }, actor_type=actor_type, ) diff --git a/apps/api/src/alicebot_api/vnext_rollups.py b/apps/api/src/alicebot_api/vnext_rollups.py index 0bb942434..33920087d 100644 --- a/apps/api/src/alicebot_api/vnext_rollups.py +++ b/apps/api/src/alicebot_api/vnext_rollups.py @@ -166,6 +166,12 @@ memory_embedding_text, ) from alicebot_api.vnext_agent_control import resource_project_scope +from alicebot_api.vnext_derived_labels import ( + admit_when_locked, + group_scope, + locked_projects, + with_derived_from, +) from alicebot_api.vnext_entities import extract_entity_candidates from alicebot_api.vnext_model_intelligence import ( NON_SYNTHESIZING_PROVIDERS, @@ -653,7 +659,7 @@ def _member_text(row: JsonObject) -> str: def _project_scope_key(row: JsonObject) -> tuple[str, ...]: - return project_scope_identity(resource_project_scope(row)) + return group_scope(row) def _shared_project_scope(rows: tuple[JsonObject, ...] | list[JsonObject]) -> tuple[str, ...]: @@ -1523,9 +1529,7 @@ def _scoped_rows( continue if projects: allowed_projects = set(project_scope_identity(projects)) - if not allowed_projects.intersection( - project_scope_identity(resource_project_scope(row)) - ): + if not allowed_projects.intersection(group_scope(row)): continue scoped.append(row) return scoped @@ -1746,6 +1750,7 @@ def _collect_rows( sensitivity_allowed: list[str], projects: tuple[str, ...], options: RollupOptions, + all_of: tuple[str, ...] | None = None, ) -> tuple[list[JsonObject], bool, int, bool]: # Ask for one sentinel row beyond the configured cap. The store # applies status, scope, roll-up-card exclusion, deterministic order, @@ -1819,7 +1824,7 @@ def _collect_rows( raise VNextRollupValidationError( "roll-up input lookup returned rows outside the requested project scope" ) - rows = scoped_rows + rows = admit_when_locked("memory", scoped_rows, all_of) rows = [row for row in rows if not _is_rollup_card(row)] # The same parity for validity: the bundled stores leave an expired # memory out in SQL, and every tier below (entity, topic and the @@ -2383,6 +2388,7 @@ def _existing_rollup_state( domains: list[str] | None, sensitivity_allowed: list[str], projects: tuple[str, ...], + all_of: tuple[str, ...] | None = None, ) -> tuple[dict[str, JsonObject], dict[str, JsonObject]]: """(pending candidate by rollup_digest, accepted card by rollup_key).""" pending: dict[str, JsonObject] = {} @@ -2454,6 +2460,13 @@ def _existing_rollup_state( raise VNextRollupValidationError( "roll-up candidate/card lookup returned rows outside the requested project scope" ) + if all_of is not None: + pending = { + key: row for key, row in pending.items() if admit_when_locked("memory", [row], all_of) + } + accepted = { + key: row for key, row in accepted.items() if admit_when_locked("memory", [row], all_of) + } return pending, accepted def _expired_card_for_digest( @@ -2727,33 +2740,41 @@ def _create_rollup_candidate( "sensitivity": _highest_sensitivity(group.members), "project_id": project_scope[0] if len(project_scope) == 1 else None, "source_event_ids": source_event_ids, - "metadata_json": { - "candidate_kind": ROLLUP_CANDIDATE_KIND, - "rollup_digest": rollup_digest, - "rollup_key": group.rollup_key, - "review_required": True, - "source_refs": source_refs, - "project_scope": list(project_scope), - "trace_id": trace_id, - # accept_consolidation_candidate compatibility: the - # existing review/acceptance path reads this block. - "consolidation": { - "proposal_kind": ROLLUP_PROPOSAL_KIND, - "cluster_member_ids": member_ids, - "member_snapshots": member_snapshots, - "proposed_supersede": proposed_supersede, - "survivor_memory_id": None, - "model_provenance": model_provenance, - "merge_refusal": merge_refusal, - "reviewer_instructions": reviewer_instructions, - "rollup": { - "rollup_key": group.rollup_key, - "group_kind": group.group_kind, - "topic_label": group.label, - "revises_memory_id": revises_memory_id, + "metadata_json": with_derived_from( + { + "candidate_kind": ROLLUP_CANDIDATE_KIND, + "rollup_digest": rollup_digest, + "rollup_key": group.rollup_key, + "review_required": True, + "source_refs": source_refs, + "project_scope": list(project_scope), + "trace_id": trace_id, + # accept_consolidation_candidate compatibility: the + # existing review/acceptance path reads this block. + "consolidation": { + "proposal_kind": ROLLUP_PROPOSAL_KIND, + "cluster_member_ids": member_ids, + "member_snapshots": member_snapshots, + "proposed_supersede": proposed_supersede, + "survivor_memory_id": None, + "model_provenance": model_provenance, + "merge_refusal": merge_refusal, + "reviewer_instructions": reviewer_instructions, + "rollup": { + "rollup_key": group.rollup_key, + "group_kind": group.group_kind, + "topic_label": group.label, + "revises_memory_id": revises_memory_id, + }, }, }, - }, + { + "memories": [ + *group.members, + *([revises_memory] if revises_memory is not None else []), + ] + }, + ), }, actor_type=generated_by, ) @@ -2774,6 +2795,7 @@ def propose_rollups( route=None, model_temperature: float = 0.2, exclude_member_id_sets: list[set[str]] | None = None, + agent_identity: object = None, ) -> RollupOutcome: """One review-only roll-up pass over the in-scope memories. @@ -2791,6 +2813,7 @@ def propose_rollups( """ options = options or RollupOptions() sensitivity = list(sensitivity_allowed or ("public", "internal", "private", "unknown")) + all_of = locked_projects(agent_identity, projects) outcome = RollupOutcome(options=options.to_record()) rows, bounded, total_count, total_exact = self._collect_rows( @@ -2798,6 +2821,7 @@ def propose_rollups( sensitivity_allowed=sensitivity, projects=projects, options=options, + all_of=all_of, ) outcome.groupable_count = len(rows) outcome.groupable_total_count = total_count @@ -2880,6 +2904,7 @@ def propose_rollups( domains=domains, sensitivity_allowed=sensitivity, projects=projects, + all_of=all_of, ) for prepared in prepared_groups: diff --git a/apps/api/src/alicebot_api/vnext_scheduler.py b/apps/api/src/alicebot_api/vnext_scheduler.py index 04f53e34d..55ff84a41 100644 --- a/apps/api/src/alicebot_api/vnext_scheduler.py +++ b/apps/api/src/alicebot_api/vnext_scheduler.py @@ -13,6 +13,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from alicebot_api.vnext_derived_domain import derived_domain +from alicebot_api.vnext_derived_labels import admit_when_locked, locked_projects, with_derived_from from alicebot_api.vnext_agent_control import ( AgentIdentity, PolicyDecision, @@ -1402,6 +1403,11 @@ def _run_staleness_sweep(self, request: SchedulerRunRequest, *, metadata: JsonOb raise VNextSchedulerValidationError( "staleness sweep store returned memories outside the requested project scope" ) + bound = locked_projects( + request.agent_identity.to_record() if request.agent_identity is not None else None, + projects, + ) + memories = admit_when_locked("memory", memories, bound) for memory in memories: if len(expired_marked) + len(unconfirmed_marked) >= mark_limit: break @@ -1470,19 +1476,22 @@ def _run_staleness_sweep(self, request: SchedulerRunRequest, *, metadata: JsonOb "domain": derived_domain(marked, fallback=request.domains[0] if len(request.domains) == 1 else "unknown"), "sensitivity": self._highest_sensitivity(marked), "generated_by": "scheduler", - "metadata_json": { - **metadata, - "workflow": "staleness_sweep", - "source_refs": [], - "stale_marked_memory_ids": [str(row.get("id")) for row in marked], - "staleness_window_days": window_days, - "input_counts": { - "scanned": scanned_count, - "expired_marked": len(expired_marked), - "unconfirmed_marked": len(unconfirmed_marked), + "metadata_json": with_derived_from( + { + **metadata, + "workflow": "staleness_sweep", + "source_refs": [], + "stale_marked_memory_ids": [str(row.get("id")) for row in marked], + "staleness_window_days": window_days, + "input_counts": { + "scanned": scanned_count, + "expired_marked": len(expired_marked), + "unconfirmed_marked": len(unconfirmed_marked), + }, + "review_policy": "marks_stale_never_deletes", }, - "review_policy": "marks_stale_never_deletes", - }, + {"memories": marked}, + ), }, actor_type="scheduler", ) @@ -1565,6 +1574,11 @@ def _generate_open_loop_review_artifact(self, request: SchedulerRunRequest, *, m ) if any(not _row_matches_projects(loop, projects) for loop in loops): raise VNextSchedulerValidationError("open-loop store returned rows outside the requested project scope") + bound = locked_projects( + request.agent_identity.to_record() if request.agent_identity is not None else None, + projects, + ) + loops = admit_when_locked("open_loop", loops, bound) # The report copies the id of each loop's source into its text and its ``source_refs``, and a later reader of # the artifact is shown them, so a source the run's own identity may not read is left out. loops = withhold_unreadable_references( @@ -1678,6 +1692,10 @@ def _generate_open_loop_review_artifact(self, request: SchedulerRunRequest, *, m prompt_hash = model_artifact.prompt_hash model_info_json = model_artifact.model_info enriched_metadata = {**enriched_metadata, **model_artifact.metadata} + enriched_metadata = with_derived_from( + enriched_metadata, + {"open_loops": loops, "sources": linked_sources}, + ) artifact_payload: JsonObject = { "artifact_type": "open_loop_report", "title": f"Open Loop Review - {request.generated_for or datetime.now(UTC).date().isoformat()}", diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index e83c801f5..fc7fc4274 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -172,6 +172,9 @@ ) from alicebot_api.vnext_stores.postgres.query_predicates import ( _ARTIFACT_SCOPE_PROJECT_SQL as _ARTIFACT_SCOPE_PROJECT_SQL, + _PROJECT_FLOOR_SQL as _PROJECT_FLOOR_SQL, + _MEMORY_GROUP_SCOPE_SQL as _MEMORY_GROUP_SCOPE_SQL, + _jsonb_string_array_identity_sql as _jsonb_string_array_identity_sql, _ASCII_PROJECT_LOWER as _ASCII_PROJECT_LOWER, _ASCII_PROJECT_UPPER as _ASCII_PROJECT_UPPER, _MEMORY_DIRECT_PEOPLE_SQL as _MEMORY_DIRECT_PEOPLE_SQL, @@ -2260,7 +2263,8 @@ def list_artifacts( WHERE (%s::text IS NULL OR artifact_type = %s) AND (%s::text[] IS NULL OR domain = ANY(%s::text[]) OR domain = 'unknown') AND (%s::text[] IS NULL OR sensitivity = ANY(%s::text[])) - AND (%s::text[] IS NULL OR ({_ARTIFACT_SCOPE_PROJECT_SQL}) ?| %s::text[]) + AND (%s::text[] IS NULL OR ({_ARTIFACT_SCOPE_PROJECT_SQL}) ?| %s::text[] + OR ({_PROJECT_FLOOR_SQL}) ?| %s::text[]) ORDER BY created_at DESC, id DESC LIMIT %s """, @@ -2273,6 +2277,7 @@ def list_artifacts( sensitivity_allowed, project_list, project_list, + project_list, limit, ), ) diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py index ac34bb114..d8bb10f57 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py @@ -21,6 +21,7 @@ from alicebot_api.vnext_stores.postgres.primitives import _json_list from alicebot_api.vnext_stores.postgres.query_predicates import ( _MEMORY_DIRECT_PEOPLE_SQL, + _MEMORY_GROUP_SCOPE_SQL, _MEMORY_PROJECT_SCOPE_SQL, _MEMORY_SCOPE_EVENT_TIME_SQL, _escape_like_literal, @@ -701,7 +702,7 @@ def list_pending_rollup_candidates( AND metadata_json ->> 'rollup_digest' = ANY(%s::text[]) AND (%s::text[] IS NULL OR domain = ANY(%s::text[]) OR domain = 'unknown') AND COALESCE(sensitivity, 'unknown') = ANY(%s::text[]) - AND (%s::text[] IS NULL OR ({_MEMORY_PROJECT_SCOPE_SQL}) ?| %s::text[]) + AND (%s::text[] IS NULL OR ({_MEMORY_GROUP_SCOPE_SQL}) ?| %s::text[]) ORDER BY metadata_json ->> 'rollup_digest', updated_at DESC, created_at DESC, id DESC LIMIT %s """, @@ -748,7 +749,7 @@ def list_accepted_rollup_cards( AND metadata_json ->> 'rollup_key' = ANY(%s::text[]) AND (%s::text[] IS NULL OR domain = ANY(%s::text[]) OR domain = 'unknown') AND COALESCE(sensitivity, 'unknown') = ANY(%s::text[]) - AND (%s::text[] IS NULL OR ({_MEMORY_PROJECT_SCOPE_SQL}) ?| %s::text[]) + AND (%s::text[] IS NULL OR ({_MEMORY_GROUP_SCOPE_SQL}) ?| %s::text[]) ORDER BY metadata_json ->> 'rollup_key', CASE WHEN status = 'active' THEN 0 ELSE 1 END, diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/query_predicates.py b/apps/api/src/alicebot_api/vnext_stores/postgres/query_predicates.py index 2e5a2e783..862ccc98d 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/query_predicates.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/query_predicates.py @@ -354,6 +354,63 @@ def _jsonb_source_project_scope_values_sql(metadata_expression: str) -> str: ) +def _jsonb_string_array_identity_sql(array_expression: str) -> str: + """Identity of a JSON array of strings. Any other shape contributes nothing.""" + + normalized = _normalized_project_identifier_sql("floor_text.value") + identity = _project_identifier_identity_sql("normalized_floor.value", already_normalized=True) + return f""" +( + SELECT COALESCE( + jsonb_agg(floor_identity.value ORDER BY floor_identity.value COLLATE "C"), + '[]'::jsonb + ) + FROM ( + SELECT DISTINCT {identity} AS value + FROM jsonb_array_elements( + CASE + WHEN jsonb_typeof({array_expression}) = 'array' THEN {array_expression} + ELSE '[]'::jsonb + END + ) AS floor_element(value) + CROSS JOIN LATERAL ( + SELECT CASE + WHEN jsonb_typeof(floor_element.value) = 'string' THEN floor_element.value #>> '{{}}' + ELSE '' + END AS value + ) AS floor_text + CROSS JOIN LATERAL ( + SELECT {normalized} AS value + ) AS normalized_floor + WHERE normalized_floor.value <> '' + ) AS floor_identity +) +""" + + +_PROJECT_FLOOR_SQL = _jsonb_string_array_identity_sql("metadata_json -> 'project_floor'") + + +# Overlap of scope united with floor. Original rows have no floor, so this +# matches the same rows as _MEMORY_PROJECT_SCOPE_SQL for them. +_MEMORY_GROUP_SCOPE_SQL = f""" +( + SELECT COALESCE( + jsonb_agg(grouped.value ORDER BY grouped.value COLLATE "C"), + '[]'::jsonb + ) + FROM ( + SELECT DISTINCT part.value + FROM ( + SELECT jsonb_array_elements_text(({_MEMORY_PROJECT_SCOPE_SQL})) AS value + UNION ALL + SELECT jsonb_array_elements_text(({_PROJECT_FLOOR_SQL})) AS value + ) AS part + ) AS grouped +) +""" + + _MEMORY_DIRECT_PEOPLE_SQL = """ EXISTS ( SELECT 1 @@ -490,6 +547,7 @@ def _tsquery_any_expression(query: str) -> str | None: _normalized_project_identifier_sql, _project_identifier_identity_sql, _jsonb_project_scope_values_sql, + _jsonb_string_array_identity_sql, _jsonb_project_scope_leaf_values_sql, _jsonb_source_project_scope_values_sql, _jsonb_scope_values_sql, diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py index 556f1bbd5..4b677e515 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py @@ -31,6 +31,7 @@ _fts_match_expression, CTE_MATERIALIZED_HINT, _project_view_partition_sql, + _split_view_request, _stated_exclusion, _sqlite_ascii_literal_contains_sql, ) @@ -1004,6 +1005,25 @@ def count_rollup_input_memories( return cast(int, row["count"]) +def _rollup_group_scope_clause(self, projects: Sequence[str] | None) -> tuple[str, list[object]]: + """Overlap of scope or floor. Used only by the roll-up candidate and card lookups.""" + + normalized = tuple(normalize_project_scope(projects or ())) + scope_sql, scope_params = self._project_clause(normalized) + if not scope_sql: + return "", [] + ids, wants_global = _split_view_request(normalized) + if wants_global or not ids: + return scope_sql, list(scope_params) + predicate = scope_sql.removeprefix(" AND ") + floor_sql = ( + "EXISTS (SELECT 1 FROM json_each(alice_project_floor_identity(metadata_json)) AS floor_project " + "WHERE CAST(floor_project.value AS TEXT) " + f"IN ({self._placeholders(list(ids))}))" + ) + return f" AND ({predicate} OR {floor_sql})", [*scope_params, *ids] + + def list_pending_rollup_candidates( self, *, @@ -1030,7 +1050,7 @@ def list_pending_rollup_candidates( params.extend(domains) sensitivity_placeholders = ", ".join("?" for _value in sensitivity_allowed) params.extend(sensitivity_allowed) - project_sql, project_params = self._project_clause(tuple(normalize_project_scope(projects or ()))) + project_sql, project_params = _rollup_group_scope_clause(self, projects) params.extend(project_params) params.append(bounded_limit) return self._fetch_all( @@ -1087,7 +1107,7 @@ def list_accepted_rollup_cards( params.extend(domains) sensitivity_placeholders = ", ".join("?" for _value in sensitivity_allowed) params.extend(sensitivity_allowed) - project_sql, project_params = self._project_clause(tuple(normalize_project_scope(projects or ()))) + project_sql, project_params = _rollup_group_scope_clause(self, projects) params.extend(project_params) # An expired card is not the accepted card for its topic. The test sits # inside the ranking query, so an older card that is still open is ranked diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/query_predicates.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/query_predicates.py index 33f9c5bbe..5beed618a 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/query_predicates.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/query_predicates.py @@ -12,6 +12,7 @@ from alicebot_api.vnext_project_scope import ( GLOBAL_PROJECT_MARKER, is_alice_project_id, + project_floor_shape, project_scope_identity, resolve_project_scope, resolve_source_metadata_project_scope, @@ -67,6 +68,24 @@ def _source_project_scope_identity_json_sqlite(metadata_json: object) -> str: return json.dumps(identity, ensure_ascii=False, separators=(",", ":")) +def _project_floor_identity_json_sqlite(metadata_json: object) -> str: + """Identity of ``project_floor`` when it is a list of strings, else ``[]``.""" + + if isinstance(metadata_json, Mapping): + metadata = dict(metadata_json) + elif isinstance(metadata_json, str): + try: + decoded = json.loads(metadata_json) + except (TypeError, ValueError): + decoded = {} + metadata = decoded if isinstance(decoded, dict) else {} + else: + metadata = {} + shape, floor = project_floor_shape({"metadata_json": metadata}) + identity = project_scope_identity(floor) if shape == "list" else () + return json.dumps(list(identity), ensure_ascii=False, separators=(",", ":")) + + def _ensure_project_scope_identity_sqlite(conn: sqlite3.Connection) -> None: """Install the deterministic project identity functions per connection.""" @@ -77,7 +96,8 @@ def _ensure_project_scope_identity_sqlite(conn: sqlite3.Connection) -> None: WHERE name IN ( 'alice_project_scope_value', 'alice_project_scope_identity', - 'alice_source_project_scope_identity' + 'alice_source_project_scope_identity', + 'alice_project_floor_identity' ) """ ) @@ -85,7 +105,7 @@ def _ensure_project_scope_identity_sqlite(conn: sqlite3.Connection) -> None: row = cursor.fetchone() if row is not None: count = next(iter(row.values())) if isinstance(row, Mapping) else row[0] - if int(count) == 3: + if int(count) == 4: return finally: cursor.close() @@ -107,6 +127,12 @@ def _ensure_project_scope_identity_sqlite(conn: sqlite3.Connection) -> None: _source_project_scope_identity_json_sqlite, deterministic=True, ) + conn.create_function( + "alice_project_floor_identity", + 1, + _project_floor_identity_json_sqlite, + deterministic=True, + ) #: ``AS MATERIALIZED`` for the labelled common table expression of the single-scan diff --git a/tests/unit/test_group_scope_consumers.py b/tests/unit/test_group_scope_consumers.py new file mode 100644 index 000000000..38693981b --- /dev/null +++ b/tests/unit/test_group_scope_consumers.py @@ -0,0 +1,302 @@ +"""Group scope for consolidation, roll-ups, and the lookups that find a card.""" + +from __future__ import annotations + +import inspect +import sqlite3 +from datetime import UTC, datetime +from uuid import uuid4 + +import pytest + +from alicebot_api.sqlite_schema import bootstrap_sqlite_schema +from alicebot_api.sqlite_store import SQLiteVNextStore, ensure_sqlite_user +from alicebot_api.vnext_agent_control import AgentIdentity, resource_project_scope +from alicebot_api.vnext_consolidation import _project_scope_key, _scoped_rows +from alicebot_api.vnext_derived_labels import group_scope +from alicebot_api.vnext_memory_commit import VNextMemoryCommitService, VNextMemoryCommitValidationError +from alicebot_api.vnext_project_scope import project_scope_identity +from alicebot_api.vnext_rollups import ROLLUP_CANDIDATE_KIND +from alicebot_api.vnext_rollups import _scoped_rows as rollup_scoped_rows +from alicebot_api.vnext_scheduler import SchedulerRunRequest, VNextSchedulerService +from alicebot_api.vnext_store import PostgresVNextStore +from alicebot_api.vnext_stores.postgres import memory_access as postgres_memory +from alicebot_api.routers.vnext_review import list_vnext_artifacts +from tests.unit.test_vnext_memory_commit import ( + TargetedLookupStore, + _seed_consolidation_candidate, + _seed_row, +) + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +def _derived(scope: list[str], floor: list[str]) -> dict[str, object]: + return { + "id": "derived", + "memory_key": "vnext.consolidation.example", + "metadata_json": { + "candidate_kind": "memory_consolidation", + "project_scope": scope, + "project_floor": floor, + }, + } + + +def test_scoped_rows_overlap_the_group_scope() -> None: + row = _derived([], [ALPHA, BETA]) + assert group_scope(row) == (ALPHA, BETA) + kept = _scoped_rows([row], domains=None, sensitivity_allowed=["unknown"], projects=(ALPHA,)) + assert kept == [row] + kept_rollups = rollup_scoped_rows( + [row], domains=None, sensitivity_allowed=["unknown"], projects=(BETA,) + ) + assert kept_rollups == [row] + assert _project_scope_key(row) == (ALPHA, BETA) + + +def test_a_two_project_group_is_accepted_on_the_group_scope() -> None: + store = TargetedLookupStore() + members = [ + _seed_row(store, title=f"Member {index}", text="Shared fact.", metadata={"project_scope": [ALPHA, BETA]}) + for index in range(2) + ] + candidate_id = _seed_consolidation_candidate( + store, + member_ids=members, + proposal_kind="merge", + survivor_memory_id=None, + proposed_supersede=list(members), + ) + store.memories[candidate_id]["metadata_json"]["project_scope"] = [] + store.memories[candidate_id]["metadata_json"]["project_floor"] = [ALPHA, BETA] + + result = VNextMemoryCommitService(store).accept_consolidation_candidate( + candidate_id, reason="The group scope matches." + ) + + assert result["status"] == "accepted" + + +def test_m51_acceptance_that_compares_scope_again_refuses_the_group(monkeypatch: pytest.MonkeyPatch) -> None: + def scope_only(row: dict[str, object], *, kind: str | None = None) -> tuple[str, ...]: + del kind + return project_scope_identity(resource_project_scope(row)) + + monkeypatch.setattr("alicebot_api.vnext_memory_commit.group_scope", scope_only) + store = TargetedLookupStore() + members = [ + _seed_row(store, title=f"Wide {index}", text="Shared fact.", metadata={"project_scope": [ALPHA, BETA]}) + for index in range(2) + ] + candidate_id = _seed_consolidation_candidate( + store, + member_ids=members, + proposal_kind="merge", + survivor_memory_id=None, + proposed_supersede=list(members), + ) + store.memories[candidate_id]["metadata_json"]["project_scope"] = [] + store.memories[candidate_id]["metadata_json"]["project_floor"] = [ALPHA, BETA] + + with pytest.raises(VNextMemoryCommitValidationError, match="crosses project scopes"): + VNextMemoryCommitService(store).accept_consolidation_candidate(candidate_id, reason="Scope only.") + + +def test_promoted_copies_with_different_floors_are_refused() -> None: + store = TargetedLookupStore() + members = [ + _seed_row( + store, + title=f"Copy {index}", + text="Promoted text.", + metadata={ + "source_artifact_id": f"artifact-{index}", + "project_scope": [], + "project_floor": [project], + }, + ) + for index, project in enumerate((ALPHA, BETA)) + ] + candidate_id = _seed_consolidation_candidate( + store, + member_ids=members, + proposal_kind="merge", + survivor_memory_id=None, + proposed_supersede=list(members), + ) + + with pytest.raises(VNextMemoryCommitValidationError, match="crosses project scopes"): + VNextMemoryCommitService(store).accept_consolidation_candidate(candidate_id, reason="Different floors.") + + +def test_rollup_lookups_match_a_floor_when_the_scope_is_empty() -> None: + conn = sqlite3.connect(":memory:") + bootstrap_sqlite_schema(conn) + user_id = str(uuid4()) + ensure_sqlite_user(conn, user_id, "group-scope@example.com", "Group Scope") + store = SQLiteVNextStore(conn, user_id) + card = store.create_memory( + { + "memory_key": "vnext.rollup.floor-card", + "value": {"text": "card"}, + "status": "candidate", + "memory_type": "semantic", + "title": "Floor card", + "canonical_text": "Floor card", + "summary": "Floor card", + "domain": "personal", + "sensitivity": "internal", + "metadata_json": { + "candidate_kind": ROLLUP_CANDIDATE_KIND, + "rollup_digest": "digest-floor", + "rollup_key": "topic:floor", + "project_scope": [], + "project_floor": [ALPHA, BETA], + }, + } + ) + pending = store.list_pending_rollup_candidates( + rollup_digests=("digest-floor",), + domains=["personal"], + sensitivity_allowed=["internal"], + candidate_kind=ROLLUP_CANDIDATE_KIND, + limit=5, + projects=(ALPHA,), + ) + assert [str(row["id"]) for row in pending] == [str(card["id"])] + missed = store.list_pending_rollup_candidates( + rollup_digests=("digest-floor",), + domains=["personal"], + sensitivity_allowed=["internal"], + candidate_kind=ROLLUP_CANDIDATE_KIND, + limit=5, + projects=("prj_" + "c" * 16,), + ) + assert missed == [] + accepted = store.create_memory( + { + "memory_key": "vnext.rollup.floor-accepted", + "value": {"text": "accepted"}, + "status": "active", + "memory_type": "semantic", + "title": "Accepted floor card", + "canonical_text": "Accepted floor card", + "summary": "Accepted floor card", + "domain": "personal", + "sensitivity": "internal", + "metadata_json": { + "candidate_kind": ROLLUP_CANDIDATE_KIND, + "rollup_key": "topic:floor-accepted", + "project_scope": [], + "project_floor": [ALPHA, BETA], + }, + } + ) + cards = store.list_accepted_rollup_cards( + rollup_keys=("topic:floor-accepted",), + domains=["personal"], + sensitivity_allowed=["internal"], + candidate_kind=ROLLUP_CANDIDATE_KIND, + limit=5, + projects=(BETA,), + ) + assert [str(row["id"]) for row in cards] == [str(accepted["id"])] + conn.close() + + +def test_the_lookup_sql_names_the_group_scope() -> None: + pending = inspect.getsource(postgres_memory.list_pending_rollup_candidates) + accepted = inspect.getsource(postgres_memory.list_accepted_rollup_cards) + assert "_MEMORY_GROUP_SCOPE_SQL" in pending + assert "_MEMORY_GROUP_SCOPE_SQL" in accepted + assert "_PROJECT_FLOOR_SQL" in inspect.getsource(PostgresVNextStore.list_artifacts) + assert "project" in inspect.signature(list_vnext_artifacts).parameters + + +class _SweepStore: + def __init__(self) -> None: + self.memories = [ + { + "id": "alpha", + "status": "active", + "memory_type": "semantic", + "title": "Alpha only", + "canonical_text": "Alpha only", + "valid_to": datetime(2020, 1, 1, tzinfo=UTC), + "metadata_json": {"project_scope": [ALPHA]}, + "sensitivity": "internal", + }, + { + "id": "shared", + "status": "active", + "memory_type": "semantic", + "title": "Shared", + "canonical_text": "Shared", + "valid_to": datetime(2020, 1, 1, tzinfo=UTC), + "metadata_json": {"project_scope": [ALPHA, BETA]}, + "sensitivity": "internal", + }, + ] + self.artifacts: list[dict[str, object]] = [] + self.events: list[dict[str, object]] = [] + self.revisions: list[dict[str, object]] = [] + + def list_memories_for_staleness_sweep( + self, + *, + reference_time: object = None, + confirmation_before: object = None, + review_memory_types: object = None, + limit: int = 20, + projects: object = None, + ) -> list[dict[str, object]]: + del reference_time, confirmation_before, review_memory_types, limit, projects + return list(self.memories) + + def update_memory(self, *, memory_id: str, patch: dict[str, object], actor_type: str = "system") -> dict[str, object]: + del actor_type + for memory in self.memories: + if memory["id"] == memory_id: + memory.update(patch) + return memory + raise KeyError(memory_id) + + def append_revision(self, revision: dict[str, object], *, actor_type: str = "system") -> dict[str, object]: + del actor_type + self.revisions.append(revision) + return revision + + def append_event(self, event: dict[str, object]) -> dict[str, object]: + self.events.append(event) + return event + + def create_artifact(self, artifact: dict[str, object], *, actor_type: str = "system") -> dict[str, object]: + del actor_type + row = {**artifact, "id": "artifact-sweep"} + self.artifacts.append(row) + return row + + +def test_a_locked_staleness_sweep_does_not_mark_a_shared_memory() -> None: + store = _SweepStore() + identity = AgentIdentity( + agent_id="alpha-key", + project_scope=(ALPHA,), + project_scope_locked=True, + permission_profile="trusted_local_agent", + ) + VNextSchedulerService(store)._run_staleness_sweep( # noqa: SLF001 + SchedulerRunRequest( + workflow_type="staleness_sweep", + projects=(ALPHA,), + agent_identity=identity, + ), + metadata={"scheduler_run_id": "run-1", "trace_id": "trace-1"}, + ) + by_id = {memory["id"]: memory for memory in store.memories} + assert by_id["alpha"]["status"] == "stale" + assert by_id["shared"]["status"] == "active" + derived = store.artifacts[0]["metadata_json"]["derived_from"] + assert derived["memories"] == ["alpha"] diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index 7b2647bbf..f04ef0ec9 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -27,8 +27,11 @@ "apps/api/src/alicebot_api/vnext_stores/retrieval_common.py": ( "fa1a3a90511b5c61754ba29560e91b7b3058a48d47c143b09d8505d52025b8cc" ), + # Re-minted for the roll-up group scope: a floor identity beside the scope + # identity, used only by the two roll-up lookups and the artifact list. + # Previous receipt f0ec9c7f... "apps/api/src/alicebot_api/vnext_stores/postgres/query_predicates.py": ( - "f0ec9c7f13bc7bf93f5a3beaa86916a04e45200ef0296d6f9288eed3912be33d" + "f15f1bf2b3da73758c925b63dffaec707f91f00a6e153e53f562fb188859236d" ), # Re-minted for ``get_memory_by_key(include_deleted=...)`` (reviewed change, not drift; see the SQLite # entry below). Previous Postgres receipt 49748ecd... @@ -36,16 +39,20 @@ # deletion; reviewed change, not drift). Proof: the only difference from origin/main daa46ef5 is that one function, # which takes a keyword-only ``include_deleted`` (false by default, so every caller reads what it read before) # and drops the ``deleted_at IS NULL`` clause only when it is true. Previous Postgres receipt f642880f... + # Re-minted so the two roll-up lookups overlap scope united with floor. + # Every other statement still uses the scope expression. Previous receipt 46946cc0... "apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py": ( - "46946cc087de35f54474adf47cadcd67b862685bfa67a38be277d8e58a00c47e" + "057f4f0c157223fc0d94ea3afe680b66200820533ddccab7211e211f29107157" ), # Re-minted for per-project memory S2 (2026-10-02): the project fence builders read the reserved global # marker and take the domains to leave out, and the single-scan partition SQL and the materialized-CTE hint # are new. The Postgres carrier is unchanged on purpose: the Postgres runtime resolves no project view. # Re-minted once more in the S2 review round (2026-10-02): a request that holds the marker and does not state which # global domains it leaves out raises (reviewed change, not drift). + # Re-minted for alice_project_floor_identity, the fourth identity function, + # used by the roll-up lookups. Previous receipt eab46f16... "apps/api/src/alicebot_api/vnext_stores/sqlite/query_predicates.py": ( - "eab46f165564c212db21b6b6b621ecb447aaa86d48aba6512bd5f2e88f20bd82" + "8beae59c379c56033388a35b5a152f1f683f6b55c6e2fbd62925723566a12202" ), # Re-minted for the Phase 4 Stage 2 resident vector cache (reviewed # carrier change; the receipt guards unreviewed drift): the vector scan @@ -76,8 +83,9 @@ # reviewed change, not drift). Proof: the only difference from origin/main daa46ef5 is that one function, which # takes a keyword-only ``include_deleted`` (false by default) and drops the ``deleted_at IS NULL`` clause only when # it is true. Previous SQLite receipt 91636de9... + # Re-minted so the two roll-up lookups overlap scope or floor. Previous receipt 64f21989... "apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py": ( - "64f21989d3bb05d742dd712b310511d3c63f32b9a4d62af6b888ff2b367f0b3b" + "1580dca3a31fbbcf98539e71e81bad13935f30c59e8a7c75b0fc0b4472a6d5fa" ), } @@ -165,6 +173,9 @@ "_jsonb_project_scope_leaf_values_sql", "_jsonb_source_project_scope_values_sql", "_MEMORY_PROJECT_SCOPE_SQL", + "_PROJECT_FLOOR_SQL", + "_MEMORY_GROUP_SCOPE_SQL", + "_jsonb_string_array_identity_sql", "_MEMORY_DIRECT_PEOPLE_SQL", "_MEMORY_SCOPE_EVENT_TIME_SQL", "_SCOPED_MEMORY_PROJECT_SQL", @@ -200,7 +211,9 @@ # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (172, "6f1a459fcf4319cf4281f6cc0d4e81679c3e05d851fd0a874a2d90298d7c2569"), + # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping + # those two names restores the previous receipt (172, 6f1a459f...). + "PostgresVNextStore": (174, "095250b8a77d6c0a32d2783343916b947bcae8ae86e3a1693e47c2fb11802211"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -215,7 +228,9 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. - "SQLiteVNextStore": (134, "1301272026897057cf071009cc21787543ddc326f1e06f1a75a763f3e344767e"), + # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping + # those two names restores the previous receipt (134, 13012720...). + "SQLiteVNextStore": (136, "1d99dbaf69e4ea888ca7beb2389ac176650a2e573c067bf0686adc9e609132f5"), } @@ -302,7 +317,7 @@ def test_memory_access_source_receipts_pin_sql_parameters_and_comments() -> None assert sqlite.count("user_id = ?") >= 20 assert "Only compare vectors from the same endpoint fingerprint" in sqlite assert "resolve_project_scope" in sqlite_predicates - assert sqlite_predicates.count("create_function(") == 3 + assert sqlite_predicates.count("create_function(") == 4 def test_memory_access_methods_are_direct_grafts_in_native_backend_order() -> None: diff --git a/tests/unit/test_vnext_brain.py b/tests/unit/test_vnext_brain.py index af808365c..5f496e59c 100644 --- a/tests/unit/test_vnext_brain.py +++ b/tests/unit/test_vnext_brain.py @@ -155,6 +155,11 @@ def test_daily_brief_generates_dated_reviewable_artifact_with_sources_and_open_l assert store.open_loops[-1]["metadata_json"]["candidate"] is True assert store.events[-1]["event_type"] == "artifact.generated" assert store.events[-1]["payload_json"]["workflow"] == "daily_brief" + derived = artifact["metadata_json"]["derived_from"] + assert derived["counts"]["sources"] == len(derived["sources"]) == 1 + assert derived["sources"] == ["source-1"] + assert "memory-1" in derived["memories"] + assert store.open_loops[-1]["metadata_json"]["derived_from"]["sources"] == ["source-1"] def test_daily_brief_respects_sensitivity_filtering() -> None: From 1d3efca6f23df5c95c79326395ae09200f3c17fd Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 18:22:08 +0000 Subject: [PATCH 006/270] Judge exact artifact and memory reads by input labels. The artifact authorization door and a write that cites a memory settle a derived row before the policy check. A locked key is refused when that row cannot be checked. The owner and an unbound admin still read the stored row. --- CHANGELOG.md | 1 + .../src/alicebot_api/routers/_vnext_shared.py | 19 +- .../api/src/alicebot_api/vnext_label_guard.py | 219 ++++++++++++++++++ .../src/alicebot_api/vnext_source_fence.py | 14 +- tests/unit/test_label_guard_exact_doors.py | 157 +++++++++++++ 5 files changed, 402 insertions(+), 8 deletions(-) create mode 100644 apps/api/src/alicebot_api/vnext_label_guard.py create mode 100644 tests/unit/test_label_guard_exact_doors.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7be7ec35d..80ac80757 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Unreleased (on main, not in v0.20.0): an exact read of a derived artifact, and a write that cites a derived memory, uses the labels of that row's inputs. v0.20.0 used the label stored on the row, so a public copy of a confidential input could be read. A row whose inputs cannot be checked is refused to a locked key. The owner and an unbound admin still read the stored row. No migration is required. - Unreleased (on main, not in v0.20.0): a locked key's consolidation, roll-up, staleness sweep and open-loop review keep only inputs whose scope and floor are both inside the binding. The overlap check stays, and the exact test runs after it. Consolidation and roll-ups compare the group scope (scope united with floor), so a card stored with an empty scope and a floor of two projects is still found and can be accepted. The operator artifact list, given a project, matches that floor as well as the scope. Each new report and aggregate records `derived_from` for the rows it used. v0.20.0 selected by overlap of the scope alone and wrote no `derived_from`. No migration is required. - Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. Raising a source or a memory walks the rows derived from it and raises those too. A candidate open loop and a generated artifact take the same insert floor. Replacing a SQLite source with a stricter one raises the memories that cite it before they are retired. Moving a source to another project answers with how many derived rows a project-bound key would lose, and waits for confirm_label_hide before it writes. A relabel that cannot finish answers 409, and one that waits more than 3 seconds for the label lock answers 503. No migration is required. - Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. The function is what the write path calls. A read that does not go through that path still behaves as it does in v0.20.0. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. diff --git a/apps/api/src/alicebot_api/routers/_vnext_shared.py b/apps/api/src/alicebot_api/routers/_vnext_shared.py index 2edc94817..7b86ca670 100644 --- a/apps/api/src/alicebot_api/routers/_vnext_shared.py +++ b/apps/api/src/alicebot_api/routers/_vnext_shared.py @@ -25,7 +25,9 @@ agent_key_from_authorization, resolve_protected_agent_identity, ) +from alicebot_api.vnext_label_guard import LabelGuard, apply_unverified_rule, policy_labels from alicebot_api.vnext_project_scope import source_project_scope +from alicebot_api.vnext_source_fence import SourceReadFence from alicebot_api.vnext_queue import VNextQueueNotFoundError from alicebot_api.vnext_store import PostgresVNextStore, is_redacted_project_update_artifact @@ -461,14 +463,16 @@ def _vnext_exact_resource_policy( resource: dict[str, object], source_resource: bool = False, ) -> PolicyDecision: - domain = " ".join(str(resource.get("domain") or "unknown").split()).strip() or "unknown" - sensitivity = " ".join(str(resource.get("sensitivity") or "unknown").split()).strip() or "unknown" + domains, sensitivity_allowed, project_scope, project_floor = policy_labels(resource) + if source_resource: + project_scope = source_project_scope(resource) decision = evaluate_agent_policy( identity=identity, action=action, - domains=(domain,), - sensitivity_allowed=(sensitivity,), - project_scope=source_project_scope(resource) if source_resource else resource_project_scope(resource), + domains=domains, + sensitivity_allowed=sensitivity_allowed, + project_scope=project_scope, + project_floor=project_floor, require_explicit_project_scope=bool(identity is not None and identity.project_scope_locked), ) if decision.decision == "allowed_with_filtering": @@ -504,11 +508,14 @@ def _vnext_authorized_artifact( raise ValueError("feedback cannot be added to a redacted artifact") _vnext_agent_record(store, identity) + guard = LabelGuard.for_fence(store, SourceReadFence.for_identity(identity)) + effective = guard.effective_row("artifact", artifact) decision = _vnext_exact_resource_policy( identity=identity, action=action, - resource=artifact, + resource=effective if isinstance(effective, dict) else artifact, ) + decision = apply_unverified_rule(decision, effective if isinstance(effective, Mapping) else None, identity) append_policy_events( store, identity=identity, diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py new file mode 100644 index 000000000..f49b3763f --- /dev/null +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -0,0 +1,219 @@ +"""Read-time check for a derived row. + +One guard per request. An inactive guard returns its input and reads nothing. +An active guard settles the row with its inputs and answers the door with that +effective label. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, replace +from typing import Any + +from alicebot_api.vnext_agent_control import ( + ALL_SENSITIVITY, + VNEXT_DOMAINS, + AgentIdentity, + PolicyDecision, +) +from alicebot_api.vnext_derived_labels import ( + HOP_BOUND, + NODE_BOUND, + canon_kind, + dependencies_of, + is_derived, + settle_labels, +) +from alicebot_api.vnext_project_scope import project_floor_shape, project_scopes_overlap, resolve_project_scope + + +_GUARD_USER = "label-guard" + + +def _filters_admit_every( + domains: Sequence[str] | None, + sensitivity_allowed: Sequence[str] | None, + projects: Sequence[str] | None, +) -> bool: + domain_list = tuple(domains or ()) + domains_open = not domain_list or set(domain_list) >= set(VNEXT_DOMAINS) + sensitivity_list = tuple(sensitivity_allowed or ()) + sensitivity_open = set(sensitivity_list) >= set(ALL_SENSITIVITY) + return domains_open and sensitivity_open and not tuple(projects or ()) + + +@dataclass +class LabelGuard: + """The labels a door may trust for one request.""" + + store: Any + active: bool + domains: tuple[str, ...] = () + sensitivity_allowed: tuple[str, ...] = () + projects: tuple[str, ...] = () + _nodes: dict[tuple[str, str], dict[str, object]] | None = None + + @classmethod + def for_fence(cls, store: Any, fence: Any) -> LabelGuard: + """Exact doors. Inactive for the owner and for an unbound admin.""" + + fenced = bool(getattr(fence, "entity_read_fenced", False)) + return cls(store=store, active=fenced) + + @classmethod + def for_filters( + cls, + store: Any, + domains: Sequence[str] | None, + sensitivity_allowed: Sequence[str] | None, + projects: Sequence[str] | None = (), + exclude_global_domains: Sequence[str] | None = None, + ) -> LabelGuard: + """List doors. Inactive when the filters admit every label.""" + + del exclude_global_domains + domain_list = tuple(domains or ()) + sensitivity_list = tuple(sensitivity_allowed or ()) + project_list = tuple(projects or ()) + return cls( + store=store, + active=not _filters_admit_every(domain_list, sensitivity_list, project_list), + domains=domain_list, + sensitivity_allowed=sensitivity_list, + projects=project_list, + ) + + def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[str, object] | None: + """A copy whose domain, sensitivity, scope and floor are effective.""" + + if row is None or not self.active or not isinstance(row, Mapping): + return row + if not is_derived(kind, row): + return row + nodes = self._collected(kind, row) + settled = settle_labels(nodes, on_cycle="unverified", max_hops=HOP_BOUND, max_nodes=NODE_BOUND) + label = settled.by_stored(kind, str(row.get("id") or ""), user_id=_GUARD_USER) + copy = dict(row) + metadata = dict(copy.get("metadata_json")) if isinstance(copy.get("metadata_json"), Mapping) else {} + if label.unverified: + copy["domain"] = label.domain + copy["sensitivity"] = "regulated" + metadata["project_scope"] = [] + metadata["project_floor"] = [] + copy["unverified"] = True + else: + copy["domain"] = label.domain + copy["sensitivity"] = label.sensitivity + metadata["project_scope"] = list(label.project_scope) + metadata["project_floor"] = list(label.project_floor) + copy["unverified"] = False + copy["metadata_json"] = metadata + return copy + + def admit_rows(self, kind: str, rows: Sequence[Mapping[str, object]]) -> list[Mapping[str, object]]: + """Rows whose effective labels pass this guard's filters. Originals of the rows, not copies.""" + + if not self.active: + return [row for row in rows if isinstance(row, Mapping)] + kept: list[Mapping[str, object]] = [] + for row in rows: + if not isinstance(row, Mapping): + continue + effective = self.effective_row(kind, row) + if isinstance(effective, Mapping) and self._admits_effective(effective): + kept.append(row) + return kept + + def admit_beliefs(self, beliefs: Sequence[Mapping[str, object]]) -> list[Mapping[str, object]]: + """Beliefs whose backing memory the filters admit. One batched read.""" + + if not self.active: + return [row for row in beliefs if isinstance(row, Mapping)] + ids = [str(row.get("memory_id")) for row in beliefs if isinstance(row, Mapping) and row.get("memory_id")] + reader = getattr(self.store, "read_label_rows", None) + found: dict[str, Mapping[str, object]] = {} + if callable(reader) and ids: + for row in reader("memory", ids): + if isinstance(row, Mapping) and row.get("id") is not None: + found[str(row.get("id"))] = row + admitted = {str(row.get("id")) for row in self.admit_rows("memory", list(found.values()))} + return [ + row + for row in beliefs + if isinstance(row, Mapping) and str(row.get("memory_id") or "") in admitted + ] + + def _admits_effective(self, row: Mapping[str, object]) -> bool: + domain = str(row.get("domain") or "unknown") + if self.domains and domain not in self.domains and domain != "unknown": + return False + sensitivity = str(row.get("sensitivity") or "unknown") + if self.sensitivity_allowed and sensitivity not in self.sensitivity_allowed: + return False + if self.projects: + scope = resolve_project_scope(row).values + _shape, floor = project_floor_shape(row) + if not project_scopes_overlap(scope, self.projects, floor=floor): + return False + return True + + def _collected(self, kind: str, row: Mapping[str, object]) -> list[dict[str, object]]: + if self._nodes is None: + self._nodes = {} + pending: list[tuple[str, Mapping[str, object]]] = [(canon_kind(kind), row)] + hops = 0 + while pending and len(self._nodes) < NODE_BOUND and hops < HOP_BOUND: + hops += 1 + name, current = pending.pop(0) + node_id = str(current.get("id") or "") + key = (name, node_id) + if key in self._nodes: + continue + node = dict(current) + node["kind"] = name + node["user_id"] = _GUARD_USER + self._nodes[key] = node + if not is_derived(name, node): + continue + grouped: dict[str, list[str]] = {} + for dep_kind, dep_id in dependencies_of(name, node): + grouped.setdefault(canon_kind(dep_kind), []).append(str(dep_id)) + reader = getattr(self.store, "read_label_rows", None) + if not callable(reader): + continue + for dep_kind, ids in grouped.items(): + missing = [item for item in ids if (dep_kind, item) not in self._nodes] + if not missing: + continue + for found in reader(dep_kind, missing): + if isinstance(found, Mapping): + pending.append((dep_kind, found)) + return list(self._nodes.values()) + + +def policy_labels( + row: Mapping[str, object], +) -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...], tuple[str, ...]]: + """Domain, sensitivity, scope and floor to hand the policy engine.""" + + domain = " ".join(str(row.get("domain") or "unknown").split()).strip() or "unknown" + sensitivity = " ".join(str(row.get("sensitivity") or "unknown").split()).strip() or "unknown" + scope = resolve_project_scope(row).values + _shape, floor = project_floor_shape(row) + return (domain,), (sensitivity,), scope, floor + + +def apply_unverified_rule( + decision: PolicyDecision, + row: Mapping[str, object] | None, + identity: AgentIdentity | None, +) -> PolicyDecision: + """A locked identity is refused an unverified derived row.""" + + unverified = isinstance(row, Mapping) and bool(row.get("unverified")) + locked = identity is not None and bool(identity.project_scope_locked) + if not unverified or not locked: + return decision + reasons = tuple(dict.fromkeys((*decision.reasons, "derived_labels_unverified"))) + return replace(decision, decision="blocked", reasons=reasons) diff --git a/apps/api/src/alicebot_api/vnext_source_fence.py b/apps/api/src/alicebot_api/vnext_source_fence.py index e251dd26f..d46d530ba 100644 --- a/apps/api/src/alicebot_api/vnext_source_fence.py +++ b/apps/api/src/alicebot_api/vnext_source_fence.py @@ -122,7 +122,7 @@ evaluate_agent_policy, resource_project_scope, ) -from alicebot_api.vnext_project_scope import source_project_scope +from alicebot_api.vnext_project_scope import project_floor_shape, source_project_scope SOURCE_REF_NOT_FOUND_MESSAGE = "the cited source was not found in the current user scope" MEMORY_REF_NOT_FOUND_MESSAGE = "the cited memory was not found in the current user scope" @@ -229,12 +229,16 @@ def _admits(self, row: Mapping[str, object], *, project_scope: tuple[str, ...]) return False if self.identity is None: return True + if row.get("unverified") and self.identity.project_scope_locked: + return False + _shape, floor = project_floor_shape(row) decision = evaluate_agent_policy( identity=self.identity, action=EXPLAIN_DISCLOSURE_ACTION, domains=(str(row.get("domain") or "unknown"),), sensitivity_allowed=(str(row.get("sensitivity") or "unknown"),), project_scope=project_scope, + project_floor=floor, require_explicit_project_scope=True, ) # "allowed_with_filtering" is a refusal here, as it is for explain: the @@ -361,7 +365,13 @@ def resolve_attachable_memory_id(store: object, memory_id: str, *, fence: Source raise MemoryRefNotFoundError() from None getter = getattr(store, "get_memory", None) row = getter(canonical) if callable(getter) else None - if not isinstance(row, Mapping) or not fence.admits_memory(row): + if not isinstance(row, Mapping): + raise MemoryRefNotFoundError() + from alicebot_api.vnext_label_guard import LabelGuard + + effective = LabelGuard.for_fence(store, fence).effective_row("memory", row) + judged = effective if isinstance(effective, Mapping) else row + if not fence.admits_memory(judged): raise MemoryRefNotFoundError() return canonical diff --git a/tests/unit/test_label_guard_exact_doors.py b/tests/unit/test_label_guard_exact_doors.py new file mode 100644 index 000000000..3e75a7d0d --- /dev/null +++ b/tests/unit/test_label_guard_exact_doors.py @@ -0,0 +1,157 @@ +"""Exact doors judge a derived row by the labels of its inputs.""" + +from __future__ import annotations + +import pytest + +from alicebot_api.routers._vnext_shared import _vnext_authorized_artifact +from alicebot_api.vnext_agent_control import AgentIdentity, AgentPolicyBlockedError +from alicebot_api.vnext_label_guard import LabelGuard, apply_unverified_rule +from alicebot_api.vnext_source_fence import ( + MemoryRefNotFoundError, + SourceReadFence, + resolve_attachable_memory_id, +) + + +ALPHA = "prj_" + "a" * 16 +SOURCE_ID = "11111111-1111-1111-1111-111111111111" +ARTIFACT_ID = "22222222-2222-2222-2222-222222222222" +MEMORY_ID = "33333333-3333-3333-3333-333333333333" + + +class _LabelStore: + def __init__(self) -> None: + self.source = { + "id": SOURCE_ID, + "user_id": "label-guard", + "domain": "health", + "sensitivity": "confidential", + "metadata_json": {"project_scope": [ALPHA]}, + } + self.artifact = { + "id": ARTIFACT_ID, + "domain": "unknown", + "sensitivity": "public", + "content_markdown": "secret health note", + "metadata_json": { + "workflow": "daily_brief", + "project_scope": [ALPHA], + "derived_from": { + "v": 1, + "sources": [SOURCE_ID], + "memories": [], + "open_loops": [], + "artifacts": [], + "beliefs": [], + "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, + }, + }, + } + self.memory = { + "id": MEMORY_ID, + "domain": "unknown", + "sensitivity": "public", + "canonical_text": "copied", + "metadata_json": { + "source_id": SOURCE_ID, + "project_scope": [ALPHA], + "derived_from": { + "v": 1, + "sources": [SOURCE_ID], + "memories": [], + "open_loops": [], + "artifacts": [], + "beliefs": [], + "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, + }, + }, + } + self.events: list[dict[str, object]] = [] + + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + if kind == "source" and SOURCE_ID in ids: + return [self.source] + return [] + + def get_artifact(self, artifact_id: str) -> dict[str, object] | None: + return self.artifact if artifact_id == ARTIFACT_ID else None + + def get_memory(self, memory_id: str) -> dict[str, object] | None: + return self.memory if memory_id == MEMORY_ID else None + + def upsert_agent_identity(self, *_args: object, **_kwargs: object) -> None: + return None + + def append_event(self, event: dict[str, object]) -> dict[str, object]: + self.events.append(event) + return event + + +def _trusted() -> AgentIdentity: + return AgentIdentity(agent_id="trusted-key", permission_profile="trusted_local_agent") + + +def _locked_admin() -> AgentIdentity: + return AgentIdentity( + agent_id="bound-admin", + permission_profile="admin_agent", + project_scope=(ALPHA,), + project_scope_locked=True, + ) + + +def test_an_exact_artifact_door_uses_the_input_label() -> None: + store = _LabelStore() + with pytest.raises(AgentPolicyBlockedError): + _vnext_authorized_artifact( + store=store, # type: ignore[arg-type] + identity=_trusted(), + artifact_id=ARTIFACT_ID, + action="artifact.read", + for_update=False, + ) + + +def test_the_owner_guard_does_not_read() -> None: + store = _LabelStore() + guard = LabelGuard.for_fence(store, SourceReadFence.for_identity(None)) + assert guard.active is False + assert guard.effective_row("artifact", store.artifact) is store.artifact + + +def test_a_locked_key_is_refused_an_unverified_row() -> None: + store = _LabelStore() + store.artifact["metadata_json"] = { + "workflow": "daily_brief", + "project_scope": [ALPHA], + "derived_from": { + "v": 1, + "sources": ["99999999-9999-9999-9999-999999999999"], + "memories": [], + "open_loops": [], + "artifacts": [], + "beliefs": [], + "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, + }, + } + identity = _locked_admin() + guard = LabelGuard.for_fence(store, SourceReadFence.for_identity(identity)) + effective = guard.effective_row("artifact", store.artifact) + assert isinstance(effective, dict) + assert effective["unverified"] is True + from alicebot_api.vnext_agent_control import PolicyDecision + + blocked = apply_unverified_rule( + PolicyDecision(decision="allowed", action="artifact.read", permission_profile="admin_agent"), + effective, + identity, + ) + assert blocked.decision == "blocked" + assert "derived_labels_unverified" in blocked.reasons + + +def test_a_cited_memory_is_judged_by_its_source() -> None: + store = _LabelStore() + with pytest.raises(MemoryRefNotFoundError): + resolve_attachable_memory_id(store, MEMORY_ID, fence=SourceReadFence.for_identity(_trusted())) From a97fbb36a656069e08906b1545531b6d5c270440 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 18:29:25 +0000 Subject: [PATCH 007/270] Pass a row floor into project view checks. A view that asks for global rows keeps a row with no Alice project id only when every Alice project id in its floor is in the view. The four Python checks and the SQLite view SQL now pass that floor. --- CHANGELOG.md | 2 +- .../src/alicebot_api/mcp/retrieval_shared.py | 8 +- apps/api/src/alicebot_api/session_briefing.py | 17 ++++- apps/api/src/alicebot_api/vnext_retrieval.py | 34 +++++++-- .../vnext_stores/sqlite/graph_open_loops.py | 1 + .../vnext_stores/sqlite/memory_access.py | 1 + .../vnext_stores/sqlite/query_predicates.py | 21 ++++++ .../unit/test_store_graph_open_loops_split.py | 19 +++-- tests/unit/test_store_memory_access_split.py | 8 +- tests/unit/test_view_floor.py | 75 +++++++++++++++++++ 10 files changed, 168 insertions(+), 18 deletions(-) create mode 100644 tests/unit/test_view_floor.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7be7ec35d..b3b183fbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Unreleased (on main, not in v0.20.0): a locked key's consolidation, roll-up, staleness sweep and open-loop review keep only inputs whose scope and floor are both inside the binding. The overlap check stays, and the exact test runs after it. Consolidation and roll-ups compare the group scope (scope united with floor), so a card stored with an empty scope and a floor of two projects is still found and can be accepted. The operator artifact list, given a project, matches that floor as well as the scope. Each new report and aggregate records `derived_from` for the rows it used. v0.20.0 selected by overlap of the scope alone and wrote no `derived_from`. No migration is required. +- Unreleased (on main, not in v0.20.0): a locked key's consolidation, roll-up, staleness sweep and open-loop review keep only inputs whose scope and floor are both inside the binding. The overlap check stays, and the exact test runs after it. Consolidation and roll-ups compare the group scope (scope united with floor), so a card stored with an empty scope and a floor of two projects is still found and can be accepted. The operator artifact list, given a project, matches that floor as well as the scope. Each new report and aggregate records `derived_from` for the rows it used. A project view that also asks for global rows keeps a row with no Alice project id only when every Alice project id in its floor is in that view. v0.20.0 selected by overlap of the scope alone and wrote no `derived_from`. No migration is required. - Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. Raising a source or a memory walks the rows derived from it and raises those too. A candidate open loop and a generated artifact take the same insert floor. Replacing a SQLite source with a stricter one raises the memories that cite it before they are retired. Moving a source to another project answers with how many derived rows a project-bound key would lose, and waits for confirm_label_hide before it writes. A relabel that cannot finish answers 409, and one that waits more than 3 seconds for the label lock answers 503. No migration is required. - Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. The function is what the write path calls. A read that does not go through that path still behaves as it does in v0.20.0. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` now scrub an open loop that names the source only in its metadata: the id, or `source:`, as the text under `source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references` or `selected_source_ids` at any depth. That is the rule the open-loop lookup of a source already used. They blanked only loops whose `source_id` column held the id, so a loop with an empty column kept its title, description and metadata, and they reached a new export. The delete preview now counts the same loops as the receipt. The rule reads the id only as stored or as `source:`: a loop that names the source in any spelling other than those two keeps its text, although the memory rule reads several other spellings. No migration is required. diff --git a/apps/api/src/alicebot_api/mcp/retrieval_shared.py b/apps/api/src/alicebot_api/mcp/retrieval_shared.py index fa1081c62..c0586a250 100644 --- a/apps/api/src/alicebot_api/mcp/retrieval_shared.py +++ b/apps/api/src/alicebot_api/mcp/retrieval_shared.py @@ -12,6 +12,7 @@ from alicebot_api.vnext_agent_control import resource_project_scope from alicebot_api.vnext_project_scope import ( is_global_scope, + project_floor_shape, project_identifier_identity, project_scopes_overlap, ) @@ -160,7 +161,12 @@ def _provenance_count(store: SQLiteVNextStore, memory_id: object) -> int: def _resource_matches_project_scope(resource: Mapping[str, object], project_scope: tuple[str, ...]) -> bool: if not project_scope: return True - return project_scopes_overlap(resource_project_scope(resource), project_scope) + shape, floor = project_floor_shape(resource) + return project_scopes_overlap( + resource_project_scope(resource), + project_scope, + floor=floor if shape == "list" else (), + ) def _resource_is_held_back_global(resource: Mapping[str, object], exclude_global_domains: frozenset[str]) -> bool: diff --git a/apps/api/src/alicebot_api/session_briefing.py b/apps/api/src/alicebot_api/session_briefing.py index 202c88a67..f297cb97b 100644 --- a/apps/api/src/alicebot_api/session_briefing.py +++ b/apps/api/src/alicebot_api/session_briefing.py @@ -59,6 +59,7 @@ ) from alicebot_api.vnext_project_scope import ( is_global_scope, + project_floor_shape, project_scope_identity, project_scopes_overlap, source_project_scope, @@ -624,7 +625,7 @@ def _memory_honours_fence( return ( _matches_domains(row, effective_domains) and _matches_sensitivity(row, effective_sensitivity_allowed) - and _matches_project_scope(resource_scope, effective_project_scope) + and _matches_project_scope(resource_scope, effective_project_scope, floor=_brief_floor(row)) and not _is_held_back(row, resource_scope, exclude_global_domains) ) @@ -676,10 +677,20 @@ def _matches_sensitivity(row: Mapping[str, object], sensitivity_allowed: tuple[s return (row.get("sensitivity") or "unknown") in sensitivity_allowed -def _matches_project_scope(resource_scope: tuple[str, ...], project_scope: tuple[str, ...]) -> bool: +def _brief_floor(row: Mapping[str, object]) -> tuple[str, ...]: + shape, floor = project_floor_shape(row) + return floor if shape == "list" else () + + +def _matches_project_scope( + resource_scope: tuple[str, ...], + project_scope: tuple[str, ...], + *, + floor: tuple[str, ...] = (), +) -> bool: if not project_scope: return True - return project_scopes_overlap(resource_scope, project_scope) + return project_scopes_overlap(resource_scope, project_scope, floor=floor) # A fact used as the excerpt query is passed to the source search whole, and diff --git a/apps/api/src/alicebot_api/vnext_retrieval.py b/apps/api/src/alicebot_api/vnext_retrieval.py index a7671d0ab..f85ffb2b8 100644 --- a/apps/api/src/alicebot_api/vnext_retrieval.py +++ b/apps/api/src/alicebot_api/vnext_retrieval.py @@ -106,6 +106,7 @@ from alicebot_api.vnext_promotion_policy import memory_write_provenance from alicebot_api.vnext_project_scope import ( is_global_scope, + project_floor_shape, project_scope_identity, project_scopes_overlap, resolve_project_scope, @@ -946,16 +947,31 @@ def _row_scope_event_time(row: Mapping[str, object]) -> datetime | None: return parse_event_datetime(row.get("captured_at")) -def _project_scope_meets(row_scope: set[str], requested: Collection[str]) -> bool: +def _row_floor(row: Mapping[str, object]) -> tuple[str, ...]: + shape, floor = project_floor_shape(row) + return floor if shape == "list" else () + + +def _project_scope_meets( + row_scope: set[str], + requested: Collection[str], + *, + floor: Collection[str] = (), +) -> bool: """Does a row's resolved project scope meet the requested tuple? The tuple may hold the reserved global marker (spec 6.1): it asks for a row whose scope holds no Alice project id. The one predicate in ``vnext_project_scope`` decides, so a request without the marker keeps the - plain intersection it always had. + plain intersection it always had. On that global branch the row's floor + must also sit inside the view. """ - return project_scopes_overlap(tuple(sorted(row_scope)), tuple(sorted(requested))) + return project_scopes_overlap( + tuple(sorted(row_scope)), + tuple(sorted(requested)), + floor=tuple(floor), + ) def _is_held_back_global( @@ -984,7 +1000,11 @@ def _row_matches_scope( if source_scope_envelope else _row_project_scope_values(row) ) - if scope.projects and not _project_scope_meets(project_scope, scope.projects): + if scope.projects and not _project_scope_meets( + project_scope, + scope.projects, + floor=_row_floor(row), + ): return False if scope.exclude_global_domains and _is_held_back_global( row, project_scope, scope.exclude_global_domains @@ -1229,7 +1249,11 @@ def _graph_memory_admissible( if memory_types and row.get("memory_type") not in memory_types: return False if projects: - if not _project_scope_meets(_row_project_scope_values(row), projects): + if not _project_scope_meets( + _row_project_scope_values(row), + projects, + floor=_row_floor(row), + ): return False if created_by_agent_ids and row.get("created_by_agent_id") not in created_by_agent_ids: return False diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py index cc43135e1..4857bcbce 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py @@ -828,6 +828,7 @@ def list_open_loops_view_partitions( text_expressions=("metadata_json", "project_id"), domain_expression="domain", global_excluded_domains=tuple(sorted(exclude_global_domains)), + floor_expression="alice_project_floor_identity(metadata_json)", ) columns = ", ".join(f"l.{column}" for column in OPEN_LOOP_COLUMNS) rows = self._fetch_all( diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py index 4b677e515..2a019adae 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py @@ -617,6 +617,7 @@ def list_memories_view_partitions( text_expressions=("metadata_json", "project_id"), domain_expression="domain", global_excluded_domains=tuple(sorted(exclude_global_domains)), + floor_expression="alice_project_floor_identity(metadata_json)", ) ordering = ("created_at",) if order_by_created_at else ("updated_at", "created_at") order_columns = ", ".join(ordering) diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/query_predicates.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/query_predicates.py index 5beed618a..f87c89bf1 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/query_predicates.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/query_predicates.py @@ -219,6 +219,7 @@ def _view_membership_sql( global_excluded_domains: tuple[str, ...], text_expressions: tuple[str, ...], partition: bool, + floor_expression: str = "'[]'", ) -> tuple[str, list[object]]: """The exact view test as one aggregate over one identity-function call. @@ -247,6 +248,18 @@ def excluded_sql() -> str: ) when_global = "" if wants_global: + floor_outside = "" + if ids: + params.extend(ids) + floor_outside = ( + " AND CAST(floor_id.value AS TEXT) NOT IN (" + f"{placeholders(list(ids))})" + ) + floor_clear = ( + "NOT EXISTS (SELECT 1 FROM json_each(" + f"{floor_expression}) AS floor_id WHERE " + f"{_sql_has_alice_id('CAST(floor_id.value AS TEXT)')}{floor_outside})" + ) if partition: if excluded: inner = f"CASE WHEN {excluded_sql()} THEN NULL ELSE 0 END" @@ -258,6 +271,7 @@ def excluded_sql() -> str: inner = "1" when_global = ( f"WHEN COALESCE(MAX({_sql_has_alice_id('CAST(scoped_project.value AS TEXT)')}), 0) = 0 " + f"AND {floor_clear} " f"THEN {inner} " ) fallback = "NULL" if partition else "0" @@ -297,6 +311,7 @@ def _project_view_sql( text_expressions: tuple[str, ...], domain_expression: str | None, global_excluded_domains: tuple[str, ...] | None, + floor_expression: str = "'[]'", ) -> tuple[str, list[object]]: """The ``AND ...`` clause for a request tuple, or ``("", [])`` when it fences nothing. @@ -340,6 +355,7 @@ def _project_view_sql( global_excluded_domains=excluded, text_expressions=text_expressions, partition=False, + floor_expression=floor_expression, ) params.extend(exact_params) return f" AND ({fast} OR {exact} = 1)", params @@ -356,6 +372,7 @@ def _project_view_sql( global_excluded_domains=(), text_expressions=text_expressions, partition=False, + floor_expression=floor_expression, ) params.extend(exact_params) return f" AND ({prefilter} AND {exact} = 1)", params @@ -369,6 +386,7 @@ def _project_view_partition_sql( text_expressions: tuple[str, ...], domain_expression: str, global_excluded_domains: tuple[str, ...], + floor_expression: str = "'[]'", ) -> tuple[str, list[object]]: """A value per row for the single-scan fill: 1 project, 0 global, NULL outside the view. @@ -395,6 +413,7 @@ def _project_view_partition_sql( global_excluded_domains=excluded, text_expressions=text_expressions, partition=True, + floor_expression=floor_expression, ) params.extend(exact_params) return f"CASE WHEN {fast} THEN 0 ELSE {exact} END", params @@ -522,6 +541,7 @@ def _project_clause( text_expressions=(f"{prefix}metadata_json", f"{prefix}project_id"), domain_expression=f"{prefix}domain", global_excluded_domains=global_excluded_domains, + floor_expression=f"alice_project_floor_identity({prefix}metadata_json)", ) @@ -665,6 +685,7 @@ def _metadata_values(keys: tuple[str, ...], values: tuple[str, ...]) -> str: text_expressions=text_expressions, domain_expression=domain_expression, global_excluded_domains=global_excluded_domains, + floor_expression=f"alice_project_floor_identity({metadata_expression})", ) clauses.append(project_sql) params.extend(project_params) diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index eae9d1a4d..36574141b 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -110,7 +110,8 @@ # (an empty ceiling returns no rows without a query). The Postgres reader takes the same two arguments so that # the shared unscoped call site can state ``None`` for both, and it refuses anything else, since the Postgres # runtime resolves no project view (reviewed change, not drift). - POSTGRES_CARRIER_PATH: "e4724ba1ec3b8917c5be74b619259ddf1c282825b8e938ce4fa9947491a90a6f", + # The file hash now matches the carrier after the label lock. Previous receipt e4724ba1... + POSTGRES_CARRIER_PATH: "87aeac394e698a0b5709fd168abfa2a8a86af674ad42370f5a40decd773554df", # The SQLite carrier is re-minted, with its method AST manifest below, for # ``list_open_loops`` and ``list_open_loop_events``: they bind a query through # ``literal_match_operand`` and so refuse one past the LIKE operand limit. @@ -128,13 +129,17 @@ # delete preview and the scrub count and blank the same loops. The rule text moved unchanged to # ``vnext_stores/sqlite/open_loop_source_reference.py``; only the reader function and the receipt of the file # change (reviewed change, not drift). - SQLITE_CARRIER_PATH: "9a2634bef621d32262b845c046820d8b19c64801ec9f9b462e978f364f16f643", + # Re-minted so the open-loop partition read passes the floor identity. + # Previous receipt 9a2634be... + SQLITE_CARRIER_PATH: "c05ac13285a25bd59a2f22d12a7f56e2ac81f063af0949aa7adc82e43f643454", POSTGRES_COLUMNS_PATH: "5b0d972a55abf8590ce14394a37fd71b9b88ba7ab3de82d61efc1bddfc022b71", SQLITE_COLUMNS_PATH: "be81b8628d0831d3d02b280b5455fb02333db5740ebef8d85d58024384ae6556", } EXPECTED_METHOD_AST_MANIFESTS = { - POSTGRES_CARRIER_PATH: "2558088459f1b9a565e1b366ffe0b7c4025c623a9e2ea78007d06a46793ce1b8", - SQLITE_CARRIER_PATH: "2850ba6057b1510759613aaa3798a226808a42470ee11cfb9c6e3afbf3e98e66", + # Postgres manifest matches the carrier after the label lock. Previous 25580884... + POSTGRES_CARRIER_PATH: "48064a91179463a20147a8e02442f3259976752000d9aafcb51647851227c46c", + # SQLite manifest includes the floor identity on the partition read. Previous 2850ba60... + SQLITE_CARRIER_PATH: "54112e01f88e048b63731252d3fc0e34918db8575f70ab6b54b0a551699d1483", } EXPECTED_METADATA_MANIFESTS = { POSTGRES_CARRIER_PATH: "6edb6a10e7a37dbbbbde97e5550422718a0112257666de8e23d49c60490fa13f", @@ -152,7 +157,8 @@ # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (172, "6f1a459fcf4319cf4281f6cc0d4e81679c3e05d851fd0a874a2d90298d7c2569"), + # lock_label_writes and read_label_rows follow __init__. Previous receipt (172, 6f1a459f...). + "PostgresVNextStore": (174, "095250b8a77d6c0a32d2783343916b947bcae8ae86e3a1693e47c2fb11802211"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -167,7 +173,8 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. - "SQLiteVNextStore": (134, "1301272026897057cf071009cc21787543ddc326f1e06f1a75a763f3e344767e"), + # lock_label_writes and read_label_rows follow __init__. Previous receipt (134, 13012720...). + "SQLiteVNextStore": (136, "1d99dbaf69e4ea888ca7beb2389ac176650a2e573c067bf0686adc9e609132f5"), } EXPECTED_COLUMN_AST = { POSTGRES_COLUMNS_PATH: { diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index f04ef0ec9..2315cb2ac 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -51,8 +51,10 @@ # global domains it leaves out raises (reviewed change, not drift). # Re-minted for alice_project_floor_identity, the fourth identity function, # used by the roll-up lookups. Previous receipt eab46f16... + # Re-minted so a global view also requires every Alice id in the floor. + # Previous receipt 8beae59c... "apps/api/src/alicebot_api/vnext_stores/sqlite/query_predicates.py": ( - "8beae59c379c56033388a35b5a152f1f683f6b55c6e2fbd62925723566a12202" + "670fd096b461c72517f3791c1f6216778f26d68838307c11d07ffcf5e7b38e79" ), # Re-minted for the Phase 4 Stage 2 resident vector cache (reviewed # carrier change; the receipt guards unreviewed drift): the vector scan @@ -84,8 +86,10 @@ # takes a keyword-only ``include_deleted`` (false by default) and drops the ``deleted_at IS NULL`` clause only when # it is true. Previous SQLite receipt 91636de9... # Re-minted so the two roll-up lookups overlap scope or floor. Previous receipt 64f21989... + # Re-minted so the memory partition read passes the floor identity. + # Previous receipt 1580dca3... "apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py": ( - "1580dca3a31fbbcf98539e71e81bad13935f30c59e8a7c75b0fc0b4472a6d5fa" + "58e22342c6aaf4ed73aa78e5f2ee85af1d8deeb7cbad6a3abe06f78fb8f9a0ac" ), } diff --git a/tests/unit/test_view_floor.py b/tests/unit/test_view_floor.py new file mode 100644 index 000000000..fa93d315f --- /dev/null +++ b/tests/unit/test_view_floor.py @@ -0,0 +1,75 @@ +"""A project view that asks for global rows also consults the floor.""" + +from __future__ import annotations + +import sqlite3 +from uuid import uuid4 + +from alicebot_api.mcp.retrieval_shared import _resource_matches_project_scope +from alicebot_api.session_briefing import _memory_honours_fence +from alicebot_api.sqlite_schema import bootstrap_sqlite_schema +from alicebot_api.sqlite_store import SQLiteVNextStore, ensure_sqlite_user +from alicebot_api.vnext_project_scope import GLOBAL_PROJECT_MARKER +from alicebot_api.vnext_retrieval import _project_scope_meets + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +def test_python_view_mirrors_hide_a_foreign_floor() -> None: + view = (ALPHA, GLOBAL_PROJECT_MARKER) + assert _project_scope_meets(set(), view, floor=(ALPHA,)) is True + assert _project_scope_meets(set(), view, floor=(BETA,)) is False + assert _project_scope_meets(set(), view, floor=()) is True + foreign = {"metadata_json": {"project_scope": [], "project_floor": [BETA]}} + home = {"metadata_json": {"project_scope": [], "project_floor": [ALPHA]}} + assert _resource_matches_project_scope(foreign, view) is False + assert _resource_matches_project_scope(home, view) is True + assert _memory_honours_fence( + {**foreign, "domain": "project", "sensitivity": "internal"}, + effective_domains=(), + effective_sensitivity_allowed=("internal",), + effective_project_scope=view, + exclude_global_domains=frozenset(), + ) is False + + +def test_sqlite_global_view_hides_a_row_whose_floor_names_another_project() -> None: + conn = sqlite3.connect(":memory:") + bootstrap_sqlite_schema(conn) + user_id = str(uuid4()) + ensure_sqlite_user(conn, user_id, "view-floor@example.com", "View Floor") + store = SQLiteVNextStore(conn, user_id) + + def add(name: str, scope: list[str], floor: list[str]) -> str: + row = store.create_memory( + { + "memory_key": f"memory.{name}", + "value": {"text": name}, + "status": "active", + "memory_type": "semantic", + "title": name, + "canonical_text": name, + "summary": name, + "domain": "project", + "sensitivity": "internal", + "metadata_json": {"project_scope": scope, "project_floor": floor}, + } + ) + return str(row["id"]) + + home = add("home", [], [ALPHA]) + foreign = add("foreign", [], [BETA]) + plain = add("plain", [], []) + listed = { + str(row["id"]) + for row in store.list_memories( + projects=(ALPHA, GLOBAL_PROJECT_MARKER), + exclude_global_domains=(), + limit=20, + ) + } + assert home in listed + assert plain in listed + assert foreign not in listed + conn.close() From 4b6087588a91a192c0019ed440ea7aba986d8035 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 18:35:56 +0000 Subject: [PATCH 008/270] Settle labels at the remaining exact doors. Explain, memory review, redaction, open-loop update, and the legacy artifact authorizer use the input labels. A registry names each exact door and the readers that are not doors yet. --- CHANGELOG.md | 2 +- .../alicebot_api/mcp/evidence_artifacts.py | 29 ++- apps/api/src/alicebot_api/mcp/memories.py | 18 +- apps/api/src/alicebot_api/mcp/policy.py | 2 + apps/api/src/alicebot_api/mcp/retrieval.py | 5 +- apps/api/src/alicebot_api/mcp/review.py | 41 ++++- .../src/alicebot_api/routers/_vnext_shared.py | 2 + .../alicebot_api/routers/vnext_memories.py | 24 ++- .../alicebot_api/routers/vnext_projects.py | 5 +- .../api/src/alicebot_api/vnext_label_guard.py | 15 ++ .../src/alicebot_api/vnext_memory_commit.py | 22 ++- tests/unit/test_label_door_registry.py | 167 ++++++++++++++++++ tests/unit/test_label_guard_exact_doors.py | 77 ++++++++ 13 files changed, 377 insertions(+), 32 deletions(-) create mode 100644 tests/unit/test_label_door_registry.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 80ac80757..1565bb272 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Unreleased (on main, not in v0.20.0): an exact read of a derived artifact, and a write that cites a derived memory, uses the labels of that row's inputs. v0.20.0 used the label stored on the row, so a public copy of a confidential input could be read. A row whose inputs cannot be checked is refused to a locked key. The owner and an unbound admin still read the stored row. No migration is required. +- Unreleased (on main, not in v0.20.0): an exact read of a derived artifact, a write that cites a derived memory, explain, memory review, redaction, and an open-loop update use the labels of that row's inputs. v0.20.0 used the label stored on the row, so a public copy of a confidential input could be read. A row whose inputs cannot be checked is refused to a locked key. The owner and an unbound admin still read the stored row. No migration is required. - Unreleased (on main, not in v0.20.0): a locked key's consolidation, roll-up, staleness sweep and open-loop review keep only inputs whose scope and floor are both inside the binding. The overlap check stays, and the exact test runs after it. Consolidation and roll-ups compare the group scope (scope united with floor), so a card stored with an empty scope and a floor of two projects is still found and can be accepted. The operator artifact list, given a project, matches that floor as well as the scope. Each new report and aggregate records `derived_from` for the rows it used. v0.20.0 selected by overlap of the scope alone and wrote no `derived_from`. No migration is required. - Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. Raising a source or a memory walks the rows derived from it and raises those too. A candidate open loop and a generated artifact take the same insert floor. Replacing a SQLite source with a stricter one raises the memories that cite it before they are retired. Moving a source to another project answers with how many derived rows a project-bound key would lose, and waits for confirm_label_hide before it writes. A relabel that cannot finish answers 409, and one that waits more than 3 seconds for the label lock answers 503. No migration is required. - Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. The function is what the write path calls. A read that does not go through that path still behaves as it does in v0.20.0. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. diff --git a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py index 0a384563a..1607cb387 100644 --- a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py +++ b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py @@ -275,18 +275,30 @@ def _authorize_explain_resource( ) -> None: """Require an unfiltered policy decision for one expanded resource.""" + from alicebot_api.vnext_label_guard import apply_unverified_rule, effective_row_for_fence, policy_labels + + judged: Mapping[str, object] = resource + judged_scope = project_scope + judged_floor: tuple[str, ...] = () + if target_type in {"memory", "source", "artifact"}: + judged = effective_row_for_fence(store, identity, target_type, resource) + _domains, _sensitivity, judged_scope, judged_floor = policy_labels(judged) + if target_type == "source": + judged_scope = project_scope _actor_type, _actor_id, decision = _policy_checked( store, # type: ignore[arg-type] identity=identity, action=EXPLAIN_DISCLOSURE_ACTION, - domains=(str(resource.get("domain") or "unknown"),), - sensitivity_allowed=(str(resource.get("sensitivity") or "unknown"),), - project_scope=project_scope, + domains=(str(judged.get("domain") or "unknown"),), + sensitivity_allowed=(str(judged.get("sensitivity") or "unknown"),), + project_scope=judged_scope, + project_floor=judged_floor, require_explicit_project_scope=True, target_type=target_type, target_id=target_id, project_view=ProjectView.unscoped(), ) + decision = apply_unverified_rule(decision, judged, identity) # ``allowed_with_filtering`` is not sufficient for an explain response: # the downstream services expand related rows and do not accept filters. if decision.decision != "allowed": @@ -740,20 +752,25 @@ def _authorize_vnext_artifact_target( artifact = store.get_artifact_for_update(artifact_id) if for_update else store.get_artifact(artifact_id) if artifact is None: raise MCPReferenceNotFoundError(f"artifact {artifact_id} was not found") + from alicebot_api.vnext_label_guard import apply_unverified_rule, effective_row_for_fence, policy_labels + judged = effective_row_for_fence(store, identity, "artifact", artifact) + domains, sensitivity_allowed, project_scope, project_floor = policy_labels(judged) actor_type, actor_id, raw_decision = _policy_checked( store, identity=identity, action=action, - domains=(str(artifact.get("domain") or "unknown"),), - sensitivity_allowed=(str(artifact.get("sensitivity") or "unknown"),), - project_scope=resource_project_scope(artifact), + domains=domains, + sensitivity_allowed=sensitivity_allowed, + project_scope=project_scope, + project_floor=project_floor, require_explicit_project_scope=True, require_unfiltered_target=True, target_type="artifact", target_id=artifact_id, project_view=ProjectView.unscoped(), ) + raw_decision = apply_unverified_rule(raw_decision, judged, identity) return artifact, actor_type, actor_id, raw_decision diff --git a/apps/api/src/alicebot_api/mcp/memories.py b/apps/api/src/alicebot_api/mcp/memories.py index 6165fe47c..0beb74c31 100644 --- a/apps/api/src/alicebot_api/mcp/memories.py +++ b/apps/api/src/alicebot_api/mcp/memories.py @@ -427,8 +427,11 @@ def redact_memory_flow( # before it is raised, and for a deleted row it is raised as a refusal the # surface answers "not found" (RefusedOnDeletedMemoryError): a plain not-found # error here would roll the audit row back with the call. + from alicebot_api.vnext_label_guard import effective_row_for_fence, policy_labels + + judged = effective_row_for_fence(store, identity, "memory", memory) try: - memory_service.refuse_unauthorized_write(identity=identity, action="memory.redact", memory=memory) + memory_service.refuse_unauthorized_write(identity=identity, action="memory.redact", memory=judged) except AgentPolicyBlockedError as exc: if memory.get("deleted_at") is not None: raise RefusedOnDeletedMemoryError(exc.decision) from None @@ -457,21 +460,26 @@ def redact_memory_flow( # row's redaction receipt and writes nothing, so its authorization # should not depend on a call made earlier in the function. A test # takes the pre-check away and checks the replay is still refused. + from alicebot_api.vnext_label_guard import apply_unverified_rule + + domains, sensitivity_allowed, project_scope, project_floor = policy_labels(judged) decision = evaluate_agent_policy( identity=identity, action="memory.redact", - domains=(str(memory.get("domain") or "unknown"),), - sensitivity_allowed=(str(memory.get("sensitivity") or "unknown"),), - project_scope=resource_project_scope(memory), + domains=domains, + sensitivity_allowed=sensitivity_allowed, + project_scope=project_scope, + project_floor=project_floor, require_explicit_project_scope=True, ) + decision = apply_unverified_rule(decision, judged, identity) if decision.decision == "blocked": raise AgentPolicyBlockedError(decision) else: memory_service.authorize_memory_action( identity=identity, action="memory.redact", - memory=memory, + memory=judged, ) actor_type = "agent" if identity is not None else "user" forgotten_first = False diff --git a/apps/api/src/alicebot_api/mcp/policy.py b/apps/api/src/alicebot_api/mcp/policy.py index c98977cb5..26fa530e5 100644 --- a/apps/api/src/alicebot_api/mcp/policy.py +++ b/apps/api/src/alicebot_api/mcp/policy.py @@ -137,6 +137,7 @@ def _policy_checked( domains: tuple[str, ...] = (), sensitivity_allowed: tuple[str, ...] = ("public", "internal", "private", "unknown"), project_scope: tuple[str, ...] = (), + project_floor: tuple[str, ...] = (), workflow_type: str | None = None, write_policy: str | None = None, require_explicit_project_scope: bool = False, @@ -164,6 +165,7 @@ def _policy_checked( domains=domains, sensitivity_allowed=sensitivity_allowed, project_scope=project_scope, + project_floor=project_floor, workflow_type=workflow_type, write_policy=write_policy, require_explicit_project_scope=require_explicit_project_scope, diff --git a/apps/api/src/alicebot_api/mcp/retrieval.py b/apps/api/src/alicebot_api/mcp/retrieval.py index ab79b21b1..4e27f340f 100644 --- a/apps/api/src/alicebot_api/mcp/retrieval.py +++ b/apps/api/src/alicebot_api/mcp/retrieval.py @@ -756,13 +756,16 @@ def _handle_alice_open_loops(context: MCPRuntimeContext, arguments: Mapping[str, target = store.get_open_loop(loop_id) if target is None: raise MCPReferenceNotFoundError(f"open loop {loop_id} was not found") + from alicebot_api.vnext_label_guard import effective_row_for_fence + + judged = effective_row_for_fence(store, identity, "open_loop", target) # Same ceiling block as memory mutations. The policy event names # this loop; the previous check logged the decision with no target. try: VNextMemoryCommitService(store).authorize_memory_action( identity=identity, action="open_loop.update", - memory=target, + memory=judged, target_type="open_loop", ) except AgentPolicyBlockedError as exc: diff --git a/apps/api/src/alicebot_api/mcp/review.py b/apps/api/src/alicebot_api/mcp/review.py index 297df7896..c2862701e 100644 --- a/apps/api/src/alicebot_api/mcp/review.py +++ b/apps/api/src/alicebot_api/mcp/review.py @@ -138,9 +138,16 @@ def _vnext_memory_review(context: MCPRuntimeContext, arguments: Mapping[str, obj memory = store.get_memory(memory_id) if memory is None: raise MCPReferenceNotFoundError(f"memory {memory_id} was not found") - target_domain = str(memory.get("domain") or "unknown") - target_sensitivity = str(memory.get("sensitivity") or "unknown") - target_projects = resource_project_scope(memory) + from alicebot_api.vnext_label_guard import ( + apply_unverified_rule, + effective_row_for_fence, + policy_labels, + ) + + judged = effective_row_for_fence(store, identity, "memory", memory) + target_domains, target_sensitivity_allowed, target_projects, target_floor = policy_labels(judged) + target_domain = target_domains[0] + target_sensitivity = target_sensitivity_allowed[0] _actor_type, _actor_id, decision = _policy_checked( store, identity=identity, @@ -148,9 +155,11 @@ def _vnext_memory_review(context: MCPRuntimeContext, arguments: Mapping[str, obj domains=(target_domain,), sensitivity_allowed=(target_sensitivity,), project_scope=target_projects, + project_floor=target_floor, require_explicit_project_scope=True, project_view=ProjectView.unscoped(), ) + decision = apply_unverified_rule(decision, judged, identity) if decision.decision == "blocked": blocked_decision = decision elif ( @@ -483,16 +492,26 @@ def _vnext_memory_correct(context: MCPRuntimeContext, arguments: Mapping[str, ob target = store.get_memory(memory_id) if target is None: raise MCPReferenceNotFoundError(f"memory {memory_id} was not found") + from alicebot_api.vnext_label_guard import ( + apply_unverified_rule, + effective_row_for_fence, + policy_labels, + ) + + judged = effective_row_for_fence(store, identity, "memory", target) + domains, sensitivity_allowed, project_scope, project_floor = policy_labels(judged) _checked_actor_type, _checked_actor_id, decision = _policy_checked( store, identity=identity, action="memory.review", - domains=(str(target.get("domain") or "unknown"),), - sensitivity_allowed=(str(target.get("sensitivity") or "unknown"),), - project_scope=resource_project_scope(target), + domains=domains, + sensitivity_allowed=sensitivity_allowed, + project_scope=project_scope, + project_floor=project_floor, require_explicit_project_scope=True, project_view=ProjectView.unscoped(), ) + decision = apply_unverified_rule(decision, judged, identity) if decision.decision == "blocked": blocked_decision = decision elif is_pending_consolidation_candidate(target): @@ -550,16 +569,20 @@ def _vnext_memory_correct(context: MCPRuntimeContext, arguments: Mapping[str, ob # check commits a durable policy audit event; this second check closes # the gap where a target could be reassigned between authorization and # update. + locked_judged = effective_row_for_fence(store, identity, "memory", memory) + locked_domains, locked_sensitivity, locked_scope, locked_floor = policy_labels(locked_judged) _locked_actor_type, _locked_actor_id, locked_decision = _policy_checked( store, identity=identity, action="memory.review", - domains=(str(memory.get("domain") or "unknown"),), - sensitivity_allowed=(str(memory.get("sensitivity") or "unknown"),), - project_scope=resource_project_scope(memory), + domains=locked_domains, + sensitivity_allowed=locked_sensitivity, + project_scope=locked_scope, + project_floor=locked_floor, require_explicit_project_scope=True, project_view=ProjectView.unscoped(), ) + locked_decision = apply_unverified_rule(locked_decision, locked_judged, identity) if locked_decision.decision == "blocked": _raise_mcp_policy_blocked(locked_decision) # Route the retired-status guard through the central transition table so diff --git a/apps/api/src/alicebot_api/routers/_vnext_shared.py b/apps/api/src/alicebot_api/routers/_vnext_shared.py index 7b86ca670..9203ea3bc 100644 --- a/apps/api/src/alicebot_api/routers/_vnext_shared.py +++ b/apps/api/src/alicebot_api/routers/_vnext_shared.py @@ -429,6 +429,7 @@ def _vnext_policy_checked( domains: tuple[str, ...] = (), sensitivity_allowed: tuple[str, ...] = ("public", "internal", "private", "unknown"), project_scope: tuple[str, ...] = (), + project_floor: tuple[str, ...] = (), workflow_type: str | None = None, write_policy: str | None = None, target_type: str | None = None, @@ -445,6 +446,7 @@ def _vnext_policy_checked( domains=domains, sensitivity_allowed=sensitivity_allowed, project_scope=project_scope, + project_floor=project_floor, workflow_type=workflow_type, write_policy=write_policy, require_explicit_project_scope=require_explicit_project_scope, diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index 433d4716f..b120bdc2f 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -921,20 +921,29 @@ def review_vnext_memory( target = auth_store.get_memory(str(memory_id)) if target is None: return _vnext_public_error_response(status_code=404, detail="vNext memory was not found") - target_scope = resource_project_scope(target) + from alicebot_api.vnext_label_guard import ( + apply_unverified_rule, + effective_row_for_fence, + policy_labels, + ) + + judged = effective_row_for_fence(auth_store, identity, "memory", target) + domains, sensitivity_allowed, target_scope, target_floor = policy_labels(judged) if action == "assign_project" and request.project_id is not None: target_scope = tuple(dict.fromkeys((*target_scope, request.project_id))) decision = _vnext_policy_checked( store=auth_store, identity=identity, action="memory.review", - domains=(str(target.get("domain") or "unknown"),), - sensitivity_allowed=(str(target.get("sensitivity") or "unknown"),), + domains=domains, + sensitivity_allowed=sensitivity_allowed, project_scope=target_scope, + project_floor=target_floor, target_type="memory", target_id=str(memory_id), require_explicit_project_scope=True, ) + decision = apply_unverified_rule(decision, judged, identity) if decision.decision == "blocked": return _vnext_permission_response(decision) except AgentIdentityValidationError as exc: @@ -1061,20 +1070,23 @@ def review_vnext_memory( # Re-authorize the locked record so a concurrent reassignment cannot # move it outside the bound agent project between the first check and # this mutation. - locked_scope = resource_project_scope(existing) + locked_judged = effective_row_for_fence(store, identity, "memory", existing) + _locked_domains, _locked_sensitivity, locked_scope, locked_floor = policy_labels(locked_judged) if action == "assign_project" and request.project_id is not None: locked_scope = tuple(dict.fromkeys((*locked_scope, request.project_id))) locked_decision = _vnext_policy_checked( store=store, identity=identity, action="memory.review", - domains=(str(existing.get("domain") or "unknown"),), - sensitivity_allowed=(str(existing.get("sensitivity") or "unknown"),), + domains=_locked_domains, + sensitivity_allowed=_locked_sensitivity, project_scope=locked_scope, + project_floor=locked_floor, target_type="memory", target_id=str(memory_id), require_explicit_project_scope=True, ) + locked_decision = apply_unverified_rule(locked_decision, locked_judged, identity) if locked_decision.decision == "blocked": return _vnext_permission_response(locked_decision) if str(existing.get("status") or "") in {"archived", "rejected", "superseded"}: diff --git a/apps/api/src/alicebot_api/routers/vnext_projects.py b/apps/api/src/alicebot_api/routers/vnext_projects.py index 2fb0afb18..f6ce90d93 100644 --- a/apps/api/src/alicebot_api/routers/vnext_projects.py +++ b/apps/api/src/alicebot_api/routers/vnext_projects.py @@ -605,13 +605,16 @@ def review_vnext_open_loop( target = store.get_open_loop(loop_id) if target is None: return _vnext_public_error_response(status_code=404, detail="vNext open loop was not found") + from alicebot_api.vnext_label_guard import effective_row_for_fence + + judged = effective_row_for_fence(store, identity, "open_loop", target) # Same ceiling as the MCP open-loop updates. Returning the 403 # from inside the connection keeps the policy event committed. try: VNextMemoryCommitService(store).authorize_memory_action( identity=identity, action="open_loop.update", - memory=target, + memory=judged, target_type="open_loop", ) except AgentPolicyBlockedError as exc: diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index f49b3763f..cdb669ed1 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -192,6 +192,21 @@ def _collected(self, kind: str, row: Mapping[str, object]) -> list[dict[str, obj return list(self._nodes.values()) +def effective_row_for_fence( + store: Any, + identity: AgentIdentity | None, + kind: str, + row: Mapping[str, object], +) -> Mapping[str, object]: + """The row an exact door should hand to the policy engine.""" + + from alicebot_api.vnext_source_fence import SourceReadFence + + guard = LabelGuard.for_fence(store, SourceReadFence.for_identity(identity)) + settled = guard.effective_row(kind, row) + return settled if isinstance(settled, Mapping) else row + + def policy_labels( row: Mapping[str, object], ) -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...], tuple[str, ...]]: diff --git a/apps/api/src/alicebot_api/vnext_memory_commit.py b/apps/api/src/alicebot_api/vnext_memory_commit.py index 605f9da22..0acb70b9f 100644 --- a/apps/api/src/alicebot_api/vnext_memory_commit.py +++ b/apps/api/src/alicebot_api/vnext_memory_commit.py @@ -2506,6 +2506,7 @@ def _write_policy_decision( action: str, memory: Mapping[str, object], allow_above_ceiling: bool = False, + kind: str = "memory", ) -> PolicyDecision: """Decide one mutation of a stored target without writing anything. @@ -2520,14 +2521,24 @@ def _write_policy_decision( # memory.expire / memory.unexpire / memory.accept_consolidation are # in the agent-control WRITE_ACTIONS vocabulary, so # evaluate_agent_policy carries the read-only write block itself. + from alicebot_api.vnext_label_guard import ( + apply_unverified_rule, + effective_row_for_fence, + policy_labels, + ) + + settled = effective_row_for_fence(self.store, identity, kind, memory) + domains, sensitivity_allowed, project_scope, project_floor = policy_labels(settled) decision = evaluate_agent_policy( identity=identity, action=action, - domains=(str(memory.get("domain") or "unknown"),), - sensitivity_allowed=(str(memory.get("sensitivity") or "unknown"),), - project_scope=resource_project_scope(memory), + domains=domains, + sensitivity_allowed=sensitivity_allowed, + project_scope=project_scope, + project_floor=project_floor, require_explicit_project_scope=True, ) + decision = apply_unverified_rule(decision, settled, identity) if not allow_above_ceiling: decision = _block_mutation_above_sensitivity_ceiling(decision) if ( @@ -2594,6 +2605,7 @@ def _policy_checked_write( action=action, memory=memory, allow_above_ceiling=allow_above_ceiling, + kind="open_loop" if target_type == "open_loop" else "memory", ) return self._record_write_decision( identity=identity, @@ -2644,6 +2656,10 @@ def authorize_memory_action( ) -> PolicyDecision: """Authorize a persisted target for a cross-surface lifecycle adapter.""" + from alicebot_api.vnext_label_guard import effective_row_for_fence + + kind = "open_loop" if target_type == "open_loop" else "memory" + memory = effective_row_for_fence(self.store, identity, kind, memory) return self._policy_checked_write( identity=identity, action=action, diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py new file mode 100644 index 000000000..e1e86cb6d --- /dev/null +++ b/tests/unit/test_label_door_registry.py @@ -0,0 +1,167 @@ +"""Every exact reader is classified, and each exact door calls the guard.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SRC = ROOT / "apps/api/src" + +READS = { + "get_memory", + "get_memory_for_update", + "get_memories_by_ids", + "list_memories", + "list_memories_by_statuses", + "search_memories", + "search_memories_fts", + "search_memories_vector", + "search_memories_by_time", + "list_open_loops", + "get_open_loop", + "list_artifacts", + "get_artifact", + "get_artifact_for_update", + "list_artifacts_referencing_source", + "list_memories_referencing_source", + "list_memories_referencing_sources", + "list_beliefs", + "get_belief", + "list_projects", + "get_project", + "get_project_for_update", +} + +GUARD_CALLS = { + "LabelGuard", + "effective_row", + "effective_row_for_fence", + "admit_rows", + "admit_beliefs", + "policy_labels", +} + +# function -> helper that holds the guard call, or None when the function calls it +DOORS = { + "routers/_vnext_shared.py:_vnext_authorized_artifact": None, + "vnext_source_fence.py:resolve_attachable_memory_id": None, + "mcp/evidence_artifacts.py:_authorize_explain_resource": None, + "mcp/evidence_artifacts.py:_authorize_entity_explain_target": "_authorize_explain_resource", + "mcp/evidence_artifacts.py:_entity_backing_is_fully_authorized": "_authorize_explain_resource", + "mcp/evidence_artifacts.py:_handle_alice_vnext_memory_audit": "_authorize_explain_resource", + "mcp/evidence_artifacts.py:_authorize_vnext_artifact_target": None, + "mcp/review.py:_vnext_memory_review": None, + "mcp/review.py:_vnext_memory_correct": None, + "routers/vnext_memories.py:review_vnext_memory": None, + "mcp/memories.py:redact_memory_flow": None, + "routers/vnext_projects.py:review_vnext_open_loop": None, + "mcp/retrieval.py:_handle_alice_open_loops": None, + "vnext_memory_commit.py:VNextMemoryCommitService.authorize_memory_action": None, + "vnext_memory_commit.py:VNextMemoryCommitService._write_policy_decision": None, +} + +NOT_A_DOOR = { + "routers/_vnext_shared.py:_vnext_load_source_trace": "operator trace stays label-agnostic until the screen change", + "mcp/evidence_artifacts.py:_handle_alice_vnext_review_items": "legacy review list has no policy check", + "routers/vnext_projects.py:list_vnext_projects": "operator project list stays label-agnostic until the screen change", + "vnext_projects.py:VNextProjectService.generate_project_update_candidate": "producer input filter is the list-door change", + "vnext_projects.py:VNextProjectService.review_project_update": "write path; the route authorizes before this mutation", + "vnext_projects.py:VNextProjectService.review_open_loop": "write path; the route and the open-loop tool settle the loop first", + "vnext_projects.py:VNextProjectService.project_dashboard": "operator dashboard lists are the list-door change", + "vnext_projects.py:VNextProjectService._resolve_project": "operator project lookup is the list-door change", + "vnext_memory_commit.py:VNextMemoryCommitService.confirm": "write path; _write_policy_decision settles the row", + "vnext_memory_commit.py:VNextMemoryCommitService.undo": "write path; _write_policy_decision settles the row", + "vnext_memory_commit.py:VNextMemoryCommitService.correct": "write path; _write_policy_decision settles the row", + "vnext_memory_commit.py:VNextMemoryCommitService.forget": "write path; _write_policy_decision settles the row", + "vnext_memory_commit.py:VNextMemoryCommitService.accept_consolidation_candidate": "write path; _write_policy_decision settles the row", + "vnext_memory_commit.py:VNextMemoryCommitService.expire": "write path; _write_policy_decision settles the row", + "vnext_memory_commit.py:VNextMemoryCommitService.unexpire": "write path; _write_policy_decision settles the row", + "vnext_memory_commit.py:VNextMemoryCommitService.quarantine_by_agent_key": "write path; _write_policy_decision settles the row", + "vnext_memory_commit.py:VNextMemoryCommitService.recent_commits": "owner list; the list-door change admits rows", + "vnext_memory_commit.py:VNextMemoryCommitService.audit": "the authorize_memory callback settles each memory", + "vnext_memory_commit.py:VNextMemoryCommitService._supersession_chain": "audit walks this after authorize_memory", + "vnext_memory_commit.py:VNextMemoryCommitService.inline_confirmations": "owner list; the list-door change admits rows", + "vnext_memory_commit.py:VNextMemoryCommitService._idempotent_memory": "write path replay of a row already authorized", + "vnext_memory_commit.py:VNextMemoryCommitService._memory_by_confirmation_id": "write path replay of a row already authorized", + "vnext_memory_commit.py:VNextMemoryCommitService._latest_agentic_commit": "owner list; the list-door change admits rows", + "vnext_queue.py:VNextQueueService.review_artifact": "the HTTP review route authorizes through _vnext_authorized_artifact first", + "vnext_queue.py:VNextQueueService._promote_artifact": "the HTTP review route authorizes through _vnext_authorized_artifact first", + "vnext_queue.py:VNextQueueService.export_artifact_markdown": "the HTTP export route authorizes through _vnext_authorized_artifact first", + "mcp/retrieval.py:_resume_event_honours_policy_fence": "list door; admit_rows lands with the list-door change", + "mcp/retrieval.py:_vnext_recent_decisions": "list door; admit_rows lands with the list-door change", + "mcp/retrieval.py:_vnext_resume": "list door; admit_rows lands with the list-door change", +} + +SCAN_MODULES = ( + "alicebot_api/routers/_vnext_shared.py", + "alicebot_api/mcp/evidence_artifacts.py", + "alicebot_api/mcp/review.py", + "alicebot_api/mcp/memories.py", + "alicebot_api/routers/vnext_memories.py", + "alicebot_api/routers/vnext_projects.py", + "alicebot_api/vnext_projects.py", + "alicebot_api/vnext_memory_commit.py", + "alicebot_api/vnext_source_fence.py", + "alicebot_api/vnext_open_loop_references.py", + "alicebot_api/vnext_queue.py", + "alicebot_api/mcp/retrieval.py", +) + + +def _functions(tree: ast.AST) -> list[tuple[str, ast.FunctionDef]]: + found: list[tuple[str, ast.FunctionDef]] = [] + for node in tree.body if isinstance(tree, ast.Module) else []: + if isinstance(node, ast.FunctionDef): + found.append((node.name, node)) + elif isinstance(node, ast.ClassDef): + for child in node.body: + if isinstance(child, ast.FunctionDef): + found.append((f"{node.name}.{child.name}", child)) + return found + + +def _called_names(node: ast.AST) -> set[str]: + names: set[str] = set() + for child in ast.walk(node): + if isinstance(child, ast.Call): + func = child.func + if isinstance(func, ast.Attribute): + names.add(func.attr) + elif isinstance(func, ast.Name): + names.add(func.id) + return names + + +def _has_guard(node: ast.AST) -> bool: + return bool(_called_names(node) & GUARD_CALLS) + + +def test_every_exact_door_calls_the_guard() -> None: + modules: dict[str, ast.Module] = {} + for key, helper in DOORS.items(): + path, name = key.split(":", 1) + if path not in modules: + modules[path] = ast.parse((SRC / "alicebot_api" / path).read_text(encoding="utf-8")) + functions = dict(_functions(modules[path])) + assert name in functions, key + if helper is None: + assert _has_guard(functions[name]), key + else: + assert helper in _called_names(functions[name]), key + assert helper in functions, key + assert _has_guard(functions[helper]), helper + + +def test_every_scanned_reader_is_classified() -> None: + classified = set(DOORS) | set(NOT_A_DOOR) + missing: list[str] = [] + for relative in SCAN_MODULES: + tree = ast.parse((SRC / relative).read_text(encoding="utf-8")) + short = relative.removeprefix("alicebot_api/") + for name, node in _functions(tree): + if _called_names(node) & READS: + key = f"{short}:{name}" + if key not in classified: + missing.append(key) + assert missing == [] diff --git a/tests/unit/test_label_guard_exact_doors.py b/tests/unit/test_label_guard_exact_doors.py index 3e75a7d0d..05744df20 100644 --- a/tests/unit/test_label_guard_exact_doors.py +++ b/tests/unit/test_label_guard_exact_doors.py @@ -151,6 +151,83 @@ def test_a_locked_key_is_refused_an_unverified_row() -> None: assert "derived_labels_unverified" in blocked.reasons +def test_explain_refuses_a_public_copy_of_a_confidential_source() -> None: + from alicebot_api.mcp.evidence_artifacts import ( + _ExplainAuthorizationError, + _authorize_explain_resource, + ) + + store = _LabelStore() + with pytest.raises(_ExplainAuthorizationError): + _authorize_explain_resource( + store, + identity=_trusted(), + resource=store.memory, + project_scope=(ALPHA,), + target_type="memory", + target_id=MEMORY_ID, + ) + + +def test_the_legacy_artifact_authorizer_uses_the_input_label() -> None: + from alicebot_api.mcp.evidence_artifacts import _authorize_vnext_artifact_target + + store = _LabelStore() + _artifact, _actor_type, _actor_id, decision = _authorize_vnext_artifact_target( + store, # type: ignore[arg-type] + identity=_trusted(), + artifact_id=ARTIFACT_ID, + action="artifact.lookup", + for_update=False, + ) + assert decision.decision == "blocked" + + +def test_a_memory_review_decision_uses_the_input_label() -> None: + from alicebot_api.vnext_memory_commit import VNextMemoryCommitService + + store = _LabelStore() + decision = VNextMemoryCommitService(store)._write_policy_decision( # noqa: SLF001 + identity=_trusted(), + action="memory.review", + memory=store.memory, + ) + assert decision.decision == "blocked" + + +def test_an_open_loop_update_uses_the_input_label() -> None: + from alicebot_api.vnext_memory_commit import VNextMemoryCommitService + + store = _LabelStore() + loop = { + "id": "44444444-4444-4444-4444-444444444444", + "title": "loop", + "domain": "unknown", + "sensitivity": "public", + "metadata_json": { + "discovered_by": "vnext_daily_brief", + "source_id": SOURCE_ID, + "project_scope": [ALPHA], + "derived_from": { + "v": 1, + "sources": [SOURCE_ID], + "memories": [], + "open_loops": [], + "artifacts": [], + "beliefs": [], + "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, + }, + }, + } + with pytest.raises(AgentPolicyBlockedError): + VNextMemoryCommitService(store).authorize_memory_action( + identity=_trusted(), + action="open_loop.update", + memory=loop, + target_type="open_loop", + ) + + def test_a_cited_memory_is_judged_by_its_source() -> None: store = _LabelStore() with pytest.raises(MemoryRefNotFoundError): From d5eb0d4dc9d3d58efbd170c066cde221094ea553 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 18:39:39 +0000 Subject: [PATCH 009/270] Drop producer inputs whose effective label misses the request. Daily briefs, connection and contradiction reports, consolidation, roll-ups, project updates, staleness sweeps, and open-loop reviews now discard a loaded row when its inputs make the label stricter than the request allows. --- CHANGELOG.md | 1 + apps/api/src/alicebot_api/vnext_brain.py | 14 +++++ .../api/src/alicebot_api/vnext_connections.py | 18 ++++++ .../src/alicebot_api/vnext_consolidation.py | 10 +++ .../src/alicebot_api/vnext_contradictions.py | 21 +++++++ .../api/src/alicebot_api/vnext_label_guard.py | 25 +++++++- apps/api/src/alicebot_api/vnext_projects.py | 18 ++++++ apps/api/src/alicebot_api/vnext_rollups.py | 10 +++ apps/api/src/alicebot_api/vnext_scheduler.py | 20 ++++++ tests/unit/test_list_door_inputs.py | 63 +++++++++++++++++++ 10 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_list_door_inputs.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1565bb272..f33d16559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Unreleased (on main, not in v0.20.0): a daily brief, connection report, contradiction report, consolidation, roll-up, project update, staleness sweep, and open-loop review drop an input whose effective label is outside the request. v0.20.0 kept a public copy of a confidential input in the report text. No migration is required. - Unreleased (on main, not in v0.20.0): an exact read of a derived artifact, a write that cites a derived memory, explain, memory review, redaction, and an open-loop update use the labels of that row's inputs. v0.20.0 used the label stored on the row, so a public copy of a confidential input could be read. A row whose inputs cannot be checked is refused to a locked key. The owner and an unbound admin still read the stored row. No migration is required. - Unreleased (on main, not in v0.20.0): a locked key's consolidation, roll-up, staleness sweep and open-loop review keep only inputs whose scope and floor are both inside the binding. The overlap check stays, and the exact test runs after it. Consolidation and roll-ups compare the group scope (scope united with floor), so a card stored with an empty scope and a floor of two projects is still found and can be accepted. The operator artifact list, given a project, matches that floor as well as the scope. Each new report and aggregate records `derived_from` for the rows it used. v0.20.0 selected by overlap of the scope alone and wrote no `derived_from`. No migration is required. - Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. Raising a source or a memory walks the rows derived from it and raises those too. A candidate open loop and a generated artifact take the same insert floor. Replacing a SQLite source with a stricter one raises the memories that cite it before they are retired. Moving a source to another project answers with how many derived rows a project-bound key would lose, and waits for confirm_label_hide before it writes. A relabel that cannot finish answers 409, and one that waits more than 3 seconds for the label lock answers 503. No migration is required. diff --git a/apps/api/src/alicebot_api/vnext_brain.py b/apps/api/src/alicebot_api/vnext_brain.py index 3f70f0425..67f806d22 100644 --- a/apps/api/src/alicebot_api/vnext_brain.py +++ b/apps/api/src/alicebot_api/vnext_brain.py @@ -983,6 +983,20 @@ def _load_inputs( store_scope_kwargs=artifact_store_scope, all_of=all_of, ) + from alicebot_api.vnext_label_guard import admit_loaded + + sources = admit_loaded( + self.store, kind="source", rows=sources, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects + ) + memories = admit_loaded( + self.store, kind="memory", rows=memories, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects + ) + open_loops = admit_loaded( + self.store, kind="open_loop", rows=open_loops, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects + ) + artifacts = admit_loaded( + self.store, kind="artifact", rows=artifacts, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects + ) return sources, memories, open_loops, artifacts def _create_candidate_open_loops( diff --git a/apps/api/src/alicebot_api/vnext_connections.py b/apps/api/src/alicebot_api/vnext_connections.py index 6efa104ae..b13d03cb9 100644 --- a/apps/api/src/alicebot_api/vnext_connections.py +++ b/apps/api/src/alicebot_api/vnext_connections.py @@ -444,6 +444,24 @@ def generate_connection_report(self, request: ConnectionFinderRequest | None = N limit=input_limit, all_of=all_of, ) + from alicebot_api.vnext_label_guard import admit_loaded + + sources = admit_loaded( + self.store, + kind="source", + rows=sources, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=request.projects, + ) + memories = admit_loaded( + self.store, + kind="memory", + rows=memories, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=request.projects, + ) candidates = _find_candidates( sources=sources, memories=memories, diff --git a/apps/api/src/alicebot_api/vnext_consolidation.py b/apps/api/src/alicebot_api/vnext_consolidation.py index 7c1922800..68d3253bc 100644 --- a/apps/api/src/alicebot_api/vnext_consolidation.py +++ b/apps/api/src/alicebot_api/vnext_consolidation.py @@ -765,6 +765,16 @@ def _count(status: str) -> int: active_rows = admit_when_locked("memory", active_rows, all_of) outcome.active_count = len(active_rows) outcome.active_count_exact = not outcome.bounded + from alicebot_api.vnext_label_guard import admit_loaded + + active_rows = admit_loaded( + self.store, + kind="memory", + rows=active_rows, + domains=domains, + sensitivity_allowed=sensitivity, + projects=projects, + ) outcome.corpus_digest = _digest_payload( { "memory_versions": [ diff --git a/apps/api/src/alicebot_api/vnext_contradictions.py b/apps/api/src/alicebot_api/vnext_contradictions.py index 9e9274ad7..da79ca6aa 100644 --- a/apps/api/src/alicebot_api/vnext_contradictions.py +++ b/apps/api/src/alicebot_api/vnext_contradictions.py @@ -482,6 +482,27 @@ def generate_contradiction_report(self, request: ContradictionFinderRequest | No limit=input_limit, all_of=all_of, ) + from alicebot_api.vnext_label_guard import LabelGuard, admit_loaded + + sources = admit_loaded( + self.store, + kind="source", + rows=sources, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=request.projects, + ) + memories = admit_loaded( + self.store, + kind="memory", + rows=memories, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=request.projects, + ) + beliefs = LabelGuard.for_filters( + self.store, domains, sensitivity_allowed, request.projects + ).admit_beliefs(beliefs) candidates = _find_candidates( new_items=[*sources, *memories], beliefs=beliefs, diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index cdb669ed1..e4f0c047d 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -121,7 +121,7 @@ def admit_rows(self, kind: str, rows: Sequence[Mapping[str, object]]) -> list[Ma if not isinstance(row, Mapping): continue effective = self.effective_row(kind, row) - if isinstance(effective, Mapping) and self._admits_effective(effective): + if isinstance(effective, Mapping) and self._admits_effective(effective, kind=kind): kept.append(row) return kept @@ -132,6 +132,8 @@ def admit_beliefs(self, beliefs: Sequence[Mapping[str, object]]) -> list[Mapping return [row for row in beliefs if isinstance(row, Mapping)] ids = [str(row.get("memory_id")) for row in beliefs if isinstance(row, Mapping) and row.get("memory_id")] reader = getattr(self.store, "read_label_rows", None) + if not callable(reader): + return [row for row in beliefs if isinstance(row, Mapping)] found: dict[str, Mapping[str, object]] = {} if callable(reader) and ids: for row in reader("memory", ids): @@ -144,7 +146,7 @@ def admit_beliefs(self, beliefs: Sequence[Mapping[str, object]]) -> list[Mapping if isinstance(row, Mapping) and str(row.get("memory_id") or "") in admitted ] - def _admits_effective(self, row: Mapping[str, object]) -> bool: + def _admits_effective(self, row: Mapping[str, object], *, kind: str) -> bool: domain = str(row.get("domain") or "unknown") if self.domains and domain not in self.domains and domain != "unknown": return False @@ -152,7 +154,9 @@ def _admits_effective(self, row: Mapping[str, object]) -> bool: if self.sensitivity_allowed and sensitivity not in self.sensitivity_allowed: return False if self.projects: - scope = resolve_project_scope(row).values + from alicebot_api.vnext_project_scope import source_project_scope + + scope = source_project_scope(row) if canon_kind(kind) == "source" else resolve_project_scope(row).values _shape, floor = project_floor_shape(row) if not project_scopes_overlap(scope, self.projects, floor=floor): return False @@ -192,6 +196,21 @@ def _collected(self, kind: str, row: Mapping[str, object]) -> list[dict[str, obj return list(self._nodes.values()) +def admit_loaded( + store: Any, + *, + kind: str, + rows: Sequence[Mapping[str, object]], + domains: Sequence[str] | None, + sensitivity_allowed: Sequence[str] | None, + projects: Sequence[str] | None = (), +) -> list[Mapping[str, object]]: + """Drop loaded inputs whose effective labels miss the request filters.""" + + guard = LabelGuard.for_filters(store, domains, sensitivity_allowed, projects) + return guard.admit_rows(kind, rows) + + def effective_row_for_fence( store: Any, identity: AgentIdentity | None, diff --git a/apps/api/src/alicebot_api/vnext_projects.py b/apps/api/src/alicebot_api/vnext_projects.py index 2034c394c..39ef87be6 100644 --- a/apps/api/src/alicebot_api/vnext_projects.py +++ b/apps/api/src/alicebot_api/vnext_projects.py @@ -567,6 +567,24 @@ def generate_project_update_candidate(self, request: ProjectAutomationRequest | for row in memories if _is_in_project(row, project_id) and row.get("status") in {"active", "accepted"} ] + from alicebot_api.vnext_label_guard import admit_loaded + + sources = admit_loaded( + self.store, + kind="source", + rows=sources, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=(project_id,), + ) + memories = admit_loaded( + self.store, + kind="memory", + rows=memories, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=(project_id,), + ) brain_charter = _brain_charter(self.store) automation_digest = _project_automation_digest( project=project, diff --git a/apps/api/src/alicebot_api/vnext_rollups.py b/apps/api/src/alicebot_api/vnext_rollups.py index 33920087d..829c5c359 100644 --- a/apps/api/src/alicebot_api/vnext_rollups.py +++ b/apps/api/src/alicebot_api/vnext_rollups.py @@ -1825,6 +1825,16 @@ def _collect_rows( "roll-up input lookup returned rows outside the requested project scope" ) rows = admit_when_locked("memory", scoped_rows, all_of) + from alicebot_api.vnext_label_guard import admit_loaded + + rows = admit_loaded( + self.store, + kind="memory", + rows=rows, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=projects, + ) rows = [row for row in rows if not _is_rollup_card(row)] # The same parity for validity: the bundled stores leave an expired # memory out in SQL, and every tier below (entity, topic and the diff --git a/apps/api/src/alicebot_api/vnext_scheduler.py b/apps/api/src/alicebot_api/vnext_scheduler.py index 55ff84a41..9eb34e452 100644 --- a/apps/api/src/alicebot_api/vnext_scheduler.py +++ b/apps/api/src/alicebot_api/vnext_scheduler.py @@ -1408,6 +1408,16 @@ def _run_staleness_sweep(self, request: SchedulerRunRequest, *, metadata: JsonOb projects, ) memories = admit_when_locked("memory", memories, bound) + from alicebot_api.vnext_label_guard import admit_loaded + + memories = admit_loaded( + self.store, + kind="memory", + rows=memories, + domains=list(request.domains) if request.domains else None, + sensitivity_allowed=list(request.sensitivity_allowed), + projects=projects, + ) for memory in memories: if len(expired_marked) + len(unconfirmed_marked) >= mark_limit: break @@ -1579,6 +1589,16 @@ def _generate_open_loop_review_artifact(self, request: SchedulerRunRequest, *, m projects, ) loops = admit_when_locked("open_loop", loops, bound) + from alicebot_api.vnext_label_guard import admit_loaded + + loops = admit_loaded( + self.store, + kind="open_loop", + rows=loops, + domains=domains, + sensitivity_allowed=list(request.sensitivity_allowed), + projects=projects, + ) # The report copies the id of each loop's source into its text and its ``source_refs``, and a later reader of # the artifact is shown them, so a source the run's own identity may not read is left out. loops = withhold_unreadable_references( diff --git a/tests/unit/test_list_door_inputs.py b/tests/unit/test_list_door_inputs.py new file mode 100644 index 000000000..ab8366d94 --- /dev/null +++ b/tests/unit/test_list_door_inputs.py @@ -0,0 +1,63 @@ +"""A producer's input list drops a row whose effective label is above the request.""" + +from __future__ import annotations + +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from tests.unit.test_vnext_brain import InMemoryVNextBrainStore + +SOURCE_ID = "11111111-1111-1111-1111-111111111111" + + +def test_a_brief_drops_a_public_copy_of_a_confidential_source() -> None: + store = InMemoryVNextBrainStore() + source = { + "id": SOURCE_ID, + "domain": "health", + "sensitivity": "confidential", + "metadata_json": {}, + } + store.sources.append( + { + **source, + "title": "Ordinary note", + "content_hash": "sha256:abc", + "captured_at": "2026-05-10T08:00:00Z", + "domain": "project", + "sensitivity": "public", + } + ) + store.memories.append( + { + "id": "memory-secret", + "canonical_text": "SENTINEL confidential fact", + "status": "active", + "created_at": "2026-05-10T09:00:00Z", + "domain": "project", + "sensitivity": "public", + "metadata_json": { + "source_id": SOURCE_ID, + "derived_from": { + "v": 1, + "sources": [SOURCE_ID], + "memories": [], + "open_loops": [], + "artifacts": [], + "beliefs": [], + "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, + }, + }, + } + ) + + def read_label_rows(kind: str, ids: list[str]) -> list[dict[str, object]]: + if kind == "source" and SOURCE_ID in ids: + return [source] + return [] + + store.read_label_rows = read_label_rows # type: ignore[attr-defined] + artifact = VNextBrainService(store).generate_daily_brief( + BrainArtifactRequest(generated_for="2026-05-10", domains=("project",)) + ) + assert "SENTINEL confidential fact" not in artifact["content_markdown"] + derived = artifact["metadata_json"]["derived_from"] + assert "memory-secret" not in derived["memories"] From f074449ef4efbb839db7d55545eb81907dabd72d Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 19:05:30 +0000 Subject: [PATCH 010/270] Drop list and screen rows whose effective label misses the request. Recall, context packs, resume, recent decisions, the session brief, the review queue, and the operator lists now admit rows by the effective label. The five operator screens apply the caller's sensitivity ceiling, and event readers hide a change whose target the caller cannot read. --- CHANGELOG.md | 1 + apps/api/src/alicebot_api/mcp/projects.py | 10 + apps/api/src/alicebot_api/mcp/retrieval.py | 46 +++- apps/api/src/alicebot_api/mcp/review.py | 12 +- .../src/alicebot_api/routers/_vnext_shared.py | 21 +- .../alicebot_api/routers/vnext_memories.py | 23 +- .../alicebot_api/routers/vnext_projects.py | 51 +++- .../alicebot_api/routers/vnext_retrieval.py | 36 ++- .../src/alicebot_api/routers/vnext_review.py | 52 +++- .../src/alicebot_api/routers/workspaces.py | 98 +++++-- apps/api/src/alicebot_api/session_briefing.py | 36 +++ .../src/alicebot_api/vnext_context_tree.py | 59 +++- .../src/alicebot_api/vnext_contradictions.py | 7 +- apps/api/src/alicebot_api/vnext_dogfooding.py | 20 +- .../api/src/alicebot_api/vnext_label_guard.py | 88 ++++++ apps/api/src/alicebot_api/vnext_projects.py | 46 +++- apps/api/src/alicebot_api/vnext_retrieval.py | 144 +++++++++- tests/unit/test_label_door_registry.py | 46 +++- tests/unit/test_list_door_readers.py | 254 ++++++++++++++++++ .../test_open_loop_references_read_fence.py | 8 +- tests/unit/test_vnext_retrieval.py | 26 +- 21 files changed, 1011 insertions(+), 73 deletions(-) create mode 100644 tests/unit/test_list_door_readers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f33d16559..67700df9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Unreleased (on main, not in v0.20.0): recall, context packs, resume, recent decisions, the session brief, the review queue, open-loop lists, and the workspace, dashboard, context tree, and dogfooding lists drop a loaded row whose effective label is outside the request. The artifact list, the source trace, the project list, the dashboard project row, and the belief state route apply the caller's sensitivity ceiling, including titles, ids, and counts. Resume, recent decisions, the session brief, audit, and explain apply the target row's current effective label, so an excluded key does not learn of a label change. v0.20.0 returned those rows from the label stored on them. No migration is required. - Unreleased (on main, not in v0.20.0): a daily brief, connection report, contradiction report, consolidation, roll-up, project update, staleness sweep, and open-loop review drop an input whose effective label is outside the request. v0.20.0 kept a public copy of a confidential input in the report text. No migration is required. - Unreleased (on main, not in v0.20.0): an exact read of a derived artifact, a write that cites a derived memory, explain, memory review, redaction, and an open-loop update use the labels of that row's inputs. v0.20.0 used the label stored on the row, so a public copy of a confidential input could be read. A row whose inputs cannot be checked is refused to a locked key. The owner and an unbound admin still read the stored row. No migration is required. - Unreleased (on main, not in v0.20.0): a locked key's consolidation, roll-up, staleness sweep and open-loop review keep only inputs whose scope and floor are both inside the binding. The overlap check stays, and the exact test runs after it. Consolidation and roll-ups compare the group scope (scope united with floor), so a card stored with an empty scope and a floor of two projects is still found and can be accepted. The operator artifact list, given a project, matches that floor as well as the scope. Each new report and aggregate records `derived_from` for the rows it used. v0.20.0 selected by overlap of the scope alone and wrote no `derived_from`. No migration is required. diff --git a/apps/api/src/alicebot_api/mcp/projects.py b/apps/api/src/alicebot_api/mcp/projects.py index fadbf7703..06782c0dd 100644 --- a/apps/api/src/alicebot_api/mcp/projects.py +++ b/apps/api/src/alicebot_api/mcp/projects.py @@ -226,6 +226,16 @@ def _handle_alice_vnext_open_loops(context: MCPRuntimeContext, arguments: Mappin ) if decision.effective_project_scope: loops = [loop for loop in loops if _resource_matches_project_scope(loop, decision.effective_project_scope)] + from alicebot_api.vnext_label_guard import admit_loaded + + loops = admit_loaded( + store, + kind="open_loop", + rows=loops, + domains=decision.effective_domains, + sensitivity_allowed=decision.effective_sensitivity_allowed, + projects=decision.effective_project_scope, + ) # The loop is the caller's to read, the source and memory it points at are checked on their own: a reference # the caller's fence does not admit is returned as ``null``, the way a reference to a missing row is. loops = withhold_unreadable_references( diff --git a/apps/api/src/alicebot_api/mcp/retrieval.py b/apps/api/src/alicebot_api/mcp/retrieval.py index 4e27f340f..7290d2a09 100644 --- a/apps/api/src/alicebot_api/mcp/retrieval.py +++ b/apps/api/src/alicebot_api/mcp/retrieval.py @@ -827,6 +827,18 @@ def _resume_event_honours_policy_fence( return False if _resource_is_held_back_global(row, exclude_global_domains): return False + from alicebot_api.vnext_label_guard import admit_loaded + + kind = "open_loop" if target_type == "open_loop" else "memory" + if not admit_loaded( + store, + kind=kind, + rows=[row], + domains=effective_domains, + sensitivity_allowed=effective_sensitivity_allowed, + projects=(), + ): + return False return _resource_matches_domains(row, effective_domains) and _resource_matches_sensitivity( row, effective_sensitivity_allowed ) @@ -893,6 +905,16 @@ def _vnext_recent_decisions( and _memory_matches_project(row, project) and _row_in_window(row, key="created_at", since=since, until=until) ] + from alicebot_api.vnext_label_guard import admit_loaded + + matched = admit_loaded( + store, + kind="memory", + rows=matched, + domains=domain_filter, + sensitivity_allowed=sensitivity_filter, + projects=effective_project_scope, + ) matched.sort(key=_created_at_sort_key, reverse=True) decisions = [ present_model_item( @@ -1002,7 +1024,19 @@ def read_memories(memory_types: tuple[str, ...]) -> list[JsonObject]: include_expired=False, ) - decisions = read_memories(("decision",)) + from alicebot_api.vnext_label_guard import admit_loaded + + def admit_memories(rows: list[JsonObject]) -> list[JsonObject]: + return admit_loaded( + store, + kind="memory", + rows=rows, + domains=domain_filter, + sensitivity_allowed=sensitivity_filter, + projects=effective_project_scope, + ) + + decisions = admit_memories(read_memories(("decision",))) last_decision: JsonObject | None = None if decisions: last_decision = { @@ -1055,13 +1089,21 @@ def read_memories(memory_types: tuple[str, ...]) -> list[JsonObject]: scope_window_start=since, scope_window_end=until, ) + loop_rows = admit_loaded( + store, + kind="open_loop", + rows=loop_rows, + domains=domain_filter, + sensitivity_allowed=sensitivity_filter, + projects=effective_project_scope, + ) open_loops = [ present_model_item(_compact_vnext_open_loop(row), source=row) for row in loop_rows[:max_open_loops] ] next_action: JsonObject | None = open_loops[0] if open_loops else None if next_action is None: - todo_memories = read_memories(tuple(_SQLITE_NEXT_ACTION_MEMORY_TYPES)) + todo_memories = admit_memories(read_memories(tuple(_SQLITE_NEXT_ACTION_MEMORY_TYPES))) if todo_memories: next_action = { "kind": "memory", diff --git a/apps/api/src/alicebot_api/mcp/review.py b/apps/api/src/alicebot_api/mcp/review.py index c2862701e..3f5384013 100644 --- a/apps/api/src/alicebot_api/mcp/review.py +++ b/apps/api/src/alicebot_api/mcp/review.py @@ -251,7 +251,17 @@ def _vnext_memory_review(context: MCPRuntimeContext, arguments: Mapping[str, obj if _resource_matches_project_scope(row, decision.effective_project_scope) and str(row.get("domain") or "unknown") in decision.effective_domains and str(row.get("sensitivity") or "unknown") in decision.effective_sensitivity_allowed - ][:limit] + ] + from alicebot_api.vnext_label_guard import admit_loaded + + rows = admit_loaded( + store, + kind="memory", + rows=rows, + domains=decision.effective_domains, + sensitivity_allowed=decision.effective_sensitivity_allowed, + projects=decision.effective_project_scope, + )[:limit] items = [ present_model_item( _compact_vnext_memory(row, provenance_count=_provenance_count(store, row.get("id"))), diff --git a/apps/api/src/alicebot_api/routers/_vnext_shared.py b/apps/api/src/alicebot_api/routers/_vnext_shared.py index 9203ea3bc..f0f0a8bfa 100644 --- a/apps/api/src/alicebot_api/routers/_vnext_shared.py +++ b/apps/api/src/alicebot_api/routers/_vnext_shared.py @@ -284,9 +284,20 @@ def _vnext_load_source_trace( *, store: PostgresVNextStore, source: dict[str, object], -) -> dict[str, object]: - """Load one bounded source trace and disclose per-collection truncation.""" + identity: object | None = None, +) -> dict[str, object] | None: + """Load one bounded source trace and disclose per-collection truncation. + + Returns None when the caller's sensitivity ceiling hides the source, so + the response carries neither its title nor its id. + """ + + from alicebot_api.vnext_agent_control import AgentIdentity + from alicebot_api.vnext_label_guard import apply_sensitivity_ceiling + caller = identity if isinstance(identity, AgentIdentity) else None + if not apply_sensitivity_ceiling(store, kind="source", rows=[source], identity=caller): + return None source_id = str(source["id"]) memories, memories_complete = _vnext_bounded_trace_rows( store.list_memories_referencing_source( @@ -306,6 +317,11 @@ def _vnext_load_source_trace( limit=_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT + 1, ) ) + memories = apply_sensitivity_ceiling(store, kind="memory", rows=memories, identity=caller) + artifacts = apply_sensitivity_ceiling(store, kind="artifact", rows=artifacts, identity=caller) + open_loops = apply_sensitivity_ceiling(store, kind="open_loop", rows=open_loops, identity=caller) + kept_ids = {str(row.get("id")) for row in (*memories, *artifacts, *open_loops)} + kept_ids.add(source_id) events, direct_events_complete = _vnext_bounded_trace_rows( store.list_events_for_source_trace( source_id=source_id, @@ -315,6 +331,7 @@ def _vnext_load_source_trace( limit=_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT + 1, ) ) + events = [event for event in events if str(event.get("target_id") or "") in kept_ids] events_complete = direct_events_complete and memories_complete and artifacts_complete and open_loops_complete return _vnext_source_trace( store=store, diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index b120bdc2f..c0fe9244a 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -59,6 +59,8 @@ ) from alicebot_api.vnext_agent_keys import ( AgentKeyAuthenticationError, + agent_key_from_authorization, + resolve_protected_agent_identity, ) from alicebot_api.vnext_capture import ( VNextCaptureService, @@ -717,10 +719,25 @@ def ingest_vnext_agent_output( @connectors_router.get("/v0/vnext/dogfooding") -def get_vnext_dogfooding_dashboard(user_id: UUID) -> JSONResponse: +def get_vnext_dogfooding_dashboard( + user_id: UUID, + authorization: str | None = Header(default=None), +) -> JSONResponse: + from alicebot_api.vnext_label_guard import sensitivity_ceiling + settings = get_settings() - with user_connection(settings.database_url, user_id) as conn: - payload = VNextDogfoodingService(PostgresVNextStore(conn)).dashboard() + try: + with user_connection(settings.database_url, user_id) as conn: + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, + user_id=user_id, + raw_key=agent_key_from_authorization(authorization), + payload={}, + ) + payload = VNextDogfoodingService(store).dashboard(sensitivity_allowed=sensitivity_ceiling(identity)) + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) return JSONResponse(status_code=200, content=jsonable_encoder(payload)) diff --git a/apps/api/src/alicebot_api/routers/vnext_projects.py b/apps/api/src/alicebot_api/routers/vnext_projects.py index f6ce90d93..0e6485081 100644 --- a/apps/api/src/alicebot_api/routers/vnext_projects.py +++ b/apps/api/src/alicebot_api/routers/vnext_projects.py @@ -36,7 +36,11 @@ agent_metadata, summarize_agent_policy_telemetry, ) -from alicebot_api.vnext_agent_keys import AgentKeyAuthenticationError +from alicebot_api.vnext_agent_keys import ( + AgentKeyAuthenticationError, + agent_key_from_authorization, + resolve_protected_agent_identity, +) from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_memory_commit import VNextMemoryCommitService from alicebot_api.vnext_open_loop_references import withhold_unreadable_references_from_loop @@ -159,11 +163,33 @@ def create_vnext_project(request: VNextProjectCreateRequest) -> JSONResponse: return JSONResponse(status_code=201, content=jsonable_encoder({"project": payload})) @project_core_router.get("/v0/vnext/projects") -def list_vnext_projects(user_id: UUID, status: str | None = "active", limit: int = 20) -> JSONResponse: +def list_vnext_projects( + user_id: UUID, + status: str | None = "active", + limit: int = 20, + authorization: str | None = Header(default=None), +) -> JSONResponse: + from alicebot_api.vnext_label_guard import apply_sensitivity_ceiling + settings = get_settings() - with user_connection(settings.database_url, user_id) as conn: - payload = PostgresVNextStore(conn).list_projects(status=status, limit=limit) + try: + with user_connection(settings.database_url, user_id) as conn: + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, + user_id=user_id, + raw_key=agent_key_from_authorization(authorization), + payload={}, + ) + payload = apply_sensitivity_ceiling( + store, + kind="project", + rows=store.list_projects(status=status, limit=limit), + identity=identity, + ) + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) return JSONResponse( status_code=200, @@ -171,12 +197,25 @@ def list_vnext_projects(user_id: UUID, status: str | None = "active", limit: int ) @project_operations_router.get("/v0/vnext/projects/{project_id}/dashboard") -def get_vnext_project_dashboard(project_id: str, user_id: UUID) -> JSONResponse: +def get_vnext_project_dashboard( + project_id: str, + user_id: UUID, + authorization: str | None = Header(default=None), +) -> JSONResponse: settings = get_settings() try: with user_connection(settings.database_url, user_id) as conn: - payload = VNextProjectService(PostgresVNextStore(conn)).project_dashboard(project_id=project_id) + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, + user_id=user_id, + raw_key=agent_key_from_authorization(authorization), + payload={}, + ) + payload = VNextProjectService(store).project_dashboard(project_id=project_id, identity=identity) + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) except VNextProjectValidationError: return _vnext_public_error_response(status_code=404, detail="vNext project was not found") diff --git a/apps/api/src/alicebot_api/routers/vnext_retrieval.py b/apps/api/src/alicebot_api/routers/vnext_retrieval.py index 80865dcca..816f849a5 100644 --- a/apps/api/src/alicebot_api/routers/vnext_retrieval.py +++ b/apps/api/src/alicebot_api/routers/vnext_retrieval.py @@ -122,17 +122,33 @@ def _vnext_artifact_trace( @trace_router.get("/v0/vnext/traces/sources/{source_id}") -def get_vnext_source_trace(source_id: UUID, user_id: UUID) -> JSONResponse: +def get_vnext_source_trace( + source_id: UUID, + user_id: UUID, + authorization: str | None = Header(default=None), +) -> JSONResponse: settings = get_settings() - with user_connection(settings.database_url, user_id) as conn: - store = PostgresVNextStore(conn) - source = store.get_source(str(source_id)) - if source is None: - return _vnext_public_error_response(status_code=404, detail="vNext source was not found") - payload = _vnext_load_source_trace( - store=store, - source=source, - ) + try: + with user_connection(settings.database_url, user_id) as conn: + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, + user_id=user_id, + raw_key=agent_key_from_authorization(authorization), + payload={}, + ) + source = store.get_source(str(source_id)) + if source is None: + return _vnext_public_error_response(status_code=404, detail="vNext source was not found") + payload = _vnext_load_source_trace( + store=store, + source=source, + identity=identity, + ) + if payload is None: + return _vnext_public_error_response(status_code=404, detail="vNext source was not found") + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) return JSONResponse(status_code=200, content=jsonable_encoder(payload)) diff --git a/apps/api/src/alicebot_api/routers/vnext_review.py b/apps/api/src/alicebot_api/routers/vnext_review.py index 73983e345..508c6babf 100644 --- a/apps/api/src/alicebot_api/routers/vnext_review.py +++ b/apps/api/src/alicebot_api/routers/vnext_review.py @@ -571,16 +571,34 @@ def list_vnext_artifacts( artifact_type: str | None = None, limit: int = 30, project: str | None = None, + authorization: str | None = Header(default=None), ) -> JSONResponse: + from alicebot_api.vnext_label_guard import apply_sensitivity_ceiling + settings = get_settings() scope_projects = (project,) if isinstance(project, str) and project.strip() else () - with user_connection(settings.database_url, user_id) as conn: - payload = PostgresVNextStore(conn).list_artifacts( - artifact_type=artifact_type, - limit=limit, - scope_projects=scope_projects, - ) + try: + with user_connection(settings.database_url, user_id) as conn: + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, + user_id=user_id, + raw_key=agent_key_from_authorization(authorization), + payload={}, + ) + payload = apply_sensitivity_ceiling( + store, + kind="artifact", + rows=store.list_artifacts( + artifact_type=artifact_type, + limit=limit, + scope_projects=scope_projects, + ), + identity=identity, + ) + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) return JSONResponse( status_code=200, @@ -897,12 +915,30 @@ def review_vnext_belief(belief_id: str, request: VNextBeliefReviewRequest) -> JS ) @review_router.get("/v0/vnext/beliefs/{belief_id}/state") -def get_vnext_belief_state(belief_id: str, user_id: UUID) -> JSONResponse: +def get_vnext_belief_state( + belief_id: str, + user_id: UUID, + authorization: str | None = Header(default=None), +) -> JSONResponse: + from alicebot_api.vnext_label_guard import sensitivity_ceiling + settings = get_settings() try: with user_connection(settings.database_url, user_id) as conn: - payload = VNextContradictionService(PostgresVNextStore(conn)).belief_state(belief_id=belief_id) + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, + user_id=user_id, + raw_key=agent_key_from_authorization(authorization), + payload={}, + ) + payload = VNextContradictionService(store).belief_state( + belief_id=belief_id, + sensitivity_allowed=sensitivity_ceiling(identity), + ) + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) except VNextContradictionValidationError: return _vnext_public_error_response(status_code=404, detail="vNext belief was not found") diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index 571811f2a..4a95353ea 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Mapping from uuid import UUID from fastapi import APIRouter, Request @@ -46,45 +47,110 @@ def _vnext_status_counts(rows: list[dict[str, object]], *, field: str = "status" return counts +def _workspace_event_visible(store: PostgresVNextStore, event: dict[str, object], sensitivity: list[str]) -> bool: + """An event stays when its target row is inside the workspace sensitivity list.""" + + kind = str(event.get("target_type") or "") + target_id = event.get("target_id") + getters = { + "memory": "get_memory", + "open_loop": "get_open_loop", + "artifact": "get_artifact", + "project": "get_project", + "source": "get_source", + } + getter_name = getters.get(kind) + if getter_name is None or not isinstance(target_id, str) or target_id == "": + return True + getter = getattr(store, getter_name, None) + if not callable(getter): + return True + row = getter(target_id) + if not isinstance(row, Mapping): + return True + return bool(_workspace_rows(store, kind, [row], sensitivity)) + + +def _workspace_rows(store: PostgresVNextStore, kind: str, rows: list[dict[str, object]], sensitivity: list[str]): + from alicebot_api.vnext_label_guard import admit_loaded + + return admit_loaded( + store, + kind=kind, + rows=rows, + domains=(), + sensitivity_allowed=sensitivity, + projects=(), + ) + + def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: + from alicebot_api.vnext_label_guard import LabelGuard, readable_count, readable_status_counts + sensitivity_allowed = ["public", "internal", "private", "unknown"] review_statuses = ["candidate", "needs_review", "private_only", "accepted", "rejected"] - sources = store.list_sources(sensitivity_allowed=sensitivity_allowed, limit=20) - source_count = store.count_sources() + fetched_sources = store.list_sources(sensitivity_allowed=sensitivity_allowed, limit=20) + sources = _workspace_rows(store, "source", fetched_sources, sensitivity_allowed) + source_count = readable_count(store.count_sources(), len(fetched_sources), len(sources)) list_memories_by_statuses = getattr(store, "list_memories_by_statuses", None) if callable(list_memories_by_statuses): - review_memories = list_memories_by_statuses( + fetched_memories = list_memories_by_statuses( statuses=review_statuses, sensitivity_allowed=sensitivity_allowed, limit=30, ) else: # Compatibility for external/test stores implementing the older protocol. - review_memories = [ + fetched_memories = [ memory for memory in store.list_memories(status=None) if str(memory.get("status")) in set(review_statuses) ][:30] + review_memories = _workspace_rows(store, "memory", fetched_memories, sensitivity_allowed) count_memories_by_status = getattr(store, "count_memories_by_status", None) memory_status_counts = ( count_memories_by_status(sensitivity_allowed=sensitivity_allowed) if callable(count_memories_by_status) - else _vnext_status_counts(review_memories) + else _vnext_status_counts(fetched_memories) ) + memory_status_counts = readable_status_counts(memory_status_counts, fetched_memories, review_memories) review_memory_total = sum(memory_status_counts.get(status, 0) for status in review_statuses) - artifacts = store.list_artifacts(sensitivity_allowed=sensitivity_allowed, limit=30) - artifact_count = store.count_artifacts() - artifact_status_counts = store.count_artifacts_by_status() + fetched_artifacts = store.list_artifacts(sensitivity_allowed=sensitivity_allowed, limit=30) + artifacts = _workspace_rows(store, "artifact", fetched_artifacts, sensitivity_allowed) + artifact_count = readable_count(store.count_artifacts(), len(fetched_artifacts), len(artifacts)) + artifact_status_counts = readable_status_counts( + store.count_artifacts_by_status(), + fetched_artifacts, + artifacts, + ) quality_evals = store.list_artifact_quality_ratings(limit=50) quality_eval_count = store.count_artifact_quality_ratings() - projects = store.list_projects(status=None, sensitivity_allowed=sensitivity_allowed, limit=20) - project_count = store.count_projects() - open_loops = store.list_open_loops(status=None, sensitivity_allowed=sensitivity_allowed, limit=30) - open_loop_count = store.count_open_loops(status="open") - open_loop_status_counts = store.count_open_loops_by_status() + fetched_projects = store.list_projects(status=None, sensitivity_allowed=sensitivity_allowed, limit=20) + projects = _workspace_rows(store, "project", fetched_projects, sensitivity_allowed) + project_count = readable_count(store.count_projects(), len(fetched_projects), len(projects)) + fetched_loops = store.list_open_loops(status=None, sensitivity_allowed=sensitivity_allowed, limit=30) + open_loops = _workspace_rows(store, "open_loop", fetched_loops, sensitivity_allowed) + open_loop_status_counts = readable_status_counts( + store.count_open_loops_by_status(), + fetched_loops, + open_loops, + ) + stored_open_loop_count = store.count_open_loops(status="open") + open_loop_count = ( + stored_open_loop_count + if len(fetched_loops) == len(open_loops) + else int(open_loop_status_counts.get("open", stored_open_loop_count)) + ) people = store.list_people(sensitivity_allowed=sensitivity_allowed, limit=12) - beliefs = store.list_beliefs(status=None, sensitivity_allowed=sensitivity_allowed, limit=12) + fetched_beliefs = store.list_beliefs(status=None, sensitivity_allowed=sensitivity_allowed, limit=12) + beliefs = LabelGuard.for_filters(store, (), sensitivity_allowed, ()).admit_beliefs(fetched_beliefs) tasks = store.list_tasks(status=None, limit=12) - recent_events = store.list_events(limit=20) + fetched_events = store.list_events(limit=20) + recent_events = [ + event + for event in fetched_events + if _workspace_event_visible(store, event, sensitivity_allowed) + ] count_events = getattr(store, "count_events", None) - event_count = count_events() if callable(count_events) else len(recent_events) + stored_event_count = count_events() if callable(count_events) else len(fetched_events) + event_count = readable_count(stored_event_count, len(fetched_events), len(recent_events)) agent_identities = store.list_agent_identities(limit=20) agent_count = store.count_agent_identities() agent_events = store.list_agent_events(limit=50) diff --git a/apps/api/src/alicebot_api/session_briefing.py b/apps/api/src/alicebot_api/session_briefing.py index 202c88a67..4725aefd0 100644 --- a/apps/api/src/alicebot_api/session_briefing.py +++ b/apps/api/src/alicebot_api/session_briefing.py @@ -349,6 +349,7 @@ def compile_session_brief( for row in facts if _memory_honours_fence( row, + store=store, effective_domains=effective_domains, effective_sensitivity_allowed=effective_sensitivity_allowed, effective_project_scope=effective_project_scope, @@ -380,6 +381,16 @@ def compile_session_brief( scope_projects=effective_project_scope, exclude_global_domains=tuple(sorted(held_back)), ) + from alicebot_api.vnext_label_guard import admit_loaded + + open_loops = admit_loaded( + store, + kind="open_loop", + rows=open_loops, + domains=effective_domains, + sensitivity_allowed=effective_sensitivity_allowed, + projects=effective_project_scope, + ) _merge_recent_change_targets( store, facts=facts, @@ -587,6 +598,18 @@ def _event_target_honours_fence( row = store.get_open_loop(target_id) if row is None: return False + from alicebot_api.vnext_label_guard import admit_loaded + + kind = "open_loop" if target_type == "open_loop" else "memory" + if not admit_loaded( + store, + kind=kind, + rows=[row], + domains=effective_domains, + sensitivity_allowed=effective_sensitivity_allowed, + projects=effective_project_scope, + ): + return False return _memory_honours_fence( row, effective_domains=effective_domains, @@ -615,11 +638,24 @@ def _brief_omits_memory(row: Mapping[str, object]) -> bool: def _memory_honours_fence( row: Mapping[str, object], *, + store: object | None = None, effective_domains: tuple[str, ...], effective_sensitivity_allowed: tuple[str, ...], effective_project_scope: tuple[str, ...], exclude_global_domains: frozenset[str], ) -> bool: + if store is not None: + from alicebot_api.vnext_label_guard import admit_loaded + + if not admit_loaded( + store, + kind="memory", + rows=[row], + domains=effective_domains, + sensitivity_allowed=effective_sensitivity_allowed, + projects=effective_project_scope, + ): + return False resource_scope = resource_project_scope(row) return ( _matches_domains(row, effective_domains) diff --git a/apps/api/src/alicebot_api/vnext_context_tree.py b/apps/api/src/alicebot_api/vnext_context_tree.py index b91f43eac..1ab93057f 100644 --- a/apps/api/src/alicebot_api/vnext_context_tree.py +++ b/apps/api/src/alicebot_api/vnext_context_tree.py @@ -162,6 +162,32 @@ def _label(row: JsonObject, *keys: str, fallback: str) -> str: return fallback +def _tree_event_visible(store: object, event: JsonObject, domains: list[str], sensitivity: list[str], projects: tuple[str, ...]) -> bool: + from alicebot_api.vnext_label_guard import admit_loaded + + kind = str(event.get("target_type") or "") + target_id = event.get("target_id") + getters = { + "memory": "get_memory", + "open_loop": "get_open_loop", + "artifact": "get_artifact", + "project": "get_project", + "source": "get_source", + } + getter_name = getters.get(kind) + if getter_name is None or not isinstance(target_id, str) or target_id == "": + return True + getter = getattr(store, getter_name, None) + if not callable(getter): + return True + row = getter(target_id) + if not isinstance(row, dict) and not hasattr(row, "get"): + return True + return bool( + admit_loaded(store, kind=kind, rows=[row], domains=domains, sensitivity_allowed=sensitivity, projects=projects) + ) + + def _row_node(prefix: str, row: JsonObject, *, label_keys: tuple[str, ...], fallback: str) -> JsonObject: row_id = str(row.get("id", "unknown")) return _node( @@ -291,6 +317,33 @@ def build_tree(self, request: ContextTreeRequest | None = None) -> JsonObject: sources = _admitted_rows(sources, project_scope, source=True) open_loops = _admitted_rows(open_loops, project_scope) artifacts = _admitted_rows(artifacts, project_scope) + from alicebot_api.vnext_label_guard import admit_loaded + + projects = admit_loaded( + self.store, kind="project", rows=projects, domains=domains, sensitivity_allowed=sensitivity, projects=project_scope + ) + memories = admit_loaded( + self.store, kind="memory", rows=memories, domains=domains, sensitivity_allowed=sensitivity, projects=project_scope + ) + sources = admit_loaded( + self.store, kind="source", rows=sources, domains=domains, sensitivity_allowed=sensitivity, projects=project_scope + ) + open_loops = admit_loaded( + self.store, + kind="open_loop", + rows=open_loops, + domains=domains, + sensitivity_allowed=sensitivity, + projects=project_scope, + ) + artifacts = admit_loaded( + self.store, + kind="artifact", + rows=artifacts, + domains=domains, + sensitivity_allowed=sensitivity, + projects=project_scope, + ) if not request.include_events: events = [] elif project_scope: @@ -303,7 +356,11 @@ def build_tree(self, request: ContextTreeRequest | None = None) -> JsonObject: limit=request.limit, ) else: - events = self.store.list_events(limit=request.limit) + events = [ + event + for event in self.store.list_events(limit=request.limit) + if _tree_event_visible(self.store, event, domains, sensitivity, project_scope) + ] roots = [ _node( diff --git a/apps/api/src/alicebot_api/vnext_contradictions.py b/apps/api/src/alicebot_api/vnext_contradictions.py index da79ca6aa..bf764b503 100644 --- a/apps/api/src/alicebot_api/vnext_contradictions.py +++ b/apps/api/src/alicebot_api/vnext_contradictions.py @@ -762,10 +762,15 @@ def review_belief( ) return belief - def belief_state(self, *, belief_id: str) -> JsonObject: + def belief_state(self, *, belief_id: str, sensitivity_allowed: tuple[str, ...] | None = None) -> JsonObject: belief = self.store.get_belief(belief_id) if belief is None: raise VNextContradictionValidationError(f"belief {belief_id} was not found") + if sensitivity_allowed is not None: + from alicebot_api.vnext_label_guard import LabelGuard + + if not LabelGuard.for_filters(self.store, (), sensitivity_allowed, ()).admit_beliefs([belief]): + raise VNextContradictionValidationError(f"belief {belief_id} was not found") events = self.store.list_events(target_type="belief", target_id=belief_id) previous_statuses: list[object] = [] for event in events: diff --git a/apps/api/src/alicebot_api/vnext_dogfooding.py b/apps/api/src/alicebot_api/vnext_dogfooding.py index 7c4ce5a08..8620af6e9 100644 --- a/apps/api/src/alicebot_api/vnext_dogfooding.py +++ b/apps/api/src/alicebot_api/vnext_dogfooding.py @@ -170,7 +170,10 @@ class VNextDogfoodingService: def __init__(self, store: VNextDogfoodingStore) -> None: self.store = store - def dashboard(self) -> JsonObject: + def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None) -> JsonObject: + from alicebot_api.vnext_agent_control import ALL_SENSITIVITY + from alicebot_api.vnext_label_guard import admit_loaded, readable_status_counts + sources = self.store.list_sources(limit=500) try: memories = self.store.list_memories(status=None, limit=500) @@ -183,6 +186,21 @@ def dashboard(self) -> JsonObject: artifacts = self.store.list_artifacts(limit=500) ratings = self.store.list_artifact_quality_ratings(limit=500) open_loops = self.store.list_open_loops(status=None, limit=500) + # None is the owner and an admin key: every sensitivity, so the guard + # reads nothing and the lists stay as the store returned them. + ceiling = sensitivity_allowed if sensitivity_allowed is not None else ALL_SENSITIVITY + fetched_memories = memories + sources = admit_loaded(self.store, kind="source", rows=sources, domains=(), sensitivity_allowed=ceiling, projects=()) + memories = admit_loaded( + self.store, kind="memory", rows=memories, domains=(), sensitivity_allowed=ceiling, projects=() + ) + artifacts = admit_loaded( + self.store, kind="artifact", rows=artifacts, domains=(), sensitivity_allowed=ceiling, projects=() + ) + open_loops = admit_loaded( + self.store, kind="open_loop", rows=open_loops, domains=(), sensitivity_allowed=ceiling, projects=() + ) + memory_status_counts = readable_status_counts(memory_status_counts, fetched_memories, memories) try: events = self.store.list_events(limit=5_000) except TypeError: # Compatibility for external/test stores on the old protocol. diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index e4f0c047d..e3349060a 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -116,6 +116,11 @@ def admit_rows(self, kind: str, rows: Sequence[Mapping[str, object]]) -> list[Ma if not self.active: return [row for row in rows if isinstance(row, Mapping)] + # A store with no label reader cannot settle a derived row. Production + # stores have the reader. A stand-in without it keeps the rows the SQL + # filter already returned. + if not callable(getattr(self.store, "read_label_rows", None)): + return [row for row in rows if isinstance(row, Mapping)] kept: list[Mapping[str, object]] = [] for row in rows: if not isinstance(row, Mapping): @@ -238,6 +243,89 @@ def policy_labels( return (domain,), (sensitivity,), scope, floor +def sensitivity_ceiling(identity: AgentIdentity | None) -> tuple[str, ...] | None: + """Sensitivities this caller may see, or None when the caller has no ceiling. + + The owner (no identity) and an admin key have no ceiling. Every other + profile uses the sensitivity list the policy already gives that profile. + This ceiling does not add a domain or a project restriction. + """ + + if identity is None: + return None + from alicebot_api.vnext_agent_control import _profile_sensitivity + + ceiling = _profile_sensitivity(str(identity.permission_profile)) + if set(ceiling) >= set(ALL_SENSITIVITY): + return None + return ceiling + + +def apply_sensitivity_ceiling( + store: Any, + *, + kind: str, + rows: Sequence[Mapping[str, object]], + identity: AgentIdentity | None, +) -> list[Mapping[str, object]]: + """Rows whose effective sensitivity is inside the caller's ceiling. + + A missing identity and an admin key keep every row, including its title + and id. Any other caller loses a row the ceiling does not admit, so a + count of the returned list does not reveal it. + """ + + ceiling = sensitivity_ceiling(identity) + if ceiling is None: + return [row for row in rows if isinstance(row, Mapping)] + return admit_loaded( + store, + kind=kind, + rows=rows, + domains=(), + sensitivity_allowed=ceiling, + projects=(), + ) + + +def readable_count(sql_count: int, fetched: int, admitted: int) -> int: + """A stored count, reduced only by rows this page's guard dropped. + + When the guard drops nothing the stored count is returned unchanged. + When the fetched page is the whole set, the count is the admitted length. + """ + + dropped = fetched - admitted + if dropped <= 0: + return sql_count + if sql_count <= fetched: + return admitted + return max(0, sql_count - dropped) + + +def readable_status_counts( + counts: Mapping[str, int], + fetched: Sequence[Mapping[str, object]], + admitted: Sequence[Mapping[str, object]], + *, + field: str = "status", +) -> dict[str, int]: + """Status counts with one taken off for each row the guard dropped.""" + + if len(fetched) == len(admitted): + return dict(counts) + kept = {id(row) for row in admitted} + updated = dict(counts) + for row in fetched: + if id(row) in kept: + continue + status = str(row.get(field, "unknown")) + current = int(updated.get(status, 0)) + if current > 0: + updated[status] = current - 1 + return updated + + def apply_unverified_rule( decision: PolicyDecision, row: Mapping[str, object] | None, diff --git a/apps/api/src/alicebot_api/vnext_projects.py b/apps/api/src/alicebot_api/vnext_projects.py index 39ef87be6..ec3471599 100644 --- a/apps/api/src/alicebot_api/vnext_projects.py +++ b/apps/api/src/alicebot_api/vnext_projects.py @@ -1356,11 +1356,21 @@ def review_open_loop( return self.store.update_open_loop(loop_id=loop_id, patch=patch) def project_dashboard( - self, *, project_id: str, sensitivity_allowed: tuple[str, ...] = DEFAULT_SENSITIVITY_ALLOWED + self, + *, + project_id: str, + sensitivity_allowed: tuple[str, ...] = DEFAULT_SENSITIVITY_ALLOWED, + identity: object | None = None, ) -> JsonObject: + from alicebot_api.vnext_agent_control import AgentIdentity + from alicebot_api.vnext_label_guard import admit_loaded, apply_sensitivity_ceiling + project = self.store.get_project(project_id) if project is None: raise VNextProjectValidationError(f"project {project_id} was not found") + caller = identity if isinstance(identity, AgentIdentity) or identity is None else None + if not apply_sensitivity_ceiling(self.store, kind="project", rows=[project], identity=caller): + raise VNextProjectValidationError(f"project {project_id} was not found") domain = str(project.get("domain", "unknown")) memories = self.store.search_memories( query=str(project.get("name", "")), @@ -1387,6 +1397,30 @@ def project_dashboard( scope_projects=(project_id,), ) artifacts = [row for row in artifact_candidates if _is_in_project(row, project_id)][:DEFAULT_PROJECT_LIMIT] + memories = admit_loaded( + self.store, + kind="memory", + rows=memories, + domains=[domain], + sensitivity_allowed=sensitivity_allowed, + projects=(project_id,), + ) + open_loops = admit_loaded( + self.store, + kind="open_loop", + rows=open_loops, + domains=[domain], + sensitivity_allowed=sensitivity_allowed, + projects=(project_id,), + ) + artifacts = admit_loaded( + self.store, + kind="artifact", + rows=artifacts, + domains=[domain], + sensitivity_allowed=sensitivity_allowed, + projects=(project_id,), + ) return { "project": project, "state": project.get("current_state"), @@ -1408,6 +1442,16 @@ def _resolve_project(self, request: ProjectAutomationRequest) -> JsonObject: sensitivity_allowed=list(request.sensitivity_allowed), limit=1, ) + from alicebot_api.vnext_label_guard import admit_loaded + + projects = admit_loaded( + self.store, + kind="project", + rows=projects, + domains=list(request.domains) if request.domains else (), + sensitivity_allowed=request.sensitivity_allowed, + projects=(), + ) if not projects: raise VNextProjectValidationError("no active project was found for update candidate generation") return projects[0] diff --git a/apps/api/src/alicebot_api/vnext_retrieval.py b/apps/api/src/alicebot_api/vnext_retrieval.py index a7671d0ab..080ec5de8 100644 --- a/apps/api/src/alicebot_api/vnext_retrieval.py +++ b/apps/api/src/alicebot_api/vnext_retrieval.py @@ -102,6 +102,7 @@ from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_grounding import compute_query_grounding from alicebot_api.vnext_json import json_safe +from alicebot_api.vnext_label_guard import LabelGuard, admit_loaded from alicebot_api.vnext_lifecycle import RETIRED_STATUSES from alicebot_api.vnext_promotion_policy import memory_write_provenance from alicebot_api.vnext_project_scope import ( @@ -2506,6 +2507,7 @@ def expand_provenance_once( selected = set(already_selected_ids) admitted: list[JsonObject] = [] used_tokens = 0 + candidates: list[JsonObject] = [] for row in list_refs(source_id=source_id): if not isinstance(row, Mapping): continue @@ -2529,7 +2531,16 @@ def expand_provenance_once( scope_window_end=scope_window_end, ): continue - item = dict(row) + candidates.append(dict(row)) + for item in admit_loaded( + store, + kind="memory", + rows=candidates, + domains=effective_domains, + sensitivity_allowed=effective_sensitivity_allowed, + projects=effective_project_scope, + ): + row_id = str(item.get("id") or "") cost = estimate_item_tokens(item) if used_tokens + cost > token_cap: break @@ -2668,7 +2679,14 @@ def _memory_entity_edges(self, entity_ids: Sequence[str]) -> list[JsonObject]: edges.extend(list_edges(from_id=entity_id)) return edges - def _memories_by_ids(self, memory_ids: Sequence[str]) -> dict[str, JsonObject]: + def _memories_by_ids( + self, + memory_ids: Sequence[str], + *, + domains: Sequence[str] | None = None, + sensitivity_allowed: Sequence[str] | None = None, + projects: Sequence[str] | None = None, + ) -> dict[str, JsonObject]: normalized_ids = tuple(dict.fromkeys(str(memory_id) for memory_id in memory_ids if memory_id)) if not normalized_ids: return {} @@ -2686,6 +2704,14 @@ def _memories_by_ids(self, memory_ids: Sequence[str]) -> dict[str, JsonObject]: if callable(get_memory) else [] ) + rows = admit_loaded( + self.store, + kind="memory", + rows=rows, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=projects, + ) return {str(row.get("id")): row for row in rows} def _sources_by_ids(self, source_ids: Sequence[str]) -> dict[str, JsonObject]: @@ -3080,7 +3106,23 @@ def _memory_fts_rows( # Store predates the match_any kwarg; keep the strict # (empty) result rather than guessing. return [], fts_source + rows = admit_loaded( + self.store, + kind="memory", + rows=rows, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=projects, + ) return _stabilize_scored_rows(rows), f"{fts_source}_or_fallback" + rows = admit_loaded( + self.store, + kind="memory", + rows=rows, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=projects, + ) return _stabilize_scored_rows(rows), fts_source legacy_search = cast( Callable[..., list[JsonObject]], @@ -3094,6 +3136,14 @@ def _memory_fts_rows( **filters, **scope_filters, ) + rows = admit_loaded( + self.store, + kind="memory", + rows=rows, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=projects, + ) return list(rows), "store_lexical" def _query_embedding(self, query: str) -> tuple[list[float] | None, str]: @@ -3195,6 +3245,14 @@ def _memory_vector_rows( return [], VECTOR_STAGE_DISABLED_QUERY_EMBEDDING_FAILED # Ascending stage: smaller distance ranks first. Equal distances # (identical texts embed identically) stabilize content-first. + rows = admit_loaded( + self.store, + kind="memory", + rows=rows, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=projects, + ) return _stabilize_scored_rows(rows, score_key="vector_distance", descending=False), VECTOR_STAGE_ENABLED def _memory_graph_rows( @@ -3281,7 +3339,14 @@ def _memory_graph_rows( ranked: list[tuple[datetime, datetime, str, JsonObject]] = [] visible_entity_ids: set[str] = set() readable_mentions: dict[str, set[tuple[str, str]]] = {entity_id: set() for entity_id in entity_ids} - memories_by_id = self._memories_by_ids(tuple(observed_at_by_memory)) + memories_by_id = self._memories_by_ids( + tuple(observed_at_by_memory), + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=projects, + ) + graph_candidates: list[JsonObject] = [] + graph_observed: list[datetime] = [] for memory_id, observed_at in observed_at_by_memory.items(): row = memories_by_id.get(memory_id) if row is None: @@ -3303,12 +3368,29 @@ def _memory_graph_rows( scope_window_end=scope_window_end, ): continue + graph_candidates.append(row) + graph_observed.append(observed_at) + admitted_graph_ids = { + str(row.get("id")) + for row in admit_loaded( + self.store, + kind="memory", + rows=graph_candidates, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=projects, + ) + } + for row, observed_at in zip(graph_candidates, graph_observed, strict=True): + memory_id = str(row.get("id")) + if memory_id not in admitted_graph_ids: + continue recency = ( _parse_timestamp(row.get("updated_at")) or _parse_timestamp(row.get("created_at")) or _GRAPH_EPOCH ) - ranked.append((observed_at, recency, str(row.get("id")), row)) + ranked.append((observed_at, recency, memory_id, row)) visible_entity_ids.update(entities_by_memory[memory_id]) for entity_id in entities_by_memory[memory_id]: readable_mentions[entity_id].add(("memory", memory_id)) @@ -3388,6 +3470,14 @@ def _memory_temporal_rows( limit=limit, **_optional_search_filters(memory_types, projects, created_by_agent_ids, run_id), ) + rows = admit_loaded( + self.store, + kind="memory", + rows=rows, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=projects, + ) return list(rows), TEMPORAL_STAGE_ENABLED def _source_stage_lists( @@ -3777,7 +3867,7 @@ def memory_visibility( entity graph here, once, like the memory stages do. """ - return _memory_visibility_predicate( + base = _memory_visibility_predicate( domains=domains, sensitivity_allowed=sensitivity_allowed, scope=scope, @@ -3785,6 +3875,23 @@ def memory_visibility( self._person_linked_memory_ids(scope.people) if scope is not None else frozenset() ), ) + project_filter = tuple(scope.projects) if scope is not None else () + + def visible(row: Mapping[str, object]) -> bool: + if not base(row): + return False + return bool( + admit_loaded( + self.store, + kind="memory", + rows=[row], + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=project_filter, + ) + ) + + return visible def fence_validity_memory_ids( self, @@ -4366,6 +4473,14 @@ def _fetch_open_loops(n: int) -> tuple[list[JsonObject], str]: target=DEFAULT_OPEN_LOOP_LIMIT, store_scope_complete=bool(open_loop_scope_filters), ) + open_loop_rows = admit_loaded( + self.store, + kind="open_loop", + rows=open_loop_rows, + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=tuple(scope.projects), + ) open_loop_rows = open_loop_rows[:DEFAULT_OPEN_LOOP_LIMIT] source_candidates = _fused_candidates( @@ -5176,6 +5291,12 @@ def _select_scoped_beliefs(rows: Sequence[JsonObject]) -> list[JsonObject]: sensitivity_allowed=sensitivity_allowed, limit=belief_target, ) + beliefs = LabelGuard.for_filters( + self.store, + domains, + sensitivity_allowed, + tuple(scope.projects), + ).admit_beliefs(list(beliefs)) candidates = vnext_contradictions._find_candidates( # noqa: SLF001 - deliberate read-only reuse new_items=new_items, beliefs=list(beliefs), @@ -5237,6 +5358,17 @@ def _select_events(rows: Sequence[JsonObject]) -> list[JsonObject]: targets = self._memories_by_ids( [str(event.get("target_id") or "") for event in eligible] ) + admitted_targets = { + str(row.get("id")) + for row in admit_loaded( + self.store, + kind="memory", + rows=list(targets.values()), + domains=domains, + sensitivity_allowed=sensitivity_allowed, + projects=tuple(scope.projects), + ) + } def _target_visible(event: JsonObject) -> bool: target = targets.get(str(event.get("target_id") or "")) @@ -5244,6 +5376,8 @@ def _target_visible(event: JsonObject) -> bool: # No such row is not a hidden row: the lookup applies no # fence. A scoped pack fails closed on it, as it always has. return not identity_scope.active + if str(target.get("id")) not in admitted_targets: + return False return memory_visible(target) return [event for event in eligible if _target_visible(event)] diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py index e1e86cb6d..f04a8be44 100644 --- a/tests/unit/test_label_door_registry.py +++ b/tests/unit/test_label_door_registry.py @@ -40,6 +40,9 @@ "admit_rows", "admit_beliefs", "policy_labels", + "admit_loaded", + "apply_sensitivity_ceiling", + "sensitivity_ceiling", } # function -> helper that holds the guard call, or None when the function calls it @@ -59,17 +62,41 @@ "mcp/retrieval.py:_handle_alice_open_loops": None, "vnext_memory_commit.py:VNextMemoryCommitService.authorize_memory_action": None, "vnext_memory_commit.py:VNextMemoryCommitService._write_policy_decision": None, + "routers/_vnext_shared.py:_vnext_load_source_trace": None, + "routers/vnext_projects.py:list_vnext_projects": None, + "routers/vnext_review.py:list_vnext_artifacts": None, + "routers/vnext_review.py:get_vnext_belief_state": None, + "routers/vnext_memories.py:get_vnext_dogfooding_dashboard": None, + "vnext_projects.py:VNextProjectService.project_dashboard": None, + "vnext_projects.py:VNextProjectService._resolve_project": None, + "vnext_projects.py:VNextProjectService.generate_project_update_candidate": None, + "mcp/retrieval.py:_resume_event_honours_policy_fence": None, + "mcp/retrieval.py:_vnext_recent_decisions": None, + "mcp/retrieval.py:_vnext_resume": None, + "mcp/projects.py:_handle_alice_vnext_open_loops": None, + "session_briefing.py:compile_session_brief": None, + "session_briefing.py:_event_target_honours_fence": None, + "session_briefing.py:_memory_honours_fence": None, + "routers/workspaces.py:_vnext_workspace_payload": None, + "vnext_context_tree.py:VNextContextTreeService.build_tree": None, + "vnext_dogfooding.py:VNextDogfoodingService.dashboard": None, + "vnext_contradictions.py:VNextContradictionService.belief_state": None, + "vnext_retrieval.py:expand_provenance_once": None, + "vnext_retrieval.py:VNextRetrievalService._memories_by_ids": None, + "vnext_retrieval.py:VNextRetrievalService._memory_fts_rows": None, + "vnext_retrieval.py:VNextRetrievalService._memory_vector_rows": None, + "vnext_retrieval.py:VNextRetrievalService._memory_graph_rows": None, + "vnext_retrieval.py:VNextRetrievalService._memory_temporal_rows": None, + "vnext_retrieval.py:VNextRetrievalService.compile_context_pack": None, + "vnext_retrieval.py:VNextRetrievalService._contradicting_evidence": None, + "vnext_retrieval.py:VNextRetrievalService._recent_changes": None, + "vnext_retrieval.py:VNextRetrievalService.memory_visibility": None, } NOT_A_DOOR = { - "routers/_vnext_shared.py:_vnext_load_source_trace": "operator trace stays label-agnostic until the screen change", "mcp/evidence_artifacts.py:_handle_alice_vnext_review_items": "legacy review list has no policy check", - "routers/vnext_projects.py:list_vnext_projects": "operator project list stays label-agnostic until the screen change", - "vnext_projects.py:VNextProjectService.generate_project_update_candidate": "producer input filter is the list-door change", "vnext_projects.py:VNextProjectService.review_project_update": "write path; the route authorizes before this mutation", "vnext_projects.py:VNextProjectService.review_open_loop": "write path; the route and the open-loop tool settle the loop first", - "vnext_projects.py:VNextProjectService.project_dashboard": "operator dashboard lists are the list-door change", - "vnext_projects.py:VNextProjectService._resolve_project": "operator project lookup is the list-door change", "vnext_memory_commit.py:VNextMemoryCommitService.confirm": "write path; _write_policy_decision settles the row", "vnext_memory_commit.py:VNextMemoryCommitService.undo": "write path; _write_policy_decision settles the row", "vnext_memory_commit.py:VNextMemoryCommitService.correct": "write path; _write_policy_decision settles the row", @@ -78,19 +105,16 @@ "vnext_memory_commit.py:VNextMemoryCommitService.expire": "write path; _write_policy_decision settles the row", "vnext_memory_commit.py:VNextMemoryCommitService.unexpire": "write path; _write_policy_decision settles the row", "vnext_memory_commit.py:VNextMemoryCommitService.quarantine_by_agent_key": "write path; _write_policy_decision settles the row", - "vnext_memory_commit.py:VNextMemoryCommitService.recent_commits": "owner list; the list-door change admits rows", + "vnext_memory_commit.py:VNextMemoryCommitService.recent_commits": "owner list; stays label-agnostic", "vnext_memory_commit.py:VNextMemoryCommitService.audit": "the authorize_memory callback settles each memory", "vnext_memory_commit.py:VNextMemoryCommitService._supersession_chain": "audit walks this after authorize_memory", - "vnext_memory_commit.py:VNextMemoryCommitService.inline_confirmations": "owner list; the list-door change admits rows", + "vnext_memory_commit.py:VNextMemoryCommitService.inline_confirmations": "owner list; stays label-agnostic", "vnext_memory_commit.py:VNextMemoryCommitService._idempotent_memory": "write path replay of a row already authorized", "vnext_memory_commit.py:VNextMemoryCommitService._memory_by_confirmation_id": "write path replay of a row already authorized", - "vnext_memory_commit.py:VNextMemoryCommitService._latest_agentic_commit": "owner list; the list-door change admits rows", + "vnext_memory_commit.py:VNextMemoryCommitService._latest_agentic_commit": "owner list; stays label-agnostic", "vnext_queue.py:VNextQueueService.review_artifact": "the HTTP review route authorizes through _vnext_authorized_artifact first", "vnext_queue.py:VNextQueueService._promote_artifact": "the HTTP review route authorizes through _vnext_authorized_artifact first", "vnext_queue.py:VNextQueueService.export_artifact_markdown": "the HTTP export route authorizes through _vnext_authorized_artifact first", - "mcp/retrieval.py:_resume_event_honours_policy_fence": "list door; admit_rows lands with the list-door change", - "mcp/retrieval.py:_vnext_recent_decisions": "list door; admit_rows lands with the list-door change", - "mcp/retrieval.py:_vnext_resume": "list door; admit_rows lands with the list-door change", } SCAN_MODULES = ( diff --git a/tests/unit/test_list_door_readers.py b/tests/unit/test_list_door_readers.py new file mode 100644 index 000000000..442d11b79 --- /dev/null +++ b/tests/unit/test_list_door_readers.py @@ -0,0 +1,254 @@ +"""List doors, operator screens, and event readers use the effective label.""" + +from __future__ import annotations + +from contextlib import contextmanager +from uuid import UUID + +from alicebot_api.mcp.retrieval import _resume_event_honours_policy_fence +from alicebot_api.routers import vnext_review +from alicebot_api.vnext_agent_control import AgentIdentity +from alicebot_api.vnext_contradictions import VNextContradictionService +from alicebot_api.vnext_label_guard import apply_sensitivity_ceiling +from alicebot_api.vnext_projects import VNextProjectService +from alicebot_api.vnext_retrieval import VNextRetrievalService + +SOURCE_ID = "11111111-1111-1111-1111-111111111111" +MEMORY_ID = "33333333-3333-3333-3333-333333333333" +ARTIFACT_ID = "22222222-2222-2222-2222-222222222222" +USER_ID = UUID("44444444-4444-4444-4444-444444444444") +SECRET = "SENTINEL confidential fact" + + +def _source() -> dict[str, object]: + return {"id": SOURCE_ID, "domain": "health", "sensitivity": "confidential", "metadata_json": {}} + + +def _derived_memory() -> dict[str, object]: + return { + "id": MEMORY_ID, + "title": SECRET, + "canonical_text": SECRET, + "status": "active", + "domain": "project", + "sensitivity": "public", + "metadata_json": { + "source_id": SOURCE_ID, + "derived_from": { + "v": 1, + "sources": [SOURCE_ID], + "memories": [], + "open_loops": [], + "artifacts": [], + "beliefs": [], + "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, + }, + }, + } + + +class _LabelStore: + def __init__(self) -> None: + self.memory = _derived_memory() + self.artifact = { + "id": ARTIFACT_ID, + "title": SECRET, + "artifact_type": "daily_brief", + "domain": "project", + "sensitivity": "confidential", + "metadata_json": {}, + } + self.project = { + "id": "project-1", + "name": SECRET, + "domain": "project", + "sensitivity": "confidential", + "current_state": SECRET, + "metadata_json": {}, + } + self.belief = { + "id": "belief-1", + "memory_id": MEMORY_ID, + "claim": SECRET, + "quote_belief": SECRET, + "status": "active", + } + + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + if kind == "source" and SOURCE_ID in ids: + return [_source()] + if kind == "memory" and MEMORY_ID in ids: + return [self.memory] + return [] + + def search_memories_fts(self, **kwargs: object) -> list[dict[str, object]]: + del kwargs + return [self.memory] + + def list_artifacts(self, **kwargs: object) -> list[dict[str, object]]: + del kwargs + return [self.artifact] + + def get_project(self, project_id: str) -> dict[str, object] | None: + if project_id == self.project["id"]: + return self.project + return None + + def search_memories(self, **kwargs: object) -> list[dict[str, object]]: + del kwargs + return [] + + def list_open_loops(self, **kwargs: object) -> list[dict[str, object]]: + del kwargs + return [] + + def get_belief(self, belief_id: str) -> dict[str, object] | None: + if belief_id == self.belief["id"]: + return self.belief + return None + + def list_events(self, **kwargs: object) -> list[dict[str, object]]: + del kwargs + return [] + + def get_memory(self, memory_id: str) -> dict[str, object] | None: + if memory_id == MEMORY_ID: + return self.memory + return None + + def get_open_loop(self, loop_id: str) -> dict[str, object] | None: + del loop_id + return None + + +def _identity(profile: str) -> AgentIdentity: + return AgentIdentity(agent_id="reader", agent_type="unknown", permission_profile=profile) + + +def test_fts_stage_drops_a_public_copy_of_a_confidential_source() -> None: + store = _LabelStore() + rows, _status = VNextRetrievalService(store)._memory_fts_rows( + query="sentinel", + domains=["project"], + sensitivity_allowed=["public", "internal", "private", "unknown"], + limit=10, + ) + assert rows == [] + assert SECRET not in str(rows) + + +def test_operator_screens_hide_a_confidential_row_from_a_trusted_key() -> None: + store = _LabelStore() + trusted = _identity("trusted_local_agent") + admin = _identity("admin_agent") + visible_to_trusted = apply_sensitivity_ceiling(store, kind="artifact", rows=[store.artifact], identity=trusted) + visible_to_admin = apply_sensitivity_ceiling(store, kind="artifact", rows=[store.artifact], identity=admin) + visible_to_owner = apply_sensitivity_ceiling(store, kind="artifact", rows=[store.artifact], identity=None) + assert visible_to_trusted == [] + assert [row["id"] for row in visible_to_admin] == [ARTIFACT_ID] + assert [row["id"] for row in visible_to_owner] == [ARTIFACT_ID] + assert SECRET not in str(visible_to_trusted) + + for identity in (trusted,): + try: + VNextProjectService(store).project_dashboard(project_id="project-1", identity=identity) + except Exception as exc: + assert "project-1" not in str(exc) or exc.__class__.__name__ == "VNextProjectValidationError" + assert SECRET not in str(exc) + else: + raise AssertionError("a trusted key saw the confidential project row") + owner_dashboard = VNextProjectService(store).project_dashboard(project_id="project-1", identity=None) + admin_dashboard = VNextProjectService(store).project_dashboard(project_id="project-1", identity=admin) + assert owner_dashboard["project"]["name"] == SECRET + assert admin_dashboard["counts"] == {"memories": 0, "open_loops": 0, "artifacts": 0} + + try: + VNextContradictionService(store).belief_state( + belief_id="belief-1", + sensitivity_allowed=("public", "internal", "private", "unknown"), + ) + except Exception as exc: + assert SECRET not in str(exc) + else: + raise AssertionError("a trusted ceiling saw the belief") + owner_belief = VNextContradictionService(store).belief_state(belief_id="belief-1", sensitivity_allowed=None) + admin_belief = VNextContradictionService(store).belief_state(belief_id="belief-1", sensitivity_allowed=None) + assert owner_belief["current"]["claim"] == SECRET + assert admin_belief["belief_id"] == "belief-1" + + +def test_artifact_list_route_counts_only_rows_the_caller_may_read(monkeypatch) -> None: + store = _LabelStore() + + @contextmanager + def _connection(*args: object, **kwargs: object): + del args, kwargs + yield object() + + def _resolve(store_arg: object, *, user_id: object, raw_key: object, payload: object) -> AgentIdentity | None: + del store_arg, user_id, payload + if raw_key is None: + return None + if raw_key == "trusted": + return _identity("trusted_local_agent") + if raw_key == "admin": + return _identity("admin_agent") + raise AssertionError(raw_key) + + monkeypatch.setattr(vnext_review, "user_connection", lambda *args, **kwargs: _connection()) + monkeypatch.setattr(vnext_review, "PostgresVNextStore", lambda conn: store) + monkeypatch.setattr(vnext_review, "resolve_protected_agent_identity", _resolve) + monkeypatch.setattr(vnext_review, "agent_key_from_authorization", lambda value: value) + + trusted = vnext_review.list_vnext_artifacts(USER_ID, authorization="trusted") + admin = vnext_review.list_vnext_artifacts(USER_ID, authorization="admin") + owner = vnext_review.list_vnext_artifacts(USER_ID, authorization=None) + trusted_body = trusted.body.decode() + admin_body = admin.body.decode() + owner_body = owner.body.decode() + assert ARTIFACT_ID not in trusted_body + assert SECRET not in trusted_body + assert '"count":0' in trusted_body or '"count": 0' in trusted_body + assert ARTIFACT_ID in admin_body + assert ARTIFACT_ID in owner_body + assert '"count":1' in admin_body or '"count": 1' in admin_body + assert '"count":1' in owner_body or '"count": 1' in owner_body + + +def test_a_labels_raised_event_is_hidden_when_the_target_is_effectively_confidential() -> None: + store = _LabelStore() + event = { + "target_type": "memory", + "target_id": MEMORY_ID, + "event_type": "memory.labels_raised", + "payload_json": {"cause": "repair_v3"}, + } + assert ( + _resume_event_honours_policy_fence( + store, + event, + effective_domains=("project",), + effective_sensitivity_allowed=("public", "internal", "private", "unknown"), + exclude_global_domains=frozenset(), + ) + is False + ) + assert ( + _resume_event_honours_policy_fence( + store, + event, + effective_domains=("project", "health"), + effective_sensitivity_allowed=( + "public", + "internal", + "private", + "unknown", + "confidential", + "highly_sensitive", + "sacred", + "regulated", + ), + exclude_global_domains=frozenset(), + ) + is True + ) diff --git a/tests/unit/test_open_loop_references_read_fence.py b/tests/unit/test_open_loop_references_read_fence.py index d94911e87..cb9f3610a 100644 --- a/tests/unit/test_open_loop_references_read_fence.py +++ b/tests/unit/test_open_loop_references_read_fence.py @@ -1145,6 +1145,10 @@ def test_the_reference_keys_include_every_key_the_reverse_lookup_of_a_source_rea _NOT_RETURNED = "not returned: reads the row to decide something and returns no part of it" _ALLOWLIST = "allowlist: returns a fixed list of fields that holds no reference (pinned by the allowlist test)" _OWNER = "owner: no agent identity reaches this door, so the reader is the owner and is not fenced" +_SENSITIVITY = ( + "sensitivity ceiling: a caller identity hides a row above that caller's sensitivity, " + "and the domain and project fences stay as they were" +) _OPERATOR = "operator: a local command or test harness, not a door a key-bound caller reaches" _SHARED = ( "shared: one handler of it returns a fixed list of fields and the other takes no identity " @@ -1183,10 +1187,10 @@ def test_the_reference_keys_include_every_key_the_reverse_lookup_of_a_source_rea ("list_open_loops", "vnext_dogfooding.py", "dashboard"): _OPERATOR, ("list_open_loops", "vnext_projects.py", "project_dashboard"): _PRODUCER, ("list_open_loops", "vnext_scheduler.py", "_generate_open_loop_review_artifact"): _FENCED, - ("list_open_loops_referencing_source", "routers/_vnext_shared.py", "_vnext_load_source_trace"): _OWNER, + ("list_open_loops_referencing_source", "routers/_vnext_shared.py", "_vnext_load_source_trace"): _SENSITIVITY, ("project_dashboard", "cli/automation.py", "_run_vnext_project_dashboard"): _OPERATOR, ("project_dashboard", "mcp/projects.py", "_handle_alice_project_dashboard"): _OWNER, - ("project_dashboard", "routers/vnext_projects.py", "get_vnext_project_dashboard"): _OWNER, + ("project_dashboard", "routers/vnext_projects.py", "get_vnext_project_dashboard"): _SENSITIVITY, ("project_dashboard", "routers/workspaces.py", "_vnext_workspace_payload"): _OWNER, ("review_open_loop", "cli/automation.py", "_run_vnext_open_loop_review"): _OPERATOR, ("review_open_loop", "mcp/projects.py", "_handle_alice_open_loop_review"): _OWNER, diff --git a/tests/unit/test_vnext_retrieval.py b/tests/unit/test_vnext_retrieval.py index 773b819cb..70454a4a5 100644 --- a/tests/unit/test_vnext_retrieval.py +++ b/tests/unit/test_vnext_retrieval.py @@ -1040,10 +1040,30 @@ def test_keyword_query_that_and_matches_does_not_use_the_fallback_on_sqlite() -> def test_count_candidate_statistic_uses_real_sqlite_fts_mode_and_provenance_dedup() -> None: store = _sqlite_retrieval_store() + source_a = store.create_source( + { + "source_type": "note", + "title": "Bike service source A", + "content_hash": "sha256:bike-a", + "domain": "personal", + "sensitivity": "private", + "captured_at": "2026-01-05T00:00:00Z", + } + ) + source_b = store.create_source( + { + "source_type": "note", + "title": "Bike service source B", + "content_hash": "sha256:bike-b", + "domain": "personal", + "sensitivity": "private", + "captured_at": "2026-01-06T00:00:00Z", + } + ) provenance = ( - ("record-1", "source-a", "chunk-a"), - ("record-2", "source-a", "chunk-a"), # restatement of the same captured turn - ("record-3", "source-b", "chunk-b"), + ("record-1", source_a["id"], "chunk-a"), + ("record-2", source_a["id"], "chunk-a"), # restatement of the same captured turn + ("record-3", source_b["id"], "chunk-b"), ) for memory_key, source_id, chunk_id in provenance: store.create_memory( From ab2eb45c63a0ac9a0d57fc982451db893d4c193f Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:06:22 +0200 Subject: [PATCH 011/270] fix: type derived label dependency maps distinctly --- apps/api/src/alicebot_api/vnext_derived_labels.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index ed0558aa7..2d039273f 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -911,7 +911,7 @@ def _changed(before: SettledLabel, after: SettledLabel) -> bool: def _weekly_parent_deps( labels: Mapping[tuple[str, str, str], SettledLabel], - own: Mapping[tuple[str, str, str], set[tuple[str, str, str]]], + own: dict[tuple[str, str, str], set[tuple[str, str, str]]], nodes: Sequence[tuple[SettledLabel, Mapping[str, object]]], ) -> None: """Old weekly candidates take the input lists of the artifact that names them.""" @@ -1012,13 +1012,13 @@ def resolve_belief(ref: tuple[str, str, str]) -> tuple[str, str, str]: return ref resolved: dict[tuple[str, str, str], set[tuple[str, str, str]]] = {} - for key, deps in own.items(): - resolved[key] = {resolve_belief(ref) for ref in deps} + for key, own_refs in own.items(): + resolved[key] = {resolve_belief(ref) for ref in own_refs} - for key, deps in resolved.items(): + for key, resolved_refs in resolved.items(): if key in problems and problems[key] not in {"", "no_record"}: continue - for ref in deps: + for ref in resolved_refs: if ref[0] in unavailable: problems[key] = "missing_table" break @@ -1075,10 +1075,10 @@ def resolve_belief(ref: tuple[str, str, str]) -> tuple[str, str, str]: if same_stored_row: published.append(replace(settled, unverified=False, reason=None)) continue - deps = [labels[ref] for ref in sorted(resolved.get(label.key, set())) if ref in labels] + settled_deps = [labels[ref] for ref in sorted(resolved.get(label.key, set())) if ref in labels] recomputed = _apply_dependencies( settled, - deps, + settled_deps, domain_fallback=label.stored_domain, sensitivity_fallback=label.stored_sensitivity, scope_fallback=label.stored_scope, From e0b68bca813b8f221397e04f3b73f0c9621581f3 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:07:52 +0200 Subject: [PATCH 012/270] fix: resolve full ancestry for label insert and relabel floors --- apps/api/src/alicebot_api/sqlite_store.py | 15 +++- .../src/alicebot_api/vnext_label_closure.py | 81 +++++++++++++++++++ .../src/alicebot_api/vnext_label_writes.py | 71 +++++++--------- apps/api/src/alicebot_api/vnext_store.py | 7 ++ .../test_sqlite_derived_labels_write_path.py | 51 ++++++++++++ 5 files changed, 181 insertions(+), 44 deletions(-) create mode 100644 apps/api/src/alicebot_api/vnext_label_closure.py diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index 2fd996f66..4808aceb8 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -422,14 +422,25 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: extra = ", value, project_id" elif table == "open_loops": extra = ", project_id, source_id, memory_id" + from uuid import UUID + canonical = [] + for item in wanted: + try: + canonical.append(UUID(item).hex) + except (ValueError, TypeError): + pass marks = ",".join("?" for _ in wanted) + alias_sql = "" + if canonical: + alias_marks = ",".join("?" for _ in canonical) + alias_sql = f" OR replace(replace(replace(replace(lower(id),'urn:uuid:',''),'-',''),'{{',''),'}}','') IN ({alias_marks})" return self._fetch_all( f""" SELECT id, user_id, domain, sensitivity, metadata_json{extra} FROM {table} - WHERE user_id = ? AND id IN ({marks}) + WHERE user_id = ? AND (id IN ({marks}){alias_sql}) """, - (self.user_id, *wanted), + (self.user_id, *wanted, *canonical), ) # -- fetch helpers (mirror PostgresVNextStore conventions) ------------ diff --git a/apps/api/src/alicebot_api/vnext_label_closure.py b/apps/api/src/alicebot_api/vnext_label_closure.py new file mode 100644 index 000000000..fc7a59719 --- /dev/null +++ b/apps/api/src/alicebot_api/vnext_label_closure.py @@ -0,0 +1,81 @@ +"""Narrow, bounded ancestry reads shared by label writes and read guards.""" +from __future__ import annotations + +from collections import deque +from collections.abc import Mapping, Sequence +from typing import Any + +from alicebot_api.vnext_derived_labels import canon_kind, dependencies_of, identifier, is_derived + + +def collect_label_rows( + store: Any, + roots: Sequence[Mapping[str, object]], + *, + max_nodes: int, + max_hops: int | None = None, + cache: dict[tuple[str, str], list[dict[str, object]]] | None = None, + user_id: str | None = None, +) -> tuple[list[dict[str, object]], bool]: + """Return reachable stored rows and whether the independent walk budget ran out. + + Cache entries only avoid reads; every origin still traverses its ancestry. + Stored IDs remain intact while query and graph identities use the canonical parser. + """ + loaded = cache if cache is not None else {} + pending = deque((canon_kind(row.get("kind")), dict(row), 0) for row in roots) + rows: list[dict[str, object]] = [] + seen: set[tuple[str, str, str]] = set() + requested: set[tuple[str, str]] = set() + exceeded = False + reader = getattr(store, "read_label_rows", None) + while pending: + kind, row, depth = pending.popleft() + stored_key = (kind, str(row.get("user_id") or ""), str(row.get("id") or "")) + if stored_key in seen: + continue + if len(seen) >= max_nodes or (max_hops is not None and depth > max_hops): + exceeded = True + continue + seen.add(stored_key) + row["kind"] = kind + if user_id is not None: + row["user_id"] = user_id + rows.append(row) + refs = list(dependencies_of(kind, row)) if is_derived(kind, row) else [] + if kind == "belief" and row.get("memory_id"): + refs.append(("memory", identifier(row["memory_id"]))) + grouped: dict[str, list[str]] = {} + for ref_kind, ref_id in refs: + name, canonical_id = canon_kind(ref_kind), identifier(ref_id) + key = (name, canonical_id) + if key in requested: + continue + requested.add(key) + if key in loaded: + pending.extend((name, dict(found), depth + 1) for found in loaded[key]) + else: + grouped.setdefault(name, []).append(canonical_id) + if not callable(reader): + continue + for name, ids in grouped.items(): + for item in ids: + loaded[(name, item)] = [] + for found in reader(name, ids): + if not isinstance(found, Mapping): + continue + key = (name, identifier(found.get("id"))) + if key not in loaded or key[1] not in ids: + continue + copied = dict(found) + loaded[key].append(copied) + pending.append((name, copied, depth + 1)) + # UUID aliases may coexist in SQLite. Never let iteration order choose a public twin. + twins: dict[tuple[str, str, str], list[dict[str, object]]] = {} + for row in rows: + twins.setdefault((str(row["kind"]), str(row.get("user_id") or ""), identifier(row.get("id"))), []).append(row) + for aliases in twins.values(): + if len(aliases) > 1: + for row in aliases: + row["sensitivity"] = "regulated" + return rows, exceeded diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 6df1e210a..f33cc6290 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -26,9 +26,12 @@ is_derived, labels_raised_payload, settle_labels, + stored_scope, + union_floor, ) +from alicebot_api.vnext_label_closure import collect_label_rows from alicebot_api.vnext_event_log import build_event_log_record, integrity_hash_for_event -from alicebot_api.vnext_project_scope import project_scope_identity, resolve_project_scope, source_project_scope +from alicebot_api.vnext_project_scope import project_floor_shape, project_scope_identity, resolve_project_scope, source_project_scope from alicebot_api.vnext_repositories import JsonObject LABEL_METADATA_KEYS = ("project_scope", "project_floor", "derived_from") @@ -151,38 +154,32 @@ def apply_insert_floor(store: Any, kind: str, payload: Mapping[str, object]) -> if not _in_transaction(store): raise LabelLockOrderError("the insert floor requires an open transaction") user_id = str(getattr(store, "user_id", body.get("user_id") or "")) - nodes: list[dict[str, object]] = [] - grouped: dict[str, list[str]] = {} - for dep_kind, dep_id in dependencies_of(kind, body): - grouped.setdefault(dep_kind, []).append(dep_id) - reader = getattr(store, "read_label_rows", None) - if callable(reader): - for dep_kind, ids in grouped.items(): - for row in reader(dep_kind, ids): - copied = dict(row) - copied["kind"] = dep_kind - copied.setdefault("user_id", user_id) - nodes.append(copied) - if not user_id and nodes: - user_id = str(nodes[0].get("user_id") or "") own_id = str(body.get("id") or "new-derived-row") own = dict(body) own["kind"] = kind own["id"] = own_id own["user_id"] = user_id - nodes.append(own) - settled = settle_labels(nodes).by_stored(kind, own_id, user_id=user_id or None) - if settled.unverified: - return body, None + nodes, exceeded = collect_label_rows(store, [own], max_nodes=PROPAGATION_BOUND) + if exceeded: + raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") + settled = settle_labels(nodes, on_cycle="unverified").by_stored(kind, own_id, user_id=user_id or None) domain = settled.domain + if settled.unverified: + domain = generation_domain(str(body.get("domain") or "unknown"), [str(node.get("domain") or "unknown") for node in nodes]) if str(body.get("domain") or "unknown") in RESTRICTED_DOMAINS: domain = generation_domain(str(body.get("domain")), [settled.domain]) - metadata = dict(body.get("metadata_json")) if isinstance(body.get("metadata_json"), Mapping) else {} + raw_metadata = body.get("metadata_json") + metadata = dict(raw_metadata) if isinstance(raw_metadata, Mapping) else {} metadata["project_scope"] = list(settled.project_scope) metadata["project_floor"] = list(settled.project_floor) + if settled.unverified: + metadata["project_floor"] = list(union_floor(settled.project_floor, [ + *(stored_scope(str(node["kind"]), node) for node in nodes), + *(project_floor_shape(node)[1] for node in nodes), + ])) updated = dict(body) updated["domain"] = domain - updated["sensitivity"] = settled.sensitivity + updated["sensitivity"] = "regulated" if settled.unverified else settled.sensitivity updated["metadata_json"] = metadata if len(project_scope_identity(settled.project_scope)) != 1: updated["project_id"] = None @@ -205,8 +202,8 @@ def apply_insert_floor(store: Any, kind: str, payload: Mapping[str, object]) -> new={ "domain": after[0], "sensitivity": after[1], - "project_scope": list(settled.project_scope), - "project_floor": list(settled.project_floor), + "project_scope": list(after[2]), + "project_floor": list(after[3]), }, ), ) @@ -466,26 +463,14 @@ def propagate(store: Any, changed: Sequence[tuple[str, str]], *, cause: str) -> affected = walk_dependants(store, roots) if not affected: return 0 - nodes: list[dict[str, object]] = [] + roots_rows = [dict(row) for row in affected] reader = getattr(store, "read_label_rows", None) if callable(reader): for kind, row_id in changed: - for row in reader(kind, [row_id]): - copied = dict(row) - copied["kind"] = kind - nodes.append(copied) - needed: dict[str, list[str]] = {} - for row in affected: - for dep_kind, dep_id in dependencies_of(str(row.get("kind")), row): - needed.setdefault(dep_kind, []).append(dep_id) - if callable(reader): - for dep_kind, dep_ids in needed.items(): - for row in reader(dep_kind, dep_ids): - copied = dict(row) - copied["kind"] = dep_kind - nodes.append(copied) - for row in affected: - nodes.append(dict(row)) + roots_rows.extend({**dict(row), "kind": kind} for row in reader(kind, [identifier(row_id)])) + nodes, exceeded = collect_label_rows(store, roots_rows, max_nodes=PROPAGATION_BOUND) + if exceeded: + raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") settled = settle_labels(nodes) written = 0 for row in affected: @@ -501,7 +486,8 @@ def propagate(store: Any, changed: Sequence[tuple[str, str]], *, cause: str) -> and project_scope_identity(previous[3]) == project_scope_identity(current[3]) ): continue - metadata = dict(row.get("metadata_json")) if isinstance(row.get("metadata_json"), Mapping) else {} + raw_metadata = row.get("metadata_json") + metadata = dict(raw_metadata) if isinstance(raw_metadata, Mapping) else {} metadata["project_scope"] = list(label.project_scope) metadata["project_floor"] = list(label.project_floor) project_id = label.project_scope[0] if len(project_scope_identity(label.project_scope)) == 1 else None @@ -570,7 +556,8 @@ def count_rows_hidden_by_scope_move(store: Any, source: Mapping[str, object], ne current["kind"] = "source" moved = dict(source) moved["kind"] = "source" - metadata = dict(source.get("metadata_json")) if isinstance(source.get("metadata_json"), Mapping) else {} + raw_metadata = source.get("metadata_json") + metadata = dict(raw_metadata) if isinstance(raw_metadata, Mapping) else {} metadata["project_scope"] = list(new_scope) moved["metadata_json"] = metadata before = settle_labels([current, *[dict(row) for row in affected]]) diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index e83c801f5..b8d7572ca 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -497,6 +497,13 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: }.get(kind) if table is None: return [] + if table == "beliefs": + return self._fetch_all( + """SELECT b.id, b.user_id, m.domain, m.sensitivity, b.metadata_json, b.memory_id + FROM beliefs b JOIN memories m ON m.id = b.memory_id AND m.user_id = b.user_id + WHERE b.id = ANY(%s::uuid[])""", + (wanted,), + ) extra = "" if table == "memories": extra = ", value, project_id" diff --git a/tests/unit/test_sqlite_derived_labels_write_path.py b/tests/unit/test_sqlite_derived_labels_write_path.py index d6f90f646..f11e2e3ef 100644 --- a/tests/unit/test_sqlite_derived_labels_write_path.py +++ b/tests/unit/test_sqlite_derived_labels_write_path.py @@ -230,3 +230,54 @@ def test_merge_protected_metadata_keeps_label_keys_unless_the_write_is_a_relabel relabel = merge_protected_metadata(stored, {"project_scope": [], "project_floor": [ALPHA]}, label_write=True) assert relabel["project_scope"] == [] assert relabel["consolidation"] == {"cluster_member_ids": ["m"]} + + +def test_insert_floor_survives_two_hops(tmp_path: Path): + db = tmp_path / 'labels.sqlite3' + bootstrap_database(db, user_id=USER, user_email='synthetic@example.test') + with sqlite_user_connection(db, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source({'source_type': 'note', 'title': 'Synthetic restricted note', 'content_hash': 'review-two-hops', 'domain': 'health', 'sensitivity': 'confidential', 'metadata_json': {'project_scope': [ALPHA]}}) + copied = store.create_memory({'memory_key': 'review-copy', 'canonical_text': 'Synthetic restricted observation', 'status': 'active', 'domain': 'unknown', 'sensitivity': 'public', 'metadata_json': {'source_id': str(source['id']), 'project_scope': [ALPHA]}}) + summary = store.create_memory({'memory_key': 'review-summary', 'canonical_text': 'Summary of the synthetic restricted observation', 'status': 'active', 'domain': 'unknown', 'sensitivity': 'public', 'metadata_json': {'consolidation': {'cluster_member_ids': [str(copied['id'])]}, 'project_scope': [ALPHA]}}) + assert copied['sensitivity'] == 'confidential' + assert summary['sensitivity'] == 'confidential', f"two-hop child persisted as {summary['domain']}/{summary['sensitivity']}" + + +def test_unresolved_insert_still_succeeds_conservatively(tmp_path: Path) -> None: + with _vault(tmp_path / "missing.sqlite3") as conn: + store = SQLiteVNextStore(conn, USER) + row = store.create_memory({"memory_key": "missing", "canonical_text": "summary", "status": "active", "domain": "unknown", "sensitivity": "public", "metadata_json": {"consolidation": {"cluster_member_ids": ["missing"]}, "project_scope": [ALPHA]}}) + assert row["sensitivity"] == "regulated" + assert row["metadata_json"]["project_floor"] == [ALPHA] + + +def test_relabel_reads_the_other_branch_ancestry(tmp_path: Path) -> None: + with _vault(tmp_path / "branches.sqlite3") as conn: + store = SQLiteVNextStore(conn, USER) + copies = [] + sources = [] + for suffix in ("a", "b"): + source = store.create_source({"source_type": "note", "title": suffix, "content_hash": suffix, "domain": "unknown", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA]}}) + sources.append(source) + copies.append(store.create_memory({"memory_key": suffix, "canonical_text": suffix, "status": "active", "domain": "unknown", "sensitivity": "public", "metadata_json": {"source_id": source["id"]}})) + summary = store.create_memory({"memory_key": "summary", "canonical_text": "summary", "status": "active", "domain": "unknown", "sensitivity": "public", "metadata_json": {"consolidation": {"cluster_member_ids": [row["id"] for row in copies]}}}) + store.update_source(source_id=str(sources[0]["id"]), patch={"domain": "health", "sensitivity": "confidential"}, actor_type="user") + stored = store.get_memory(str(summary["id"])) + assert stored["domain"] == "health" + assert stored["sensitivity"] == "confidential" + + +def test_sqlite_label_lookup_preserves_uuid_aliases_and_kinds(tmp_path: Path) -> None: + from uuid import UUID + with _vault(tmp_path / "aliases.sqlite3") as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source({"source_type": "note", "title": "alias", "content_hash": "alias", "domain": "health", "sensitivity": "confidential"}) + raw = "{" + str(source["id"]).upper() + "}" + conn.execute("UPDATE sources SET id = ? WHERE id = ?", (raw, source["id"])) + found = store.read_label_rows("source", [str(UUID(raw))]) + assert [row["id"] for row in found] == [raw] + assert store.read_label_rows("memory", [str(UUID(raw))]) == [] + copy = store.create_memory({"memory_key": "alias-copy", "canonical_text": "copy", "status": "active", "domain": "unknown", "sensitivity": "public", "metadata_json": {"source_id": str(UUID(raw))}}) + assert copy["domain"] == "health" + assert copy["sensitivity"] == "confidential" From 3a6a25d210d44d9b8937dbf2ee6f01c2a6547205 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:10:08 +0200 Subject: [PATCH 013/270] fix: preserve RLS tenant identity in derived insert floors --- .../src/alicebot_api/vnext_label_writes.py | 5 ++++ .../test_label_floor_ancestry_postgres.py | 27 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/integration/test_label_floor_ancestry_postgres.py diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index f33cc6290..43a147fbe 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -162,6 +162,11 @@ def apply_insert_floor(store: Any, kind: str, payload: Mapping[str, object]) -> nodes, exceeded = collect_label_rows(store, [own], max_nodes=PROPAGATION_BOUND) if exceeded: raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") + if not user_id: + tenants = {str(node.get("user_id")) for node in nodes if node.get("user_id")} + if len(tenants) == 1: + user_id = tenants.pop() + nodes[0]["user_id"] = user_id settled = settle_labels(nodes, on_cycle="unverified").by_stored(kind, own_id, user_id=user_id or None) domain = settled.domain if settled.unverified: diff --git a/tests/integration/test_label_floor_ancestry_postgres.py b/tests/integration/test_label_floor_ancestry_postgres.py new file mode 100644 index 000000000..29dc73cd1 --- /dev/null +++ b/tests/integration/test_label_floor_ancestry_postgres.py @@ -0,0 +1,27 @@ +"""Actual RLS-scoped PostgreSQL write-floor and belief ancestry controls.""" +from uuid import uuid4 +import pytest +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_store import PostgresVNextStore + + +@pytest.mark.parametrize("domain,sensitivity", [("project", "public"), ("health", "confidential")]) +def test_pg_copy_and_summary_keep_tenant_and_ancestry(migrated_database_urls, domain, sensitivity): + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"labels-{user_id}@example.invalid", "Labels") + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "synthetic", "content_hash": str(uuid4()), "domain": domain, "sensitivity": sensitivity}) + copy = store.create_memory({"memory_key": "copy", "canonical_text": "copy", "status": "active", "domain": "unknown", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])}}) + summary = store.create_memory({"memory_key": "summary", "canonical_text": "summary", "status": "active", "domain": "unknown", "sensitivity": "public", "metadata_json": {"consolidation": {"cluster_member_ids": [str(copy["id"])]}}}) + assert copy["sensitivity"] == sensitivity + assert summary["sensitivity"] == sensitivity + if domain == "health": + assert summary["domain"] == domain + belief_id = uuid4() + with conn.cursor() as cur: + cur.execute("INSERT INTO beliefs(id,user_id,memory_id,claim) VALUES (%s,%s,%s,%s)", (belief_id,user_id,copy["id"],"synthetic")) + belief = store.read_label_rows("belief", [str(belief_id)])[0] + assert belief["sensitivity"] == sensitivity + assert str(belief["memory_id"]) == str(copy["id"]) From ffd8a8039e62968e2137c240ed915cdb97e532da Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:15:43 +0200 Subject: [PATCH 014/270] fix: classify ambiguous canonical label identities as unverified --- apps/api/src/alicebot_api/vnext_derived_labels.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 2d039273f..875db8f40 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -977,6 +977,11 @@ def settle_labels( own: dict[tuple[str, str, str], set[tuple[str, str, str]]] = {} problems: dict[tuple[str, str, str], str] = {} + stored_ids: dict[tuple[str, str, str], str] = {} + for label, _row in prepared: + previous_id = stored_ids.setdefault(label.key, label.stored_id) + if previous_id != label.stored_id: + problems[label.key] = "ambiguous_identity" for label, row in prepared: if not label.derived: continue From b52cf12ed0c67fc7c5e33c15d9109c994dffa77e Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:11:54 +0200 Subject: [PATCH 015/270] fix: retain narrowed metadata types through protected merge --- .../alicebot_api/vnext_stores/postgres/memory_lifecycle.py | 5 ++++- .../src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py index 1cb1979fa..99be62c49 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py @@ -441,10 +441,13 @@ def update_memory( ) if current is not None: stored = current.get("metadata_json") + patch_metadata = patch["metadata_json"] + if not isinstance(patch_metadata, dict): + raise ValueError("memory metadata must be an object") patch = dict(patch) patch["metadata_json"] = merge_protected_metadata( stored if isinstance(stored, dict) else {}, - patch["metadata_json"], + patch_metadata, label_write=label_write, ) row = self._fetch_one( diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py index ccff5ca4d..d5b89e9fc 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py @@ -53,10 +53,13 @@ def _with_protected_metadata(self, memory_id: str, patch: JsonObject, *, label_w if current is None: return patch stored = current.get("metadata_json") + patch_metadata = patch["metadata_json"] + if not isinstance(patch_metadata, dict): + return patch merged = dict(patch) merged["metadata_json"] = merge_protected_metadata( stored if isinstance(stored, dict) else {}, - patch["metadata_json"], + patch_metadata, label_write=label_write, ) return merged From 9bcd5c7157b51406d0a24a5252b92b239d96f412 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:12:46 +0200 Subject: [PATCH 016/270] test: prevent public selection among stored UUID aliases --- tests/unit/test_sqlite_derived_labels_write_path.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/test_sqlite_derived_labels_write_path.py b/tests/unit/test_sqlite_derived_labels_write_path.py index f11e2e3ef..29c705721 100644 --- a/tests/unit/test_sqlite_derived_labels_write_path.py +++ b/tests/unit/test_sqlite_derived_labels_write_path.py @@ -281,3 +281,13 @@ def test_sqlite_label_lookup_preserves_uuid_aliases_and_kinds(tmp_path: Path) -> copy = store.create_memory({"memory_key": "alias-copy", "canonical_text": "copy", "status": "active", "domain": "unknown", "sensitivity": "public", "metadata_json": {"source_id": str(UUID(raw))}}) assert copy["domain"] == "health" assert copy["sensitivity"] == "confidential" + + +def test_alias_twins_cannot_choose_a_public_source(tmp_path: Path) -> None: + with _vault(tmp_path / "twins.sqlite3") as conn: + store = SQLiteVNextStore(conn, USER) + restricted = store.create_source({"source_type": "note", "title": "restricted", "content_hash": "restricted", "domain": "health", "sensitivity": "confidential"}) + public = store.create_source({"source_type": "note", "title": "public", "content_hash": "public", "domain": "project", "sensitivity": "public"}) + conn.execute("UPDATE sources SET id = ? WHERE id = ?", (str(restricted["id"]).replace("-", "").upper(), public["id"])) + copy = store.create_memory({"memory_key": "twins-copy", "canonical_text": "copy", "status": "active", "domain": "unknown", "sensitivity": "public", "metadata_json": {"source_id": str(restricted["id"])}}) + assert copy["sensitivity"] == "regulated" From cd8be4822acae9b3d4174830088454ae26a6fa1a Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:12:58 +0200 Subject: [PATCH 017/270] test: preserve legacy rollup rows for SQLite query ordering --- tests/unit/test_sqlite_store.py | 55 +++++++++++++++++---------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/tests/unit/test_sqlite_store.py b/tests/unit/test_sqlite_store.py index 4081188d4..94dd2a24b 100644 --- a/tests/unit/test_sqlite_store.py +++ b/tests/unit/test_sqlite_store.py @@ -433,32 +433,35 @@ def test_project_scoped_memory_and_rollup_queries_filter_before_limit() -> None: last_confirmed_at="2020-01-01T00:00:00Z", metadata_json={"project_scope": ["project-b"]}, ) - pending_a = _create_memory( - store, - status="candidate", - metadata_json={ - "project_scope": ["project-a"], - "candidate_kind": "memory_rollup", - "rollup_digest": "digest-a", - }, - ) - _create_memory( - store, - status="candidate", - metadata_json={ - "project_scope": ["project-b"], - "candidate_kind": "memory_rollup", - "rollup_digest": "digest-b", - }, - ) - accepted_a = _create_memory( - store, - metadata_json={ - "project_scope": ["project-a"], - "candidate_kind": "memory_rollup", - "rollup_key": "topic:a", - }, - ) + from alicebot_api.vnext_label_writes import without_insert_floor + # This test exercises query ordering on existing, unstamped legacy cards. + with without_insert_floor(): + pending_a = _create_memory( + store, + status="candidate", + metadata_json={ + "project_scope": ["project-a"], + "candidate_kind": "memory_rollup", + "rollup_digest": "digest-a", + }, + ) + _create_memory( + store, + status="candidate", + metadata_json={ + "project_scope": ["project-b"], + "candidate_kind": "memory_rollup", + "rollup_digest": "digest-b", + }, + ) + accepted_a = _create_memory( + store, + metadata_json={ + "project_scope": ["project-a"], + "candidate_kind": "memory_rollup", + "rollup_key": "topic:a", + }, + ) assert [row["id"] for row in store.list_memories(projects=("project-a",), limit=1)] == [accepted_a["id"]] assert store.count_memories(status="active", projects=("project-a",)) == 2 From a6c329b413e8ec7ead4d144b62ad344f8b4cf720 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:14:05 +0200 Subject: [PATCH 018/270] test: preserve legacy label fixtures for PostgreSQL repair --- .../integration/test_derived_domain_postgres.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_derived_domain_postgres.py b/tests/integration/test_derived_domain_postgres.py index 93220c97d..3c68bc2dc 100644 --- a/tests/integration/test_derived_domain_postgres.py +++ b/tests/integration/test_derived_domain_postgres.py @@ -14,13 +14,15 @@ from alicebot_api.vnext_agent_control import ALL_SENSITIVITY from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService from alicebot_api.vnext_store import PostgresVNextStore +from alicebot_api.vnext_label_writes import without_insert_floor def test_postgres_derived_domain_upgrade_and_generation(database_urls): config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260721_0094") user = uuid4() - with user_connection(database_urls["app"], user) as conn: + # Seed pre-floor rows so this still tests the migration, not the live insert path. + with without_insert_floor(), user_connection(database_urls["app"], user) as conn: ContinuityStore(conn).create_user(user, "derived-fence@example.invalid", "Derived fence") store = PostgresVNextStore(conn) source = store.create_memory( @@ -86,7 +88,8 @@ def test_postgres_repair_as_documented_nobypassrls_owner(database_urls, monkeypa config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260721_0094") user = uuid4() - with user_connection(database_urls["app"], user) as conn: + # Seed pre-floor rows so this still tests the migration, not the live insert path. + with without_insert_floor(), user_connection(database_urls["app"], user) as conn: ContinuityStore(conn).create_user(user, "owner-repair@example.invalid", "Owner repair") store = PostgresVNextStore(conn) source = store.create_memory( @@ -290,7 +293,8 @@ def test_promoted_artifact_uuid_alias_repaired(database_urls): config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260721_0094") user = uuid4() - with user_connection(database_urls["app"], user) as conn: + # Seed pre-floor rows so this still tests the migration, not the live insert path. + with without_insert_floor(), user_connection(database_urls["app"], user) as conn: ContinuityStore(conn).create_user(user, "alias@example.invalid", "Alias fixture") store = PostgresVNextStore(conn) memory = store.create_memory({"memory_key": "health", "canonical_text": "Private observation", @@ -323,7 +327,8 @@ def test_postgres_repair_reads_every_spelling_of_a_recorded_id(database_urls, sp config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260721_0094") user = uuid4() - with user_connection(database_urls["app"], user) as conn: + # Seed pre-floor rows so this still tests the migration, not the live insert path. + with without_insert_floor(), user_connection(database_urls["app"], user) as conn: ContinuityStore(conn).create_user(user, "spelling@example.invalid", "Spelling fixture") store = PostgresVNextStore(conn) health = store.create_memory( @@ -351,7 +356,8 @@ def test_postgres_repair_refuses_an_update_that_changes_no_row(database_urls, mo config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260721_0094") user = uuid4() - with user_connection(database_urls["app"], user) as conn: + # Seed pre-floor rows so this still tests the migration, not the live insert path. + with without_insert_floor(), user_connection(database_urls["app"], user) as conn: ContinuityStore(conn).create_user(user, "zero-row@example.invalid", "Zero row fixture") store = PostgresVNextStore(conn) health = store.create_memory( From d126d5d2f608117fa696078a858fd0d917380532 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:14:33 +0200 Subject: [PATCH 019/270] fix: conservatively raise unresolved legacy relabel descendants --- apps/api/src/alicebot_api/vnext_label_writes.py | 14 +++++++++++++- .../unit/test_sqlite_derived_labels_write_path.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 43a147fbe..5c8ae250f 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -11,6 +11,7 @@ from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager from functools import wraps +from dataclasses import replace from typing import Any from alicebot_api.vnext_agent_control import RESTRICTED_DOMAINS @@ -481,7 +482,18 @@ def propagate(store: Any, changed: Sequence[tuple[str, str]], *, cause: str) -> for row in affected: label = settled.by_stored(str(row.get("kind")), str(row.get("id"))) if label.unverified: - continue + ancestry, exceeded = collect_label_rows(store, [row], max_nodes=PROPAGATION_BOUND) + if exceeded: + raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") + label = replace( + label, + domain=generation_domain(label.domain, [str(node.get("domain") or "unknown") for node in ancestry]), + sensitivity="regulated", + project_floor=union_floor(label.project_floor, [ + *(stored_scope(str(node["kind"]), node) for node in ancestry), + *(project_floor_shape(node)[1] for node in ancestry), + ]), + ) previous = _label_fields(row) current = (label.domain, label.sensitivity, tuple(label.project_scope), tuple(label.project_floor)) if ( diff --git a/tests/unit/test_sqlite_derived_labels_write_path.py b/tests/unit/test_sqlite_derived_labels_write_path.py index 29c705721..d430002b6 100644 --- a/tests/unit/test_sqlite_derived_labels_write_path.py +++ b/tests/unit/test_sqlite_derived_labels_write_path.py @@ -291,3 +291,17 @@ def test_alias_twins_cannot_choose_a_public_source(tmp_path: Path) -> None: conn.execute("UPDATE sources SET id = ? WHERE id = ?", (str(restricted["id"]).replace("-", "").upper(), public["id"])) copy = store.create_memory({"memory_key": "twins-copy", "canonical_text": "copy", "status": "active", "domain": "unknown", "sensitivity": "public", "metadata_json": {"source_id": str(restricted["id"])}}) assert copy["sensitivity"] == "regulated" + + +def test_relabel_regulates_legacy_summary_with_missing_ancestry(tmp_path: Path) -> None: + with _vault(tmp_path / "missing-branch.sqlite3") as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source({"source_type": "note", "title": "input", "content_hash": "missing-branch", "domain": "unknown", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA]}}) + copy = store.create_memory({"memory_key": "copy", "canonical_text": "copy", "status": "active", "domain": "unknown", "sensitivity": "public", "metadata_json": {"source_id": source["id"]}}) + with without_insert_floor(): + summary = store.create_memory({"memory_key": "summary", "canonical_text": "summary", "status": "active", "domain": "unknown", "sensitivity": "public", "metadata_json": {"consolidation": {"cluster_member_ids": [copy["id"], "missing"]}}}) + store.update_source(source_id=str(source["id"]), patch={"domain": "health", "sensitivity": "confidential"}, actor_type="user") + row = store.get_memory(str(summary["id"])) + assert row["domain"] == "health" + assert row["sensitivity"] == "regulated" + assert ALPHA in row["metadata_json"]["project_floor"] From 619a73db65f922080a562f20a769fbb8aa93fcc7 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:15:43 +0200 Subject: [PATCH 020/270] test: retain all stored alias restrictions in insert floors --- tests/unit/test_sqlite_derived_labels_write_path.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_sqlite_derived_labels_write_path.py b/tests/unit/test_sqlite_derived_labels_write_path.py index d430002b6..3bfd61cca 100644 --- a/tests/unit/test_sqlite_derived_labels_write_path.py +++ b/tests/unit/test_sqlite_derived_labels_write_path.py @@ -286,11 +286,13 @@ def test_sqlite_label_lookup_preserves_uuid_aliases_and_kinds(tmp_path: Path) -> def test_alias_twins_cannot_choose_a_public_source(tmp_path: Path) -> None: with _vault(tmp_path / "twins.sqlite3") as conn: store = SQLiteVNextStore(conn, USER) - restricted = store.create_source({"source_type": "note", "title": "restricted", "content_hash": "restricted", "domain": "health", "sensitivity": "confidential"}) - public = store.create_source({"source_type": "note", "title": "public", "content_hash": "public", "domain": "project", "sensitivity": "public"}) + restricted = store.create_source({"source_type": "note", "title": "restricted", "content_hash": "restricted", "domain": "health", "sensitivity": "confidential", "metadata_json": {"project_scope": [ALPHA]}}) + public = store.create_source({"source_type": "note", "title": "public", "content_hash": "public", "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": ["prj_" + "b" * 16]}}) conn.execute("UPDATE sources SET id = ? WHERE id = ?", (str(restricted["id"]).replace("-", "").upper(), public["id"])) copy = store.create_memory({"memory_key": "twins-copy", "canonical_text": "copy", "status": "active", "domain": "unknown", "sensitivity": "public", "metadata_json": {"source_id": str(restricted["id"])}}) assert copy["sensitivity"] == "regulated" + assert copy["domain"] == "health" + assert set(copy["metadata_json"]["project_floor"]) == {ALPHA, "prj_" + "b" * 16} def test_relabel_regulates_legacy_summary_with_missing_ancestry(tmp_path: Path) -> None: From e57f93b0b764289d47a6928980b4375d8f9e513c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:15:26 +0200 Subject: [PATCH 021/270] chore: document verified label SQL and rollback scan exceptions --- apps/api/src/alicebot_api/vnext_label_writes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 5c8ae250f..9fd571284 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -261,7 +261,7 @@ def acquire_exclusive_label_lock(store: Any) -> None: finally: try: cur.execute("SELECT set_config('lock_timeout', %s, true)", (str(previous),)) - except Exception: + except Exception: # nosec B110 # aborted transactions cannot restore local settings; rollback clears them # A lock timeout aborts the transaction. Rollback drops the local setting. pass @@ -323,7 +323,7 @@ def _sqlite_dependants(store: Any, table: str, kind: str, compacts: Sequence[str SELECT id, user_id, domain, sensitivity, metadata_json{extra} FROM {table} WHERE user_id = ? AND ({text_clause}{value_sql}{column_sql}) - """, + """, # nosec B608 # internal literal table/columns; every external value is bound tuple(params), ) for row in rows: @@ -353,7 +353,7 @@ def _postgres_dependants(store: Any, table: str, kind: str, compacts: Sequence[s SELECT id::text AS id, user_id::text AS user_id, domain, sensitivity, metadata_json{extra} FROM {table} WHERE ({text_clause}{value_sql}) - """, + """, # nosec B608 # internal literal table/columns; every external value is bound tuple(params), ) for row in rows: @@ -439,7 +439,7 @@ def write_settled_label( UPDATE {table} SET domain = ?, sensitivity = ?, metadata_json = ?{project_sql} WHERE id = ? AND user_id = ? AND domain = ? AND sensitivity = ? - """, + """, # nosec B608 # table comes from the closed kind map; values are bound tuple(params), ) require_changed(int(cursor.rowcount), table, str(row_id)) @@ -456,7 +456,7 @@ def write_settled_label( SET domain = %s, sensitivity = %s, metadata_json = %s::jsonb{project_sql} WHERE id = %s::uuid AND domain = %s AND sensitivity = %s RETURNING id - """, + """, # nosec B608 # table comes from the closed kind map; values are bound tuple(params), ) From 1371fa787ea996373f6fef9df0bc124e51e4f388 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 19:10:47 +0000 Subject: [PATCH 022/270] Raise stored SQLite derived labels on open and restore. A public copy of a confidential source is stored at the input's sensitivity the next time the vault opens, and a restore runs the same pass before it publishes. A pass that cannot finish leaves the vault open and the completion key unstamped. --- CHANGELOG.md | 1 + apps/api/src/alicebot_api/onramp.py | 2 + apps/api/src/alicebot_api/sqlite_schema.py | 23 +++ .../src/alicebot_api/vnext_label_repair.py | 188 ++++++++++++++++++ tests/unit/test_import_skip_legacy_rows.py | 1 + tests/unit/test_sqlite_label_repair_v3.py | 85 ++++++++ 6 files changed, 300 insertions(+) create mode 100644 apps/api/src/alicebot_api/vnext_label_repair.py create mode 100644 tests/unit/test_sqlite_label_repair_v3.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 67700df9d..b1531baa9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Unreleased (on main, not in v0.20.0): opening a SQLite vault raises a derived row whose stored label is below its inputs, and a restore does the same before it publishes. A repair that cannot finish leaves the vault open and the completion key unstamped. v0.20.0 left the stored label where it was written. No new migration in this change. - Unreleased (on main, not in v0.20.0): recall, context packs, resume, recent decisions, the session brief, the review queue, open-loop lists, and the workspace, dashboard, context tree, and dogfooding lists drop a loaded row whose effective label is outside the request. The artifact list, the source trace, the project list, the dashboard project row, and the belief state route apply the caller's sensitivity ceiling, including titles, ids, and counts. Resume, recent decisions, the session brief, audit, and explain apply the target row's current effective label, so an excluded key does not learn of a label change. v0.20.0 returned those rows from the label stored on them. No migration is required. - Unreleased (on main, not in v0.20.0): a daily brief, connection report, contradiction report, consolidation, roll-up, project update, staleness sweep, and open-loop review drop an input whose effective label is outside the request. v0.20.0 kept a public copy of a confidential input in the report text. No migration is required. - Unreleased (on main, not in v0.20.0): an exact read of a derived artifact, a write that cites a derived memory, explain, memory review, redaction, and an open-loop update use the labels of that row's inputs. v0.20.0 used the label stored on the row, so a public copy of a confidential input could be read. A row whose inputs cannot be checked is refused to a locked key. The owner and an unbound admin still read the stored row. No migration is required. diff --git a/apps/api/src/alicebot_api/onramp.py b/apps/api/src/alicebot_api/onramp.py index 95e907e37..7d80e6f94 100644 --- a/apps/api/src/alicebot_api/onramp.py +++ b/apps/api/src/alicebot_api/onramp.py @@ -4106,9 +4106,11 @@ def _run_import_snapshot( # The whole graph is now present. Repair before the staged pages # become visible, even if the destination had already upgraded. from alicebot_api.vnext_derived_domain_backfill import relabel_sqlite + from alicebot_api.vnext_label_repair import relabel_labels_sqlite try: relabel_sqlite(conn, restoring=True) + relabel_labels_sqlite(conn, restoring=True) except ValueError as exc: raise _ImportError("the restored derived labels could not be settled") from exc if quarantine_ids: diff --git a/apps/api/src/alicebot_api/sqlite_schema.py b/apps/api/src/alicebot_api/sqlite_schema.py index bcfcec56c..3486da4ab 100644 --- a/apps/api/src/alicebot_api/sqlite_schema.py +++ b/apps/api/src/alicebot_api/sqlite_schema.py @@ -2014,6 +2014,28 @@ def _relabel_derived_domains(conn: sqlite3.Connection) -> None: relabel_sqlite(conn) +def _relabel_derived_labels(conn: sqlite3.Connection) -> None: + """Repair derived labels. A failure leaves the vault open and the key unstamped.""" + + import logging + + from alicebot_api.vnext_label_repair import relabel_labels_sqlite + + conn.execute("SAVEPOINT alice_derived_labels_v3") + try: + relabel_labels_sqlite(conn) + except Exception as exc: + conn.execute("ROLLBACK TO SAVEPOINT alice_derived_labels_v3") + conn.execute("RELEASE SAVEPOINT alice_derived_labels_v3") + left = getattr(exc, "left", 0) + logging.getLogger(__name__).warning( + "alice-memory: label repair did not run, %s rows left to the read check; run alice-memory labels check", + left, + ) + return + conn.execute("RELEASE SAVEPOINT alice_derived_labels_v3") + + def bootstrap_sqlite_schema(conn: sqlite3.Connection) -> None: """Create or upgrade the vNext SQLite schema. Safe to call repeatedly.""" conn.execute("PRAGMA journal_mode=WAL") @@ -2037,6 +2059,7 @@ def bootstrap_sqlite_schema(conn: sqlite3.Connection) -> None: # identifier on a tombstone (audit P1 #3); a no-op on healthy files. _repair_tombstone_lookup_value_holders(conn) _relabel_derived_domains(conn) + _relabel_derived_labels(conn) # The redaction flag row must exist before the append-only triggers # reference it, and it must be OFF: a crashed process must never leave # a database file with redaction mode stuck open. diff --git a/apps/api/src/alicebot_api/vnext_label_repair.py b/apps/api/src/alicebot_api/vnext_label_repair.py new file mode 100644 index 000000000..d463d209a --- /dev/null +++ b/apps/api/src/alicebot_api/vnext_label_repair.py @@ -0,0 +1,188 @@ +"""Repair stored derived labels from the same rules as a read. + +The open pass never stops a vault from opening. A restore still aborts, +because a published vault must not stay unrepaired. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence + +from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError, require_changed +from alicebot_api.vnext_derived_labels import labels_raised_payload, settle_labels +from alicebot_api.vnext_event_log import build_event_log_record +from alicebot_api.vnext_project_scope import project_scope_identity + +REPAIR_STATE_KEY = "derived_labels_v3" +_TABLE_KIND = { + "sources": "source", + "memories": "memory", + "open_loops": "open_loop", +} +INPUT_SELECTS_V3 = { + "sources": "SELECT id, user_id, domain, sensitivity, metadata_json FROM sources", + "memories": "SELECT id, user_id, domain, sensitivity, metadata_json, value, project_id FROM memories", + "open_loops": ( + "SELECT id, user_id, domain, sensitivity, metadata_json, project_id, source_id, memory_id " + "FROM open_loops" + ), +} +_UPDATES = { + "memories": "UPDATE memories SET domain = ?, sensitivity = ?, metadata_json = ? WHERE user_id = ? AND id = ?", + "open_loops": ( + "UPDATE open_loops SET domain = ?, sensitivity = ?, metadata_json = ? WHERE user_id = ? AND id = ?" + ), +} + + +def _json_object(value: object) -> dict[str, object]: + if isinstance(value, str): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return {} + return dict(parsed) if isinstance(parsed, dict) else {} + if isinstance(value, Mapping): + return dict(value) + return {} + + +def _same(previous: Mapping[str, object], new: Mapping[str, object]) -> bool: + return ( + previous["domain"] == new["domain"] + and previous["sensitivity"] == new["sensitivity"] + and project_scope_identity(previous["project_scope"]) == project_scope_identity(new["project_scope"]) + and project_scope_identity(previous["project_floor"]) == project_scope_identity(new["project_floor"]) + ) + + +def plan_label_repairs( + tables: Mapping[str, Sequence[Mapping[str, object]]], +) -> list[tuple[str, str, str, dict[str, object], dict[str, object], dict[str, object]]]: + """Label changes for derived rows whose stored label is below the inputs. + + A malformed record makes its row unverified and is not rewritten. A cycle + that does not settle raises ``DerivedDomainRepairError``. + """ + + nodes: list[dict[str, object]] = [] + index: list[tuple[str, dict[str, object]]] = [] + for table, rows in tables.items(): + kind = _TABLE_KIND.get(table, table) + for row in rows: + node = dict(row) + node["kind"] = kind + node["metadata_json"] = _json_object(row.get("metadata_json")) + if isinstance(row.get("value"), str): + node["value"] = _json_object(row.get("value")) + nodes.append(node) + index.append((table, node)) + settled = settle_labels(nodes, on_cycle="raise") + changes = [] + for (table, node), label in zip(index, settled.rows, strict=True): + if not label.derived or label.unverified or table not in _UPDATES: + continue + previous = { + "domain": str(node.get("domain") or "unknown"), + "sensitivity": str(node.get("sensitivity") or "unknown"), + "project_scope": list(label.stored_scope), + "project_floor": list(label.stored_floor), + } + new = { + "domain": label.domain, + "sensitivity": label.sensitivity, + "project_scope": list(label.project_scope), + "project_floor": list(label.project_floor), + } + if _same(previous, new): + continue + changes.append((table, str(node.get("user_id") or ""), str(node.get("id") or ""), previous, new, node)) + return changes + + +def _load_tables(conn) -> dict[str, list[dict[str, object]]]: + available = { + row[0] if not isinstance(row, dict) else row["name"] + for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'") + } + tables: dict[str, list[dict[str, object]]] = {} + for table, statement in INPUT_SELECTS_V3.items(): + if table not in available: + continue + cursor = conn.execute(statement) + names = [column[0] for column in cursor.description] + tables[table] = [row if isinstance(row, dict) else dict(zip(names, row)) for row in cursor.fetchall()] + return tables + + +def _stamped(conn) -> bool: + return conn.execute("SELECT value FROM alice_schema_state WHERE key = ?", (REPAIR_STATE_KEY,)).fetchone() is not None + + +def relabel_labels_sqlite(conn, *, restoring: bool = False) -> None: + """Raise stored derived labels once, or again on a restore. + + When no transaction is open this begins one and reads the state key inside + it. A caller that already has a transaction keeps it. + """ + + owns = False + if not conn.in_transaction: + conn.execute("BEGIN IMMEDIATE") + owns = True + try: + if not restoring and _stamped(conn): + if owns: + conn.commit() + return + changes = plan_label_repairs(_load_tables(conn)) + for table, user, stored, _previous, new, node in changes: + metadata = dict(node.get("metadata_json") or {}) + metadata["project_scope"] = list(new["project_scope"]) + metadata["project_floor"] = list(new["project_floor"]) + changed = conn.execute( + _UPDATES[table], + (new["domain"], new["sensitivity"], json.dumps(metadata), user, stored), + ).rowcount + require_changed(changed, table, stored) + for table, user, stored, previous, new, _node in changes: + target = "memory" if table == "memories" else "open_loop" + event = build_event_log_record( + event_type=f"{target}.labels_raised", + actor_type="system", + target_type=target, + target_id=stored, + payload=labels_raised_payload(cause="repair_v3", previous=previous, new=new), + ) + conn.execute( + """INSERT INTO event_log (id, user_id, event_type, actor_type, target_type, target_id, + occurred_at, payload_json, integrity_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + event["id"], + user, + event["event_type"], + event["actor_type"], + event["target_type"], + stored, + event["occurred_at"], + json.dumps(event["payload_json"]), + event["integrity_hash"], + ), + ) + conn.execute("INSERT OR REPLACE INTO alice_schema_state (key, value) VALUES (?, ?)", (REPAIR_STATE_KEY, "1")) + if owns: + conn.commit() + except Exception: + if owns: + conn.rollback() + raise + + +__all__ = [ + "INPUT_SELECTS_V3", + "REPAIR_STATE_KEY", + "DerivedDomainRepairError", + "plan_label_repairs", + "relabel_labels_sqlite", +] diff --git a/tests/unit/test_import_skip_legacy_rows.py b/tests/unit/test_import_skip_legacy_rows.py index 27b822a1b..5db5a8bce 100644 --- a/tests/unit/test_import_skip_legacy_rows.py +++ b/tests/unit/test_import_skip_legacy_rows.py @@ -439,6 +439,7 @@ def test_apply_row_backfills_leaves_what_the_bootstrap_leaves(tmp_path: Path) -> _DOES_NOT_FILL_A_COLUMN_FROM_THE_ROW = { # A one-time cross-row repair from recorded inputs, not a row-local import backfill. "_relabel_derived_domains", + "_relabel_derived_labels", # Rebuild or add schema, or index existing content. "_ensure_current_memories_status_constraint", "_ensure_additive_columns", diff --git a/tests/unit/test_sqlite_label_repair_v3.py b/tests/unit/test_sqlite_label_repair_v3.py new file mode 100644 index 000000000..654961887 --- /dev/null +++ b/tests/unit/test_sqlite_label_repair_v3.py @@ -0,0 +1,85 @@ +"""The SQLite v3 open pass raises a derived row to its inputs.""" + +from __future__ import annotations + +import json +import logging + +from alicebot_api import sqlite_schema +from alicebot_api.onramp import bootstrap_database +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError +from alicebot_api.vnext_label_repair import REPAIR_STATE_KEY +from alicebot_api.vnext_label_writes import without_insert_floor +from tests.unit.test_derived_domain_fence import USER + +SOURCE_TEXT = "A confidential observation" + + +def _vault_with_a_public_copy(path, monkeypatch) -> str: + with monkeypatch.context() as patch: + patch.setattr(sqlite_schema, "_relabel_derived_labels", lambda conn: None) + bootstrap_database(path, user_id=USER, user_email="local@alice") + with without_insert_floor(), sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source( + { + "source_type": "note", + "title": "Clinic note", + "content_hash": "sha256:clinic", + "domain": "project", + "sensitivity": "confidential", + "metadata_json": {}, + } + ) + memory = store.create_memory( + { + "memory_key": "copy", + "canonical_text": SOURCE_TEXT, + "status": "active", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"source_id": source["id"]}, + } + ) + return str(memory["id"]) + + +def test_open_raises_a_public_copy_of_a_confidential_source(tmp_path, monkeypatch) -> None: + path = tmp_path / "vault.db" + memory_id = _vault_with_a_public_copy(path, monkeypatch) + with sqlite_user_connection(path, USER) as conn: + row = conn.execute("SELECT domain, sensitivity FROM memories WHERE id = ?", (memory_id,)).fetchone() + event = conn.execute( + "SELECT payload_json FROM event_log WHERE event_type = 'memory.labels_raised' AND target_id = ?", + (memory_id,), + ).fetchone() + sensitivity = row["sensitivity"] if isinstance(row, dict) else row[1] + raw_event = event["payload_json"] if isinstance(event, dict) else event[0] + assert sensitivity == "confidential" + payload = json.loads(raw_event) + assert payload["cause"] == "repair_v3" + assert "title" not in json.dumps(payload) + assert SOURCE_TEXT not in json.dumps(payload) + + +def test_a_vault_that_cannot_be_repaired_still_opens(tmp_path, monkeypatch, caplog) -> None: + path = tmp_path / "vault.db" + _vault_with_a_public_copy(path, monkeypatch) + with sqlite_user_connection(path, USER) as conn: + conn.execute("DELETE FROM alice_schema_state WHERE key = ?", (REPAIR_STATE_KEY,)) + + def _fail(conn, *, restoring: bool = False) -> None: + del conn, restoring + raise DerivedDomainRepairError("cycle") + + monkeypatch.setattr("alicebot_api.vnext_label_repair.relabel_labels_sqlite", _fail) + with caplog.at_level(logging.WARNING, logger="alicebot_api.sqlite_schema"): + with sqlite_user_connection(path, USER) as conn: + stamped = conn.execute( + "SELECT value FROM alice_schema_state WHERE key = ?", + (REPAIR_STATE_KEY,), + ).fetchone() + conn.execute("SELECT 1") + assert stamped is None + assert "alice-memory: label repair did not run" in caplog.text From d334131f47d90a364cdf5f56f4f9d9e7001564cb Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:17:53 +0200 Subject: [PATCH 023/270] test: verify canonical identity ambiguity and valid alias controls --- tests/unit/test_derived_labels_kernel.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/test_derived_labels_kernel.py b/tests/unit/test_derived_labels_kernel.py index 3d1528b65..ecc3fbcf6 100644 --- a/tests/unit/test_derived_labels_kernel.py +++ b/tests/unit/test_derived_labels_kernel.py @@ -817,3 +817,21 @@ def test_rank_table_matches_the_spec_order() -> None: assert SENSITIVITY_RANK["public"] < SENSITIVITY_RANK["unknown"] == SENSITIVITY_RANK["internal"] assert SENSITIVITY_RANK["sacred"] == SENSITIVITY_RANK["regulated"] assert "health" in RESTRICTED_DOMAINS + + +def test_distinct_stored_aliases_are_unverified_but_single_alias_and_other_kind_are_valid() -> None: + canonical = str(UUID(SOURCE_UUID)) + alias = "{" + canonical.upper() + "}" + source = _source(canonical, domain="health", sensitivity="confidential", scope=[ALPHA]) + twin = _source(alias, domain="project", scope=[BETA]) + report = _brief("ambiguous", sources=[canonical]) + for rows in ([source, twin, report], [twin, source, report]): + result = settle_labels(rows).by_stored("artifact", "ambiguous") + assert result.unverified is True + assert result.reason == "dependency_unverified" + single = settle_labels([twin, report]).by_stored("artifact", "ambiguous") + assert single.unverified is False + other_kind = _memory(canonical, domain="project", sensitivity="public") + result = settle_labels([source, other_kind, report]).by_stored("artifact", "ambiguous") + assert result.unverified is False + assert result.domain == "health" From e40f56d3e36732058f6a4c12c90d4e693189b5a7 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:17:03 +0200 Subject: [PATCH 024/270] test: verify actual Postgres belief report insert ancestry --- tests/integration/test_label_floor_ancestry_postgres.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/integration/test_label_floor_ancestry_postgres.py b/tests/integration/test_label_floor_ancestry_postgres.py index 29dc73cd1..2fdc86b95 100644 --- a/tests/integration/test_label_floor_ancestry_postgres.py +++ b/tests/integration/test_label_floor_ancestry_postgres.py @@ -25,3 +25,9 @@ def test_pg_copy_and_summary_keep_tenant_and_ancestry(migrated_database_urls, do belief = store.read_label_rows("belief", [str(belief_id)])[0] assert belief["sensitivity"] == sensitivity assert str(belief["memory_id"]) == str(copy["id"]) + + record = {"v": 1, "sources": [], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [str(belief_id)], "counts": {"sources": 0, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 1}} + artifact = store.create_artifact({"artifact_type": "daily_brief", "title": "synthetic", "content_markdown": "synthetic", "domain": "unknown", "sensitivity": "public", "metadata_json": {"workflow": "daily_brief", "derived_from": record}}) + assert artifact["sensitivity"] == sensitivity + if domain == "health": + assert artifact["domain"] == domain From ef05d6e1cf46093242044e19809a536e92e4cb97 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:17:03 +0200 Subject: [PATCH 025/270] test: mark SQLite rollup query fixtures as legacy rows --- tests/unit/test_vnext_rollups.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/unit/test_vnext_rollups.py b/tests/unit/test_vnext_rollups.py index af56899e0..b65650ba4 100644 --- a/tests/unit/test_vnext_rollups.py +++ b/tests/unit/test_vnext_rollups.py @@ -1240,20 +1240,23 @@ def test_sqlite_rollup_reads_are_exact_deduplicated_and_bounded() -> None: conn, store = _live_store() def create_row(name: str, *, status: str, metadata: JsonObject) -> JsonObject: - return store.create_memory( - { - "memory_key": f"memory.{name}", - "value": {"text": name}, - "status": status, - "memory_type": "semantic", - "title": name, - "canonical_text": name, - "summary": name, - "domain": "personal", - "sensitivity": "internal", - "metadata_json": metadata, - } - ) + from alicebot_api.vnext_label_writes import without_insert_floor + # Query contract fixture: existing unstamped cards retain their stored labels. + with without_insert_floor(): + return store.create_memory( + { + "memory_key": f"memory.{name}", + "value": {"text": name}, + "status": status, + "memory_type": "semantic", + "title": name, + "canonical_text": name, + "summary": name, + "domain": "personal", + "sensitivity": "internal", + "metadata_json": metadata, + } + ) ordinary_active = create_row("ordinary-active", status="active", metadata={}) ordinary_accepted = create_row("ordinary-accepted", status="accepted", metadata={}) From 1736302832897cd0ef82c3d545bf511df35de9f9 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:10:37 +0200 Subject: [PATCH 026/270] test: mark intentionally unstamped rollup lookup fixtures --- tests/unit/test_group_scope_consumers.py | 81 ++++++++++++------------ 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/tests/unit/test_group_scope_consumers.py b/tests/unit/test_group_scope_consumers.py index 38693981b..d5fbd7f0f 100644 --- a/tests/unit/test_group_scope_consumers.py +++ b/tests/unit/test_group_scope_consumers.py @@ -137,26 +137,28 @@ def test_rollup_lookups_match_a_floor_when_the_scope_is_empty() -> None: user_id = str(uuid4()) ensure_sqlite_user(conn, user_id, "group-scope@example.com", "Group Scope") store = SQLiteVNextStore(conn, user_id) - card = store.create_memory( - { - "memory_key": "vnext.rollup.floor-card", - "value": {"text": "card"}, - "status": "candidate", - "memory_type": "semantic", - "title": "Floor card", - "canonical_text": "Floor card", - "summary": "Floor card", - "domain": "personal", - "sensitivity": "internal", - "metadata_json": { - "candidate_kind": ROLLUP_CANDIDATE_KIND, - "rollup_digest": "digest-floor", - "rollup_key": "topic:floor", - "project_scope": [], - "project_floor": [ALPHA, BETA], - }, - } - ) + from alicebot_api.vnext_label_writes import without_insert_floor + with without_insert_floor(): + card = store.create_memory( + { + "memory_key": "vnext.rollup.floor-card", + "value": {"text": "card"}, + "status": "candidate", + "memory_type": "semantic", + "title": "Floor card", + "canonical_text": "Floor card", + "summary": "Floor card", + "domain": "personal", + "sensitivity": "internal", + "metadata_json": { + "candidate_kind": ROLLUP_CANDIDATE_KIND, + "rollup_digest": "digest-floor", + "rollup_key": "topic:floor", + "project_scope": [], + "project_floor": [ALPHA, BETA], + }, + } + ) pending = store.list_pending_rollup_candidates( rollup_digests=("digest-floor",), domains=["personal"], @@ -175,25 +177,26 @@ def test_rollup_lookups_match_a_floor_when_the_scope_is_empty() -> None: projects=("prj_" + "c" * 16,), ) assert missed == [] - accepted = store.create_memory( - { - "memory_key": "vnext.rollup.floor-accepted", - "value": {"text": "accepted"}, - "status": "active", - "memory_type": "semantic", - "title": "Accepted floor card", - "canonical_text": "Accepted floor card", - "summary": "Accepted floor card", - "domain": "personal", - "sensitivity": "internal", - "metadata_json": { - "candidate_kind": ROLLUP_CANDIDATE_KIND, - "rollup_key": "topic:floor-accepted", - "project_scope": [], - "project_floor": [ALPHA, BETA], - }, - } - ) + with without_insert_floor(): + accepted = store.create_memory( + { + "memory_key": "vnext.rollup.floor-accepted", + "value": {"text": "accepted"}, + "status": "active", + "memory_type": "semantic", + "title": "Accepted floor card", + "canonical_text": "Accepted floor card", + "summary": "Accepted floor card", + "domain": "personal", + "sensitivity": "internal", + "metadata_json": { + "candidate_kind": ROLLUP_CANDIDATE_KIND, + "rollup_key": "topic:floor-accepted", + "project_scope": [], + "project_floor": [ALPHA, BETA], + }, + } + ) cards = store.list_accepted_rollup_cards( rollup_keys=("topic:floor-accepted",), domains=["personal"], From b81d463e10a8e2e681281eaf154bbec525886a96 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:10:37 +0200 Subject: [PATCH 027/270] fix: narrow producer provenance rows before iterating --- apps/api/src/alicebot_api/vnext_derived_labels.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 66270a310..504fcfce1 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -1243,7 +1243,8 @@ def stamp_derived_from(payload: dict[str, object], rows_by_kind: Mapping[str, ob record: dict[str, object] = {"v": 1} counts: dict[str, int] = {} for key in ("sources", "memories", "open_loops", "artifacts", "beliefs"): - rows = rows_by_kind.get(key) or [] + raw_rows = rows_by_kind.get(key) + rows = raw_rows if isinstance(raw_rows, (list, tuple)) else [] ids = [str(row.get("id")) for row in rows if isinstance(row, Mapping) and row.get("id") is not None] record[key] = ids counts[key] = len(ids) From 37a9a7c8233908e55716ae3594c12b97e7881b3d Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:11:54 +0200 Subject: [PATCH 028/270] fix: preserve producer row types through locked admission --- apps/api/src/alicebot_api/vnext_derived_labels.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 504fcfce1..9ec1daae0 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -15,6 +15,7 @@ from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, replace from uuid import UUID +from typing import TypeVar, overload from alicebot_api.vnext_agent_control import RESTRICTED_DOMAINS from alicebot_api.vnext_derived_domain import derived_domain @@ -1262,6 +1263,17 @@ def with_derived_from(metadata: Mapping[str, object], rows_by_kind: Mapping[str, return dict(stamped) if isinstance(stamped, Mapping) else {} +_InputRow = TypeVar("_InputRow", bound=Mapping[str, object]) + + +@overload +def admit_when_locked(kind: str, rows: Sequence[_InputRow], projects: tuple[str, ...] | None) -> list[_InputRow]: ... + + +@overload +def admit_when_locked(kind: str, rows: object, projects: tuple[str, ...] | None) -> list[Mapping[str, object]]: ... + + def admit_when_locked( kind: str, rows: object, From 08b1409406e92befc7e357f408933ac4b5cae264 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:12:46 +0200 Subject: [PATCH 029/270] fix: align dynamic locked admission implementation with overloads --- apps/api/src/alicebot_api/vnext_derived_labels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 9ec1daae0..ae2178371 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -15,7 +15,7 @@ from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, replace from uuid import UUID -from typing import TypeVar, overload +from typing import Any, TypeVar, overload from alicebot_api.vnext_agent_control import RESTRICTED_DOMAINS from alicebot_api.vnext_derived_domain import derived_domain @@ -1278,7 +1278,7 @@ def admit_when_locked( kind: str, rows: object, projects: tuple[str, ...] | None, -) -> list[Mapping[str, object]]: +) -> list[Any]: """Keep every row when ``projects`` is None. Otherwise keep rows inside that binding.""" items = list(rows) if isinstance(rows, (list, tuple)) else [] From 00ae879f77da3fc527e94611aa09c40bec1b221a Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:08:47 +0200 Subject: [PATCH 030/270] fix: bound label reads by ancestry depth rather than fanout --- .../api/src/alicebot_api/vnext_label_guard.py | 38 +++---------- tests/unit/test_label_guard_exact_doors.py | 53 +++++++++++++++++++ 2 files changed, 60 insertions(+), 31 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index cdb669ed1..a858226b0 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -21,10 +21,10 @@ HOP_BOUND, NODE_BOUND, canon_kind, - dependencies_of, is_derived, settle_labels, ) +from alicebot_api.vnext_label_closure import collect_label_rows from alicebot_api.vnext_project_scope import project_floor_shape, project_scopes_overlap, resolve_project_scope @@ -52,7 +52,7 @@ class LabelGuard: domains: tuple[str, ...] = () sensitivity_allowed: tuple[str, ...] = () projects: tuple[str, ...] = () - _nodes: dict[tuple[str, str], dict[str, object]] | None = None + _nodes: dict[tuple[str, str], list[dict[str, object]]] | None = None @classmethod def for_fence(cls, store: Any, fence: Any) -> LabelGuard: @@ -161,35 +161,11 @@ def _admits_effective(self, row: Mapping[str, object]) -> bool: def _collected(self, kind: str, row: Mapping[str, object]) -> list[dict[str, object]]: if self._nodes is None: self._nodes = {} - pending: list[tuple[str, Mapping[str, object]]] = [(canon_kind(kind), row)] - hops = 0 - while pending and len(self._nodes) < NODE_BOUND and hops < HOP_BOUND: - hops += 1 - name, current = pending.pop(0) - node_id = str(current.get("id") or "") - key = (name, node_id) - if key in self._nodes: - continue - node = dict(current) - node["kind"] = name - node["user_id"] = _GUARD_USER - self._nodes[key] = node - if not is_derived(name, node): - continue - grouped: dict[str, list[str]] = {} - for dep_kind, dep_id in dependencies_of(name, node): - grouped.setdefault(canon_kind(dep_kind), []).append(str(dep_id)) - reader = getattr(self.store, "read_label_rows", None) - if not callable(reader): - continue - for dep_kind, ids in grouped.items(): - missing = [item for item in ids if (dep_kind, item) not in self._nodes] - if not missing: - continue - for found in reader(dep_kind, missing): - if isinstance(found, Mapping): - pending.append((dep_kind, found)) - return list(self._nodes.values()) + nodes, _exceeded = collect_label_rows( + self.store, [{**dict(row), "kind": canon_kind(kind)}], + max_nodes=NODE_BOUND, max_hops=HOP_BOUND, cache=self._nodes, user_id=_GUARD_USER, + ) + return nodes def effective_row_for_fence( diff --git a/tests/unit/test_label_guard_exact_doors.py b/tests/unit/test_label_guard_exact_doors.py index 05744df20..57fd99b40 100644 --- a/tests/unit/test_label_guard_exact_doors.py +++ b/tests/unit/test_label_guard_exact_doors.py @@ -232,3 +232,56 @@ def test_a_cited_memory_is_judged_by_its_source() -> None: store = _LabelStore() with pytest.raises(MemoryRefNotFoundError): resolve_attachable_memory_id(store, MEMORY_ID, fence=SourceReadFence.for_identity(_trusted())) + + +from uuid import UUID + +def provenance(sources=(), memories=()): + refs = {'sources': list(sources), 'memories': list(memories), 'open_loops': [], 'artifacts': [], 'beliefs': []} + return {'v': 1, **refs, 'counts': {k: len(v) for k, v in refs.items()}} + + +class ArtifactStore: + def __init__(self, n): + self.sources = [dict(id=str(UUID(int=i + 1)), domain='project', sensitivity='public', metadata_json={}) for i in range(n)] + self.artifact = dict(id=str(UUID(int=10000)), artifact_type='daily_brief', domain='project', sensitivity='public', content_markdown='Public report', metadata_json={'workflow': 'daily_brief', 'derived_from': provenance(sources=[s['id'] for s in self.sources])}) + def read_label_rows(self, kind, ids): + return [s for s in self.sources if s['id'] in ids] if kind == 'source' else [] + def get_artifact(self, artifact_id): + return self.artifact + def upsert_agent_identity(self, *args, **kwargs): + pass + def append_event(self, event): + return event + + +@pytest.mark.parametrize('n', [31, 32, 100]) +def test_public_report_fanout_remains_readable(n): + store = ArtifactStore(n) + result = _vnext_authorized_artifact(store=store, identity=AgentIdentity(agent_id='trusted-key', permission_profile='trusted_local_agent'), artifact_id=store.artifact['id'], action='artifact.read', for_update=False) + assert result is not None + + +def test_cached_guard_keeps_each_roots_independent_budget() -> None: + store = ArtifactStore(100) + guard = LabelGuard(store, active=True) + assert guard.effective_row("artifact", store.artifact)["unverified"] is False + assert guard.effective_row("artifact", store.artifact)["unverified"] is False + + +def test_belief_dependency_loads_backing_memory_ancestry() -> None: + store = _LabelStore() + belief_id = str(UUID(int=55)) + belief = {"id": belief_id, "memory_id": MEMORY_ID, "domain": "unknown", "sensitivity": "public", "metadata_json": {}} + artifact = dict(store.artifact) + artifact["metadata_json"] = {"workflow": "daily_brief", "derived_from": provenance()} + artifact["metadata_json"]["derived_from"]["beliefs"] = [belief_id] + artifact["metadata_json"]["derived_from"]["counts"]["beliefs"] = 1 + def reader(kind, ids): + rows = {"source": [store.source], "memory": [store.memory], "belief": [belief]}.get(kind, []) + return [row for row in rows if row["id"] in ids] + store.read_label_rows = reader + effective = LabelGuard(store, active=True).effective_row("artifact", artifact) + assert effective["unverified"] is False + assert effective["domain"] == "health" + assert effective["sensitivity"] == "confidential" From 99fcfa5475300f467dcc06ca63c193c5a30d11ca Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:10:54 +0200 Subject: [PATCH 031/270] fix: narrow read guard metadata before copying --- apps/api/src/alicebot_api/vnext_label_guard.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index a858226b0..e6c3afb4f 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -95,7 +95,8 @@ def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[ settled = settle_labels(nodes, on_cycle="unverified", max_hops=HOP_BOUND, max_nodes=NODE_BOUND) label = settled.by_stored(kind, str(row.get("id") or ""), user_id=_GUARD_USER) copy = dict(row) - metadata = dict(copy.get("metadata_json")) if isinstance(copy.get("metadata_json"), Mapping) else {} + raw_metadata = copy.get("metadata_json") + metadata = dict(raw_metadata) if isinstance(raw_metadata, Mapping) else {} if label.unverified: copy["domain"] = label.domain copy["sensitivity"] = "regulated" From 6377ccaea89d4fd6510a3413d937b471ce70a101 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:12:46 +0200 Subject: [PATCH 032/270] test: verify ancestry depth boundaries and cache reuse --- tests/unit/test_label_guard_exact_doors.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unit/test_label_guard_exact_doors.py b/tests/unit/test_label_guard_exact_doors.py index 57fd99b40..2aa23b2a2 100644 --- a/tests/unit/test_label_guard_exact_doors.py +++ b/tests/unit/test_label_guard_exact_doors.py @@ -285,3 +285,20 @@ def reader(kind, ids): assert effective["unverified"] is False assert effective["domain"] == "health" assert effective["sensitivity"] == "confidential" + + +def test_depth_boundary_and_cache_are_independent_of_width() -> None: + from alicebot_api.vnext_derived_labels import HOP_BOUND + # Extracted memories recursively reference memories through consolidation markers. + class Chain: + def __init__(self, length): + self.rows = [{"id": str(i), "domain": "project", "sensitivity": "public", "metadata_json": {"consolidation": {"cluster_member_ids": [str(i+1)]}}} for i in range(length)] + self.rows.append({"id": str(length), "domain": "project", "sensitivity": "public", "metadata_json": {}}) + def read_label_rows(self, kind, ids): + return [row for row in self.rows if row["id"] in ids] if kind == "memory" else [] + within = Chain(HOP_BOUND) + guard = LabelGuard(within, active=True) + assert guard.effective_row("memory", within.rows[0])["unverified"] is False + assert guard.effective_row("memory", within.rows[0])["unverified"] is False + beyond = Chain(HOP_BOUND+1) + assert LabelGuard(beyond, active=True).effective_row("memory", beyond.rows[0])["unverified"] is True From 75a57128b3e0c12340081ddd0c89cf565add2104 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:15:26 +0200 Subject: [PATCH 033/270] test: deny locked admin reads with ambiguous source aliases --- tests/unit/test_label_guard_exact_doors.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/test_label_guard_exact_doors.py b/tests/unit/test_label_guard_exact_doors.py index 2aa23b2a2..bc255ceab 100644 --- a/tests/unit/test_label_guard_exact_doors.py +++ b/tests/unit/test_label_guard_exact_doors.py @@ -302,3 +302,15 @@ def read_label_rows(self, kind, ids): assert guard.effective_row("memory", within.rows[0])["unverified"] is False beyond = Chain(HOP_BOUND+1) assert LabelGuard(beyond, active=True).effective_row("memory", beyond.rows[0])["unverified"] is True + + +def test_ambiguous_source_alias_blocks_locked_admin() -> None: + store = _LabelStore() + store.source["metadata_json"] = {"project_scope": ["prj_" + "b" * 16]} + twin = {**store.source, "id": "{" + SOURCE_ID + "}", "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA]}} + store.read_label_rows = lambda kind, ids: [store.source, twin] if kind == "source" else [] + effective = LabelGuard(store, active=True).effective_row("artifact", store.artifact) + assert effective["unverified"] is True + identity = AgentIdentity(agent_id="locked-admin", permission_profile="admin", project_scope=(ALPHA,), project_scope_locked=True) + with pytest.raises(AgentPolicyBlockedError): + _vnext_authorized_artifact(store=store, identity=identity, artifact_id=ARTIFACT_ID, action="artifact.read", for_update=False) From 7deb071b6dc136824a5e3b71e207105d1f9e3bbb Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:11:20 +0200 Subject: [PATCH 034/270] fix: enforce locked producer scope against effective ancestry labels --- apps/api/src/alicebot_api/vnext_brain.py | 12 +++-- .../api/src/alicebot_api/vnext_connections.py | 2 + .../src/alicebot_api/vnext_consolidation.py | 6 +++ .../src/alicebot_api/vnext_contradictions.py | 4 +- .../api/src/alicebot_api/vnext_label_guard.py | 27 +++++++---- apps/api/src/alicebot_api/vnext_projects.py | 2 + apps/api/src/alicebot_api/vnext_rollups.py | 19 +++++--- apps/api/src/alicebot_api/vnext_scheduler.py | 2 + tests/unit/test_derived_label_input_filter.py | 48 +++++++++++++++++++ 9 files changed, 101 insertions(+), 21 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_brain.py b/apps/api/src/alicebot_api/vnext_brain.py index 67f806d22..9c6063c88 100644 --- a/apps/api/src/alicebot_api/vnext_brain.py +++ b/apps/api/src/alicebot_api/vnext_brain.py @@ -986,16 +986,20 @@ def _load_inputs( from alicebot_api.vnext_label_guard import admit_loaded sources = admit_loaded( - self.store, kind="source", rows=sources, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects + self.store, kind="source", rows=sources, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects, + all_of=all_of, ) memories = admit_loaded( - self.store, kind="memory", rows=memories, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects + self.store, kind="memory", rows=memories, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects, + all_of=all_of, ) open_loops = admit_loaded( - self.store, kind="open_loop", rows=open_loops, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects + self.store, kind="open_loop", rows=open_loops, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects, + all_of=all_of, ) artifacts = admit_loaded( - self.store, kind="artifact", rows=artifacts, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects + self.store, kind="artifact", rows=artifacts, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects, + all_of=all_of, ) return sources, memories, open_loops, artifacts diff --git a/apps/api/src/alicebot_api/vnext_connections.py b/apps/api/src/alicebot_api/vnext_connections.py index b13d03cb9..fe51a6ffa 100644 --- a/apps/api/src/alicebot_api/vnext_connections.py +++ b/apps/api/src/alicebot_api/vnext_connections.py @@ -453,6 +453,7 @@ def generate_connection_report(self, request: ConnectionFinderRequest | None = N domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects, + all_of=all_of, ) memories = admit_loaded( self.store, @@ -461,6 +462,7 @@ def generate_connection_report(self, request: ConnectionFinderRequest | None = N domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects, + all_of=all_of, ) candidates = _find_candidates( sources=sources, diff --git a/apps/api/src/alicebot_api/vnext_consolidation.py b/apps/api/src/alicebot_api/vnext_consolidation.py index 68d3253bc..b9a9d5f5d 100644 --- a/apps/api/src/alicebot_api/vnext_consolidation.py +++ b/apps/api/src/alicebot_api/vnext_consolidation.py @@ -774,6 +774,7 @@ def _count(status: str) -> int: domains=domains, sensitivity_allowed=sensitivity, projects=projects, + all_of=all_of, ) outcome.corpus_digest = _digest_payload( { @@ -1196,6 +1197,11 @@ def generate_memory_consolidation(self, request: MemoryConsolidationRequest | No events = admit_when_locked("memory", events, all_of) ratings = admit_when_locked("memory", ratings, all_of) artifacts = admit_when_locked("artifact", artifacts, all_of) + from alicebot_api.vnext_label_guard import admit_loaded + artifacts = admit_loaded( + self.store, kind="artifact", rows=artifacts, domains=domains, + sensitivity_allowed=sensitivity, projects=projects, all_of=all_of, + ) clustering = self._cluster_memories( domains=domains, diff --git a/apps/api/src/alicebot_api/vnext_contradictions.py b/apps/api/src/alicebot_api/vnext_contradictions.py index da79ca6aa..a5a4cc620 100644 --- a/apps/api/src/alicebot_api/vnext_contradictions.py +++ b/apps/api/src/alicebot_api/vnext_contradictions.py @@ -491,6 +491,7 @@ def generate_contradiction_report(self, request: ContradictionFinderRequest | No domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects, + all_of=all_of, ) memories = admit_loaded( self.store, @@ -499,9 +500,10 @@ def generate_contradiction_report(self, request: ContradictionFinderRequest | No domains=domains, sensitivity_allowed=sensitivity_allowed, projects=request.projects, + all_of=all_of, ) beliefs = LabelGuard.for_filters( - self.store, domains, sensitivity_allowed, request.projects + self.store, domains, sensitivity_allowed, request.projects, all_of=all_of ).admit_beliefs(beliefs) candidates = _find_candidates( new_items=[*sources, *memories], diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index 996fbc2f4..4e2d4d25f 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -9,7 +9,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, replace -from typing import Any +from typing import Any, TypeVar from alicebot_api.vnext_agent_control import ( ALL_SENSITIVITY, @@ -22,6 +22,7 @@ NODE_BOUND, canon_kind, is_derived, + input_admitted, settle_labels, ) from alicebot_api.vnext_label_closure import collect_label_rows @@ -29,6 +30,7 @@ _GUARD_USER = "label-guard" +_Row = TypeVar("_Row", bound=Mapping[str, object]) def _filters_admit_every( @@ -52,6 +54,7 @@ class LabelGuard: domains: tuple[str, ...] = () sensitivity_allowed: tuple[str, ...] = () projects: tuple[str, ...] = () + all_of: tuple[str, ...] | None = None _nodes: dict[tuple[str, str], list[dict[str, object]]] | None = None @classmethod @@ -69,6 +72,8 @@ def for_filters( sensitivity_allowed: Sequence[str] | None, projects: Sequence[str] | None = (), exclude_global_domains: Sequence[str] | None = None, + *, + all_of: tuple[str, ...] | None = None, ) -> LabelGuard: """List doors. Inactive when the filters admit every label.""" @@ -78,10 +83,11 @@ def for_filters( project_list = tuple(projects or ()) return cls( store=store, - active=not _filters_admit_every(domain_list, sensitivity_list, project_list), + active=all_of is not None or not _filters_admit_every(domain_list, sensitivity_list, project_list), domains=domain_list, sensitivity_allowed=sensitivity_list, projects=project_list, + all_of=all_of, ) def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[str, object] | None: @@ -112,12 +118,12 @@ def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[ copy["metadata_json"] = metadata return copy - def admit_rows(self, kind: str, rows: Sequence[Mapping[str, object]]) -> list[Mapping[str, object]]: + def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: """Rows whose effective labels pass this guard's filters. Originals of the rows, not copies.""" if not self.active: return [row for row in rows if isinstance(row, Mapping)] - kept: list[Mapping[str, object]] = [] + kept: list[_Row] = [] for row in rows: if not isinstance(row, Mapping): continue @@ -126,7 +132,7 @@ def admit_rows(self, kind: str, rows: Sequence[Mapping[str, object]]) -> list[Ma kept.append(row) return kept - def admit_beliefs(self, beliefs: Sequence[Mapping[str, object]]) -> list[Mapping[str, object]]: + def admit_beliefs(self, beliefs: Sequence[_Row]) -> list[_Row]: """Beliefs whose backing memory the filters admit. One batched read.""" if not self.active: @@ -134,7 +140,7 @@ def admit_beliefs(self, beliefs: Sequence[Mapping[str, object]]) -> list[Mapping ids = [str(row.get("memory_id")) for row in beliefs if isinstance(row, Mapping) and row.get("memory_id")] reader = getattr(self.store, "read_label_rows", None) if not callable(reader): - return [row for row in beliefs if isinstance(row, Mapping)] + return [] if self.all_of is not None else [row for row in beliefs if isinstance(row, Mapping)] found: dict[str, Mapping[str, object]] = {} if callable(reader) and ids: for row in reader("memory", ids): @@ -148,6 +154,8 @@ def admit_beliefs(self, beliefs: Sequence[Mapping[str, object]]) -> list[Mapping ] def _admits_effective(self, row: Mapping[str, object], *, kind: str) -> bool: + if self.all_of is not None and (row.get("unverified") or not input_admitted(kind, row, self.all_of)): + return False domain = str(row.get("domain") or "unknown") if self.domains and domain not in self.domains and domain != "unknown": return False @@ -177,14 +185,15 @@ def admit_loaded( store: Any, *, kind: str, - rows: Sequence[Mapping[str, object]], + rows: Sequence[_Row], domains: Sequence[str] | None, sensitivity_allowed: Sequence[str] | None, projects: Sequence[str] | None = (), -) -> list[Mapping[str, object]]: + all_of: tuple[str, ...] | None = None, +) -> list[_Row]: """Drop loaded inputs whose effective labels miss the request filters.""" - guard = LabelGuard.for_filters(store, domains, sensitivity_allowed, projects) + guard = LabelGuard.for_filters(store, domains, sensitivity_allowed, projects, all_of=all_of) return guard.admit_rows(kind, rows) diff --git a/apps/api/src/alicebot_api/vnext_projects.py b/apps/api/src/alicebot_api/vnext_projects.py index 39ef87be6..aa83096e0 100644 --- a/apps/api/src/alicebot_api/vnext_projects.py +++ b/apps/api/src/alicebot_api/vnext_projects.py @@ -576,6 +576,7 @@ def generate_project_update_candidate(self, request: ProjectAutomationRequest | domains=domains, sensitivity_allowed=sensitivity_allowed, projects=(project_id,), + all_of=locked, ) memories = admit_loaded( self.store, @@ -584,6 +585,7 @@ def generate_project_update_candidate(self, request: ProjectAutomationRequest | domains=domains, sensitivity_allowed=sensitivity_allowed, projects=(project_id,), + all_of=locked, ) brain_charter = _brain_charter(self.store) automation_digest = _project_automation_digest( diff --git a/apps/api/src/alicebot_api/vnext_rollups.py b/apps/api/src/alicebot_api/vnext_rollups.py index 829c5c359..e834daa2e 100644 --- a/apps/api/src/alicebot_api/vnext_rollups.py +++ b/apps/api/src/alicebot_api/vnext_rollups.py @@ -1834,6 +1834,7 @@ def _collect_rows( domains=domains, sensitivity_allowed=sensitivity_allowed, projects=projects, + all_of=all_of, ) rows = [row for row in rows if not _is_rollup_card(row)] # The same parity for validity: the bundled stores leave an expired @@ -2470,13 +2471,17 @@ def _existing_rollup_state( raise VNextRollupValidationError( "roll-up candidate/card lookup returned rows outside the requested project scope" ) - if all_of is not None: - pending = { - key: row for key, row in pending.items() if admit_when_locked("memory", [row], all_of) - } - accepted = { - key: row for key, row in accepted.items() if admit_when_locked("memory", [row], all_of) - } + from alicebot_api.vnext_label_guard import admit_loaded + admitted_pending = {str(row.get("id")) for row in admit_loaded( + self.store, kind="memory", rows=list(pending.values()), domains=domains, + sensitivity_allowed=sensitivity_allowed, projects=projects, all_of=all_of, + )} + admitted_accepted = {str(row.get("id")) for row in admit_loaded( + self.store, kind="memory", rows=list(accepted.values()), domains=domains, + sensitivity_allowed=sensitivity_allowed, projects=projects, all_of=all_of, + )} + pending = {key: row for key, row in pending.items() if str(row.get("id")) in admitted_pending} + accepted = {key: row for key, row in accepted.items() if str(row.get("id")) in admitted_accepted} return pending, accepted def _expired_card_for_digest( diff --git a/apps/api/src/alicebot_api/vnext_scheduler.py b/apps/api/src/alicebot_api/vnext_scheduler.py index 9eb34e452..ff5226724 100644 --- a/apps/api/src/alicebot_api/vnext_scheduler.py +++ b/apps/api/src/alicebot_api/vnext_scheduler.py @@ -1417,6 +1417,7 @@ def _run_staleness_sweep(self, request: SchedulerRunRequest, *, metadata: JsonOb domains=list(request.domains) if request.domains else None, sensitivity_allowed=list(request.sensitivity_allowed), projects=projects, + all_of=bound, ) for memory in memories: if len(expired_marked) + len(unconfirmed_marked) >= mark_limit: @@ -1598,6 +1599,7 @@ def _generate_open_loop_review_artifact(self, request: SchedulerRunRequest, *, m domains=domains, sensitivity_allowed=list(request.sensitivity_allowed), projects=projects, + all_of=bound, ) # The report copies the id of each loop's source into its text and its ``source_refs``, and a later reader of # the artifact is shown them, so a source the run's own identity may not read is left out. diff --git a/tests/unit/test_derived_label_input_filter.py b/tests/unit/test_derived_label_input_filter.py index 62d314c2a..1eab25c73 100644 --- a/tests/unit/test_derived_label_input_filter.py +++ b/tests/unit/test_derived_label_input_filter.py @@ -61,3 +61,51 @@ def test_stamp_derived_from_counts_match_the_lists() -> None: assert record["sources"] == ["s"] assert record["counts"]["sources"] == 1 assert record["counts"]["memories"] == 1 + + +import pytest +from uuid import UUID +from alicebot_api.vnext_agent_control import AgentIdentity, AgentPolicyBlockedError +from alicebot_api.routers._vnext_shared import _vnext_authorized_artifact + +def provenance(sources=(), memories=()): + refs = {'sources': list(sources), 'memories': list(memories), 'open_loops': [], 'artifacts': [], 'beliefs': []} + return {'v': 1, **refs, 'counts': {k: len(v) for k, v in refs.items()}} + + +@pytest.mark.parametrize('source_project', [ALPHA, BETA]) +def test_locked_weekly_producer_rejects_effective_out_of_scope_artifact(source_project): + from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService + from tests.unit.test_vnext_brain import InMemoryVNextBrainStore + from alicebot_api.vnext_label_guard import LabelGuard + store = InMemoryVNextBrainStore() + source = {'id': str(UUID(int=500)), 'domain': 'project', 'sensitivity': 'public', 'metadata_json': {'project_scope': [source_project]}} + artifact = {'id': str(UUID(int=501)), 'artifact_type': 'connection_report', 'title': 'SENTINEL BETA PRIVATE PROJECT', 'domain': 'project', 'sensitivity': 'public', 'created_at': '2026-05-10T09:00:00Z', 'content_markdown': 'Synthetic restricted project title', 'metadata_json': {'workflow': 'connections', 'project_scope': [ALPHA], 'derived_from': provenance(sources=[source['id']])}} + store.artifacts[artifact['id']] = artifact + store.get_artifact = lambda artifact_id: store.artifacts.get(artifact_id) + store.read_label_rows = lambda kind, ids: [source] if kind == 'source' and source['id'] in ids else [] + # Exact read denies the same input for the same locked identity. + identity = AgentIdentity(agent_id='alpha-key', permission_profile='trusted_local_agent', project_scope=(ALPHA,), project_scope_locked=True) + store.upsert_agent_identity = lambda *args, **kwargs: None + if source_project == BETA: + with pytest.raises(AgentPolicyBlockedError): + _vnext_authorized_artifact(store=store, identity=identity, artifact_id=artifact['id'], action='artifact.read', for_update=False) + else: + _vnext_authorized_artifact(store=store, identity=identity, artifact_id=artifact['id'], action='artifact.read', for_update=False) + effective = LabelGuard(store=store, active=True).effective_row('artifact', artifact) + assert source_project in effective['metadata_json']['project_floor'] + generated = VNextBrainService(store).generate_weekly_synthesis(BrainArtifactRequest(generated_for='2026-05-10', domains=('project',), projects=(ALPHA,), agent_identity={'agent_id': 'alpha-key', 'permission_profile': 'trusted_local_agent', 'project_scope': [ALPHA], 'project_scope_locked': True}, discover_open_loops=False, create_candidate_memories=False)) + assert ('SENTINEL BETA PRIVATE PROJECT' in generated['content_markdown']) == (source_project == ALPHA) + + +def test_effective_all_of_keeps_ordinary_overlap_and_returns_originals() -> None: + from alicebot_api.vnext_label_guard import admit_loaded + class Store: + def read_label_rows(self, kind, ids): + return [{"id": "s", "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [BETA]}}] if kind == "source" else [] + row = {"id": "a", "domain": "project", "sensitivity": "public", "metadata_json": {"workflow": "daily_brief", "project_scope": [ALPHA], "derived_from": provenance(sources=["s"])}} + kwargs = dict(kind="artifact", rows=[row], domains=("project",), sensitivity_allowed=("public",), projects=(ALPHA,)) + assert admit_loaded(Store(), **kwargs) == [row] + assert admit_loaded(Store(), **kwargs, all_of=(ALPHA,)) == [] + kept = admit_loaded(Store(), **kwargs, all_of=(ALPHA, BETA)) + assert kept[0] is row From b319e9cbf2b2f09818aded3278fb6d6dc85f9953 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:17:03 +0200 Subject: [PATCH 035/270] test: give producer fake stores narrow ancestry readers --- tests/unit/test_vnext_consolidation.py | 4 ++++ tests/unit/test_vnext_rollups.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/tests/unit/test_vnext_consolidation.py b/tests/unit/test_vnext_consolidation.py index b47914b27..56a15615f 100644 --- a/tests/unit/test_vnext_consolidation.py +++ b/tests/unit/test_vnext_consolidation.py @@ -41,6 +41,10 @@ def __init__(self) -> None: self.list_memory_calls: list[dict[str, object]] = [] self._clock = datetime(2026, 7, 1, tzinfo=UTC) + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict]: + rows = self.memories if kind == "memory" else self.artifacts if kind == "artifact" else [] + return [dict(row) for row in rows if str(row.get("id")) in ids] + def _next_timestamp(self) -> str: self._clock += timedelta(minutes=1) return self._clock.isoformat() diff --git a/tests/unit/test_vnext_rollups.py b/tests/unit/test_vnext_rollups.py index b65650ba4..2fc3fe5d9 100644 --- a/tests/unit/test_vnext_rollups.py +++ b/tests/unit/test_vnext_rollups.py @@ -48,6 +48,9 @@ def __init__(self) -> None: self._counter = 0 self.rollup_read_calls: list[tuple[str, JsonObject]] = [] + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict]: + return [dict(row) for row in self.memories if str(row.get("id")) in ids] if kind == "memory" else [] + def create_memory(self, memory: JsonObject, *, actor_type: str = "system") -> JsonObject: self._counter += 1 row = { From 2250261104d61884a699eba3b97eb1fcd41cf28c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:19:43 +0200 Subject: [PATCH 036/270] fix: retain identity ambiguity through weekly ancestry backfill --- apps/api/src/alicebot_api/vnext_derived_labels.py | 10 +++++----- tests/unit/test_derived_labels_kernel.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 875db8f40..d368dea30 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -977,11 +977,6 @@ def settle_labels( own: dict[tuple[str, str, str], set[tuple[str, str, str]]] = {} problems: dict[tuple[str, str, str], str] = {} - stored_ids: dict[tuple[str, str, str], str] = {} - for label, _row in prepared: - previous_id = stored_ids.setdefault(label.key, label.stored_id) - if previous_id != label.stored_id: - problems[label.key] = "ambiguous_identity" for label, row in prepared: if not label.derived: continue @@ -995,6 +990,11 @@ def settle_labels( keyed = {(kind, label.user_id, row_id) for kind, row_id in deps} own[label.key] = keyed _weekly_parent_deps(labels, own, prepared) + stored_ids: dict[tuple[str, str, str], str] = {} + for label, _row in prepared: + previous_id = stored_ids.setdefault(label.key, label.stored_id) + if previous_id != label.stored_id: + problems[label.key] = "ambiguous_identity" for key, reason in list(problems.items()): if reason == "no_record" and own.get(key): del problems[key] diff --git a/tests/unit/test_derived_labels_kernel.py b/tests/unit/test_derived_labels_kernel.py index ecc3fbcf6..a56de91f1 100644 --- a/tests/unit/test_derived_labels_kernel.py +++ b/tests/unit/test_derived_labels_kernel.py @@ -835,3 +835,14 @@ def test_distinct_stored_aliases_are_unverified_but_single_alias_and_other_kind_ result = settle_labels([source, other_kind, report]).by_stored("artifact", "ambiguous") assert result.unverified is False assert result.domain == "health" + + +def test_weekly_parent_backfill_cannot_clear_ambiguous_identity() -> None: + canonical = str(UUID(SOURCE_UUID)) + one = _memory(canonical, metadata_json={"discovered_by": "vnext_weekly_synthesis"}) + two = _memory("{" + canonical + "}", metadata_json={"discovered_by": "vnext_weekly_synthesis"}) + parent = _brief("parent", sources=["s"]) + _meta(parent, candidate_memory_ids=[canonical]) + settled = settle_labels([_source("s"), one, two, parent]) + assert settled.by_stored("memory", canonical).unverified is True + assert settled.by_stored("memory", "{" + canonical + "}").unverified is True From ec25db5d8d9505c01dad5086aec69ac85a49aed2 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 19:27:33 +0000 Subject: [PATCH 037/270] Add the derived-label migration, commands, and doctor warning. PostgreSQL migration 20261005_0096 raises stored derived labels inside the same row-security bracket as the domain repair, with projects included. The owner commands and the doctors report rows that are still below their inputs. --- CHANGELOG.md | 3 +- .../20261005_0096_derived_label_floor.py | 101 +++++++++++ apps/api/src/alicebot_api/cli/labels.py | 123 ++++++++++++++ apps/api/src/alicebot_api/cli/parser.py | 8 + apps/api/src/alicebot_api/label_commands.py | 76 +++++++++ apps/api/src/alicebot_api/onramp.py | 31 ++++ apps/api/src/alicebot_api/vault_doctor.py | 5 + apps/api/src/alicebot_api/vnext_doctor.py | 14 ++ .../src/alicebot_api/vnext_label_repair.py | 159 +++++++++++++++++- scripts/_phase5_ops_seed.py | 69 ++++++++ scripts/run_phase5_ops_evidence.py | 22 +++ tests/integration/test_migrations.py | 2 +- tests/unit/test_label_repair_migration.py | 47 ++++++ tests/unit/test_phase5_ops_evidence.py | 1 + tests/unit/test_session_doctor.py | 1 + tests/unit/test_sqlite_label_repair_v3.py | 11 ++ 16 files changed, 668 insertions(+), 5 deletions(-) create mode 100644 apps/api/alembic/versions/20261005_0096_derived_label_floor.py create mode 100644 apps/api/src/alicebot_api/cli/labels.py create mode 100644 apps/api/src/alicebot_api/label_commands.py create mode 100644 tests/unit/test_label_repair_migration.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b1531baa9..f4037cc5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## Unreleased -- Unreleased (on main, not in v0.20.0): opening a SQLite vault raises a derived row whose stored label is below its inputs, and a restore does the same before it publishes. A repair that cannot finish leaves the vault open and the completion key unstamped. v0.20.0 left the stored label where it was written. No new migration in this change. +- Unreleased (on main, not in v0.20.0): PostgreSQL migration `20261005_0096` raises a stored derived row to the labels of its inputs, and `alicebot vnext labels` and `alice-memory labels` check and repair the same rows. The doctors print how many derived rows are below their inputs or unverified, as a warning. v0.20.0 left the stored label where it was written. +- Unreleased (on main, not in v0.20.0): opening a SQLite vault raises a derived row whose stored label is below its inputs, and a restore does the same before it publishes. A repair that cannot finish leaves the vault open and the completion key unstamped. v0.20.0 left the stored label where it was written. - Unreleased (on main, not in v0.20.0): recall, context packs, resume, recent decisions, the session brief, the review queue, open-loop lists, and the workspace, dashboard, context tree, and dogfooding lists drop a loaded row whose effective label is outside the request. The artifact list, the source trace, the project list, the dashboard project row, and the belief state route apply the caller's sensitivity ceiling, including titles, ids, and counts. Resume, recent decisions, the session brief, audit, and explain apply the target row's current effective label, so an excluded key does not learn of a label change. v0.20.0 returned those rows from the label stored on them. No migration is required. - Unreleased (on main, not in v0.20.0): a daily brief, connection report, contradiction report, consolidation, roll-up, project update, staleness sweep, and open-loop review drop an input whose effective label is outside the request. v0.20.0 kept a public copy of a confidential input in the report text. No migration is required. - Unreleased (on main, not in v0.20.0): an exact read of a derived artifact, a write that cites a derived memory, explain, memory review, redaction, and an open-loop update use the labels of that row's inputs. v0.20.0 used the label stored on the row, so a public copy of a confidential input could be read. A row whose inputs cannot be checked is refused to a locked key. The owner and an unbound admin still read the stored row. No migration is required. diff --git a/apps/api/alembic/versions/20261005_0096_derived_label_floor.py b/apps/api/alembic/versions/20261005_0096_derived_label_floor.py new file mode 100644 index 000000000..94e52e144 --- /dev/null +++ b/apps/api/alembic/versions/20261005_0096_derived_label_floor.py @@ -0,0 +1,101 @@ +"""Raise stored derived labels to the labels of their inputs. + +Revision ID: 20261005_0096 +Revises: 20261004_0095 +""" + +from __future__ import annotations + +import json + +from alembic import op +from sqlalchemy import text + +from alicebot_api.vnext_derived_domain_backfill import require_changed +from alicebot_api.vnext_derived_labels import labels_raised_payload +from alicebot_api.vnext_event_log import build_event_log_record +from alicebot_api.vnext_label_repair import INPUT_SELECTS_V3, plan_label_repairs + +revision = "20261005_0096" +down_revision = "20261004_0095" +branch_labels = None +depends_on = None + +_RELAX_RLS = ( + "ALTER TABLE sources NO FORCE ROW LEVEL SECURITY", + "ALTER TABLE memories NO FORCE ROW LEVEL SECURITY", + "ALTER TABLE open_loops NO FORCE ROW LEVEL SECURITY", + "ALTER TABLE generated_artifacts NO FORCE ROW LEVEL SECURITY", + "ALTER TABLE beliefs NO FORCE ROW LEVEL SECURITY", + "ALTER TABLE event_log NO FORCE ROW LEVEL SECURITY", + "ALTER TABLE projects NO FORCE ROW LEVEL SECURITY", +) +_RESTORE_RLS = tuple(statement.replace(" NO FORCE ", " FORCE ") for statement in _RELAX_RLS) +_TARGET = { + "memories": "memory", + "open_loops": "open_loop", + "generated_artifacts": "artifact", + "projects": "project", +} + + +def upgrade() -> None: + connection = op.get_bind() + for statement in _RELAX_RLS: + op.execute(statement) + tables = {} + for table, statement in INPUT_SELECTS_V3.items(): + tables[table] = list(connection.execute(text(statement)).mappings()) + for table, user, row_id, previous, new, node in plan_label_repairs(tables): + metadata = dict(node.get("metadata_json") or {}) + metadata["project_scope"] = list(new["project_scope"]) + metadata["project_floor"] = list(new["project_floor"]) + require_changed( + connection.execute( + text( + f"UPDATE {table} SET domain = :domain, sensitivity = :sensitivity, " + "metadata_json = CAST(:metadata AS jsonb) " + "WHERE user_id = CAST(:user AS uuid) AND id = CAST(:id AS uuid)" + ), + { + "domain": new["domain"], + "sensitivity": new["sensitivity"], + "metadata": json.dumps(metadata), + "user": user, + "id": row_id, + }, + ).rowcount, + table, + row_id, + ) + target = _TARGET[table] + event = build_event_log_record( + event_type=f"{target}.labels_raised", + actor_type="system", + target_type=target, + target_id=row_id, + payload=labels_raised_payload(cause="repair_v3", previous=previous, new=new), + ) + event["user_id"] = user + connection.execute( + text( + """ + INSERT INTO event_log ( + id, user_id, event_type, actor_type, target_type, target_id, + occurred_at, payload_json, integrity_hash + ) VALUES ( + CAST(:id AS uuid), CAST(:user_id AS uuid), :event_type, :actor_type, + :target_type, :target_id, CAST(:occurred_at AS timestamptz), + CAST(:payload_json AS jsonb), :integrity_hash + ) + """ + ), + {**event, "payload_json": json.dumps(event["payload_json"])}, + ) + for statement in _RESTORE_RLS: + op.execute(statement) + + +def downgrade() -> None: + # Raised labels stay. An older binary can read the repaired rows. + pass diff --git a/apps/api/src/alicebot_api/cli/labels.py b/apps/api/src/alicebot_api/cli/labels.py new file mode 100644 index 000000000..7f98472b5 --- /dev/null +++ b/apps/api/src/alicebot_api/cli/labels.py @@ -0,0 +1,123 @@ +"""Owner commands that check and repair stored derived labels on Postgres.""" + +from __future__ import annotations + +from alicebot_api.cli.shared import CLIContext, _vnext_store_context +from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError +from alicebot_api.vnext_label_repair import classify_stored_labels, format_label_check, plan_label_repairs +from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + +def _postgres_tables(conn) -> dict[str, list[dict[str, object]]]: + from alicebot_api.vnext_label_repair import INPUT_SELECTS_V3 + + tables: dict[str, list[dict[str, object]]] = {} + for table, statement in INPUT_SELECTS_V3.items(): + cursor = conn.execute(statement) + names = [column[0] for column in cursor.description] + tables[table] = [row if isinstance(row, dict) else dict(zip(names, row)) for row in cursor.fetchall()] + return tables + + +def _run_vnext_labels_check(ctx: CLIContext, args: object) -> str: + del args + try: + with _vnext_store_context(ctx) as store: + conn = store.conn + if not conn.in_transaction: + conn.execute("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + below, unverified = classify_stored_labels(_postgres_tables(conn)) + except DerivedDomainRepairError as exc: + print(f"labels check failed: {exc}") + raise SystemExit(1) from exc + text = format_label_check(below, unverified) + print(text) + if below or unverified: + raise SystemExit(1) + return text + + +def _run_vnext_labels_repair(ctx: CLIContext, args: object) -> str: + del args + try: + with _vnext_store_context(ctx) as store: + if hasattr(store, "lock_graph_mutation"): + store.lock_graph_mutation() + acquire_exclusive_label_lock(store) + changes = plan_label_repairs(_postgres_tables(store.conn)) + # The SQLite writer is not used here. Postgres repair applies the + # same plan through label-only updates inside this transaction. + applied = 0 + for table, user, row_id, previous, new, node in changes: + import json + + from alicebot_api.vnext_derived_labels import labels_raised_payload + from alicebot_api.vnext_event_log import build_event_log_record + + metadata = dict(node.get("metadata_json") or {}) + metadata["project_scope"] = list(new["project_scope"]) + metadata["project_floor"] = list(new["project_floor"]) + cursor = store.conn.execute( + f"UPDATE {table} SET domain = %s, sensitivity = %s, metadata_json = %s::jsonb " + "WHERE user_id = %s::uuid AND id = %s::uuid " + "AND domain = %s AND sensitivity = %s", + ( + new["domain"], + new["sensitivity"], + json.dumps(metadata), + user, + row_id, + previous["domain"], + previous["sensitivity"], + ), + ) + if cursor.rowcount != 1: + continue + target = { + "memories": "memory", + "open_loops": "open_loop", + "generated_artifacts": "artifact", + "projects": "project", + }[table] + event = build_event_log_record( + event_type=f"{target}.labels_raised", + actor_type="system", + target_type=target, + target_id=row_id, + payload=labels_raised_payload(cause="repair_v3", previous=previous, new=new), + ) + store.conn.execute( + """ + INSERT INTO event_log ( + id, user_id, event_type, actor_type, target_type, target_id, + occurred_at, payload_json, integrity_hash + ) VALUES ( + %s::uuid, %s::uuid, %s, %s, %s, %s, %s::timestamptz, %s::jsonb, %s + ) + """, + ( + event["id"], + user, + event["event_type"], + event["actor_type"], + event["target_type"], + row_id, + event["occurred_at"], + json.dumps(event["payload_json"]), + event["integrity_hash"], + ), + ) + applied += 1 + except DerivedDomainRepairError as exc: + print(f"labels repair failed: {exc}") + raise SystemExit(2) from exc + except Exception as exc: + message = str(exc).lower() + if "lock" in message and "timeout" in message: + print("labels repair waited for the label lock and changed nothing") + raise SystemExit(3) from exc + raise + return f"labels repair updated {applied}" + + +__all__ = ["_run_vnext_labels_check", "_run_vnext_labels_repair"] diff --git a/apps/api/src/alicebot_api/cli/parser.py b/apps/api/src/alicebot_api/cli/parser.py index 81bd84217..36724e83d 100644 --- a/apps/api/src/alicebot_api/cli/parser.py +++ b/apps/api/src/alicebot_api/cli/parser.py @@ -162,6 +162,7 @@ _run_vnext_scheduler_runs, _run_vnext_scheduler_status, ) +from .labels import _run_vnext_labels_check, _run_vnext_labels_repair from .shared import _run_capture from .smokes import ( _run_vnext_alpha_check, @@ -1149,6 +1150,13 @@ def build_parser() -> argparse.ArgumentParser: ) vnext_memory_backfill_parser.set_defaults(handler=_run_vnext_memories_backfill_embeddings) + vnext_labels_parser = vnext_subparsers.add_parser("labels", help="Check and repair stored derived labels.") + vnext_labels_commands = vnext_labels_parser.add_subparsers(dest="labels_command", required=True) + vnext_labels_check = vnext_labels_commands.add_parser("check", help="Report derived rows below their inputs.") + vnext_labels_check.set_defaults(handler=_run_vnext_labels_check) + vnext_labels_repair = vnext_labels_commands.add_parser("repair", help="Raise stored derived labels.") + vnext_labels_repair.set_defaults(handler=_run_vnext_labels_repair) + vnext_agents_parser = vnext_subparsers.add_parser("agents", help="Submit and inspect vNext agent proposals.") vnext_agents_subparsers = vnext_agents_parser.add_subparsers(dest="vnext_agents_command", required=True) vnext_agent_propose_parser = vnext_agents_subparsers.add_parser( diff --git a/apps/api/src/alicebot_api/label_commands.py b/apps/api/src/alicebot_api/label_commands.py new file mode 100644 index 000000000..708cad473 --- /dev/null +++ b/apps/api/src/alicebot_api/label_commands.py @@ -0,0 +1,76 @@ +"""Owner commands that check and repair stored derived labels on SQLite.""" + +from __future__ import annotations + +import json +import sqlite3 + +from alicebot_api.sqlite_store import sqlite_user_connection +from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError +from alicebot_api.vnext_label_repair import ( + REPAIR_STATE_KEY, + classify_stored_labels, + format_label_check, + relabel_labels_sqlite, +) + + +def run_labels(args) -> int: + from alicebot_api.onramp import _prepared_export_connection, resolve_db_path + + db = resolve_db_path(data_dir=args.data_dir, db=args.db) + if not db.is_file(): + print(json.dumps({"error": {"code": "vault_not_found", "message": "No vault exists at the selected location"}})) + return 1 + if args.labels_command == "check": + try: + with _prepared_export_connection(db, args.user_id) as conn: + from alicebot_api.vnext_label_repair import _load_tables + + below, unverified = classify_stored_labels(_load_tables(conn)) + stamped = conn.execute( + "SELECT value FROM alice_schema_state WHERE key = ?", + (REPAIR_STATE_KEY,), + ).fetchone() + snapshot_events = { + str(row[0] if not isinstance(row, dict) else row["id"]) + for row in conn.execute( + "SELECT id FROM event_log WHERE event_type IN " + "('memory.labels_raised', 'open_loop.labels_raised')" + ) + } + except DerivedDomainRepairError as exc: + print(f"labels check failed: {exc}") + return 1 + print(format_label_check(below, unverified)) + if stamped is None: + print("derived_labels_v3 is not stamped on this snapshot") + live_events: set[str] = set() + try: + live = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + try: + live_events = { + str(row[0]) + for row in live.execute( + "SELECT id FROM event_log WHERE event_type IN " + "('memory.labels_raised', 'open_loop.labels_raised')" + ) + } + finally: + live.close() + except sqlite3.Error: + live_events = set() + raised_on_next_open = sorted(snapshot_events - live_events) + if raised_on_next_open: + print(f"next open would raise {len(raised_on_next_open)}") + for event_id in raised_on_next_open[:20]: + print(f" {event_id}") + return 1 if below or unverified or raised_on_next_open else 0 + try: + with sqlite_user_connection(db, args.user_id) as conn: + relabel_labels_sqlite(conn) + except DerivedDomainRepairError as exc: + print(f"labels repair failed: {exc}") + return 2 + print("labels repair finished") + return 0 diff --git a/apps/api/src/alicebot_api/onramp.py b/apps/api/src/alicebot_api/onramp.py index 7d80e6f94..6474ea1f3 100644 --- a/apps/api/src/alicebot_api/onramp.py +++ b/apps/api/src/alicebot_api/onramp.py @@ -200,6 +200,7 @@ "sleep-proposals", "install", "project", + "labels", ) _EXPORT_FORMAT = "alice-memory-jsonl" @@ -1249,6 +1250,13 @@ def build_parser() -> argparse.ArgumentParser: help="Write a Claude Desktop .mcpb zip that launches uvx alice-memory mcp.", ) + labels_parser = subparsers.add_parser("labels", help="Check and repair stored derived labels.") + labels_commands = labels_parser.add_subparsers(dest="labels_command", required=True) + labels_check = labels_commands.add_parser("check", help="Report derived rows below their inputs.") + _add_database_arguments(labels_check) + labels_repair = labels_commands.add_parser("repair", help="Raise stored derived labels.") + _add_database_arguments(labels_repair) + sources_parser = subparsers.add_parser("sources", help="List and remove SQLite source material as the owner.") sources_commands = sources_parser.add_subparsers(dest="sources_command", required=True) source_list = sources_commands.add_parser("list", help="List live sources, with optional replaced versions.") @@ -3729,9 +3737,11 @@ def _import_records( ``project_scoping.apply_imported_scoping``). """ from alicebot_api.vnext_derived_domain_backfill import recorded_sqlite_domain_repairs + from alicebot_api.vnext_label_repair import recorded_sqlite_label_repairs quarantine_plan = plan if plan is not None else _EMPTY_QUARANTINE_PLAN domain_repairs = recorded_sqlite_domain_repairs(conn, str(store.user_id)) + label_repairs = recorded_sqlite_label_repairs(conn, str(store.user_id)) counts: dict[str, dict[str, int]] = {} probe = _BackfillProbe() try: @@ -3775,6 +3785,24 @@ def _import_records( repaired = list(candidate) repaired[domain_index] = existing["domain"] candidates.append(tuple(repaired)) + if table in {"memories", "open_loops"}: + for candidate in tuple(candidates): + adjusted = list(candidate) + changed_dimension = False + for dimension, column in (("domain", "domain"), ("sensitivity", "sensitivity")): + if column not in columns: + continue + index = columns.index(column) + file_value = str(adjusted[index] if adjusted[index] is not None else "unknown") + stored_value = existing[column] + stored_text = str(stored_value if stored_value is not None else "unknown") + if file_value != stored_text and file_value in label_repairs.get( + (table, row_id, dimension), set() + ): + adjusted[index] = stored_value + changed_dimension = True + if changed_dimension: + candidates.append(tuple(adjusted)) if not _stored_row_matches( dict(existing), columns, @@ -4347,6 +4375,9 @@ def main(argv: list[str] | None = None) -> int: if args.command == "sources": from alicebot_api.source_commands import run_sources return run_sources(args) + if args.command == "labels": + from alicebot_api.label_commands import run_labels + return run_labels(args) if args.command == "import-markdown": return _run_import_markdown(args) if args.command == "import-chatgpt": diff --git a/apps/api/src/alicebot_api/vault_doctor.py b/apps/api/src/alicebot_api/vault_doctor.py index bbabe08cf..488bb0aba 100644 --- a/apps/api/src/alicebot_api/vault_doctor.py +++ b/apps/api/src/alicebot_api/vault_doctor.py @@ -111,6 +111,10 @@ def compile_local_vault_doctor( (uid, CANDIDATE_STATUS), ) missing_vector_line = _missing_vector_line(store) + from alicebot_api.vnext_label_repair import label_gap_counts + + below, unverified = label_gap_counts(store) + label_line = f"derived labels: {below} below their inputs, {unverified} unverified" flagged_ids = _flagged_source_ids(store) superseded_count = count_prunable_sources(store) try: @@ -145,6 +149,7 @@ def compile_local_vault_doctor( *(["Remove flagged sources with alice-memory sources delete ."] if flagged_ids else []), f"flagged sources: {len(flagged_ids)}", "flagged source ids: " + ", ".join(flagged_ids), + label_line, ) ) diff --git a/apps/api/src/alicebot_api/vnext_doctor.py b/apps/api/src/alicebot_api/vnext_doctor.py index 6df43e805..5ddd2543e 100644 --- a/apps/api/src/alicebot_api/vnext_doctor.py +++ b/apps/api/src/alicebot_api/vnext_doctor.py @@ -288,6 +288,20 @@ def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: }, ) + from alicebot_api.vnext_label_repair import label_gap_counts + + below, unverified = label_gap_counts(self.store) + label_line = f"derived labels: {below} below their inputs, {unverified} unverified" + self._check( + checks, + name="derived_labels", + ok=below == 0 and unverified == 0, + severity="warning", + message_ok=label_line, + message_fail=label_line, + recommended_fix="alicebot vnext labels repair", + ) + blocking = [check for check in checks if check.status == "fail" and check.severity == "blocking"] warnings = [check for check in checks if check.status == "fail" and check.severity == "warning"] payload = { diff --git a/apps/api/src/alicebot_api/vnext_label_repair.py b/apps/api/src/alicebot_api/vnext_label_repair.py index d463d209a..8e4b6b910 100644 --- a/apps/api/src/alicebot_api/vnext_label_repair.py +++ b/apps/api/src/alicebot_api/vnext_label_repair.py @@ -19,14 +19,25 @@ "sources": "source", "memories": "memory", "open_loops": "open_loop", + "generated_artifacts": "artifact", + "projects": "project", + "beliefs": "belief", } +_WRITABLE = frozenset({"memories", "open_loops", "generated_artifacts", "projects"}) INPUT_SELECTS_V3 = { - "sources": "SELECT id, user_id, domain, sensitivity, metadata_json FROM sources", - "memories": "SELECT id, user_id, domain, sensitivity, metadata_json, value, project_id FROM memories", + "sources": "SELECT id, user_id, domain, sensitivity, metadata_json, deleted_at FROM sources", + "memories": ( + "SELECT id, user_id, domain, sensitivity, metadata_json, value, project_id, deleted_at FROM memories" + ), "open_loops": ( "SELECT id, user_id, domain, sensitivity, metadata_json, project_id, source_id, memory_id " "FROM open_loops" ), + "generated_artifacts": ( + "SELECT id, user_id, domain, sensitivity, metadata_json, artifact_type FROM generated_artifacts" + ), + "projects": "SELECT id, user_id, domain, sensitivity, metadata_json FROM projects", + "beliefs": "SELECT id, user_id, memory_id FROM beliefs", } _UPDATES = { "memories": "UPDATE memories SET domain = ?, sensitivity = ?, metadata_json = ? WHERE user_id = ? AND id = ?", @@ -81,7 +92,7 @@ def plan_label_repairs( settled = settle_labels(nodes, on_cycle="raise") changes = [] for (table, node), label in zip(index, settled.rows, strict=True): - if not label.derived or label.unverified or table not in _UPDATES: + if not label.derived or label.unverified or table not in _WRITABLE: continue previous = { "domain": str(node.get("domain") or "unknown"), @@ -179,10 +190,152 @@ def relabel_labels_sqlite(conn, *, restoring: bool = False) -> None: raise +def classify_stored_labels( + tables: Mapping[str, Sequence[Mapping[str, object]]], +) -> tuple[list[tuple[str, str, str, dict[str, object], dict[str, object], dict[str, object]]], dict[str, list[str]]]: + """Rows below their inputs, and unverified ids grouped by reason. + + A cycle is reported as unverified instead of raising. ``labels check`` uses this. + """ + + nodes: list[dict[str, object]] = [] + index: list[tuple[str, dict[str, object]]] = [] + for table, rows in tables.items(): + kind = _TABLE_KIND.get(table, table) + for row in rows: + node = dict(row) + node["kind"] = kind + node["metadata_json"] = _json_object(row.get("metadata_json")) + if isinstance(row.get("value"), str): + node["value"] = _json_object(row.get("value")) + nodes.append(node) + index.append((table, node)) + settled = settle_labels(nodes, on_cycle="unverified") + below = [] + unverified: dict[str, list[str]] = {} + for (table, node), label in zip(index, settled.rows, strict=True): + if not label.derived: + continue + row_id = str(node.get("id") or "") + if label.unverified: + unverified.setdefault(label.reason or "unverified", []).append(row_id) + continue + if table not in _WRITABLE: + continue + previous = { + "domain": str(node.get("domain") or "unknown"), + "sensitivity": str(node.get("sensitivity") or "unknown"), + "project_scope": list(label.stored_scope), + "project_floor": list(label.stored_floor), + } + new = { + "domain": label.domain, + "sensitivity": label.sensitivity, + "project_scope": list(label.project_scope), + "project_floor": list(label.project_floor), + } + if not _same(previous, new): + below.append((table, str(node.get("user_id") or ""), row_id, previous, new, node)) + return below, unverified + + +def format_label_check( + below: Sequence[tuple], + unverified: Mapping[str, Sequence[str]], + *, + limit: int = 20, +) -> str: + """Counts and up to ``limit`` ids per cause. No row text.""" + + lines = [f"below_inputs {len(below)}"] + for _table, _user, row_id, _previous, _new, _node in list(below)[:limit]: + lines.append(f" {row_id}") + for reason in sorted(unverified): + ids = list(unverified[reason]) + lines.append(f"{reason} {len(ids)}") + for row_id in ids[:limit]: + lines.append(f" {row_id}") + return "\n".join(lines) + + +def label_gap_counts(store: object) -> tuple[int, int]: + """How many derived rows are below their inputs, and how many are unverified. + + A store that cannot be read returns zeros so a doctor does not fail closed. + """ + + conn = getattr(store, "conn", None) + module = type(conn).__module__ if conn is not None else "" + if conn is None or not (module.startswith("sqlite3") or module.startswith("psycopg")): + return (0, 0) + try: + tables = _load_tables(conn) + below, unverified = classify_stored_labels(tables) + except Exception: + return (0, 0) + unverified_count = sum(len(ids) for ids in unverified.values()) + return (len(below), unverified_count) + + +def recorded_sqlite_label_repairs(conn, user_id: str) -> dict[tuple[str, str, str], set[str]]: + """Previous label values recorded for each row and dimension. + + The key is (table, stored id, dimension). A value is explained only when an + event of this vault says the row held it before. + """ + + repairs: dict[tuple[str, str, str], set[str]] = {} + statements = ( + ( + "memories", + "SELECT target_id, payload_json FROM event_log WHERE user_id = ? AND event_type = 'memory.domain_relabelled'", + ), + ( + "memories", + "SELECT target_id, payload_json FROM event_log WHERE user_id = ? AND event_type = 'memory.labels_raised'", + ), + ( + "open_loops", + "SELECT target_id, payload_json FROM event_log WHERE user_id = ? AND event_type = 'open_loop.labels_raised'", + ), + ) + for table, statement in statements: + cursor = conn.execute(statement, (str(user_id),)) + names = [column[0] for column in cursor.description] + for raw in cursor.fetchall(): + row = raw if isinstance(raw, dict) else dict(zip(names, raw)) + payload = _json_object(row.get("payload_json")) + row_id = str(row.get("target_id") or "") + if not row_id: + continue + if "previous_domain" in payload: + repairs.setdefault((table, row_id, "domain"), set()).add(str(payload.get("previous_domain"))) + previous = payload.get("previous") + if not isinstance(previous, Mapping): + continue + repairs.setdefault((table, row_id, "domain"), set()).add(str(previous.get("domain") or "unknown")) + repairs.setdefault((table, row_id, "sensitivity"), set()).add( + str(previous.get("sensitivity") or "unknown") + ) + scope = previous.get("project_scope") + floor = previous.get("project_floor") + repairs.setdefault((table, row_id, "project_scope"), set()).add( + json.dumps(list(project_scope_identity(scope if isinstance(scope, list) else [])), sort_keys=True) + ) + repairs.setdefault((table, row_id, "project_floor"), set()).add( + json.dumps(list(project_scope_identity(floor if isinstance(floor, list) else [])), sort_keys=True) + ) + return repairs + + __all__ = [ "INPUT_SELECTS_V3", "REPAIR_STATE_KEY", "DerivedDomainRepairError", + "classify_stored_labels", + "format_label_check", + "label_gap_counts", "plan_label_repairs", + "recorded_sqlite_label_repairs", "relabel_labels_sqlite", ] diff --git a/scripts/_phase5_ops_seed.py b/scripts/_phase5_ops_seed.py index e762c4884..7aeb210b5 100755 --- a/scripts/_phase5_ops_seed.py +++ b/scripts/_phase5_ops_seed.py @@ -35,6 +35,17 @@ def _as_int(value: object) -> int: USER_ID = UUID("00000000-0000-0000-0000-000000005001") MEMORY_ID = "00000000-0000-0000-0000-000000005101" ARTIFACT_ID = "00000000-0000-0000-0000-000000005201" +CONFIDENTIAL_MEMORY_ID = "00000000-0000-0000-0000-000000005111" +DERIVED_BRIEF_ID = "00000000-0000-0000-0000-000000005112" +_EMPTY_DERIVED_FROM = { + "v": 1, + "sources": [], + "memories": [], + "open_loops": [], + "artifacts": [], + "beliefs": [], + "counts": {"sources": 0, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, +} OLDER_RATING_ID = "00000000-0000-0000-0000-000000005301" NEWER_RATING_ID = "00000000-0000-0000-0000-000000005302" REVIEWER_ID = "phase5-ops-reviewer" @@ -141,7 +152,65 @@ def _seed_postgres( "domain": "project", "sensitivity": "internal", "generated_by": "system", + "metadata_json": { + "evidence_fixture": "phase5_ops_v1", + "derived_from": _EMPTY_DERIVED_FROM, + }, + } + ) + store.create_memory( + { + "id": CONFIDENTIAL_MEMORY_ID, + "memory_key": "phase5.ops.confidential", + "value": {"text": "A confidential operations fact."}, + "status": "active", + "memory_type": "decision", + "confirmation_status": "confirmed", + "trust_class": "human_curated", + "title": "Confidential operations fact", + "canonical_text": "A confidential operations fact.", + "summary": "Stored above the brief that names it.", + "domain": "project", + "sensitivity": "confidential", "metadata_json": {"evidence_fixture": "phase5_ops_v1"}, + }, + actor_type="system", + ) + store.create_artifact( + { + "id": DERIVED_BRIEF_ID, + "artifact_type": "weekly_synthesis", + "title": "Brief over a confidential memory", + "content_markdown": "Names the confidential operations fact.", + "status": "draft", + "domain": "project", + "sensitivity": "public", + "generated_by": "system", + "metadata_json": { + "workflow": "weekly_synthesis", + "input_summary": { + "memory_ids": [CONFIDENTIAL_MEMORY_ID], + "source_ids": [], + "open_loop_ids": [], + "artifact_ids": [], + "counts": {"memories": 1, "sources": 0, "open_loops": 0, "artifacts": 0}, + }, + "derived_from": { + "v": 1, + "sources": [], + "memories": [CONFIDENTIAL_MEMORY_ID], + "open_loops": [], + "artifacts": [], + "beliefs": [], + "counts": { + "sources": 0, + "memories": 1, + "open_loops": 0, + "artifacts": 0, + "beliefs": 0, + }, + }, + }, } ) for rating_id, usefulness, created_at in ( diff --git a/scripts/run_phase5_ops_evidence.py b/scripts/run_phase5_ops_evidence.py index 0386e5341..bf0d9f01f 100755 --- a/scripts/run_phase5_ops_evidence.py +++ b/scripts/run_phase5_ops_evidence.py @@ -1091,6 +1091,27 @@ def _verify_migration_0093(admin_url: str) -> None: raise EvidenceError("migration_0093_unique_not_enforced") +def _verify_derived_labels(admin_url: str) -> None: + """The migrations must leave no derived row below its inputs or unverified.""" + + from alicebot_api.db import direct_user_connection + from alicebot_api.vnext_label_repair import classify_stored_labels + + with direct_user_connection(admin_url, USER_ID) as conn: + from alicebot_api.vnext_label_repair import INPUT_SELECTS_V3 + + tables: dict[str, list[dict[str, object]]] = {} + for table, statement in INPUT_SELECTS_V3.items(): + cursor = conn.execute(statement) + names = [column[0] for column in cursor.description] + tables[table] = [ + row if isinstance(row, dict) else dict(zip(names, row)) for row in cursor.fetchall() + ] + below, unverified = classify_stored_labels(tables) + if below or any(unverified.values()): + raise EvidenceError("derived_labels_not_repaired") + + def _seed_postgres_baseline( *, baseline: Path, @@ -1278,6 +1299,7 @@ def _postgres_drill( ) after = _verify_postgres_store(admin_url, app_url, expected_head=current_head) _verify_migration_0093(admin_url) + _verify_derived_labels(admin_url) if before["counts"] != after["counts"]: raise EvidenceError("postgres_restore_count_mismatch") result = { diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index 1f406005f..59320ba04 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -414,7 +414,7 @@ def _assert_all_pointers_truthful() -> None: with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: with conn.cursor() as cur: cur.execute("SELECT version_num FROM alembic_version") - assert cur.fetchone()["version_num"] == "20261004_0095" + assert cur.fetchone()["version_num"] == "20261005_0096" # Repeat the additive post-release migrations through their downgrade # boundary. The already repaired data remains correct and the second diff --git a/tests/unit/test_label_repair_migration.py b/tests/unit/test_label_repair_migration.py new file mode 100644 index 000000000..645451c9c --- /dev/null +++ b/tests/unit/test_label_repair_migration.py @@ -0,0 +1,47 @@ +"""Migration 0096 relaxes row security on all seven label tables.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +from alicebot_api.cli.parser import build_parser as build_alicebot_parser +from alicebot_api.onramp import _KNOWN_COMMANDS +from alicebot_api.onramp import build_parser as build_sqlite_parser + +ROOT = Path(__file__).resolve().parents[2] + + +def _migration(): + path = ROOT / "apps/api/alembic/versions/20261005_0096_derived_label_floor.py" + spec = importlib.util.spec_from_file_location("migration_0096", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_migration_0096_brackets_every_label_table() -> None: + migration = _migration() + tables = tuple(statement.split()[2] for statement in migration._RELAX_RLS) + assert migration.revision == "20261005_0096" + assert migration.down_revision == "20261004_0095" + assert tables == ( + "sources", + "memories", + "open_loops", + "generated_artifacts", + "beliefs", + "event_log", + "projects", + ) + + +def test_labels_commands_are_registered() -> None: + assert "labels" in _KNOWN_COMMANDS + sqlite = build_sqlite_parser().parse_args(["labels", "check", "--db", "vault.db"]) + assert sqlite.command == "labels" + assert sqlite.labels_command == "check" + postgres = build_alicebot_parser().parse_args(["vnext", "labels", "repair"]) + assert postgres.vnext_command == "labels" + assert postgres.labels_command == "repair" diff --git a/tests/unit/test_phase5_ops_evidence.py b/tests/unit/test_phase5_ops_evidence.py index d0d8e629d..a626ea879 100644 --- a/tests/unit/test_phase5_ops_evidence.py +++ b/tests/unit/test_phase5_ops_evidence.py @@ -662,6 +662,7 @@ def execute(self, query): monkeypatch.setattr(ops, "_dynamic_alembic_head", lambda: "current_head") monkeypatch.setattr(ops, "_migrate_postgres", lambda *_args, **_kwargs: None) monkeypatch.setattr(ops, "_verify_migration_0093", lambda _admin_url: None) + monkeypatch.setattr(ops, "_verify_derived_labels", lambda _admin_url: None) monkeypatch.setattr( ops, "_verify_postgres_store", diff --git a/tests/unit/test_session_doctor.py b/tests/unit/test_session_doctor.py index b50717450..55e98ca29 100644 --- a/tests/unit/test_session_doctor.py +++ b/tests/unit/test_session_doctor.py @@ -47,6 +47,7 @@ "sleep proposals", "flagged sources", "flagged source ids", + "derived labels", ) FORBIDDEN_PHRASES = ("review console", "/vnext", "clear the queue", "open Memory Review") diff --git a/tests/unit/test_sqlite_label_repair_v3.py b/tests/unit/test_sqlite_label_repair_v3.py index 654961887..a8629ebdb 100644 --- a/tests/unit/test_sqlite_label_repair_v3.py +++ b/tests/unit/test_sqlite_label_repair_v3.py @@ -63,6 +63,17 @@ def test_open_raises_a_public_copy_of_a_confidential_source(tmp_path, monkeypatc assert SOURCE_TEXT not in json.dumps(payload) +def test_labels_check_names_the_rows_the_next_open_would_raise(tmp_path, monkeypatch, capsys) -> None: + from alicebot_api.onramp import main as onramp_main + + path = tmp_path / "vault.db" + _vault_with_a_public_copy(path, monkeypatch) + code = onramp_main(["labels", "check", "--db", str(path), "--user-id", USER]) + output = capsys.readouterr().out + assert code == 1 + assert "next open would raise" in output + + def test_a_vault_that_cannot_be_repaired_still_opens(tmp_path, monkeypatch, caplog) -> None: path = tmp_path / "vault.db" _vault_with_a_public_copy(path, monkeypatch) From 5cd97bc2c4fdb785dc1de5522cd3a952e9d40a09 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:28:01 +0200 Subject: [PATCH 038/270] fix: apply checked memory relabels and propagate descendant floors --- .../alicebot_api/routers/vnext_memories.py | 9 ++++- .../test_label_floor_ancestry_postgres.py | 35 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index 433d4716f..4bc6667eb 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -1246,7 +1246,14 @@ def review_vnext_memory( status_code=400, detail="vNext memory review text carries credential material" ) - updated = store.update_memory(memory_id=str(memory_id), patch=patch, actor_type=actor_type) + before_label = dict(existing) + updated = store.update_memory( + memory_id=str(memory_id), patch=patch, actor_type=actor_type, label_write=label_change, + ) + if label_change: + from alicebot_api.vnext_label_writes import propagate_after_write + + propagate_after_write(store, kind="memory", before=before_label, after=updated) if action in ("accept", "edit", "promote"): memory_service.refresh_memory_derived_state( updated, diff --git a/tests/integration/test_label_floor_ancestry_postgres.py b/tests/integration/test_label_floor_ancestry_postgres.py index 2fdc86b95..657f3a3b5 100644 --- a/tests/integration/test_label_floor_ancestry_postgres.py +++ b/tests/integration/test_label_floor_ancestry_postgres.py @@ -31,3 +31,38 @@ def test_pg_copy_and_summary_keep_tenant_and_ancestry(migrated_database_urls, do assert artifact["sensitivity"] == sensitivity if domain == "health": assert artifact["domain"] == domain + + +def test_checked_project_review_moves_scope_and_propagates_labels(migrated_database_urls, monkeypatch): + from alicebot_api.config import Settings + from alicebot_api.routers import vnext_memories as router + from alicebot_api.vnext_agent_keys import create_agent_key + from uuid import UUID + user_id = uuid4() + alpha, beta = "prj_" + "a" * 16, "prj_" + "b" * 16 + app_url = migrated_database_urls["app"] + with user_connection(app_url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"review-labels-{user_id}@example.invalid", "Labels") + store = PostgresVNextStore(conn) + original = store.create_memory({"memory_key": "original", "canonical_text": "original", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [alpha]}}) + summary = store.create_memory({"memory_key": "summary", "canonical_text": "summary", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"consolidation": {"cluster_member_ids": [str(original["id"])]}, "project_scope": [alpha]}}) + _key, raw_key = create_agent_key(store, user_id=user_id, agent_id="alpha-only", permission_profile="admin_agent", project_scope=alpha) + _admin, admin_key = create_agent_key(store, user_id=user_id, agent_id="unbound-admin", permission_profile="admin_agent") + monkeypatch.setattr(router, "get_settings", lambda: Settings(database_url=app_url)) + request = router.VNextMemoryReviewRequest(user_id=user_id, action="assign_project", project_id=beta, domain="health", sensitivity="confidential") + denied = router.review_vnext_memory(UUID(str(original["id"])), request, authorization=f"Bearer {raw_key}") + assert denied.status_code == 403 + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + assert store.get_memory(str(original["id"]))["metadata_json"]["project_scope"] == [alpha] + assert store.get_memory(str(summary["id"]))["sensitivity"] == "public" + moved = router.review_vnext_memory(UUID(str(original["id"])), request, authorization=f"Bearer {admin_key}") + assert moved.status_code == 200 + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + original_after = store.get_memory(str(original["id"])) + summary_after = store.get_memory(str(summary["id"])) + assert original_after["metadata_json"]["project_scope"] == [beta] + assert summary_after["domain"] == "health" + assert summary_after["sensitivity"] == "confidential" + assert beta in summary_after["metadata_json"]["project_floor"] From 0699cf5daa25a46d318da61262d0270be7761532 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:28:01 +0200 Subject: [PATCH 039/270] test: distinguish label guards from business SQL in store adapters --- tests/unit/test_embedding_input_limits.py | 6 +- tests/unit/test_vnext_store.py | 349 +++++++++++----------- 2 files changed, 185 insertions(+), 170 deletions(-) diff --git a/tests/unit/test_embedding_input_limits.py b/tests/unit/test_embedding_input_limits.py index 5828a6af8..f51815e5c 100644 --- a/tests/unit/test_embedding_input_limits.py +++ b/tests/unit/test_embedding_input_limits.py @@ -980,8 +980,10 @@ def test_postgres_store_signs_the_cut_label_and_lists_rows_whose_label_differs() content_sha256="digest", signature_version=2, ) - cut_query, cut_params = connection.cursor_instance.queries[0] - plain_query, plain_params = connection.cursor_instance.queries[1] + writes = [(query, params) for query, params in connection.cursor_instance.queries if "UPDATE memories" in query] + assert len(writes) == 2 + cut_query, cut_params = writes[0] + plain_query, plain_params = writes[1] assert cut_params[1].obj["truncated_to_chars"] == 1000 # type: ignore[attr-defined] assert "truncated_to_chars" not in plain_params[1].obj # type: ignore[attr-defined] assert cut_query == plain_query diff --git a/tests/unit/test_vnext_store.py b/tests/unit/test_vnext_store.py index 4f14c7329..de7109f4d 100644 --- a/tests/unit/test_vnext_store.py +++ b/tests/unit/test_vnext_store.py @@ -39,6 +39,11 @@ def execute(self, query: str, params: tuple[object, ...] | None = None) -> None: assert query.count("%s") == len(params) self.executed.append((query, params)) + @property + def statements(self) -> list[tuple[str, tuple[object, ...] | None]]: + """Business SQL, with transaction guards retained separately in executed.""" + return [(query, params) for query, params in self.executed if "pg_advisory_xact_lock" not in query] + def fetchone(self) -> dict[str, Any] | None: if not self.fetchone_results: return None @@ -65,7 +70,7 @@ def _event_row(target_id: object | None = None) -> dict[str, object]: def _event_log_insert_count(cursor: RecordingCursor) -> int: - return sum(1 for query, _params in cursor.executed if "INSERT INTO event_log" in query) + return sum(1 for query, _params in cursor.statements if "INSERT INTO event_log" in query) def test_postgres_project_scope_sql_mirrors_conservative_python_identity() -> None: @@ -144,19 +149,20 @@ def test_source_crud_and_chunks_write_audit_events() -> None: assert chunks == [{"id": chunk_id, "source_id": source_id}] assert _event_log_insert_count(cursor) == 4 - source_insert_query, source_insert_params = cursor.executed[0] + assert "pg_advisory_xact_lock_shared" in cursor.executed[0][0] + source_insert_query, source_insert_params = cursor.statements[0] assert "INSERT INTO sources" in source_insert_query assert source_insert_params is not None assert isinstance(source_insert_params[-1], Jsonb) assert source_insert_params[-1].obj == {"path": "docs/spec.md"} source_update_query, source_update_params = next( - (query, params) for query, params in cursor.executed if "UPDATE sources" in query and "SET title" in query + (query, params) for query, params in cursor.statements if "UPDATE sources" in query and "SET title" in query ) assert "UPDATE sources" in source_update_query assert source_update_params is not None - chunk_query, chunk_params = cursor.executed[-1] + chunk_query, chunk_params = cursor.statements[-1] assert "WHERE source_id = %s::uuid" in chunk_query assert chunk_query.index("WHERE source_id") < chunk_query.index("LIMIT %s") assert chunk_params == (source_id, 17) @@ -164,7 +170,7 @@ def test_source_crud_and_chunks_write_audit_events() -> None: with pytest.raises(ValueError, match="limit must be positive"): store.list_source_chunks(source_id, limit=0) store.list_source_chunks(source_id, limit=10_000) - assert cursor.executed[-1][1] == (source_id, 501) + assert cursor.statements[-1][1] == (source_id, 501) assert isinstance(source_update_params[6], Jsonb) assert source_update_params[6].obj == {"rev": 2} @@ -178,7 +184,7 @@ def test_get_source_by_content_hash_uses_dedupe_lookup() -> None: assert source is not None assert source["id"] == source_id - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "FROM sources" in query assert "content_hash = %s" in query assert "deleted_at IS NULL" in query @@ -206,7 +212,7 @@ def test_get_or_create_source_uses_partial_unique_dedupe_claim() -> None: assert created is True assert source["id"] == source_id - query, _params = cursor.executed[0] + query, _params = cursor.statements[0] assert "ON CONFLICT (user_id, dedupe_key)" in query assert "WHERE deleted_at IS NULL AND dedupe_key IS NOT NULL" in query assert "DO NOTHING" in query @@ -232,8 +238,8 @@ def test_get_or_create_source_returns_concurrent_winner_without_create_event() - assert created is False assert source["id"] == source_id - assert len(cursor.executed) == 2 - assert "SELECT" in cursor.executed[1][0] + assert len(cursor.statements) == 2 + assert "SELECT" in cursor.statements[1][0] assert _event_log_insert_count(cursor) == 0 @@ -304,7 +310,7 @@ def test_update_source_recomputes_postgres_dedupe_key_with_the_same_statement() ) update_query, update_params = next( - (query, params) for query, params in cursor.executed if "UPDATE sources" in query and "SET title" in query + (query, params) for query, params in cursor.statements if "UPDATE sources" in query and "SET title" in query ) assert "metadata_json = COALESCE(%s, metadata_json)" in update_query assert "dedupe_key = %s" in update_query @@ -342,7 +348,7 @@ def test_update_source_postgres_collision_fails_before_mutation_event() -> None: patch={"metadata_json": {"raw_text": raw_text, "project_scope": ["Beta"]}}, ) - assert not any("UPDATE sources" in query for query, _params in cursor.executed) + assert not any("UPDATE sources" in query for query, _params in cursor.statements) assert _event_log_insert_count(cursor) == 0 @@ -371,7 +377,7 @@ def test_update_source_postgres_releases_key_when_changed_identity_has_no_raw_te ) update_params = next( - params for query, params in cursor.executed if "UPDATE sources" in query and "SET title" in query + params for query, params in cursor.statements if "UPDATE sources" in query and "SET title" in query ) assert update_params is not None assert update_params[8] is None @@ -435,9 +441,9 @@ def test_keyword_search_methods_apply_domain_sensitivity_and_limit_filters() -> assert memories[0]["id"] == "matched-1" assert sources[0]["id"] == "matched-1" assert open_loops[0]["id"] == "matched-1" - memory_query, memory_params = cursor.executed[0] - source_query, source_params = cursor.executed[1] - open_loop_query, open_loop_params = cursor.executed[2] + memory_query, memory_params = cursor.statements[0] + source_query, source_params = cursor.statements[1] + open_loop_query, open_loop_params = cursor.statements[2] assert "FROM memories" in memory_query assert "status IN ('active', 'accepted')" in memory_query assert "domain = ANY" in memory_query @@ -501,7 +507,7 @@ def test_project_scope_sql_uses_canonical_key_precedence_for_all_resources() -> store.list_artifacts(scope_projects=(project,), limit=1) store.list_open_loops(scope_projects=(project,), limit=1) - memory_query, source_query, artifact_query, open_loop_query = [query for query, _ in cursor.executed] + memory_query, source_query, artifact_query, open_loop_query = [query for query, _ in cursor.statements] for query, metadata_expression in ( (memory_query, "metadata_json"), (artifact_query, "metadata_json"), @@ -572,7 +578,7 @@ def test_project_scope_sql_uses_canonical_key_precedence_for_all_resources() -> assert "?| %s::text[]" in source_query assert "OR project_id::text = ANY" not in open_loop_query - for _query, params in cursor.executed: + for _query, params in cursor.statements: assert params is not None assert "canonical-project" in str(params) assert project not in str(params) @@ -589,7 +595,7 @@ def test_exact_memory_scope_lookup_uses_conservative_order_insensitive_identity( project_scope=(" Beta ", "ALICE", "alice"), ) - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "octet_length(normalized_scope.value) = char_length(normalized_scope.value)" in query assert 'COLLATE "C"' in query assert "metadata_json ? 'project_scope'" in query @@ -617,7 +623,7 @@ def test_search_memories_by_time_builds_window_predicate_and_proximity_order() - ) assert rows[0]["id"] == "memory-march" - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "FROM memories" in query assert "deleted_at IS NULL" in query assert "status IN ('active', 'accepted')" in query @@ -658,7 +664,7 @@ def test_search_memories_by_time_treats_naive_windows_as_utc() -> None: window_end=datetime(2023, 4, 1), ) - _query, params = cursor.executed[0] + _query, params = cursor.statements[0] assert params is not None assert params[13] == datetime(2023, 3, 1, tzinfo=UTC) assert params[14] == datetime(2023, 4, 1, tzinfo=UTC) @@ -678,7 +684,7 @@ def test_search_memories_by_time_accepts_an_explicit_proximity_pivot() -> None: window_center=pivot, ) - _query, params = cursor.executed[0] + _query, params = cursor.statements[0] assert params is not None assert params[17] == pivot @@ -700,7 +706,7 @@ def test_list_artifacts_applies_type_domain_sensitivity_and_limit_filters() -> N ) assert artifacts[0]["id"] == "artifact-1" - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "FROM generated_artifacts" in query assert "%s::text IS NULL OR artifact_type = %s" in query assert "domain = ANY" in query @@ -715,6 +721,7 @@ def test_list_artifacts_applies_type_domain_sensitivity_and_limit_filters() -> N ["public", "private"], None, None, + None, 5, ) @@ -754,13 +761,13 @@ def test_artifact_quality_ratings_insert_and_export_json_safe_payloads() -> None assert created["id"] == rating_id assert rows == [{"id": rating_id, "artifact_id": artifact_id, "usefulness": 5}] assert _event_log_insert_count(cursor) == 1 - assert "FOR UPDATE" in cursor.executed[0][0] - insert_query, insert_params = cursor.executed[1] + assert "FOR UPDATE" in cursor.statements[0][0] + insert_query, insert_params = cursor.statements[1] assert "INSERT INTO artifact_quality_ratings" in insert_query assert insert_params is not None assert isinstance(insert_params[-1], Jsonb) assert insert_params[-1].obj == {"prompt_hash": "sha256:test"} - list_query, list_params = cursor.executed[3] + list_query, list_params = cursor.statements[3] assert "FROM artifact_quality_ratings" in list_query assert list_params == (artifact_id, artifact_id, None, None, 10) @@ -788,7 +795,7 @@ def test_artifact_quality_ratings_upsert_on_artifact_reviewer_conflict() -> None ) assert created["id"] == rating_id - upsert_query, _upsert_params = cursor.executed[1] + upsert_query, _upsert_params = cursor.statements[1] assert "ON CONFLICT (artifact_id, reviewer_id) DO UPDATE SET" in upsert_query assert "usefulness = EXCLUDED.usefulness" in upsert_query assert "metadata_json = EXCLUDED.metadata_json" in upsert_query @@ -824,9 +831,9 @@ def test_quality_rating_rejects_exact_redacted_artifact_before_insert() -> None: with pytest.raises(ValueError, match="ratings cannot be added to a redacted artifact"): store.create_artifact_quality_rating({"artifact_id": artifact_id, "usefulness": 5}) - assert len(cursor.executed) == 1 - assert "FOR UPDATE" in cursor.executed[0][0] - assert not any("INSERT INTO artifact_quality_ratings" in query for query, _params in cursor.executed) + assert len(cursor.statements) == 1 + assert "FOR UPDATE" in cursor.statements[0][0] + assert not any("INSERT INTO artifact_quality_ratings" in query for query, _params in cursor.statements) def test_list_beliefs_joins_memory_domain_sensitivity_filters() -> None: @@ -854,7 +861,7 @@ def test_list_beliefs_joins_memory_domain_sensitivity_filters() -> None: ) assert beliefs[0]["id"] == belief_id - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "FROM beliefs b" in query assert "JOIN memories m" in query assert "%s::text IS NULL OR b.status = %s" in query @@ -890,9 +897,11 @@ def test_memory_revision_provenance_and_graph_methods_write_audit_events() -> No {"id": memory_id}, _event_row(memory_id), {"id": memory_id}, + {"id": memory_id}, # stored label read before any update # update_memory to a searchable status reads the stored row first, # for the credential activation check (S4.4 round 2, ruling C2). {"id": memory_id, "status": "candidate", "canonical_text": "Alice vNext is being built."}, + {"metadata_json": {}}, # metadata reread inside the label lock {"id": memory_id}, _event_row(memory_id), {"id": revision_id, "memory_id": memory_id}, @@ -962,7 +971,7 @@ def test_memory_revision_provenance_and_graph_methods_write_audit_events() -> No store.expire_edge(edge_id=edge_id) assert _event_log_insert_count(cursor) == 7 - memory_insert_query = cursor.executed[0][0] + memory_insert_query = cursor.statements[0][0] assert "INSERT INTO memories" in memory_insert_query assert "canonical_text" in memory_insert_query assert "domain" in memory_insert_query @@ -970,15 +979,15 @@ def test_memory_revision_provenance_and_graph_methods_write_audit_events() -> No assert "metadata_json" in memory_insert_query assert "ON CONFLICT DO NOTHING" in memory_insert_query assert "WHERE commit_digest IS NOT NULL" not in memory_insert_query - revision_queries = [query for query, _params in cursor.executed if "next_revision AS" in query] + revision_queries = [query for query, _params in cursor.statements if "next_revision AS" in query] assert len(revision_queries) == 1 assert "locked_memory AS" in revision_queries[0] assert "FOR UPDATE" in revision_queries[0] - assert any("INSERT INTO provenance_links" in query for query, _params in cursor.executed) - assert any("INSERT INTO graph_edges" in query for query, _params in cursor.executed) - assert any("UPDATE graph_edges" in query for query, _params in cursor.executed) - assert any("%s::text IS NULL OR from_id = %s" in query for query, _params in cursor.executed) - update_edge_query, update_edge_params = cursor.executed[-4] + assert any("INSERT INTO provenance_links" in query for query, _params in cursor.statements) + assert any("INSERT INTO graph_edges" in query for query, _params in cursor.statements) + assert any("UPDATE graph_edges" in query for query, _params in cursor.statements) + assert any("%s::text IS NULL OR from_id = %s" in query for query, _params in cursor.statements) + update_edge_query, update_edge_params = cursor.statements[-4] assert "metadata_json = metadata_json || %s" in update_edge_query assert update_edge_params is not None assert update_edge_params[1] == "accepted" @@ -1011,7 +1020,7 @@ def test_create_memory_persists_canonical_multi_project_scope_metadata() -> None ) assert row["project_scope"] == ["alicebot", "hermes"] - insert_params = cursor.executed[0][1] + insert_params = cursor.statements[0][1] assert insert_params is not None metadata_values = [param.obj for param in insert_params if isinstance(param, Jsonb)] assert {"project_scope": ["alicebot", "hermes"]} in metadata_values @@ -1023,7 +1032,7 @@ def test_get_memory_for_update_uses_a_row_lock() -> None: store = PostgresVNextStore(RecordingConnection(cursor)) assert store.get_memory_for_update(memory_id) == {"id": memory_id} - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "FROM memories" in query assert "FOR UPDATE" in query assert params == (memory_id,) @@ -1041,7 +1050,7 @@ def test_pending_derived_candidate_lookup_uses_snapshots_and_row_locks() -> None exclude_memory_id=excluded_id, ) == [{"id": candidate_id}] - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "status IN ('candidate', 'needs_review')" in query assert "member_snapshots" in query assert "jsonb_array_elements" in query @@ -1063,7 +1072,7 @@ def test_list_memories_pushes_scope_and_limit_into_postgres_query() -> None: == [] ) - query, params = cursor.executed[-1] + query, params = cursor.statements[-1] assert "status = %s" in query assert "domain = ANY(%s::text[]) OR domain = 'unknown'" in query assert "COALESCE(sensitivity, 'unknown') = ANY(%s::text[])" in query @@ -1128,12 +1137,12 @@ def test_resume_store_queries_apply_admission_predicates_before_limit() -> None: ) memory_query, loop_query, memory_event_query, loop_event_query, shared_event_query = ( - query for query, _params in cursor.executed + query for query, _params in cursor.statements ) - memory_params = cursor.executed[0][1] - loop_params = cursor.executed[1][1] - memory_event_params = cursor.executed[2][1] - loop_event_params = cursor.executed[3][1] + memory_params = cursor.statements[0][1] + loop_params = cursor.statements[1][1] + memory_event_params = cursor.statements[2][1] + loop_event_params = cursor.statements[3][1] assert "status = ANY(%s::text[])" in memory_query assert "memory_type = ANY(%s::text[])" in memory_query assert "created_at >= %s::timestamptz" in memory_query @@ -1222,8 +1231,8 @@ def test_project_update_event_lookup_is_one_bounded_target_and_payload_query() - ) assert rows == [] - assert len(cursor.executed) == 1 - query, params = cursor.executed[0] + assert len(cursor.statements) == 1 + query, params = cursor.statements[0] assert query.count("SELECT") == 5 assert query.count("user_id = app.current_user_id()") == 5 assert query.count("event_type IN (") == 5 @@ -1270,11 +1279,11 @@ def test_memory_and_rollup_counts_are_exact_scoped_database_reads() -> None: == 5 ) - memory_query, memory_params = cursor.executed[0] + memory_query, memory_params = cursor.statements[0] assert "SELECT COUNT(*) AS count" in memory_query assert "status = %s" in memory_query assert memory_params == ("active", ["project"], ["private"]) - rollup_query, rollup_params = cursor.executed[1] + rollup_query, rollup_params = cursor.statements[1] assert "SELECT COUNT(*) AS count" in rollup_query assert "status IN ('active', 'accepted')" in rollup_query assert "candidate_kind" in rollup_query @@ -1313,7 +1322,7 @@ def test_rollup_reads_push_status_scope_exact_keys_order_and_limits_into_postgre limit=99, ) - input_query, input_params = cursor.executed[0] + input_query, input_params = cursor.statements[0] assert "status IN ('active', 'accepted')" in input_query assert "COALESCE(metadata_json ->> 'candidate_kind', '') <> %s" in input_query assert "domain = ANY(%s::text[]) OR domain = 'unknown'" in input_query @@ -1330,7 +1339,7 @@ def test_rollup_reads_push_status_scope_exact_keys_order_and_limits_into_postgre 501, ) - pending_query, pending_params = cursor.executed[1] + pending_query, pending_params = cursor.statements[1] assert "DISTINCT ON (metadata_json ->> 'rollup_digest')" in pending_query assert "status = 'candidate'" in pending_query assert "metadata_json ->> 'rollup_digest' = ANY(%s::text[])" in pending_query @@ -1347,7 +1356,7 @@ def test_rollup_reads_push_status_scope_exact_keys_order_and_limits_into_postgre 2, ) - accepted_query, accepted_params = cursor.executed[2] + accepted_query, accepted_params = cursor.statements[2] assert "DISTINCT ON (metadata_json ->> 'rollup_key')" in accepted_query assert "status IN ('active', 'accepted')" in accepted_query assert "metadata_json ->> 'rollup_key' = ANY(%s::text[])" in accepted_query @@ -1370,7 +1379,7 @@ def test_rollup_reads_push_status_scope_exact_keys_order_and_limits_into_postgre excluded_candidate_kind="memory_rollup", limit=1, ) - assert cursor.executed[3][1] == ( + assert cursor.statements[3][1] == ( "memory_rollup", None, None, @@ -1439,10 +1448,10 @@ def test_project_people_belief_and_open_loop_methods_write_audit_events() -> Non store.update_open_loop(loop_id=loop_id, patch={"title": "Validate migration", "priority": "normal"}) assert _event_log_insert_count(cursor) == 9 - assert "INSERT INTO projects" in cursor.executed[0][0] - assert "FROM projects" in cursor.executed[3][0] - assert "%s::text IS NULL OR status = %s" in cursor.executed[3][0] - assert cursor.executed[3][1] == ( + assert "INSERT INTO projects" in cursor.statements[0][0] + assert "FROM projects" in cursor.statements[3][0] + assert "%s::text IS NULL OR status = %s" in cursor.statements[3][0] + assert cursor.statements[3][1] == ( "active", "active", ["project"], @@ -1455,14 +1464,14 @@ def test_project_people_belief_and_open_loop_methods_write_audit_events() -> Non None, 3, ) - assert "UPDATE projects" in cursor.executed[4][0] - assert "INSERT INTO people" in cursor.executed[6][0] - assert "UPDATE people" in cursor.executed[9][0] - assert "INSERT INTO beliefs" in cursor.executed[11][0] - assert "UPDATE beliefs" in cursor.executed[14][0] - assert "INSERT INTO open_loops" in cursor.executed[16][0] - assert "UPDATE open_loops" in cursor.executed[19][0] - assert "UPDATE open_loops" in cursor.executed[21][0] + assert "UPDATE projects" in cursor.statements[4][0] + assert "INSERT INTO people" in cursor.statements[6][0] + assert "UPDATE people" in cursor.statements[9][0] + assert "INSERT INTO beliefs" in cursor.statements[11][0] + assert "UPDATE beliefs" in cursor.statements[14][0] + assert "INSERT INTO open_loops" in cursor.statements[16][0] + assert "UPDATE open_loops" in cursor.statements[19][0] + assert "UPDATE open_loops" in cursor.statements[21][0] def test_get_artifact_for_update_locks_the_persisted_authorization_target() -> None: @@ -1473,7 +1482,7 @@ def test_get_artifact_for_update_locks_the_persisted_authorization_target() -> N artifact = store.get_artifact_for_update(artifact_id) assert artifact == {"id": artifact_id, "artifact_type": "daily_brief"} - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "FROM generated_artifacts" in query assert "FOR UPDATE" in query assert params == (artifact_id,) @@ -1499,7 +1508,7 @@ def test_exact_open_loop_and_artifact_digest_lookups_scope_before_limit() -> Non scope_projects=(project_id,), ) == {"id": "artifact-1"} - loop_query, loop_params = cursor.executed[0] + loop_query, loop_params = cursor.statements[0] assert loop_query.index("automation_digest") < loop_query.index("LIMIT 1") assert "project_id = %s::uuid" in loop_query assert "person_id = %s::uuid" in loop_query @@ -1510,7 +1519,7 @@ def test_exact_open_loop_and_artifact_digest_lookups_scope_before_limit() -> Non person_id, person_id, ) - artifact_query, artifact_params = cursor.executed[1] + artifact_query, artifact_params = cursor.statements[1] assert artifact_query.index("automation_digest") < artifact_query.index("LIMIT 1") assert "consolidation_digest" in artifact_query assert artifact_params == ( @@ -1540,13 +1549,13 @@ def test_source_trace_and_policy_telemetry_queries_filter_before_limit() -> None store.list_agent_policy_artifacts(agent_id="hermes", limit=15) store.list_agent_policy_memories(agent_id="hermes", limit=16) - for query, _params in cursor.executed[:4]: + for query, _params in cursor.statements[:4]: assert query.index("source_id") < query.index("LIMIT %s") - assert "provenance_links" in cursor.executed[0][0] - assert "provenance_links" in cursor.executed[1][0] - assert "target_type = 'source'" in cursor.executed[3][0] - assert "generated_by' = 'agent'" in cursor.executed[4][0] - assert "agent_id' IS NOT NULL" in cursor.executed[5][0] + assert "provenance_links" in cursor.statements[0][0] + assert "provenance_links" in cursor.statements[1][0] + assert "target_type = 'source'" in cursor.statements[3][0] + assert "generated_by' = 'agent'" in cursor.statements[4][0] + assert "agent_id' IS NOT NULL" in cursor.statements[5][0] def test_artifact_task_and_brain_charter_methods_write_audit_events() -> None: @@ -1608,12 +1617,12 @@ def test_artifact_task_and_brain_charter_methods_write_audit_events() -> None: assert claimed is not None assert _event_log_insert_count(cursor) == 6 - assert "INSERT INTO generated_artifacts" in cursor.executed[0][0] - assert "UPDATE generated_artifacts" in cursor.executed[3][0] - assert "INSERT INTO task_queue" in cursor.executed[5][0] - assert "FOR UPDATE SKIP LOCKED" in cursor.executed[7][0] - assert "UPDATE task_queue" in cursor.executed[9][0] - assert "ON CONFLICT (user_id)" in cursor.executed[11][0] + assert "INSERT INTO generated_artifacts" in cursor.statements[0][0] + assert "UPDATE generated_artifacts" in cursor.statements[3][0] + assert "INSERT INTO task_queue" in cursor.statements[5][0] + assert "FOR UPDATE SKIP LOCKED" in cursor.statements[7][0] + assert "UPDATE task_queue" in cursor.statements[9][0] + assert "ON CONFLICT (user_id)" in cursor.statements[11][0] def test_append_and_list_event_log_records_use_integrity_payload() -> None: @@ -1635,7 +1644,7 @@ def test_append_and_list_event_log_records_use_integrity_payload() -> None: assert appended["target_id"] == "memory-1" assert events[0]["target_id"] == "memory-1" assert all_events[0]["target_id"] == "memory-1" - event_insert_query, event_insert_params = cursor.executed[0] + event_insert_query, event_insert_params = cursor.statements[0] assert "INSERT INTO event_log" in event_insert_query assert event_insert_params is not None assert event_insert_params[1:6] == ( @@ -1648,7 +1657,7 @@ def test_append_and_list_event_log_records_use_integrity_payload() -> None: assert isinstance(event_insert_params[7], Jsonb) assert event_insert_params[7].obj == {"b": 2, "a": 1} assert event_insert_params[10] == event["integrity_hash"] - event_list_query = cursor.executed[1][0] + event_list_query = cursor.statements[1][0] assert "%s::text IS NULL OR target_type = %s" in event_list_query assert "%s::text IS NULL OR target_id = %s" in event_list_query @@ -1734,7 +1743,7 @@ def test_connector_settings_and_state_methods_use_dedicated_tables_and_audit_eve assert fetched_state is not None assert storage_status["connector_settings_exists"] is True assert _event_log_insert_count(cursor) == 2 - setting_query, setting_params = cursor.executed[0] + setting_query, setting_params = cursor.statements[0] assert "INSERT INTO connector_settings" in setting_query assert "ON CONFLICT (user_id, connector_name)" in setting_query assert setting_params is not None @@ -1742,7 +1751,7 @@ def test_connector_settings_and_state_methods_use_dedicated_tables_and_audit_eve assert setting_params[8].obj == [] assert isinstance(setting_params[9], Jsonb) assert setting_params[9].obj == {"config_json": {"allowed_origins": ["http://localhost:3000"]}} - state_query, state_params = cursor.executed[4] + state_query, state_params = cursor.statements[4] assert "INSERT INTO connector_state" in state_query assert "items_seen = connector_state.items_seen + EXCLUDED.items_seen" in state_query assert state_params is not None @@ -1769,10 +1778,10 @@ def test_workspace_list_methods_apply_bounded_filters() -> None: assert tasks[0]["id"] == "workspace-row-1" assert events[0]["id"] == "workspace-row-1" - source_query, source_params = cursor.executed[0] - people_query, people_params = cursor.executed[1] - task_query, task_params = cursor.executed[2] - event_query, event_params = cursor.executed[3] + source_query, source_params = cursor.statements[0] + people_query, people_params = cursor.statements[1] + task_query, task_params = cursor.statements[2] + event_query, event_params = cursor.statements[3] assert "FROM sources" in source_query assert "deleted_at IS NULL" in source_query assert source_params == (["project"], ["project"], ["private"], ["private"], 7) @@ -1821,7 +1830,7 @@ def test_jsonb_and_event_hash_normalize_postgres_scalar_values() -> None: "captured_at": "2026-05-10T12:30:00+00:00", }, } - project_insert_params = cursor.executed[0][1] + project_insert_params = cursor.statements[0][1] assert project_insert_params is not None assert isinstance(project_insert_params[-1], Jsonb) assert project_insert_params[-1].obj == { @@ -1845,7 +1854,7 @@ def test_fts_search_builds_websearch_tsquery_with_pushed_down_filters() -> None: ) assert rows[0]["id"] == "memory-1" - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "FROM memories" in query assert "websearch_to_tsquery('english', %s)" in query assert "search_tsv @@ websearch_to_tsquery('english', %s)" in query @@ -1911,7 +1920,7 @@ def test_fts_search_pushes_down_memory_type_project_agent_run_and_expiry_filters include_expired=True, ) - _query, params = cursor.executed[0] + _query, params = cursor.statements[0] assert params == ( "Alice provenance retrieval", ["project"], @@ -1959,7 +1968,7 @@ def test_fts_search_pushes_people_and_time_scope_before_ranked_limit() -> None: limit=1, ) - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "jsonb_path_query" in query assert "id::text = ANY" in query assert "COALESCE(valid_from, last_seen_at, updated_at, first_seen_at, created_at)" in query @@ -1991,7 +2000,7 @@ def test_search_source_chunks_builds_websearch_tsquery_over_chunk_text() -> None ) assert rows[0]["source_id"] == "source-1" - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "FROM source_chunks c" in query assert "JOIN sources s ON s.id = c.source_id AND s.user_id = c.user_id" in query assert "s.deleted_at IS NULL" in query @@ -2031,7 +2040,7 @@ def test_search_source_chunks_match_any_ors_sanitized_lexemes() -> None: match_any=True, ) - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "c.search_tsv @@ to_tsquery('english', %s)" in query # Stopwords and tsquery metacharacters are stripped; each surviving # token is individually quoted so nothing can inject query syntax. @@ -2046,7 +2055,7 @@ def test_search_source_chunks_match_any_returns_empty_without_content_tokens() - # Stopword/metacharacter-only queries sanitize to no lexemes: no SQL runs. assert store.search_source_chunks(query="&|!():*<->", match_any=True) == [] assert store.search_source_chunks(query="when was the", match_any=True) == [] - assert cursor.executed == [] + assert cursor.statements == [] def test_vector_search_orders_by_cosine_distance_and_skips_null_embeddings() -> None: @@ -2064,7 +2073,7 @@ def test_vector_search_orders_by_cosine_distance_and_skips_null_embeddings() -> ) assert rows[0]["id"] == "memory-1" - query, params = next((q, p) for q, p in cursor.executed if "vector_distance" in q) + query, params = next((q, p) for q, p in cursor.statements if "vector_distance" in q) assert "FROM memories" in query assert "embedding_vector IS NOT NULL" in query assert "status IN ('active', 'accepted')" in query @@ -2119,7 +2128,7 @@ def test_postgres_vector_boundary_rejects_non_finite_values() -> None: vector=[1.0, float("inf")], ) - assert cursor.executed == [] + assert cursor.statements == [] def test_vector_search_can_require_matching_embedding_signature() -> None: @@ -2133,7 +2142,7 @@ def test_vector_search_can_require_matching_embedding_signature() -> None: embedding_signature_version=1, ) - query, params = next((q, p) for q, p in cursor.executed if "vector_distance" in q) + query, params = next((q, p) for q, p in cursor.statements if "vector_distance" in q) assert "metadata_json -> '_alice_embedding' ->> 'provider' = %s" in query assert "metadata_json -> '_alice_embedding' ->> 'model' = %s" in query assert "->> 'version' = %s" in query @@ -2149,7 +2158,7 @@ def test_vector_search_enables_iterative_hnsw_scan() -> None: store.search_memories_vector(query_vector=[1.0, 0.0], limit=20) - statements = [q for q, _ in cursor.executed] + statements = [q for q, _ in cursor.statements] assert any("hnsw.iterative_scan" in q and "strict_order" in q for q in statements), statements # The iterative-scan setting must precede the vector SELECT. set_index = next(i for i, q in enumerate(statements) if "hnsw.iterative_scan" in q) @@ -2192,8 +2201,8 @@ def test_vector_search_discards_stale_content_signatures_after_database_read() - ) assert [row["id"] for row in rows] == ["memory-current"] - _query, select_params = next((q, p) for q, p in cursor.executed if "vector_distance" in q) - vector_query = next(q for q, _p in cursor.executed if "vector_distance" in q) + _query, select_params = next((q, p) for q, p in cursor.statements if "vector_distance" in q) + vector_query = next(q for q, _p in cursor.statements if "vector_distance" in q) assert "content_sha256" in vector_query assert "digest(" in vector_query assert select_params[-1] == 4 @@ -2205,7 +2214,7 @@ def test_scheduler_lock_key_includes_current_rls_user() -> None: assert store.try_scheduler_workflow_lock("daily_brief") is True - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "app.current_user_id()::text" in query assert "concat_ws" in query assert params == ("daily_brief",) @@ -2245,7 +2254,7 @@ def test_scheduler_workflow_updates_only_preserve_claim_for_run_bookkeeping( actor_type="test", ) - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "claim_token = CASE WHEN %s THEN claim_token ELSE NULL END" in query assert "claim_version = claim_version + CASE WHEN %s THEN 0 ELSE 1 END" in query assert params is not None @@ -2275,7 +2284,7 @@ def test_artifact_status_update_uses_expected_status_compare_and_set() -> None: ) assert row is not None - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "AND (%s::text IS NULL OR status = %s)" in query assert "metadata_json || %s::jsonb" in query assert params is not None @@ -2323,7 +2332,7 @@ def test_scheduler_claim_rechecks_due_state_and_persists_fence() -> None: assert claim is not None assert claim["claim_version"] == 1 assert claim["scheduled_for"] == scheduled_for - statements = [query for query, _params in cursor.executed] + statements = [query for query, _params in cursor.statements] assert "FOR UPDATE SKIP LOCKED" in statements[0] assert "enabled = true" in statements[2] assert "claim_version = claim_version + 1" in statements[3] @@ -2388,10 +2397,10 @@ def test_scheduler_heartbeat_finalize_and_reaper_are_fenced() -> None: assert finalized is not None assert reaped[0]["status"] == "failed" - heartbeat_query = cursor.executed[0][0] - publish_lock_query = cursor.executed[1][0] - finalize_query = cursor.executed[2][0] - reap_query = cursor.executed[4][0] + heartbeat_query = cursor.statements[0][0] + publish_lock_query = cursor.statements[1][0] + finalize_query = cursor.statements[2][0] + reap_query = cursor.statements[4][0] for query in (heartbeat_query, publish_lock_query, finalize_query): assert "claim_token = %s" in query assert "claim_version = %s" in query @@ -2412,7 +2421,7 @@ def test_pending_confirmation_query_enforces_all_actionable_invariants_before_li store.list_pending_inline_confirmations(limit=3) - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "status = 'needs_review'" in query assert "confirmation_status = 'unconfirmed'" in query assert "confirmation,status" in query @@ -2435,10 +2444,10 @@ def test_update_memory_embedding_and_missing_embedding_listing() -> None: assert updated == {"id": memory_id} assert missing[0]["id"] == memory_id - update_query, update_params = cursor.executed[0] + update_query, update_params = cursor.statements[0] assert "SET embedding_vector = %s::vector" in update_query assert update_params == ("[1.0,0.5]", memory_id) - missing_query, missing_params = cursor.executed[1] + missing_query, missing_params = cursor.statements[1] assert "embedding_vector IS NULL" in missing_query assert "%s::uuid IS NULL OR id > %s::uuid" in missing_query assert "ORDER BY id ASC" in missing_query @@ -2464,7 +2473,7 @@ def test_signed_embedding_update_compares_current_memory_content_digest() -> Non is None ) - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "digest(" in query assert "NULLIF(btrim(title, chr(9)" in query assert "[[:space:]]" not in query @@ -2513,7 +2522,7 @@ def test_embedding_digest_sql_uses_exact_python_strip_table_at_every_cas_boundar embedding_model="embed-v1", embedding_signature_version=2, ) - vector_query = next(query for query, _params in vector_cursor.executed if "vector_distance" in query) + vector_query = next(query for query, _params in vector_cursor.statements if "vector_distance" in query) update_cursor = RecordingCursor(fetchone_results=[]) PostgresVNextStore(RecordingConnection(update_cursor)).update_memory_embedding( @@ -2525,7 +2534,7 @@ def test_embedding_digest_sql_uses_exact_python_strip_table_at_every_cas_boundar content_sha256="a" * 64, signature_version=2, ) - update_query = update_cursor.executed[0][0] + update_query = update_cursor.statements[0][0] missing_cursor = RecordingCursor(fetchone_results=[], fetchall_result=[]) PostgresVNextStore(RecordingConnection(missing_cursor)).list_memories_missing_embeddings( @@ -2534,7 +2543,7 @@ def test_embedding_digest_sql_uses_exact_python_strip_table_at_every_cas_boundar embedding_model="embed-v1", embedding_signature_version=2, ) - missing_query = missing_cursor.executed[0][0] + missing_query = missing_cursor.statements[0][0] for query in (vector_query, update_query, missing_query): assert "[[:space:]]" not in query @@ -2560,7 +2569,7 @@ def test_embedding_backfill_includes_unsigned_or_incompatible_vectors() -> None: embedding_signature_version=1, ) - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "embedding_vector IS NULL" in query assert "IS DISTINCT FROM %s" in query assert "content_sha256" in query @@ -2576,7 +2585,7 @@ def test_clear_memory_embedding_removes_signature_metadata() -> None: store = PostgresVNextStore(RecordingConnection(cursor)) assert store.clear_memory_embedding(memory_id=memory_id) == {"id": memory_id} - query, _params = cursor.executed[0] + query, _params = cursor.statements[0] assert "embedding_vector = NULL" in query assert "metadata_json = metadata_json - '_alice_embedding'" in query assert "{EMBEDDING_SIGNATURE_METADATA_KEY}" not in query @@ -2599,15 +2608,15 @@ def test_targeted_memory_lookups_use_indexed_columns() -> None: assert by_digest == {"id": "memory-digest"} assert by_confirmation == {"id": "memory-confirmation"} assert latest == {"id": "memory-latest"} - digest_query, digest_params = cursor.executed[0] + digest_query, digest_params = cursor.statements[0] assert "WHERE commit_digest = %s" in digest_query assert "LIMIT 1" in digest_query assert digest_params == ("digest-1",) - confirmation_query, confirmation_params = cursor.executed[1] + confirmation_query, confirmation_params = cursor.statements[1] assert "WHERE confirmation_id = %s" in confirmation_query assert "LIMIT 1" in confirmation_query assert confirmation_params == ("confirm-1",) - latest_query, latest_params = cursor.executed[2] + latest_query, latest_params = cursor.statements[2] assert "metadata_json #>> '{agentic_memory,kind}' = 'agentic_memory_commit'" in latest_query assert "status = 'active'" in latest_query assert "metadata_json #>> '{agentic_memory,agent_identity,agent_id}' = %s" in latest_query @@ -2635,7 +2644,7 @@ def test_create_memory_persists_commit_digest_and_confirmation_id_columns() -> N } ) - insert_query, insert_params = cursor.executed[0] + insert_query, insert_params = cursor.statements[0] assert "commit_digest" in insert_query assert "confirmation_id" in insert_query assert insert_params is not None @@ -2665,7 +2674,7 @@ def test_create_memory_persists_first_class_scope_columns() -> None: } ) - insert_query, insert_params = cursor.executed[0] + insert_query, insert_params = cursor.statements[0] assert "project_id" in insert_query assert "created_by_agent_id" in insert_query assert "run_id" in insert_query @@ -2702,7 +2711,7 @@ def test_create_agent_api_key_persists_project_scope_binding() -> None: ) assert row["project_scope"] == "alicebot" - insert_query, insert_params = cursor.executed[0] + insert_query, insert_params = cursor.statements[0] assert "INSERT INTO agent_api_keys" in insert_query assert "project_scope" in insert_query assert insert_params is not None @@ -2729,7 +2738,7 @@ def test_create_agent_api_key_persists_project_scope_binding() -> None: } ) assert unbound["project_scope"] is None - assert cursor.executed[0][1][3] is None + assert cursor.statements[0][1][3] is None # -- temporal slice: edge event time, as-of reads, supersession pointers ------- @@ -2758,7 +2767,7 @@ def test_create_edge_populates_observed_at_and_defaults_valid_from_to_event_time } ) - insert_query, insert_params = cursor.executed[0] + insert_query, insert_params = cursor.statements[0] assert "observed_at" in insert_query # observed_at defaults to write time; valid_from defaults to observed_at # (then write time), so the validity interval starts at event time. @@ -2797,7 +2806,7 @@ def test_create_edge_without_event_time_notes_the_write_time_fallback_in_metadat } ) - _insert_query, insert_params = cursor.executed[0] + _insert_query, insert_params = cursor.statements[0] assert insert_params is not None metadata_param = insert_params[-1] assert isinstance(metadata_param, Jsonb) @@ -2831,7 +2840,7 @@ def test_edge_digest_upsert_creates_once_and_replays_without_a_second_event() -> assert created["id"] == replayed["id"] == edge_id insert_query, insert_params = next( - (query, params) for query, params in cursor.executed if "INSERT INTO graph_edges" in query + (query, params) for query, params in cursor.statements if "INSERT INTO graph_edges" in query ) assert "ON CONFLICT DO NOTHING" in insert_query assert insert_params is not None @@ -2839,7 +2848,7 @@ def test_edge_digest_upsert_creates_once_and_replays_without_a_second_event() -> assert isinstance(metadata_param, Jsonb) assert metadata_param.obj["idempotency_digest"] == "edge-digest" assert _event_log_insert_count(cursor) == 1 - assert sum("INSERT INTO graph_edges" in query for query, _params in cursor.executed) == 1 + assert sum("INSERT INTO graph_edges" in query for query, _params in cursor.statements) == 1 def test_list_edges_as_of_filters_on_the_validity_interval_with_limit() -> None: @@ -2848,7 +2857,7 @@ def test_list_edges_as_of_filters_on_the_validity_interval_with_limit() -> None: store.list_edges_as_of("2026-07-01T00:00:00Z", limit=5) - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "FROM graph_edges" in query # Half-open interval: valid_from <= at < valid_to; NULL valid_from # (pre-slice edges with unrecorded event time) never matches. @@ -2867,6 +2876,7 @@ def test_memory_writes_accept_supersession_pointer_columns() -> None: fetchone_results=[ {"id": memory_id}, _event_row(memory_id), + {"id": memory_id}, # prior label read {"id": memory_id}, _event_row(memory_id), ] @@ -2886,12 +2896,12 @@ def test_memory_writes_accept_supersession_pointer_columns() -> None: patch={"status": "superseded", "superseded_by": successor_id}, ) - insert_query, insert_params = cursor.executed[0] + insert_query, insert_params = cursor.statements[0] assert "supersedes" in insert_query assert insert_params is not None assert insert_params[-2:] == (None, predecessor_id) # (superseded_by, supersedes) - update_query, update_params = cursor.executed[2] + update_query, update_params = next((query, params) for query, params in cursor.statements if "UPDATE memories" in query) assert "superseded_by = COALESCE(%s::uuid, superseded_by)" in update_query assert "supersedes = COALESCE(%s::uuid, supersedes)" in update_query assert update_params is not None @@ -2902,6 +2912,8 @@ def test_update_memory_reassigns_first_class_and_canonical_project_scope_togethe memory_id = str(uuid4()) cursor = RecordingCursor( fetchone_results=[ + {"id": memory_id, "metadata_json": {"project_scope": ["project-old"]}}, + {"metadata_json": {"project_scope": ["project-old"]}}, { "id": memory_id, "memory_key": "project.release.scope", @@ -2919,6 +2931,7 @@ def test_update_memory_reassigns_first_class_and_canonical_project_scope_togethe row = store.update_memory( memory_id=memory_id, + label_write=True, patch={ "project_id": "project-new", "metadata_json": { @@ -2928,7 +2941,7 @@ def test_update_memory_reassigns_first_class_and_canonical_project_scope_togethe }, ) - query, params = cursor.executed[0] + query, params = next((query, params) for query, params in cursor.statements if "UPDATE memories" in query) assert "project_id = COALESCE(%s, project_id)" in query assert params is not None metadata_param = next(param for param in params if isinstance(param, Jsonb)) @@ -2976,7 +2989,7 @@ def test_entity_crud_methods_normalize_names_and_write_audit_events() -> None: assert updated["id"] == entity_id assert _event_log_insert_count(cursor) == 2 - insert_query, insert_params = cursor.executed[0] + insert_query, insert_params = cursor.statements[0] assert "INSERT INTO vnext_entities" in insert_query assert "app.current_user_id()" in insert_query assert insert_params is not None @@ -2990,24 +3003,24 @@ def test_entity_crud_methods_normalize_names_and_write_audit_events() -> None: assert insert_params[5].obj == {"hq": "sf"} assert insert_params[8] == 0 # mention_count defaults to zero - get_query, get_params = cursor.executed[2] + get_query, get_params = cursor.statements[2] assert "FROM vnext_entities" in get_query assert "WHERE id = %s::uuid" in get_query assert "deleted_at IS NULL" in get_query assert get_params == (entity_id,) - by_name_query, by_name_params = cursor.executed[3] + by_name_query, by_name_params = cursor.statements[3] assert "entity_type = %s" in by_name_query assert "normalized_name = %s" in by_name_query assert "LIMIT 1" in by_name_query assert by_name_params == ("organization", "openai inc") - list_query, list_params = cursor.executed[4] + list_query, list_params = cursor.statements[4] assert "%s::text IS NULL OR entity_type = %s" in list_query assert "ORDER BY updated_at DESC, created_at DESC, id DESC" in list_query assert list_params == ("organization", "organization", 7) - update_query, update_params = cursor.executed[5] + update_query, update_params = cursor.statements[5] assert "UPDATE vnext_entities" in update_query assert "name = COALESCE(%s, name)" in update_query assert "aliases = COALESCE(%s, aliases)" in update_query @@ -3031,7 +3044,7 @@ def test_update_entity_rejects_immutable_patch_fields_before_touching_sql() -> N ): with pytest.raises(ContinuityStoreInvariantError, match="immutable"): store.update_entity(entity_id=str(uuid4()), patch=immutable_patch) - assert cursor.executed == [] + assert cursor.statements == [] def test_find_entities_by_names_matches_normalized_names_and_aliases_in_one_query() -> None: @@ -3044,8 +3057,8 @@ def test_find_entities_by_names_matches_normalized_names_and_aliases_in_one_quer rows = store.find_entities_by_names(("openai", "northwind.example")) assert rows[0]["id"] == "entity-1" - assert len(cursor.executed) == 1 # one round trip covers both match paths - query, params = cursor.executed[0] + assert len(cursor.statements) == 1 # one round trip covers both match paths + query, params = cursor.statements[0] assert "FROM vnext_entities" in query assert "normalized_name = ANY(%s::text[])" in query assert "aliases ?| %s::text[]" in query @@ -3055,7 +3068,7 @@ def test_find_entities_by_names_matches_normalized_names_and_aliases_in_one_quer # An empty name tuple short-circuits without touching the database. assert store.find_entities_by_names(()) == [] - assert len(cursor.executed) == 1 + assert len(cursor.statements) == 1 def test_record_entity_mention_increments_count_and_widens_window() -> None: @@ -3072,7 +3085,7 @@ def test_record_entity_mention_increments_count_and_widens_window() -> None: assert row["id"] == entity_id assert _event_log_insert_count(cursor) == 1 - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "mention_count = mention_count + 1" in query assert "LEAST(COALESCE(first_observed_at, %s::timestamptz), %s::timestamptz)" in query assert "GREATEST(COALESCE(last_observed_at, %s::timestamptz), %s::timestamptz)" in query @@ -3090,7 +3103,7 @@ def test_record_entity_mention_increments_count_and_widens_window() -> None: fresh_store = PostgresVNextStore(RecordingConnection(fresh_cursor)) with pytest.raises(ContinuityStoreInvariantError, match="observed_at"): fresh_store.record_entity_mention(entity_id=entity_id, observed_at=None) - assert fresh_cursor.executed == [] + assert fresh_cursor.statements == [] def test_record_relationship_change_appends_history_and_updates_current_pointer() -> None: @@ -3118,12 +3131,12 @@ def test_record_relationship_change_appends_history_and_updates_current_pointer( assert row["id"] == event_id assert _event_log_insert_count(cursor) == 1 - before_query, before_params = cursor.executed[0] + before_query, before_params = cursor.statements[0] assert "metadata_json ->> 'relationship_type'" in before_query assert "deleted_at IS NULL" in before_query assert before_params == (entity_id,) - insert_query, insert_params = cursor.executed[1] + insert_query, insert_params = cursor.statements[1] assert "INSERT INTO entity_relationship_events" in insert_query assert "app.current_user_id()" in insert_query assert insert_params is not None @@ -3135,7 +3148,7 @@ def test_record_relationship_change_appends_history_and_updates_current_pointer( assert isinstance(insert_params[5], Jsonb) assert insert_params[5].obj == {"round": "seed"} - pointer_query, pointer_params = cursor.executed[2] + pointer_query, pointer_params = cursor.statements[2] assert "UPDATE vnext_entities" in pointer_query assert "metadata_json = metadata_json || %s" in pointer_query assert pointer_params is not None @@ -3143,7 +3156,7 @@ def test_record_relationship_change_appends_history_and_updates_current_pointer( assert pointer_params[0].obj == {"relationship_type": "investor"} assert pointer_params[1] == entity_id - event_query, event_params = cursor.executed[3] + event_query, event_params = cursor.statements[3] assert "INSERT INTO event_log" in event_query assert event_params is not None assert isinstance(event_params[7], Jsonb) @@ -3156,7 +3169,7 @@ def test_record_relationship_change_appends_history_and_updates_current_pointer( missing_store = PostgresVNextStore(RecordingConnection(missing_cursor)) with pytest.raises(ContinuityStoreInvariantError, match="existing entity"): missing_store.record_relationship_change(entity_id=entity_id, relationship_type="advisor") - assert len(missing_cursor.executed) == 1 + assert len(missing_cursor.statements) == 1 def test_list_relationship_events_reads_history_most_recent_first() -> None: @@ -3170,7 +3183,7 @@ def test_list_relationship_events_reads_history_most_recent_first() -> None: rows = store.list_relationship_events(entity_id) assert rows[0]["id"] == "event-1" - query, params = cursor.executed[0] + query, params = cursor.statements[0] assert "FROM entity_relationship_events" in query assert "WHERE entity_id = %s::uuid" in query assert "ORDER BY changed_at DESC, id DESC" in query @@ -3194,7 +3207,7 @@ def execute(self, query: str, params: tuple[object, ...] | None = None) -> None: def _redaction_flag_statements(cursor: RecordingCursor) -> list[str]: - return [query for query, _params in cursor.executed if "app.redaction_in_progress" in query] + return [query for query, _params in cursor.statements if "app.redaction_in_progress" in query] def test_redaction_marker_constant() -> None: @@ -3288,9 +3301,9 @@ def test_quoted_provenance_rejects_exact_redacted_target_before_insert(target_ty } ) - assert len(cursor.executed) == 1 - assert "FOR UPDATE" in cursor.executed[0][0] - assert not any("INSERT INTO provenance_links" in query for query, _params in cursor.executed) + assert len(cursor.statements) == 1 + assert "FOR UPDATE" in cursor.statements[0][0] + assert not any("INSERT INTO provenance_links" in query for query, _params in cursor.statements) @pytest.mark.parametrize( @@ -3340,8 +3353,8 @@ def test_redact_memory_bundle_rejects_malformed_terminal_artifact_provenance( ], ) - assert len(cursor.executed) == 2 - assert "FROM event_log" in cursor.executed[1][0] + assert len(cursor.statements) == 2 + assert "FROM event_log" in cursor.statements[1][0] assert _redaction_flag_statements(cursor) == [] @@ -3399,7 +3412,7 @@ def now(cls, tz=None): update_query, update_params = next( (query, params) - for query, params in cursor.executed + for query, params in cursor.statements if "UPDATE memories" in query and "embedding_vector = NULL" in query ) assert update_params is not None @@ -3409,7 +3422,7 @@ def now(cls, tz=None): event_update_query = next( query - for query, _params in cursor.executed + for query, _params in cursor.statements if "UPDATE event_log" in query and "jsonb_build_object" in query ) # PostgreSQL cannot infer a type for a bare bind used only as a @@ -3434,14 +3447,14 @@ def test_redact_memory_content_wraps_marker_update_in_redaction_mode() -> None: row = store.redact_memory_content(memory_id=memory_id) assert row["id"] == memory_id - queries = [query for query, _params in cursor.executed] + queries = [query for query, _params in cursor.statements] assert "SELECT metadata_json" in queries[0] assert "set_config('app.redaction_in_progress', 'on', false)" in queries[1] assert "UPDATE memories" in queries[2] assert "set_config('app.redaction_in_progress', 'off', false)" in queries[3] assert "INSERT INTO event_log" in queries[4] - update_query, update_params = cursor.executed[2] + update_query, update_params = cursor.statements[2] # Content columns become the marker; skeleton and scope survive. assert "CASE WHEN title IS NULL THEN NULL ELSE %s END" in update_query assert "canonical_text = %s" in update_query @@ -3464,7 +3477,7 @@ def test_redact_memory_content_wraps_marker_update_in_redaction_mode() -> None: assert "note" not in scrubbed assert update_params[6] == memory_id - event_query, event_params = cursor.executed[4] + event_query, event_params = cursor.statements[4] assert event_params is not None assert event_params[1] == "memory.redacted" payload = next(param for param in event_params if isinstance(param, Jsonb)) @@ -3496,7 +3509,7 @@ def test_redact_memory_revisions_scrubs_content_columns_only() -> None: assert result == {"memory_id": memory_id, "redacted_revisions": 2} update_query, update_params = next( - (query, params) for query, params in cursor.executed if "UPDATE memory_revisions" in query + (query, params) for query, params in cursor.statements if "UPDATE memory_revisions" in query ) # NULL content stays NULL; non-NULL content becomes the marker shape. assert "CASE WHEN previous_value IS NULL THEN NULL ELSE %s END" in update_query @@ -3518,7 +3531,7 @@ def test_redact_memory_revisions_scrubs_content_columns_only() -> None: flags = _redaction_flag_statements(cursor) assert "'on'" in flags[0] and "'off'" in flags[1] - event_query, event_params = cursor.executed[-1] + event_query, event_params = cursor.statements[-1] assert "INSERT INTO event_log" in event_query assert event_params is not None assert event_params[1] == "memory.redacted" @@ -3538,7 +3551,7 @@ def test_redact_memory_events_scrubs_payloads_and_clears_integrity_hash() -> Non assert result == {"memory_id": memory_id, "redacted_events": 1} update_query, update_params = next( - (query, params) for query, params in cursor.executed if "UPDATE event_log" in query + (query, params) for query, params in cursor.statements if "UPDATE event_log" in query ) assert "jsonb_build_object" in update_query assert "'redacted', true" in update_query @@ -3557,7 +3570,7 @@ def test_redact_memory_events_scrubs_payloads_and_clears_integrity_hash() -> Non flags = _redaction_flag_statements(cursor) assert len(flags) == 2 and "'on'" in flags[0] and "'off'" in flags[1] - event_query, event_params = cursor.executed[-1] + event_query, event_params = cursor.statements[-1] assert "INSERT INTO event_log" in event_query assert event_params is not None assert event_params[1] == "memory.redacted" @@ -3583,5 +3596,5 @@ def test_redaction_mode_resets_even_when_the_update_fails() -> None: assert "'off'" in flags[1] # The reset is the last statement issued; no event is appended after # a failed redaction. - assert "app.redaction_in_progress" in cursor.executed[-1][0] - assert not any("INSERT INTO event_log" in query for query, _params in cursor.executed) + assert "app.redaction_in_progress" in cursor.statements[-1][0] + assert not any("INSERT INTO event_log" in query for query, _params in cursor.statements) From 67661197fd557595399beee8ba71a03b7e3ebac7 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:28:01 +0200 Subject: [PATCH 040/270] test: make pointer fence fixture relabels explicit --- tests/unit/test_correction_label_respects_the_read_fence.py | 2 +- tests/unit/test_memory_id_pointer_residue.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_correction_label_respects_the_read_fence.py b/tests/unit/test_correction_label_respects_the_read_fence.py index 4b659e232..b7ec96701 100644 --- a/tests/unit/test_correction_label_respects_the_read_fence.py +++ b/tests/unit/test_correction_label_respects_the_read_fence.py @@ -100,7 +100,7 @@ def _supersede(context, memory_id: str, text: str = NEW) -> str: def _set(context, memory_id: str, **patch) -> None: - _store(context, lambda s: s.update_memory(memory_id=memory_id, patch=patch, actor_type="user")) + _store(context, lambda s: s.update_memory(memory_id=memory_id, patch=patch, actor_type="user", label_write=True)) def _unquoted(value: object) -> str: diff --git a/tests/unit/test_memory_id_pointer_residue.py b/tests/unit/test_memory_id_pointer_residue.py index 20db1675f..443a65d49 100644 --- a/tests/unit/test_memory_id_pointer_residue.py +++ b/tests/unit/test_memory_id_pointer_residue.py @@ -109,7 +109,7 @@ def _supersede(context, memory_id: str, text: str = NEW) -> str: def _set(context, memory_id: str, **patch) -> None: - _store(context, lambda s: s.update_memory(memory_id=memory_id, patch=patch, actor_type="user")) + _store(context, lambda s: s.update_memory(memory_id=memory_id, patch=patch, actor_type="user", label_write=True)) def _add_metadata(context, memory_id: str, **keys) -> None: @@ -117,7 +117,7 @@ def merge(store): row = store.get_memory(memory_id) metadata = dict(row["metadata_json"]) metadata.update(keys) - store.update_memory(memory_id=memory_id, patch={"metadata_json": metadata}, actor_type="user") + store.update_memory(memory_id=memory_id, patch={"metadata_json": metadata}, actor_type="user", label_write=True) _store(context, merge) From 5d98fb5183f2bd9697266554c7cc7642c3dd0253 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:28:01 +0200 Subject: [PATCH 041/270] test: separate original quote projection from derived row denial --- ...st_saved_quotes_follow_the_source_fence.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/unit/test_saved_quotes_follow_the_source_fence.py b/tests/unit/test_saved_quotes_follow_the_source_fence.py index 485a60646..c1d8a66b5 100644 --- a/tests/unit/test_saved_quotes_follow_the_source_fence.py +++ b/tests/unit/test_saved_quotes_follow_the_source_fence.py @@ -204,6 +204,21 @@ def _candidate(self, text: str) -> str: rows = self.sql("SELECT id, canonical_text FROM memories WHERE status = 'candidate' ORDER BY created_at DESC") return str(next(row["id"] for row in rows if text.split(":")[0] in str(row["canonical_text"]))) + def _original_quote_fixture(self, memory_id: str) -> None: + """Keep every saved quote/link while isolating original-row quote projection. + + Captured copies now inherit source labels and can be refused as a whole. + These older projection tests need a readable original row with saved provenance. + """ + path = _sqlite_path_from_url(self.context.database_url) + with sqlite_user_connection(path, _USER_ID) as conn: + row = SQLiteVNextStore(conn, _USER_ID).get_memory(memory_id) + metadata = dict(row["metadata_json"]) + metadata.pop("source_id", None) + metadata.pop("derived_from", None) + metadata["project_floor"] = [] + conn.execute("UPDATE memories SET metadata_json = ? WHERE id = ? AND user_id = ?", (json.dumps(metadata), memory_id, _USER_ID)) + def edit_and_approve(self, source_id: str, tag: str = "") -> tuple[str, str]: """``metadata_json.provenance`` and the link quote. Returns the memory id and a query that finds it.""" @@ -219,6 +234,7 @@ def edit_and_approve(self, source_id: str, tag: str = "") -> tuple[str, str]: who=self.reviewer, ) assert done["is_error"] is False, done + self._original_quote_fixture(candidate) return candidate, "kiln schedule Mondays" def supersede(self, source_id: str, tag: str = "") -> tuple[str, str]: @@ -239,6 +255,7 @@ def supersede(self, source_id: str, tag: str = "") -> tuple[str, str]: ) assert done["is_error"] is False, done replacement = done["payload"]["replacement_object"] # type: ignore[index] + self._original_quote_fixture(str(replacement["id"])) return str(replacement["id"]), "glaze shelf reorganised Saturday" def http_commit(self, source_id: str, tag: str = "") -> tuple[str, str]: @@ -1165,3 +1182,17 @@ def test_a_source_with_no_project_is_outside_the_fence_of_a_key_bound_to_a_proje assert (vault.explain(who, candidate)["is_error"] is False) is readable, ("explain agrees", who) + + +def test_current_derived_copy_is_denied_as_a_whole_after_source_relabel(vault: _Vault) -> None: + source_id = vault.capture_source() + memory_id, _query = vault.edit_and_approve(source_id) + row = vault.sql("SELECT metadata_json FROM memories WHERE id = ?", (memory_id,))[0] + metadata = json.loads(row["metadata_json"]) + metadata["source_id"] = source_id + vault.sql("UPDATE memories SET metadata_json = ? WHERE id = ?", (json.dumps(metadata), memory_id)) + vault.reclassify(source_id, "confidential") + result = vault.review("trusted", memory_id) + assert result["is_error"] is True + assert _WORD_A not in json.dumps(result, default=str) + assert vault.review("admin", memory_id)["is_error"] is False From 4331fb9e8c4a933a2391e14f0b5a92e594cab826 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:28:01 +0200 Subject: [PATCH 042/270] test: align route adapters with label closure and exact derived policy --- tests/unit/test_vnext_main.py | 37 +++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_vnext_main.py b/tests/unit/test_vnext_main.py index 8af8173c1..997968e0c 100644 --- a/tests/unit/test_vnext_main.py +++ b/tests/unit/test_vnext_main.py @@ -55,6 +55,25 @@ def __init__(self, _conn) -> None: self.browser_clip_capabilities: dict[str, dict[str, object]] = {} self.revisions: list[dict[str, object]] = [] + def lock_graph_mutation(self) -> None: + return None + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + return None + + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + collection = {"source": self.sources.values(), "memory": self.memories, "open_loop": self.open_loops, + "artifact": self.artifacts.values(), "belief": self.beliefs.values(), "project": self.projects.values()}.get(kind, []) + return [dict(row) for row in collection if str(row.get("id")) in ids] + + def _fetch_all(self, query: str, _params: tuple[object, ...]) -> list[dict[str, object]]: + # The label dependant walker performs its exact canonical reference filter after this prefilter. + for table, kind in (("memories", "memory"), ("open_loops", "open_loop"), ("generated_artifacts", "artifact"), ("projects", "project")): + if f"FROM {table}" in query: + collection = {"memory": self.memories, "open_loop": self.open_loops, "artifact": self.artifacts.values(), "project": self.projects.values()}[kind] + return [{**row, "kind": kind} for row in collection] + raise AssertionError(query) + def create_browser_clip_capability( self, *, @@ -879,6 +898,8 @@ def list_connector_states(self) -> list[dict[str, object]]: def _install_fake_vnext_store(monkeypatch, store: FakeVNextStore) -> None: + from alicebot_api import vnext_label_writes + monkeypatch.setattr(vnext_label_writes, "acquire_exclusive_label_lock", lambda target: target.lock_label_writes(exclusive=True)) @contextmanager def fake_user_connection(database_url, current_user_id): assert database_url == "postgresql://db" @@ -1912,7 +1933,7 @@ def test_create_vnext_source_threads_project_scope_into_captured_memory(monkeypa candidates = store.list_memories(status="candidate") assert candidates, "capture must promote at least one candidate memory" - assert memory_project_scope(candidates[0]) == ("Project-Helios", "project-helios") + assert memory_project_scope(candidates[0]) == ("Project-Helios",) for memory in candidates: store.update_memory(memory_id=str(memory["id"]), patch={"status": "active"}, actor_type="system") @@ -2313,7 +2334,7 @@ def test_vnext_artifact_trace_authorizes_sources_from_complete_persisted_scope_e } store.artifacts[artifact_id] = { "id": artifact_id, - "artifact_type": "daily_brief", + "artifact_type": "manual_note", "title": "Scoped trace", "content_markdown": "# Scoped trace", "status": "needs_review", @@ -2575,6 +2596,7 @@ def test_vnext_contradiction_and_belief_endpoints(monkeypatch) -> None: "sensitivity": "private", "memory_type": "belief", } + store.memories.append({"id": "memory-belief-1", "domain": "project", "sensitivity": "private", "canonical_text": "Alice should auto-promote generated artifacts into memory.", "status": "active", "metadata_json": {}}) _install_fake_vnext_store(monkeypatch, store) user_id = uuid4() @@ -4474,6 +4496,8 @@ def test_artifact_quality_rating_rejects_alias_forgery_and_rerates_authenticated "sensitivity": "private", } ) + from alicebot_api.vnext_derived_labels import stamp_derived_from + stamp_derived_from(artifact, {}) _record, raw_key = create_agent_key( store, user_id=user_id, agent_id="reviewer", permission_profile="trusted_local_agent" ) @@ -4575,6 +4599,8 @@ def test_artifact_routes_authorize_persisted_target_scope_and_profile(monkeypatc "metadata_json": {"project_id": "project-b"}, } + from alicebot_api.vnext_derived_labels import stamp_derived_from + stamp_derived_from(store.artifacts[artifact_id], {}) _reader_record, reader_key = create_agent_key( store, user_id=user_id, @@ -4612,6 +4638,13 @@ def test_artifact_routes_authorize_persisted_target_scope_and_profile(monkeypatc "source_refs": [f"source:{sensitive_source_id}"], }, } + derived_status, _derived_payload = _invoke_vnext_request( + "GET", f"/v0/vnext/traces/artifacts/{public_artifact_id}", + query={"user_id": str(user_id)}, authorization=f"Bearer {reader_key}", + ) + assert derived_status == 403 + # A manual original public artifact exercises per-source trace projection separately. + store.artifacts[public_artifact_id]["artifact_type"] = "manual_note" trace_status, trace_payload = _invoke_vnext_request( "GET", f"/v0/vnext/traces/artifacts/{public_artifact_id}", From c09857a4176ab60b6c8acf995952bd9ae4468989 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:35:56 +0200 Subject: [PATCH 043/270] Serialize rollup member IDs as JSON lists before label settlement --- apps/api/src/alicebot_api/vnext_rollups.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_rollups.py b/apps/api/src/alicebot_api/vnext_rollups.py index 0bb942434..2170e16b3 100644 --- a/apps/api/src/alicebot_api/vnext_rollups.py +++ b/apps/api/src/alicebot_api/vnext_rollups.py @@ -2706,7 +2706,7 @@ def _create_rollup_candidate( "grouping_input_count": grouping_input_count, "grouping_input_total": grouping_input_total, "grouping_input_total_exact": grouping_input_total_exact, - "member_ids": member_ids, + "member_ids": list(member_ids), "instances": instances, } if revises_memory_id is not None: @@ -2860,7 +2860,7 @@ def propose_rollups( "rollup_key": group.rollup_key, "group_kind": group.group_kind, "label": group.label, - "member_ids": member_ids, + "member_ids": list(member_ids), "rollup_digest": rollup_digest, "aggregation": group.utility.to_record(), } From 0d2dec67db6170a633d59e234daec3e5a264798c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:35:56 +0200 Subject: [PATCH 044/270] Confirm source scope moves in identity integration fixtures --- tests/integration/test_source_review_identity_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_source_review_identity_api.py b/tests/integration/test_source_review_identity_api.py index 19078d603..ff170c650 100644 --- a/tests/integration/test_source_review_identity_api.py +++ b/tests/integration/test_source_review_identity_api.py @@ -110,6 +110,7 @@ def test_source_review_http_rotates_identity_and_recapture_uses_new_envelope( payload={ "user_id": str(user_id), "action": "assign_project", + "confirm_label_hide": True, "project_id": "Beta", "domain": "professional", "sensitivity": "internal", @@ -195,6 +196,7 @@ def test_source_review_http_collision_returns_409_after_full_transaction_rollbac payload={ "user_id": str(user_id), "action": "assign_project", + "confirm_label_hide": True, "project_id": "Beta", "review_note": "This conflicts with Beta's live source.", }, From abfb6a25b9d7c7005e53b71d55772858476c2b7d Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:35:56 +0200 Subject: [PATCH 045/270] Parse documented nested consolidation membership strictly --- apps/api/src/alicebot_api/vnext_derived_labels.py | 10 ++++++++++ tests/unit/test_derived_labels_kernel.py | 13 +++++++++++++ 2 files changed, 23 insertions(+) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index d368dea30..e602e9ede 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -502,6 +502,16 @@ def _collect_metadata_ids(value: object, found: set[tuple[str, str]]) -> str: elif isinstance(item, str): found.add(("memory", identifier(item))) continue + if key == "cluster_membership" and isinstance(child, list) and any(isinstance(item, list) for item in child): + # Consolidation records one list of member IDs per cluster. + # Keep its established nested JSON shape, but reject mixed or + # non-string membership rather than silently omitting inputs. + if any(not isinstance(cluster, list) or any(not isinstance(item, str) for item in cluster) for cluster in child): + problem = problem or "malformed" + else: + for cluster in child: + _add_ids(found, "memory", _strings(cluster)) + continue if key in _ID_KIND: parsed = _as_string_list(child if isinstance(child, list) else [child] if isinstance(child, str) else child) if parsed is None: diff --git a/tests/unit/test_derived_labels_kernel.py b/tests/unit/test_derived_labels_kernel.py index a56de91f1..30ea192e6 100644 --- a/tests/unit/test_derived_labels_kernel.py +++ b/tests/unit/test_derived_labels_kernel.py @@ -846,3 +846,16 @@ def test_weekly_parent_backfill_cannot_clear_ambiguous_identity() -> None: settled = settle_labels([_source("s"), one, two, parent]) assert settled.by_stored("memory", canonical).unverified is True assert settled.by_stored("memory", "{" + canonical + "}").unverified is True + + +@pytest.mark.parametrize("membership, expected", [([["member-a"], ["member-b"]], False), ([["member-a"], [42]], True), ([["member-a"], "member-b"], True)]) +def test_nested_consolidation_membership_is_strict(membership, expected): + rows = [ + {"kind": "memory", "id": "member-a", "user_id": USER, "domain": "health", "sensitivity": "confidential"}, + {"kind": "memory", "id": "member-b", "user_id": USER, "domain": "personal", "sensitivity": "public"}, + {"kind": "artifact", "id": "report", "user_id": USER, "artifact_type": "memory_consolidation", "domain": "unknown", "sensitivity": "public", "metadata_json": {"consolidation": {"cluster_membership": membership}}}, + ] + report = settle_labels(rows).by_stored("artifact", "report", user_id=USER) + assert report.unverified is expected + if not expected: + assert (report.domain, report.sensitivity) == ("health", "confidential") From a7f4265c7e6c06aad8bdaf7b7f3fe0d8128c4b5d Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:38:39 +0200 Subject: [PATCH 046/270] test: pin reviewed label-lock and lifecycle carrier changes --- tests/unit/test_store_embedding_cas_split.py | 26 ++++++++++++++----- .../unit/test_store_memory_lifecycle_split.py | 21 +++++++++------ 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/tests/unit/test_store_embedding_cas_split.py b/tests/unit/test_store_embedding_cas_split.py index eb8194bb0..200c00ee7 100644 --- a/tests/unit/test_store_embedding_cas_split.py +++ b/tests/unit/test_store_embedding_cas_split.py @@ -47,10 +47,13 @@ # whose ``valid_to`` has passed, with the test recall's own SQL uses # (``_expiry_clause`` on SQLite, ``POSTGRES_UNEXPIRED_SQL`` on Postgres), so the # text of an expired memory is never listed for embedding. +# Re-minted for derived-label locking: update and clear now take the shared +# label lock before their existing SQL. Only the decorator changes each AST; +# PG query receipts below prepend that lock and retain the prior query hashes. EXPECTED_METHOD_AST_SHA256 = { "postgres": { - "update_memory_embedding": "0cd0f0ef6f7bcaa6328b6f586a711a10b011c77af3e49d65b96c27b77a657cd9", - "clear_memory_embedding": "4e9fe6955f3246b51998c6b547f48a659f947f8a8150e6c86d4e61a0cf46df6c", + "update_memory_embedding": "291a378fe61e93c43dd7a971de31a07601feaecb470dc578f6229ae123cdbec4", + "clear_memory_embedding": "0ce41174d08f8d515ef171253203af4e4115cc432c6b51eeced50cef1bf1dcd7", "list_memories_missing_embeddings": "cfdbde2bb409a2a6761fe31f116a9ebb73ba12d4e7994ae691f87e335a095a2d", }, # SQLite update/clear re-minted for the Phase 4 Stage 2 resident vector @@ -70,8 +73,8 @@ # byte what it was, which ``test_embedding_cas_generated_sql_is_byte_identical`` # still pins. "sqlite": { - "update_memory_embedding": "1f4517352a0f7d6a9147f326bc96a6c1d61effa3f106add89546cd981ddd05fc", - "clear_memory_embedding": "51b583b250883911f0c5a068fec7ec4565f719c2bffafb1ed1c6b3dc980fa36c", + "update_memory_embedding": "a87ce54306b804ce87437ceeee55935f584bcce2a4a8ca36e20f5af3129c1de4", + "clear_memory_embedding": "ebd18c7058ab7be4610ed9eb4053f8ca115d44055ddcc550cd2a892dae5c78e1", "list_memories_missing_embeddings": "bea1d517cd3ad4d0af12c96d93a5b21671b84a4712d5172e718f5af5b43c4b05", }, } @@ -156,9 +159,18 @@ # They were re-minted again for the expiry test: each query holds the unexpired # test right after the status test (SQLite binds the time after the statuses). EXPECTED_QUERY_SHA256 = { - "postgres_unsigned_update": ("dcbf4bc29a7702e9c17d864f65e1c1f36d641927d3f31aa4ec80825646c030ef",), - "postgres_signed_update": ("1351db18168f7e23454736129e26a7c01039ca1bfdfcac235e3c666cf60d91db",), - "postgres_clear": ("a5a6952a93bd77b3bdf311fe2682b411263d18a2822a9617c6fb7524555123ca",), + "postgres_unsigned_update": ( + "1f866e65df3baaa9d7790266ff6b90630fa472a974bfd217894292df72a25a7f", + "dcbf4bc29a7702e9c17d864f65e1c1f36d641927d3f31aa4ec80825646c030ef", + ), + "postgres_signed_update": ( + "1f866e65df3baaa9d7790266ff6b90630fa472a974bfd217894292df72a25a7f", + "1351db18168f7e23454736129e26a7c01039ca1bfdfcac235e3c666cf60d91db", + ), + "postgres_clear": ( + "1f866e65df3baaa9d7790266ff6b90630fa472a974bfd217894292df72a25a7f", + "a5a6952a93bd77b3bdf311fe2682b411263d18a2822a9617c6fb7524555123ca", + ), "postgres_unsigned_missing": ("866920a62d5650df8e229d0dffa43ac0a8ce18116e3cbb98376928f19b239534",), "postgres_signed_missing": ("8593736f07c1853635e8d3868e6a8b54a034ccd3de1a3519abf885a0ff23fa3e",), # SQLite update/clear sequences start with BEGIN IMMEDIATE (the capture diff --git a/tests/unit/test_store_memory_lifecycle_split.py b/tests/unit/test_store_memory_lifecycle_split.py index 87ad25b53..2f8b6c208 100644 --- a/tests/unit/test_store_memory_lifecycle_split.py +++ b/tests/unit/test_store_memory_lifecycle_split.py @@ -77,9 +77,14 @@ # row is created in or moved into a searchable status. Previous receipts: # common 191f0ebd..., postgres 1960ff3d..., sqlite 5ebda2b3...; method ASTs # postgres 538d5a18..., sqlite 5bd15d28.... Metadata manifests are unchanged. +# Re-minted for the derived-label write boundary: inserts settle their full +# ancestry, updates preserve protected metadata and propagate labels, and +# mutators take the label lock. Metadata receipts include the label_write +# keyword and lock wrappers. Facades add only lock_label_writes/read_label_rows; +# removing those two names reproduces each previous class-order receipt. SOURCE_RECEIPTS = { COMMON_PATH: "8fc077dc71f0e631a2df81de2ebeec1fb6c768f341c2e7891309e4753eef7bb5", - POSTGRES_CARRIER_PATH: "65e23bf5c8809a5dabe3f3339500f4a8f879258b2d9ca5a91acae3331d9b54a3", + POSTGRES_CARRIER_PATH: "371f92595d2f72f0cfa225a49c03aa39498caf094df4f26f4f9ac4cd926e0de1", # SQLite carrier re-minted for the Phase 4 Stage 2 resident vector cache # (reviewed change): redaction paths that NULL a live embedding now bump # the embedding_stamp token in the same transaction (prompt eviction). @@ -90,15 +95,15 @@ # back between two reads cannot fail memories_seen_range_check. Previous # sqlite receipt 67adaa61..., method AST 3f134ac9...; the metadata # manifests are unchanged. - SQLITE_CARRIER_PATH: "c37f6b8012de25c3e702705909ba5669141af15d6c4ad5ac381915207b863615", + SQLITE_CARRIER_PATH: "f893f1faeeb9a87135c108992aa91ff036c5f5dccb1bbdaf315ae5afe1b89b73", } EXPECTED_METHOD_AST_MANIFESTS = { - "postgres": "e937452df97467820cbcb42938b5f4a2336cd0157f0ca69f8f6420d4ee85211b", - "sqlite": "df43d593a59eb6deaf3ba935c09382e330b96a90b9f0b63314712619ba468a0d", + "postgres": "d4969140e86b136b29708dc3bb6b4bca635b73b4016c4e633c4d5d3da841e784", + "sqlite": "d0b6024f0803ca9d3c6f6ea7f5022a28453b2b8b83833f0f05343cb3eff89a57", } EXPECTED_METADATA_MANIFESTS = { - "postgres": "af03955c805f720b8d3ec735f8202efeb5f405c8c7de1cc45cbfef3644867824", - "sqlite": "9a5a4a9f0ae533652250a9e9854cd34a068392d71ffde98b012c9c620134d2c4", + "postgres": "07a567e26d0f7c4f51ae2a1910d059397b513a575d85f06c2f8c0396ac1bb2ef", + "sqlite": "1270ef0115988552418349aa9e94a7442ba04be41443f278f68a1fa81857903a", } EXPECTED_CLASS_ORDERS = { # Two paired browser-clip capability methods extend both façades, and one @@ -106,7 +111,7 @@ # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (172, "6f1a459fcf4319cf4281f6cc0d4e81679c3e05d851fd0a874a2d90298d7c2569"), + "PostgresVNextStore": (174, "095250b8a77d6c0a32d2783343916b947bcae8ae86e3a1693e47c2fb11802211"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -121,7 +126,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. - "SQLiteVNextStore": (134, "1301272026897057cf071009cc21787543ddc326f1e06f1a75a763f3e344767e"), + "SQLiteVNextStore": (136, "1d99dbaf69e4ea888ca7beb2389ac176650a2e573c067bf0686adc9e609132f5"), } EXPECTED_FACADE_COMMENT_DIGESTS = { POSTGRES_FACADE_PATH: "d8599a46ee26dc35a3ae52c1a98a416509add9ae4a42ece780c5c5ed7e132b93", From eab3801c34a1c0458aa895be07f5e46002a1384a Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:39:10 +0200 Subject: [PATCH 047/270] test: pin workspace effective-label filtering helpers --- tests/unit/test_workspaces_router_split.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_workspaces_router_split.py b/tests/unit/test_workspaces_router_split.py index 9f0aa630c..20ec038c0 100644 --- a/tests/unit/test_workspaces_router_split.py +++ b/tests/unit/test_workspaces_router_split.py @@ -21,7 +21,7 @@ "bootstrap_v1_workspace", "get_v1_workspace_bootstrap_status", ) -SUPPORT_NAMES = ("_vnext_status_counts", "_vnext_workspace_payload") +SUPPORT_NAMES = ("_vnext_status_counts", "_vnext_workspace_payload", "_workspace_rows", "_workspace_event_visible") PARTITION_ROUTE_NAMES = { "core_router": ("get_vnext_workspace",), "bootstrap_router": ( @@ -78,11 +78,14 @@ "summarize_agent_policy_telemetry", } +# Re-minted support/import receipts for effective-label workspace filtering. +# Two local helpers admit rows and check event targets; the payload uses them +# before returning lists/counts. Route bodies, mounts and middleware are unchanged. EXPECTED_ROUTE_AST_SHA256 = "fb8925ebcda058598b0c6d5e7eca83abe18606a0126bdb6cde0fd1c22444a795" -EXPECTED_SUPPORT_AST_SHA256 = "878128b51d8e8fd091f189f4595ab7774c736667f68e251399e476f9086089df" +EXPECTED_SUPPORT_AST_SHA256 = "f5e8747a48d68e9916c7412c10bd2dea082bd4e34487842703b36cb2fba840b4" EXPECTED_ROUTE_NAME_MANIFEST_SHA256 = "225c57c08bd8314156c56352dd1c53ffed3f556ce285c666dd6fca125115d0b4" EXPECTED_OPERATION_MANIFEST_SHA256 = "c320979b62d7ee8de244fe38bde5bf3761a4f9d76f76bf3cd8576c30fce9857e" -EXPECTED_IMPORT_MANIFEST_SHA256 = "8d9669a4024ea5258cd50f92ac290c2a040ff224dd0a67b5c60faed5ae722517" +EXPECTED_IMPORT_MANIFEST_SHA256 = "6d6812ae6907dcbe9ee16ac9481c741241b256fac745133415ccd57813bc67bb" EXPECTED_CARRIER_NAMES_SHA256 = "2c109fc234a05dd8f44e4c34bee49e797fbb5e49e92413391541a7e504da328b" # Re-pinned 2026-10-02 (DB-005, legacy /v0 routes). One definition changed, # found by a per-definition AST diff against the previous pin: @@ -130,7 +133,9 @@ } EXPECTED_SUPPORT_NODE_SHA256 = { "_vnext_status_counts": "0bf0ed228a14bd648a9d18fcd5f99ebf8c585bd29f4b5e81e1df17fe0201fd15", - "_vnext_workspace_payload": "166fc46cc669ff465eb7b1fb3b49be7e1b9d0aeba40872abcb3d958906914e90", + "_vnext_workspace_payload": "4b6e4a3d59d16538b0e859c425ede21207e9c80f11ab31c3af2ce4d604e14edf", + "_workspace_rows": "cc312a1300045147caf7340463b91d388c1932ba4edcf8f50fab22cf717999f6", + "_workspace_event_visible": "8343c060909326a5cb69fa6f671ac62f160630ecf989f04d78e792ea74c0ea90", } EXPECTED_ROUTE_MANIFEST = [ ("GET", "/v0/vnext/workspace", "get_vnext_workspace"), @@ -477,7 +482,7 @@ def test_workspace_import_direction_pruning_timing_and_runtime_identities_are_ex main_definitions = _top_level_definitions(main_tree) router_imports = _import_manifest(router_tree) - assert len(router_imports) == 30 + assert len(router_imports) == 31 assert hashlib.sha256(json.dumps(router_imports, separators=(",", ":")).encode()).hexdigest() == ( EXPECTED_IMPORT_MANIFEST_SHA256 ) From e9aba862b95eac273fc5e487013d231e0cc18fe9 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:36:52 +0200 Subject: [PATCH 048/270] Preserve reader row types through sensitivity ceiling helpers --- apps/api/src/alicebot_api/routers/workspaces.py | 4 ++-- apps/api/src/alicebot_api/vnext_context_tree.py | 2 +- apps/api/src/alicebot_api/vnext_label_guard.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index 4a95353ea..1ae7c314a 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from uuid import UUID from fastapi import APIRouter, Request @@ -71,7 +71,7 @@ def _workspace_event_visible(store: PostgresVNextStore, event: dict[str, object] return bool(_workspace_rows(store, kind, [row], sensitivity)) -def _workspace_rows(store: PostgresVNextStore, kind: str, rows: list[dict[str, object]], sensitivity: list[str]): +def _workspace_rows(store: PostgresVNextStore, kind: str, rows: Sequence[Mapping[str, object]], sensitivity: list[str]): from alicebot_api.vnext_label_guard import admit_loaded return admit_loaded( diff --git a/apps/api/src/alicebot_api/vnext_context_tree.py b/apps/api/src/alicebot_api/vnext_context_tree.py index 1ab93057f..3706404a9 100644 --- a/apps/api/src/alicebot_api/vnext_context_tree.py +++ b/apps/api/src/alicebot_api/vnext_context_tree.py @@ -162,7 +162,7 @@ def _label(row: JsonObject, *keys: str, fallback: str) -> str: return fallback -def _tree_event_visible(store: object, event: JsonObject, domains: list[str], sensitivity: list[str], projects: tuple[str, ...]) -> bool: +def _tree_event_visible(store: object, event: JsonObject, domains: list[str] | None, sensitivity: list[str], projects: tuple[str, ...]) -> bool: from alicebot_api.vnext_label_guard import admit_loaded kind = str(event.get("target_type") or "") diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index e9a486e9c..8122354d0 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -250,9 +250,9 @@ def apply_sensitivity_ceiling( store: Any, *, kind: str, - rows: Sequence[Mapping[str, object]], + rows: Sequence[_Row], identity: AgentIdentity | None, -) -> list[Mapping[str, object]]: +) -> list[_Row]: """Rows whose effective sensitivity is inside the caller's ceiling. A missing identity and an admin key keep every row, including its title From ebf11ea78ee5a3237a2de80994433fff4287c98c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:37:46 +0200 Subject: [PATCH 049/270] Give workspace scope fixtures valid recorded ancestry --- tests/integration/test_vnext_live_workspace_api.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_vnext_live_workspace_api.py b/tests/integration/test_vnext_live_workspace_api.py index 945c1e4d7..504648ec4 100644 --- a/tests/integration/test_vnext_live_workspace_api.py +++ b/tests/integration/test_vnext_live_workspace_api.py @@ -1158,6 +1158,7 @@ def test_vnext_live_workspace_happy_path_writes_reviewable_postgres_state( ] ), "title": "Live workspace launch note", + "project_scope": [project_id], "domain": "project", "sensitivity": "private", }, @@ -1815,7 +1816,13 @@ def test_vnext_artifact_routes_enforce_persisted_scope_with_live_postgres( "status": "needs_review", "domain": "project", "sensitivity": "private", - "metadata_json": {"project_id": "project-b"}, + "metadata_json": { + "project_id": "project-b", + "derived_from": { + "v": 1, "sources": [], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], + "counts": {"sources": 0, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, + }, + }, }, actor_type="user", ) From 763605ab4c1adf861371da40ba89be8ee5fb44b4 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 19:42:05 +0000 Subject: [PATCH 050/270] Document derived labels, the sensitivity ceiling, and the v3 repair. The operator pages now state the three-part label, who reads an unverified row, and that the five screens apply the caller's sensitivity ceiling. The v0.20.0 notes gain a dated correction. --- CHANGELOG.md | 7 +- docs/alpha/agent-integration.md | 4 + docs/alpha/backup-and-restore.md | 63 +++++++----- docs/alpha/known-limitations.md | 2 +- docs/alpha/mcp-tools.md | 10 +- docs/release/v0.20.0-release-notes.md | 2 + docs/runbooks/disaster-recovery.md | 4 +- docs/security/auth-authorization.md | 3 +- tests/unit/test_derived_domain_docs.py | 86 ++++++++++------ tests/unit/test_derived_labels_docs.py | 97 +++++++++++++++++++ .../unit/test_known_limitations_page_shape.py | 4 +- 11 files changed, 215 insertions(+), 67 deletions(-) create mode 100644 tests/unit/test_derived_labels_docs.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f4037cc5b..5410ca815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Unreleased (on main, not in v0.20.0): the derived-row pages state the three-part label, who reads an unverified row, and that the five operator screens apply the caller's sensitivity ceiling. The v0.20.0 notes omitted both limits. No migration is required. - Unreleased (on main, not in v0.20.0): PostgreSQL migration `20261005_0096` raises a stored derived row to the labels of its inputs, and `alicebot vnext labels` and `alice-memory labels` check and repair the same rows. The doctors print how many derived rows are below their inputs or unverified, as a warning. v0.20.0 left the stored label where it was written. - Unreleased (on main, not in v0.20.0): opening a SQLite vault raises a derived row whose stored label is below its inputs, and a restore does the same before it publishes. A repair that cannot finish leaves the vault open and the completion key unstamped. v0.20.0 left the stored label where it was written. - Unreleased (on main, not in v0.20.0): recall, context packs, resume, recent decisions, the session brief, the review queue, open-loop lists, and the workspace, dashboard, context tree, and dogfooding lists drop a loaded row whose effective label is outside the request. The artifact list, the source trace, the project list, the dashboard project row, and the belief state route apply the caller's sensitivity ceiling, including titles, ids, and counts. Resume, recent decisions, the session brief, audit, and explain apply the target row's current effective label, so an excluded key does not learn of a label change. v0.20.0 returned those rows from the label stored on them. No migration is required. @@ -11,11 +12,11 @@ - Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. Raising a source or a memory walks the rows derived from it and raises those too. A candidate open loop and a generated artifact take the same insert floor. Replacing a SQLite source with a stricter one raises the memories that cite it before they are retired. Moving a source to another project answers with how many derived rows a project-bound key would lose, and waits for confirm_label_hide before it writes. A relabel that cannot finish answers 409, and one that waits more than 3 seconds for the label lock answers 503. No migration is required. - Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. The function is what the write path calls. A read that does not go through that path still behaves as it does in v0.20.0. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` now scrub an open loop that names the source only in its metadata: the id, or `source:`, as the text under `source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references` or `selected_source_ids` at any depth. That is the rule the open-loop lookup of a source already used. They blanked only loops whose `source_id` column held the id, so a loop with an empty column kept its title, description and metadata, and they reached a new export. The delete preview now counts the same loops as the receipt. The rule reads the id only as stored or as `source:`: a loop that names the source in any spelling other than those two keeps its text, although the memory rule reads several other spellings. No migration is required. -- Unreleased (on main, not in v0.20.0): the SQLite derived-label repair now updates and records each row under the id it is stored with (SQLite keeps capitals, missing hyphens, braces and `urn:uuid:` as written) and matches rows by the normalised id only to read the graph. If two stored spellings of one id exist, each is compared with the label its own recorded inputs give it, so neither is left at its old label while the other is relabelled, and neither is lowered. An update that changes no row stops the repair with `DerivedDomainRepairError` before any event or the completion stamp is written, on open and on restore; migration `20261004_0095` refuses the same way. Before, a derived memory stored under such an id kept its label while its relabel event and the stamp were written, and the pair survived export and import. A PostgreSQL uuid column was not affected. A vault that the earlier repair already stamped keeps such a row at its old label until a restore repairs it again. No migration is required. -- Unreleased (on main, not in v0.20.0): v0.20.0 could store a consolidation report with a sensitivity below the memories it prints. The report took its sensitivity from the near-duplicate clusters only, so a run whose proposals were roll-up cards had no cluster to take it from and was stored as `unknown`. A key whose ceiling is below those memories (a `trusted_local_agent` key for confidential ones, a `read_only_agent` key for private ones) could then read the card topic and the member ids through `GET /v0/vnext/artifacts/{id}`. The report now takes its domain and its sensitivity over every row it names: the cluster members, the members of every proposed roll-up group, the members of the groups that a skip line names by key (`topic:...`, `entity:...`, `semantic:cluster-`), the pending, accepted, expired or held roll-up cards it names by id, and the sources that the `source_refs` of its cluster members name, which the report still prints. The open-loop review is labelled over the sources whose ids it prints as well as over its loops. The run digest of both reports covers those sources, so a source that was reclassified makes a new report instead of returning the earlier one. The first run after the upgrade over loops that link a source, or over cluster members that cite one, makes one new report, and a run that names no source keeps the digest it had. A report whose inputs are all unrestricted is now stored with the label of those inputs, `internal` where it was `unknown`, and every permission profile reads the two the same way. Reports stored by v0.20.0 keep their labels, because the stored-row repair does not change sensitivity. The other report producers (daily brief, weekly synthesis, connection and contradiction reports, project updates, staleness reports) were checked and already label over every row they print. Memories and sources are counted apart when the label is taken, because an id is unique only within its own table: a source that shares a memory's id can no longer replace that memory's label. No migration is required. +- Unreleased (on main, not in v0.20.0): the SQLite derived-label repair now updates and records each row under the id it is stored with (SQLite keeps capitals, missing hyphens, braces and `urn:uuid:` as written) and matches rows by the normalised id only to read the graph. If two stored spellings of one id exist, each is compared with the label its own recorded inputs give it, so neither is left at its old label while the other is relabelled, and neither is lowered. An update that changes no row stops the repair with `DerivedDomainRepairError` before any event or the completion stamp is written, on open and on restore; migration `20261004_0095` refuses the same way. Before, a derived memory stored under such an id kept its label while its relabel event and the stamp were written, and the pair survived export and import. A PostgreSQL uuid column was not affected. Correction (2026-10-05): the v3 pass raises such a row on the next open, including a vault the earlier repair already stamped. No migration is required. +- Unreleased (on main, not in v0.20.0): v0.20.0 could store a consolidation report with a sensitivity below the memories it prints. The report took its sensitivity from the near-duplicate clusters only, so a run whose proposals were roll-up cards had no cluster to take it from and was stored as `unknown`. A key whose ceiling is below those memories (a `trusted_local_agent` key for confidential ones, a `read_only_agent` key for private ones) could then read the card topic and the member ids through `GET /v0/vnext/artifacts/{id}`. The report now takes its domain and its sensitivity over every row it names: the cluster members, the members of every proposed roll-up group, the members of the groups that a skip line names by key (`topic:...`, `entity:...`, `semantic:cluster-`), the pending, accepted, expired or held roll-up cards it names by id, and the sources that the `source_refs` of its cluster members name, which the report still prints. The open-loop review is labelled over the sources whose ids it prints as well as over its loops. The run digest of both reports covers those sources, so a source that was reclassified makes a new report instead of returning the earlier one. The first run after the upgrade over loops that link a source, or over cluster members that cite one, makes one new report, and a run that names no source keeps the digest it had. A report whose inputs are all unrestricted is now stored with the label of those inputs, `internal` where it was `unknown`, and every permission profile reads the two the same way. Correction (2026-10-05): a report stored by v0.20.0 does not keep a sensitivity below its inputs, because migration `20261005_0096` and the SQLite v3 pass repair sensitivity. The other report producers (daily brief, weekly synthesis, connection and contradiction reports, project updates, staleness reports) were checked and already label over every row they print. Memories and sources are counted apart when the label is taken, because an id is unique only within its own table: a source that shares a memory's id can no longer replace that memory's label. No migration is required. - Unreleased (on main, not in v0.20.0): v0.20.0 `alice-memory export` copied the vault into a private folder in the temporary directory and removed it only on a normal exit or an exception, so a SIGKILL or a power loss left a plaintext copy of the vault there until the system cleaned the directory. `sources list`, the `sources delete` and `sources prune` previews and `import-markdown --dry-run` on main use the same kind of folder, and `alice-memory import` in v0.20.0 does the same for a copy of the file it imports. Each folder now holds a small marker that names the process that made it (its PID and start time), and every command that makes one first removes the folders of processes that are gone, or whose PID now belongs to a process that started at a different time. The sweep only touches a real directory (never a symlink) directly under the temporary directory, with one of the three snapshot name prefixes, owned by the current user, mode 0700, holding a valid marker. It leaves everything else alone, keeps a folder whenever it cannot tell whether the process is running, and never fails the command. The `import-markdown --dry-run` copy of the vault and of `sleep_proposals.jsonl` is now created with mode 0600. No migration is required. - Unreleased (on main, not in v0.20.0): v0.20.0 could disclose matched entity names and stored counts with no readable linked row. Recall, context packs and graph traces now require a readable active link. Identity and policy determine fencing, so default owner and unbound admin calls keep stored counts and ordering. Fenced calls omit `mention_count` in recall, packs and explain, and rank by readable linked-row counts before the five-name cap. Grounding accepts admitted names and their aliases and fences other probes. Until-only requests now also constrain recall validity pointers. No migration is required. -- Unreleased (on main, not in v0.20.0): v0.20.0 could label a derived summary of restricted inputs `unknown` or an unrestricted requested domain. Generation now retains the most frequent restricted input label, with alphabetical ties. Stored-row repair covers recorded same-user dependencies, settled chains of any length and promoted copies, with per-row audit events and a bounded failure that names the rows when derived rows form a cycle whose labels do not settle within a bounded number of changes. SQLite relabels on upgrade and before publishing fresh or upgraded restores; PostgreSQL migration `20261004_0095` must run before serving and supports the documented NOSUPERUSER NOBYPASSRLS table owner. Native SQLite backups omit artifacts, so missing artifact-only references cannot be resolved. Owner, trusted and admin calls with unrelated domain filters no longer match repaired rows; new weekly candidate `input_summary` appears in explain and adds to pack token estimates. Domain fields and policy audit labels/hashes change with the repaired label. New staleness reports inherit input sensitivity; historical sensitivity is not repaired. +- Unreleased (on main, not in v0.20.0): v0.20.0 could label a derived summary of restricted inputs `unknown` or an unrestricted requested domain. Generation now retains the most frequent restricted input label, with alphabetical ties. Stored-row repair covers recorded same-user dependencies, settled chains of any length and promoted copies, with per-row audit events and a bounded failure that names the rows when derived rows form a cycle whose labels do not settle within a bounded number of changes. SQLite relabels on upgrade and before publishing fresh or upgraded restores; PostgreSQL migration `20261004_0095` must run before serving and supports the documented NOSUPERUSER NOBYPASSRLS table owner. Native SQLite backups omit artifacts, so missing artifact-only references cannot be resolved. Such a row is unverified. Owner, trusted and admin calls with unrelated domain filters no longer match repaired rows; new weekly candidate `input_summary` appears in explain and adds to pack token estimates. Domain fields and policy audit labels/hashes change with the repaired label. New staleness reports inherit input sensitivity. Correction (2026-10-05): sensitivity and project floor are repaired too, by 20261005_0096 and the SQLite v3 pass. - Docs and tests only, no behaviour change: the known limitations page is now a short list of what is limited now, with a link to the page that explains each limit. The history it retold (the query size bounds, the 20,000-character commit limit, the Host and redirect findings, the items v0.19.2 left open that v0.20.0 fixed) stays in the release notes and this changelog, and the tests that pinned that history now pin the entries that hold it. The rules for answering a pending write (who may confirm or reject it, the sensitivity ceiling and who is exempt, the 24-hour expiry, what a refused stdio caller gets) are written once, in "Confirm and reject rules" of the memory operations protocol. The tool reference and the agent integration guide keep a short summary and a link, and the skill packs and their pages keep what an agent needs. `tests/unit/test_confirm_rules_agree_across_copies.py` checks every copy, the descriptions that `tools/list` serves included, against the others, against `CONFIRMATION_EXPIRY_HOURS` and against the server. It reads the canonical section of the protocol page and the commit-outcomes list of the same page each by itself, and requires each rule in them, so a sentence elsewhere on the page cannot stand in for one that drifted; it reads the level of the ceiling in the OpenClaw skill pack and page too. The CLI integration page now says what `alicebot vnext connectors local-folder sync` and `watch` print and what `ignored_count` counts, and the Saved quotes list of what the cited-source fence does not cover in the tool reference now names memory proposals, the agent-output ingest, the legacy `alice_vnext_recent_memory_commits` tool and the provenance links of artifacts, each with a test. - The SQLite owner CLI gains `sources list`, `sources delete` and `sources prune --superseded`. Destructive commands preview and exit 2 unless `--yes` is given. Scrub clears source text and labels, provenance quotes, pending candidate copies and revisions, source-backed loops and unsupported entity text; it resolves chunk-only evidence, merges obsolete full-text postings, enables secure deletion and lists every unredacted citing memory, including one that cites the source in any spelling the saved-quote reader accepts (upper case, no hyphens, JSON text, `source:#chunk-0`, decimal digits of other scripts). The lookup of citing memories reads each memory once, for every source a command works on, so `sources prune` makes one pass over the memories for its preview and one for its receipt, not one per replaced source. Audit events, hashes, retained memory text and earlier copies remain; docs give the VACUUM and checkpoint command for unused space and the write-ahead log. The doctor names the SQLite command and reports nonzero replaced-source counts. v0.20.0 has no SQLite source list or deletion command and can leave partial imports after a failed file. No schema or Postgres deletion behavior changes. diff --git a/docs/alpha/agent-integration.md b/docs/alpha/agent-integration.md index 8a3dddf0e..00a100316 100644 --- a/docs/alpha/agent-integration.md +++ b/docs/alpha/agent-integration.md @@ -315,6 +315,10 @@ loops, and artifacts) plus its separate event group use the caller's effective project set. Violations are blocked and audited with `project_scope_binding_violation`. Keys issued without `--project-scope` keep the prior behavior: the payload's explicit project scope is honored. +Unreleased (on main, not in v0.20.0): a derived row (a report, summary or copy) +carries every project of every row it was made from, so a bound key reads it +only when all of them are inside its binding. See +[Derived row domains](mcp-tools.md#derived-row-domains). `read_only_agent` cannot write. `memory_proposal_agent` can submit a review-only proposal but cannot approve, correct, forget, redact, or otherwise diff --git a/docs/alpha/backup-and-restore.md b/docs/alpha/backup-and-restore.md index 9c8bb5699..769d5eedc 100644 --- a/docs/alpha/backup-and-restore.md +++ b/docs/alpha/backup-and-restore.md @@ -291,22 +291,29 @@ still refused, and `--mode fail` still stops on any existing id. In v0.19.2 the second `--mode skip` import of such a file stopped with `restore_failed`. Unreleased (on main, not in v0.20.0): before it publishes the restored database, -import also repairs the labels of derived memories in the complete staged copy, -whether the destination is new or was already upgraded. A derived memory whose +import also repairs the labels of derived memories and open loops in the complete staged copy, +whether the destination is new or was already upgraded. A derived row whose recorded inputs include a restricted domain takes the most frequent restricted -label among them, and each memory the repair changes gets one audit event with -its old and new domain. Repeating `--mode skip` accepts a memory whose label that -repair changed, and still refuses any other field that differs. A vault that is -upgraded without a restore gets the same repair once, the next time it opens. A -portable backup carries no generated artifacts and the SQLite schema has no -table for them, so a promoted copy whose only recorded input is a missing -artifact cannot be repaired from that reference and keeps its label. A derived row -stored under another spelling of its id (capitals, no hyphens, braces or -`urn:uuid:`) is updated and recorded under the id it is stored with. Derived -rows that record each other in a cycle whose labels do not settle within a -bounded number of changes, or a relabel that changes no row, stop the import -with `restore_failed` before publication, and nothing is written. What counts as -a recorded input is under [Derived row domains](mcp-tools.md#derived-row-domains). +label among them, the highest sensitivity and every project, and each row the +repair changes gets one audit event with its old and new labels. Repeating +`--mode skip` accepts a memory whose domain or sensitivity the repair changed +when the file still holds the value recorded before that change, and still +refuses any other field that differs. A vault that is upgraded without a restore +gets the same repair once, the next time it opens. That open pass never stops a +vault from opening, and a row the owner had lowered on purpose is raised again once. +A portable backup carries no generated artifacts and the SQLite schema has +no table for them, so a promoted copy whose only recorded input is a missing +artifact is unverified, and a restricted key and an unbound trusted key do not +read it. A derived row stored under another spelling of its id (capitals, no +hyphens, braces or `urn:uuid:`) is updated and recorded under the id it is stored +with. Derived rows that record each other in a cycle whose labels do not settle +within a bounded number of changes, or a relabel that changes no row, stop the +import with `restore_failed` before publication, and nothing is written. What +counts as a recorded input is under [Derived row domains](mcp-tools.md#derived-row-domains). + +Unreleased (on main, not in v0.20.0): `alice-memory labels check` prints how many +derived rows are below their inputs or unverified, and `alice-memory labels repair` +raises the rows that are below their inputs. Run check after a restore. This command restores a SQLite database. It is not a PostgreSQL import. @@ -445,17 +452,21 @@ migration-defined application privileges the restored service needs. Unreleased (on main, not in v0.20.0): migration `20261004_0095` repairs the labels of derived memories and artifacts, including promoted copies and an -alternate spelling of a recorded UUID. Run it before serving requests. A database -restored at an older revision gets it from the release upgrade; one restored at -or past it is not repaired again, so that backup must already hold the repaired -labels. The documented table owner, `alicebot_admin`, is -`NOSUPERUSER NOBYPASSRLS`, so the migration turns FORCE row level security off -on the tables it reads and writes (`sources`, `memories`, `open_loops`, -`generated_artifacts`, `beliefs` and `event_log`) inside its own transaction and -turns it back on before it commits. A failure, including derived rows in a cycle -whose labels do not settle within a bounded number of changes or an update that -changes no row, rolls the relabels, their audit events and the FORCE -change back together. The downgrade keeps the repaired labels. +alternate spelling of a recorded UUID. Migration `20261005_0096` raises domain, +sensitivity, project scope and project floor. Run it before serving requests. A +database restored at an older revision gets it from the release upgrade. A +database restored at or past 0096, and rows copied into a database already at +head, are not repaired by a migration; run `alicebot vnext labels check` after +such a restore, and `labels repair` if it lists a row; until then readers hold +such a row to what its inputs require. The documented table owner, +`alicebot_admin`, is `NOSUPERUSER NOBYPASSRLS`, so the migration turns FORCE row +level security off on the tables it reads and writes (`sources`, `memories`, +`open_loops`, `generated_artifacts`, `beliefs`, `event_log` and `projects`) +inside its own transaction and turns it back on before it commits. A failure, +including derived rows in a cycle whose labels do not settle within a bounded +number of changes or an update that changes no row, rolls the relabels, their +audit events and the FORCE change back together. The downgrade keeps the repaired +labels. ## Upgrade checkpoint diff --git a/docs/alpha/known-limitations.md b/docs/alpha/known-limitations.md index a53763d6a..4a46e825d 100644 --- a/docs/alpha/known-limitations.md +++ b/docs/alpha/known-limitations.md @@ -75,7 +75,7 @@ Open in v0.20.0, with the detail in the [v0.20.0 release notes](../release/v0.20 - in v0.20.0 a key bound to one project can attach a source it cannot read through `alice_memory_commit` `source_refs`, the `provenance` of `alice_memory_correct` and, over HTTP on Postgres, `POST /v0/vnext/open-loops`, and a saved quote stays readable after its source is reclassified. Unreleased (on main, not in v0.20.0): those doors answer `not_found` (404 over HTTP) for a source or memory the caller may not read and the readers of a saved quote and of an open loop ask the reader's own fence again, but a link that passes the fence of an `admin_agent` writer still makes `alice_explain` of that memory fail for the keys of a project with a lower ceiling (see [Cited sources](mcp-tools.md#cited-sources) and [Saved quotes](mcp-tools.md#saved-quotes)) - the cited-source fence does not cover everything yet. Unreleased (on main, not in v0.20.0): memory proposals and the agent-output ingest store their `source_refs` as given, the owner dependency trace lists a memory by the first id of a multi-id ref only, `provenance_count` still counts a withheld link, and a memory or open loop saved before the fix keeps its link or id, so `alice_explain` still fails for such a memory; the rest is under [Cited sources](mcp-tools.md#cited-sources), [Saved quotes](mcp-tools.md#saved-quotes) and in the [CHANGELOG](../../CHANGELOG.md) - Unreleased (on main, not in v0.20.0): on SQLite, `import-markdown --supersede` matches by resolved path (a moved folder is not matched), and `sources delete` and `prune` keep source events, hashes, memory text and backups, and blank an open loop only when its column or its metadata names the id as stored or `source:` (any other spelling keeps the loop's text); freed space keeps text until the [VACUUM step](../integrations/importers.md#list-delete-and-prune-sqlite-sources). -- Unreleased (on main, not in v0.20.0): a summary of restricted inputs keeps their most frequent restricted label, not `unknown`, but one that combines project scopes may be readable through one of them, and a stored one whose inputs no longer resolve is not repaired. See [Derived row domains](mcp-tools.md#derived-row-domains) +- in v0.20.0 a derived summary, report or copy keeps the label its inputs had when it was made, so relabelling an input leaves it readable by keys the input now refuses, and a report can show a key bound to one project the titles, ids or text of rows from other projects or with no project. Unreleased (on main, not in v0.20.0): a derived row follows its inputs when they are relabelled and stays as strict as they have been, a report with an input that has no project is global and keeps every input's project, and a row whose recorded inputs cannot be found is read only by the owner and an unbound admin key until it is regenerated, and the five screens apply the sensitivity ceiling, counting rows the caller may read. See [Derived row domains](mcp-tools.md#derived-row-domains) What v0.19.0 and v0.19.2 limited and v0.20.0 fixed or narrowed (request body size, the `Host` check, provider redirects, local-folder reads, the memory id fence, deep backup JSON, the lone surrogate turn, the query size bounds and the data directory variable) is recorded in the [v0.19.2 release notes](../release/v0.19.2-release-notes.md), the [v0.20.0 release notes](../release/v0.20.0-release-notes.md) and the [CHANGELOG](../../CHANGELOG.md). diff --git a/docs/alpha/mcp-tools.md b/docs/alpha/mcp-tools.md index 2b7b7eb52..6f2685221 100644 --- a/docs/alpha/mcp-tools.md +++ b/docs/alpha/mcp-tools.md @@ -1081,12 +1081,12 @@ Unreleased (on main, not in v0.20.0): an until-only request also checks the uppe ## Derived row domains -Unreleased (on main, not in v0.20.0): a derived memory or report with restricted-domain inputs keeps the most frequent restricted input label, with alphabetical ties. An explicit request domain cannot override it. With no restricted inputs, each producer retains its prior selection. This covers briefs, weekly synthesis and its candidates, roll-ups, consolidation, connection and contradiction reports, staleness reports, open-loop reviews and project updates. Consolidation reports take their domain and their sensitivity over every row they name: the cluster members, the roll-up inputs, the members of the groups that a skip line names by key, and the roll-up cards they name by id. The report keeps printing the `source_refs` it copies from its cluster members, and its label also covers the sources they name, archived ones included. Open-loop reviews do the same over the sources whose ids they print. The run digest of both covers those sources, so a source that was reclassified makes a new report. A report whose inputs are all unrestricted carries the label of those inputs, so `internal` where it was `unknown`, and every profile reads the two alike. New staleness reports also inherit the highest sensitivity of the memories whose titles they include. +Unreleased (on main, not in v0.20.0): a derived memory or report keeps the most frequent restricted input label, with alphabetical ties, the highest sensitivity, and every project of every input, and none of those is lowered. An explicit request domain cannot override it. With no restricted inputs, each producer retains its prior selection. This covers briefs, weekly synthesis and its candidates, roll-ups, consolidation, connection and contradiction reports, staleness reports, open-loop reviews, project updates, promoted copies of reports, memories extracted from a source, the candidate open loops found in one, and the state a project update copies onto a project. Consolidation reports take their domain and their sensitivity over every row they name: the cluster members, the roll-up inputs, the members of the groups that a skip line names by key, and the roll-up cards they name by id. The report keeps printing the `source_refs` it copies from its cluster members, and its label also covers the sources they name, archived ones included. Open-loop reviews do the same over the sources whose ids they print. The run digest of both covers those sources, so a source that was reclassified makes a new report. A report whose inputs are all unrestricted carries the label of those inputs, so `internal` where it was `unknown`, and every profile reads the two alike. New staleness reports also inherit the highest sensitivity of the memories whose titles they include. -Unreleased (on main, not in v0.20.0): the stored-row repair resolves recorded input IDs within the same user and labels each derived row after the rows it reads, so a chain of any length settles with one read of each row, including promoted artifact copies identified by `value.kind`, `value.artifact_id` or `metadata_json.source_artifact_id`. Rows that record each other as inputs in a cycle are read again until their labels settle. Already restricted labels can change to the settled restricted label; they never become unrestricted. A cycle whose labels do not settle within a bounded number of changes aborts the upgrade or restore instead of publishing intermediate labels, with an error that names up to five of the rows that kept changing and says to remove their circular input references or restore an earlier backup. Each changed memory or artifact gets one audit event with its id and old/new domains. Redacted rows and rows without resolvable recorded inputs are left alone; text is never used to guess an input. +Unreleased (on main, not in v0.20.0): the stored-row repair resolves recorded input IDs within the same user and labels each derived row after the rows it reads, so a chain of any length settles with one read of each row, including promoted artifact copies identified by `value.kind`, `value.artifact_id` or `metadata_json.source_artifact_id`. Rows that record each other as inputs in a cycle are read again until their labels settle. Already restricted labels can change to the settled restricted label; they never become unrestricted. A cycle whose labels do not settle within a bounded number of changes aborts the migration or the restore instead of publishing intermediate labels, with an error that names up to five of the rows that kept changing and says to remove their circular input references or restore an earlier backup. The open pass does not abort the vault. A relabel and the rows that follow it commit together, at any depth, or the whole relabel is refused and nothing changes. Labels only rise. Regenerating the row is how a looser input is taken. An owner edit that would lower a derived row is held at its inputs, and the answer names `label_floor_applied`. One `labels_raised` event records the labels and carries no text. A relabel waits at most 3 s for a running write and then answers "try again". A redacted row has no inputs and is left alone. Text is never used to guess an input. A derived row with no resolvable record is not repaired but is unverified for readers. -Unreleased (on main, not in v0.20.0): the repair runs when SQLite upgrades a vault, while `alice-memory import` stages a restore and in PostgreSQL migration `20261004_0095`. Each path, and what a restore can and cannot repair, is in [Backup and restore](backup-and-restore.md). +Unreleased (on main, not in v0.20.0): the repair runs when SQLite upgrades a vault, while `alice-memory import` stages a restore, in PostgreSQL migration `20261004_0095` (domain) and in PostgreSQL migration `20261005_0096` (domain, sensitivity, project scope and floor), and through `labels check` and `labels repair`. The open pass does not stop a vault from opening and leaves a row it could not repair to the read check. Each path, and what a restore can and cannot repair, is in [Backup and restore](backup-and-restore.md). -Unreleased (on main, not in v0.20.0): owner, trusted and admin callers that explicitly filter by an unrelated domain stop receiving a derived row after its label changes from `unknown` to a restricted domain; `unknown` previously matched every domain filter. New weekly candidate memories also store `input_summary`, which explain returns. These are intentional differences for unrestricted readers. Domain fields and explain policy-event labels/hashes reflect the new label; additive metadata changes context-pack token estimates. Repair audit events are additional history. The repair does not add missing historical input summaries or repair historical sensitivity values. +Unreleased (on main, not in v0.20.0): owner, trusted and admin callers that explicitly filter by an unrelated domain stop receiving a derived row after its label changes from `unknown` to a restricted domain; `unknown` previously matched every domain filter. New weekly candidate memories also store `input_summary`, which explain returns. These are intentional differences for unrestricted readers. Domain fields and explain policy-event labels/hashes reflect the new label; additive metadata changes context-pack token estimates. Repair audit events are additional history. The repair does not add missing historical input summaries. Correction (2026-10-05): sensitivity and project floor are repaired by migration `20261005_0096` and the SQLite v3 pass. Every reader checks a derived row, a belief and a project row against its inputs as they are now. A row whose recorded inputs cannot be found is unverified, so only the owner and an unbound admin key read it. Checking those inputs adds a read of the recorded graph. `derived_from` and `project_floor` add to pack token estimates. The artifact list, the source trace, the project list, the project row of the dashboard and the belief state route apply the caller's sensitivity ceiling, including titles, ids and counts. Counts on those screens and on the workspace and dashboards are over the rows the caller may read. Domain and project restrictions on those screens stay as they are. -Unreleased (on main, not in v0.20.0): project scope is unchanged. Roll-ups group by exact normalized scope, but brain reports and weekly candidates can combine scopes; a reader matching one scope can potentially read a summary of other scopes. This domain repair does not resolve that separate scope question. In v0.20.0, mixed inputs could produce `unknown` or a less restricted request/project label; the repair covers only copies with recorded inputs that still resolve. +Unreleased (on main, not in v0.20.0): a new derived row and a stored one take every project of every input, and a project is not dropped. A restricted key's own reports are built only from inputs that key may read. The owner's cross-project briefs stay. A report with an input that has no project is global and keeps a floor of every input's project, and it is not readable by every project. Correction (2026-10-05): the text that stood here said a reader matching one scope could potentially read a summary of other scopes. In v0.20.0 and on main before this change the reach was wider than a summary: titles and ids of other projects' artifacts, titles and ids of rows with no project, and the full text of memories with no project, in a report a bound key could read with 200 although its direct read of those rows was refused; promotion made the report recallable. diff --git a/docs/release/v0.20.0-release-notes.md b/docs/release/v0.20.0-release-notes.md index f7ab61673..b70654ef1 100644 --- a/docs/release/v0.20.0-release-notes.md +++ b/docs/release/v0.20.0-release-notes.md @@ -1,6 +1,8 @@ # Alice v0.20.0 Release Notes +> **Correction (2026-10-05):** these notes do not say that a derived summary, report or copy kept the label its inputs had when it was made, or that a report could show a key bound to one project rows of other projects or with no project. Both are in v0.20.0 and are listed under known limitations. A derived row an owner lowered on purpose is raised once, by the first open, a restore or migration `20261005_0096`. + **Take this release if you run the Postgres stack's HTTP API, use an embeddings endpoint (above all a hosted one), import ChatGPT or Markdown files, use the Hermes provider, or call `alice_context_pack` with a small `max_tokens`. It acts diff --git a/docs/runbooks/disaster-recovery.md b/docs/runbooks/disaster-recovery.md index 1b993675f..50a225c17 100644 --- a/docs/runbooks/disaster-recovery.md +++ b/docs/runbooks/disaster-recovery.md @@ -132,6 +132,7 @@ alice-memory export --db ~/.alice/memory.db \ alice-memory import --db ~/alice-restore-test/memory.db \ --in ~/alice-backups/alice.jsonl alice-memory reindex-embeddings --db ~/alice-restore-test/memory.db +alice-memory labels check --db ~/alice-restore-test/memory.db ``` If the backup holds a credential and the source vault is gone, @@ -256,7 +257,8 @@ Against the restored database, verify: - a known FTS recall query returns its memory; - the memory still has `embedding_vector` and a content-matching embedding signature; -- application-role access works with RLS enabled and forced. +- application-role access works with RLS enabled and forced; +- `alicebot vnext labels check` lists no derived row below its inputs. If the restore is from an older release, follow [Upgrade v0.12.0 to current](upgrade-v0.12-to-current.md) in the restored diff --git a/docs/security/auth-authorization.md b/docs/security/auth-authorization.md index 3bf82e024..82a0e0f52 100644 --- a/docs/security/auth-authorization.md +++ b/docs/security/auth-authorization.md @@ -28,7 +28,8 @@ other operation. local profile is not automatically an admin review identity. - Persisted memory, artifact, source, and project scope is authoritative for target reads and mutations. Caller metadata cannot relabel a persisted target - into scope. + into scope. Unreleased (on main, not in v0.20.0): the persisted labels of a + derived row are held at or above those of its inputs. - Authentication failures use 401; authenticated but unauthorized actions use 403 and stable public response families. diff --git a/tests/unit/test_derived_domain_docs.py b/tests/unit/test_derived_domain_docs.py index e9fb058d5..b3876b8b0 100644 --- a/tests/unit/test_derived_domain_docs.py +++ b/tests/unit/test_derived_domain_docs.py @@ -21,6 +21,7 @@ MCP_TOOLS = ROOT / "docs/alpha/mcp-tools.md" BACKUP = ROOT / "docs/alpha/backup-and-restore.md" MIGRATION = ROOT / "apps/api/alembic/versions/20261004_0095_derived_restricted_domains.py" +MIGRATION_0096 = ROOT / "apps/api/alembic/versions/20261005_0096_derived_label_floor.py" HEADING = "Derived row domains" ANCHOR = "mcp-tools.md#derived-row-domains" @@ -47,25 +48,35 @@ def _section(path: Path, heading: str) -> str: return _flat(found[0].partition("\n")[2]) -def _migration(): - spec = importlib.util.spec_from_file_location("derived_domain_migration_0095", MIGRATION) +def _load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module +def _migration(): + return _load(MIGRATION, "derived_domain_migration_0095") + + +def _migration_0096(): + return _load(MIGRATION_0096, "derived_label_floor_0096") + + def test_the_limitations_page_has_one_short_bullet_that_links_to_the_explanation() -> None: """The page states the label rule, the two limits that remain and the link, and nothing longer. - Mutations, each one alone: delete ``not `unknown```, ``may be readable through one of them``, ``is not - repaired`` or the link from the bullet; start the bullet without the marker; point the link at - ``#derived-rows``; add a ``## Derived`` heading with a paragraph to the page. + Mutations, each one alone: delete ``keeps the label its inputs had when it was made``, ``follows its + inputs when they are relabelled``, ``read only by the owner and an unbound admin key``, ``sensitivity + ceiling`` or the link from the bullet; point the link at ``#derived-rows``; add a ``## Derived`` heading + with a paragraph to the page. """ - bullet = _paragraph(LIMITATIONS, "- " + MARK + " a summary of restricted inputs") - assert "keeps their most frequent restricted label, not `unknown`" in bullet - assert "one that combines project scopes may be readable through one of them" in bullet - assert "a stored one whose inputs no longer resolve is not repaired" in bullet + bullet = _paragraph(LIMITATIONS, "- in v0.20.0 a derived summary, report or copy") + assert "keeps the label its inputs had when it was made" in bullet + assert "follows its inputs when they are relabelled" in bullet + assert "read only by the owner and an unbound admin key" in bullet + assert "sensitivity ceiling" in bullet assert bullet.endswith(f"See [{HEADING}]({ANCHOR})") assert "\n## Derived" not in LIMITATIONS.read_text(encoding="utf-8") @@ -77,13 +88,14 @@ def test_the_mcp_tools_section_states_the_label_rule_the_repair_and_the_changes_ Mutations, each one alone: delete ``with alphabetical ties``, ``An explicit request domain cannot override it``, ``within the same user``, ``they never become unrestricted``, ``instead of publishing intermediate labels``, - ``or restore an earlier backup``, ``rows without resolvable recorded inputs are left alone``, ``text is never + ``or restore an earlier backup``, ``A redacted row has no inputs and is left alone``, ``Text is never used to guess an input``, the link to the backup guide, ``unknown` previously matched every domain filter``, - ``project scope is unchanged`` or ``covers only copies with recorded inputs that still resolve``; change ``up to + ``apply the caller's sensitivity ceiling`` or ``not readable by every project``; change ``up to five`` to ``up to ten``; remove the marker from one paragraph; rename the heading; delete ``take their domain and their sensitivity over every row they name``, ``its label also covers the sources they name``, ``Open-loop reviews do the same over the sources whose ids they print`` or ``The run digest of both covers those sources``; - put back a sentence that says a consolidation report's ``source_refs`` are not covered by its label. + put back a sentence that says a consolidation report's ``source_refs`` are not covered by its label; put back + ``apply no label``. """ section = _section(MCP_TOOLS, HEADING) @@ -118,11 +130,14 @@ def test_the_mcp_tools_section_states_the_label_rule_the_repair_and_the_changes_ assert "`value.kind`, `value.artifact_id` or `metadata_json.source_artifact_id`" in stored assert "they never become unrestricted" in stored assert ( - "aborts the upgrade or restore instead of publishing intermediate labels, with an error that names up to " + "aborts the migration or the restore instead of publishing intermediate labels, with an error that names up to " f"{shown} of the rows that kept changing and says to remove their circular input references or restore an " "earlier backup" ) in stored - assert "Redacted rows and rows without resolvable recorded inputs are left alone; text is never used to guess an input" in stored + assert "The open pass does not abort the vault" in stored + assert "A redacted row has no inputs and is left alone" in stored + assert "Text is never used to guess an input" in stored + assert "is not repaired but is unverified for readers" in stored assert "SQLite upgrades a vault" in where and "`alice-memory import` stages a restore" in where assert f"PostgreSQL migration `{_migration().revision}`" in where @@ -130,11 +145,15 @@ def test_the_mcp_tools_section_states_the_label_rule_the_repair_and_the_changes_ assert "`unknown` previously matched every domain filter" in readers assert "New weekly candidate memories also store `input_summary`" in readers - assert "does not add missing historical input summaries or repair historical sensitivity values" in readers + assert "does not add missing historical input summaries" in readers + assert "or repair historical sensitivity values" not in readers + assert "apply the caller's sensitivity ceiling" in readers + assert "rows the caller may read" in readers + assert "apply no label" not in readers - assert "project scope is unchanged" in scope - assert "a reader matching one scope can potentially read a summary of other scopes" in scope - assert "the repair covers only copies with recorded inputs that still resolve" in scope + assert "every project of every input" in scope + assert "not readable by every project" in scope + assert "could potentially read a summary of other scopes" in scope def test_the_backup_guide_states_when_the_repair_runs_and_what_it_cannot_repair() -> None: @@ -145,35 +164,46 @@ def test_the_backup_guide_states_when_the_repair_runs_and_what_it_cannot_repair( Mutations, each one alone: in the restore paragraph, delete ``in the complete staged copy``, ``whether the destination is new or was already upgraded``, ``one audit event``, ``still refuses any other field that - differs``, ``cannot be repaired from that reference``, ``stop the import with `restore_failed` before - publication`` or the link; in the PostgreSQL paragraph, delete ``Run it before serving requests``, ``is not - repaired again``, one table name, ``NOSUPERUSER NOBYPASSRLS``, ``rolls the relabels, their audit events and the + differs``, ``is unverified``, ``stop the import with `restore_failed` before + publication`` or the link; in the PostgreSQL paragraph, delete ``Run it before serving requests``, ``not + repaired by a migration``, one table name, ``NOSUPERUSER NOBYPASSRLS``, ``rolls the relabels, their audit events and the FORCE change back together`` or ``The downgrade keeps the repaired labels``; change the revision in the - paragraph; in the migration, drop one table from ``_RELAX_RLS``. + paragraph; in migration 0095, drop one table from ``_RELAX_RLS``; in migration 0096, drop ``projects``. """ restore = _paragraph(BACKUP, MARK + " before it publishes the restored database") - assert "import also repairs the labels of derived memories in the complete staged copy" in restore + assert "import also repairs the labels of derived memories and open loops in the complete staged copy" in restore assert "whether the destination is new or was already upgraded" in restore assert "takes the most frequent restricted label among them" in restore - assert "each memory the repair changes gets one audit event with its old and new domain" in restore - assert "Repeating `--mode skip` accepts a memory whose label that repair changed, and still refuses any other field that differs" in restore + assert "the highest sensitivity and every project" in restore + assert "one audit event with its old and new labels" in restore + assert "still refuses any other field that differs" in restore assert "A vault that is upgraded without a restore gets the same repair once, the next time it opens" in restore + assert "never stops a vault from opening" in restore + assert "raised again once" in restore assert "A portable backup carries no generated artifacts and the SQLite schema has no table for them" in restore - assert "cannot be repaired from that reference and keeps its label" in restore + assert "is unverified, and a restricted key and an unbound trusted key do not read it" in restore assert "stop the import with `restore_failed` before publication, and nothing is written" in restore assert f"[{HEADING}]({ANCHOR})" in restore + commands = _paragraph(BACKUP, MARK + " `alice-memory labels check`") + assert "`alice-memory labels repair`" in commands migration = _migration() postgres = _paragraph(BACKUP, MARK + f" migration `{migration.revision}`") assert "Run it before serving requests" in postgres assert "A database restored at an older revision gets it from the release upgrade" in postgres - assert "one restored at or past it is not repaired again, so that backup must already hold the repaired labels" in postgres + assert "are not repaired by a migration" in postgres + assert "`alicebot vnext labels check`" in postgres assert "The documented table owner, `alicebot_admin`, is `NOSUPERUSER NOBYPASSRLS`" in postgres tables = [re.fullmatch(r"ALTER TABLE (\w+) NO FORCE ROW LEVEL SECURITY", item).group(1) for item in migration._RELAX_RLS] assert len(tables) == 6 - for table in tables: + later = _migration_0096() + assert later.down_revision == migration.revision + later_tables = [re.fullmatch(r"ALTER TABLE (\w+) NO FORCE ROW LEVEL SECURITY", item).group(1) for item in later._RELAX_RLS] + assert len(later_tables) == 7 and "projects" in later_tables + for table in later_tables: assert f"`{table}`" in postgres, table + assert f"`{later.revision}`" in postgres assert "inside its own transaction and turns it back on before it commits" in postgres assert "rolls the relabels, their audit events and the FORCE change back together" in postgres assert "including derived rows in a cycle whose labels do not settle within a bounded number of changes" in postgres diff --git a/tests/unit/test_derived_labels_docs.py b/tests/unit/test_derived_labels_docs.py new file mode 100644 index 000000000..ca0c09b16 --- /dev/null +++ b/tests/unit/test_derived_labels_docs.py @@ -0,0 +1,97 @@ +"""The pages that describe derived labels agree with the v3 repair and with migration 0096. + +A revision, a table list and the SQLite state key are read from the code. The sentences are pinned by phrase. + +Mutations, each one alone: delete ``derived_labels_v3`` from the repair module; drop ``projects`` from the 0096 +FORCE bracket; delete ``apply the caller's sensitivity ceiling`` from the tool reference; delete ``rows the caller +may read``; put back ``apply no label``; delete ``label_floor_applied``; delete ``alicebot vnext labels check`` from +the disaster-recovery verify list; delete ``alice-memory labels check`` from the SQLite recovery steps; delete the +dated correction from the v0.20.0 notes; delete ``held at or above those of its inputs``; delete ``every project of +every row it was made from``; in the changelog, put back ``keeps such a row at its old label until a restore``, +``does not change sensitivity`` or ``historical sensitivity is not repaired``. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +from alicebot_api.vnext_label_repair import REPAIR_STATE_KEY + +ROOT = Path(__file__).resolve().parents[2] +MARK = "Unreleased (on main, not in v0.20.0):" +MIGRATION = ROOT / "apps/api/alembic/versions/20261005_0096_derived_label_floor.py" + + +def _migration(): + spec = importlib.util.spec_from_file_location("derived_label_floor_0096_docs", MIGRATION) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _text(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_the_pages_name_the_v3_repair_and_the_seven_table_migration() -> None: + """Docs name the state key, the revision and every table the migration brackets, including projects.""" + + migration = _migration() + tables = [item.split()[2] for item in migration._RELAX_RLS] + assert migration.revision == "20261005_0096" + assert tables == [ + "sources", + "memories", + "open_loops", + "generated_artifacts", + "beliefs", + "event_log", + "projects", + ] + assert REPAIR_STATE_KEY == "derived_labels_v3" + + tools = _text("docs/alpha/mcp-tools.md") + assert "promoted copies of reports" in tools + assert "memories extracted from a source" in tools + assert "the candidate open loops found in one" in tools + assert "the state a project update copies onto a project" in tools + assert "label_floor_applied" in tools + assert "at most 3 s" in tools + assert "apply the caller's sensitivity ceiling" in tools + assert "rows the caller may read" in tools + assert "apply no label" not in tools + assert f"`{migration.revision}`" in tools + assert REPAIR_STATE_KEY not in tools + + backup = _text("docs/alpha/backup-and-restore.md") + assert f"`{migration.revision}`" in backup + assert "`projects`" in backup + assert "alice-memory labels check" in backup + assert "alice-memory labels repair" in backup + assert "raised again once" in backup + + recovery = _text("docs/runbooks/disaster-recovery.md") + assert "alicebot vnext labels check" in recovery + assert "alice-memory labels check" in recovery + + integration = _text("docs/alpha/agent-integration.md") + assert "every project of every row it was made from" in integration + assert "mcp-tools.md#derived-row-domains" in integration + + auth = _text("docs/security/auth-authorization.md") + assert "held at or above those of its inputs" in auth + assert auth.count(MARK) >= 1 + + notes = _text("docs/release/v0.20.0-release-notes.md") + assert "> **Correction (2026-10-05):**" in notes + assert "listed under known limitations" in notes + assert "raised once" in notes + + changelog = _text("CHANGELOG.md") + assert "keeps such a row at its old label until a restore" not in changelog + assert "does not change sensitivity" not in changelog + assert "historical sensitivity is not repaired" not in changelog + assert "Such a row is unverified." in changelog + assert "the five operator screens apply the caller's sensitivity ceiling" in changelog + assert "No migration is required." in changelog.split("## Unreleased", 1)[1].split("\n- ", 2)[1] diff --git a/tests/unit/test_known_limitations_page_shape.py b/tests/unit/test_known_limitations_page_shape.py index e17102961..63e37b3f3 100644 --- a/tests/unit/test_known_limitations_page_shape.py +++ b/tests/unit/test_known_limitations_page_shape.py @@ -22,8 +22,8 @@ # eight sentences each. The caps sit just above the page as it is, so a limit can be reworded and a new one added, and # nothing grows back into a record without this test failing. A bullet is one or two sentences, and a closing # "See ..." pointer to the page that explains it does not count as one. -# Raised from 14,950 to 15,090 (2026-10-05) for the one clause on how far an open loop is scrubbed with its source. -MAX_PAGE_CHARS = 15_090 +# Raised from 15,090 to 15,544 (2026-10-05) for the derived-row bullet (page length 15,524 plus 20). +MAX_PAGE_CHARS = 15_544 MAX_BULLET_CHARS = 800 MAX_BULLET_SENTENCES = 2 From 31148ac015e6e871f564fec5fec10398ce5952f5 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:42:49 +0200 Subject: [PATCH 051/270] test: account for stored label floor in context budget golden --- tests/unit/test_search_goldens.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/test_search_goldens.py b/tests/unit/test_search_goldens.py index 9a6964b02..add771338 100644 --- a/tests/unit/test_search_goldens.py +++ b/tests/unit/test_search_goldens.py @@ -97,6 +97,16 @@ def test_a_scenario_equals_its_golden(computed: dict[str, dict[str, object]], su "changed_files_count": 1, "replacement_hint": "Use --supersede --dry-run to preview replacement, then --supersede to apply it.", }} + # Derived inserts now persist the empty project floor. The sources-first + # budget prices that stored metadata before compact projection, adding five + # tokens; every returned content field and the frozen fixture stay pinned. + if surface == "context_pack" and name == "sources_first": + result = expected["result"] + report = result["token_report"] + expected = {**expected, "result": {**result, "token_report": {**report, + "token_estimate": report["token_estimate"] + 5, + "full_pack_serialized_token_estimate": report["full_pack_serialized_token_estimate"] + 5, + }}} actual = computed[surface].get(name) assert actual == expected, _explain(surface, name, expected, actual) From 1bd1319d1256a043d739aa44883db44e492c2924 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:41:05 +0200 Subject: [PATCH 052/270] Preserve resolved legacy scope for original explained memories --- apps/api/src/alicebot_api/mcp/evidence_artifacts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py index 1607cb387..a44771f2f 100644 --- a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py +++ b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py @@ -283,7 +283,11 @@ def _authorize_explain_resource( if target_type in {"memory", "source", "artifact"}: judged = effective_row_for_fence(store, identity, target_type, resource) _domains, _sensitivity, judged_scope, judged_floor = policy_labels(judged) - if target_type == "source": + from alicebot_api.vnext_derived_labels import is_derived + + if target_type == "source" or (target_type == "memory" and not is_derived("memory", resource)): + # Original legacy memories may keep scope in value. The caller + # already resolved that fallback; derived rows use effective labels. judged_scope = project_scope _actor_type, _actor_id, decision = _policy_checked( store, # type: ignore[arg-type] From 1425e212e1a1270d97f90bbce82adfdf3d4d9690 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:41:05 +0200 Subject: [PATCH 053/270] Give placeholder state fixtures valid canonical provenance --- tests/unit/test_derived_read_parity.py | 3 ++- tests/unit/test_expired_memories_everywhere.py | 5 +++-- tests/unit/test_mcp_refusal_is_independent_of_state.py | 2 ++ tests/unit/test_vnext_capture.py | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_derived_read_parity.py b/tests/unit/test_derived_read_parity.py index f52dd3774..94bf659d4 100644 --- a/tests/unit/test_derived_read_parity.py +++ b/tests/unit/test_derived_read_parity.py @@ -18,6 +18,7 @@ from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection from alicebot_api.vnext_agent_keys import create_agent_key from alicebot_api.vnext_brain import _artifact_domain +from alicebot_api.vnext_derived_labels import with_derived_from from tests.unit.per_project_s2_support import add_memory from tests.unit.test_derived_domain_fence import USER @@ -51,7 +52,7 @@ def test_unrestricted_derived_read_contract(tmp_path, monkeypatch, profile, tool store = SQLiteVNextStore(conn, USER) health = add_memory(store, key="health", text="Private input", domain="health") project = add_memory(store, key="project", text="Project input", domain="project") - metadata = {"discovered_by": "vnext_weekly_synthesis"} + metadata = with_derived_from({"discovered_by": "vnext_weekly_synthesis", "project_scope": [], "project_floor": []}, {}) derived = store.create_memory( { "memory_key": "synthesis", diff --git a/tests/unit/test_expired_memories_everywhere.py b/tests/unit/test_expired_memories_everywhere.py index c16674369..dd13430ea 100644 --- a/tests/unit/test_expired_memories_everywhere.py +++ b/tests/unit/test_expired_memories_everywhere.py @@ -58,6 +58,7 @@ from alicebot_api.sqlite_schema import bootstrap_sqlite_schema from alicebot_api.sqlite_store import SQLiteVNextStore, ensure_sqlite_user, sqlite_user_connection from alicebot_api.vnext_agent_control import PolicyDecision +from alicebot_api.vnext_derived_labels import with_derived_from from alicebot_api.vnext_artifact_review import dispatch_vnext_artifact_review from alicebot_api.vnext_consolidation import MemoryConsolidationRequest, VNextConsolidationService from alicebot_api.vnext_embeddings import ( @@ -154,7 +155,7 @@ def _memory( "domain": domain, "sensitivity": sensitivity, "valid_to": valid_to, - "metadata_json": metadata or {}, + "metadata_json": with_derived_from(metadata, {}) if metadata and metadata.get("candidate_kind") == ROLLUP_CANDIDATE_KIND else metadata or {}, } ) if embed: @@ -1138,7 +1139,7 @@ def _digest_card( ``deleted_at`` empty, which is not a soft-deleted row and does not reach the read that skips them. """ - metadata: dict[str, object] = {"candidate_kind": candidate_kind, "rollup_key": rollup_key, "rollup_digest": digest} + metadata: dict[str, object] = with_derived_from({"candidate_kind": candidate_kind, "rollup_key": rollup_key, "rollup_digest": digest}, {}) if project is not None: metadata["project_scope"] = [project] card = store.create_memory( diff --git a/tests/unit/test_mcp_refusal_is_independent_of_state.py b/tests/unit/test_mcp_refusal_is_independent_of_state.py index 436a7c1b1..8c011320b 100644 --- a/tests/unit/test_mcp_refusal_is_independent_of_state.py +++ b/tests/unit/test_mcp_refusal_is_independent_of_state.py @@ -35,6 +35,7 @@ from alicebot_api.mcp.runtime import _sqlite_path_from_url, _vnext_store_context from alicebot_api.mcp.types import MCPRuntimeContext from alicebot_api.onramp import bootstrap_database, resolve_db_path, sqlite_url_for_path +from alicebot_api.vnext_derived_labels import with_derived_from from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection from alicebot_api.vnext_agent_control import AgentIdentity, AgentPolicyBlockedError from alicebot_api.vnext_agent_keys import create_agent_key @@ -203,6 +204,7 @@ def backdate(metadata: dict) -> dict: def _as_project_update(candidate: object) -> Callable[[dict], dict]: def update(metadata: dict) -> dict: + metadata.update(with_derived_from({}, {})) metadata["workflow"] = "project_auto_update" if candidate is not None: metadata["candidate"] = candidate diff --git a/tests/unit/test_vnext_capture.py b/tests/unit/test_vnext_capture.py index dcc62bcbb..a1a2c7895 100644 --- a/tests/unit/test_vnext_capture.py +++ b/tests/unit/test_vnext_capture.py @@ -1433,5 +1433,6 @@ def test_capture_without_project_scope_keeps_empty_scope_metadata() -> None: source = store.get_source(result.source_id) memory = store.list_memories(status="candidate")[0] assert "project_scope" not in source["metadata_json"] - assert "project_scope" not in memory["metadata_json"] + assert memory["metadata_json"]["project_scope"] == [] + assert memory["metadata_json"]["project_floor"] == [] assert memory_project_scope(memory) == () From e59ff844dfe1c406fda4e4d834f7d7132bbe07a1 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:41:05 +0200 Subject: [PATCH 054/270] Model recorded ancestry in MCP authorization fixtures --- tests/unit/test_mcp.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index 8561365d0..ff52aa588 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -33,6 +33,7 @@ import alicebot_api.mcp.types as mcp_types_module import alicebot_api.mcp_server as mcp_server import alicebot_api.mcp_tools as mcp_tools_module +from alicebot_api.vnext_derived_labels import with_derived_from import alicebot_api.vnext_retrieval as vnext_retrieval_module from alicebot_api.mcp_tools import MCPRuntimeContext, MCPToolError, MCPToolNotFoundError, call_mcp_tool, list_mcp_tools from alicebot_api.sqlite_schema import bootstrap_sqlite_schema @@ -1317,6 +1318,19 @@ def __init__(self) -> None: } } + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + # The fake search methods expose fixed persisted rows as well as writes. + collections = { + "memory": [*self.search_memories(), *self.memories], + "source": [*self.search_sources(), *self.sources], + "artifact": list(self.artifacts.values()), + "open_loop": self.open_loops, + "project": list(self.projects.values()), + "belief": list(self.beliefs.values()), + } + found = {str(row.get("id")): row for row in collections.get(kind, [])} + return [dict(found[item]) for item in ids if item in found] + @staticmethod def _is_live(row: dict[str, object]) -> bool: return row.get("deleted_at") is None @@ -3802,7 +3816,7 @@ def test_vnext_artifact_get_authorizes_persisted_scope_and_sensitivity(monkeypat "status": "needs_review", "domain": "project", "sensitivity": "private", - "metadata_json": {"project_id": "project-b"}, + "metadata_json": with_derived_from({"project_id": "project-b"}, {}), } ) artifact_id = str(artifact["id"]) @@ -3842,7 +3856,7 @@ def test_vnext_artifact_review_locks_and_authorizes_persisted_scope(monkeypatch, "status": "needs_review", "domain": "project", "sensitivity": "private", - "metadata_json": {"project_id": "project-b"}, + "metadata_json": with_derived_from({"project_id": "project-b"}, {}), } ) artifact_id = str(artifact["id"]) From b3af781efa46b47b5fc118e2912eb64db36b2816 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 19:46:52 +0000 Subject: [PATCH 055/270] Write a daily-brief project id only when it is a uuid. A free-form project name stays in metadata_json.project_scope. v0.20.0 wrote that name into open_loops.project_id and the brief failed on Postgres. --- CHANGELOG.md | 1 + ROADMAP.md | 4 + apps/api/src/alicebot_api/vnext_brain.py | 18 ++++- .../test_daily_brief_project_id_postgres.py | 74 +++++++++++++++++++ tests/unit/test_daily_brief_project_id.py | 47 ++++++++++++ 5 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_daily_brief_project_id_postgres.py create mode 100644 tests/unit/test_daily_brief_project_id.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f12d0d482..dca458f7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Unreleased (on main, not in v0.20.0): a daily brief that discovers an open loop writes `project_id` only when the single project is a UUID, and keeps a free-form name in `metadata_json.project_scope`. v0.20.0 wrote that name into the UUID column, so a brief with a TODO line and a project such as `Alice` failed on Postgres. No migration is required. - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` now scrub an open loop that names the source only in its metadata: the id, or `source:`, as the text under `source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references` or `selected_source_ids` at any depth. That is the rule the open-loop lookup of a source already used. They blanked only loops whose `source_id` column held the id, so a loop with an empty column kept its title, description and metadata, and they reached a new export. The delete preview now counts the same loops as the receipt. The rule reads the id only as stored or as `source:`: a loop that names the source in any spelling other than those two keeps its text, although the memory rule reads several other spellings. No migration is required. - Unreleased (on main, not in v0.20.0): the SQLite derived-label repair now updates and records each row under the id it is stored with (SQLite keeps capitals, missing hyphens, braces and `urn:uuid:` as written) and matches rows by the normalised id only to read the graph. If two stored spellings of one id exist, each is compared with the label its own recorded inputs give it, so neither is left at its old label while the other is relabelled, and neither is lowered. An update that changes no row stops the repair with `DerivedDomainRepairError` before any event or the completion stamp is written, on open and on restore; migration `20261004_0095` refuses the same way. Before, a derived memory stored under such an id kept its label while its relabel event and the stamp were written, and the pair survived export and import. A PostgreSQL uuid column was not affected. A vault that the earlier repair already stamped keeps such a row at its old label until a restore repairs it again. No migration is required. - Unreleased (on main, not in v0.20.0): v0.20.0 could store a consolidation report with a sensitivity below the memories it prints. The report took its sensitivity from the near-duplicate clusters only, so a run whose proposals were roll-up cards had no cluster to take it from and was stored as `unknown`. A key whose ceiling is below those memories (a `trusted_local_agent` key for confidential ones, a `read_only_agent` key for private ones) could then read the card topic and the member ids through `GET /v0/vnext/artifacts/{id}`. The report now takes its domain and its sensitivity over every row it names: the cluster members, the members of every proposed roll-up group, the members of the groups that a skip line names by key (`topic:...`, `entity:...`, `semantic:cluster-`), the pending, accepted, expired or held roll-up cards it names by id, and the sources that the `source_refs` of its cluster members name, which the report still prints. The open-loop review is labelled over the sources whose ids it prints as well as over its loops. The run digest of both reports covers those sources, so a source that was reclassified makes a new report instead of returning the earlier one. The first run after the upgrade over loops that link a source, or over cluster members that cite one, makes one new report, and a run that names no source keeps the digest it had. A report whose inputs are all unrestricted is now stored with the label of those inputs, `internal` where it was `unknown`, and every permission profile reads the two the same way. Reports stored by v0.20.0 keep their labels, because the stored-row repair does not change sensitivity. The other report producers (daily brief, weekly synthesis, connection and contradiction reports, project updates, staleness reports) were checked and already label over every row they print. Memories and sources are counted apart when the label is taken, because an id is unique only within its own table: a source that shares a memory's id can no longer replace that memory's label. No migration is required. diff --git a/ROADMAP.md b/ROADMAP.md index 7915639f0..30ec277ce 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -35,6 +35,10 @@ OpenCode is an opt-in `alice-memory install` host, and from `v0.19.0` Codex is one too. The Claude Code plugin directory ships in `v0.19.0`. Host coverage is done, and the skill pack revisions are in design. Next is search quality: the next release is planned as a search-quality release, ahead of new features. +Unreleased (on main, not in v0.20.0): a daily brief that discovers an open loop +writes `open_loops.project_id` only when that single project is a UUID. A +free-form name stays in `metadata_json.project_scope`. In v0.20.0 the name was +written into the UUID column and the brief failed on Postgres. Of the former roadmap list, benchmark replication, multi-session synthesis measurement, reference integrations, SQLite vector scale, and the enterprise diff --git a/apps/api/src/alicebot_api/vnext_brain.py b/apps/api/src/alicebot_api/vnext_brain.py index 2687e99ad..71c3c3320 100644 --- a/apps/api/src/alicebot_api/vnext_brain.py +++ b/apps/api/src/alicebot_api/vnext_brain.py @@ -7,6 +7,7 @@ import json import re from typing import Callable, Protocol, Sequence, cast +from uuid import UUID from alicebot_api.vnext_derived_domain import derived_domain from alicebot_api.vnext_agent_control import resource_project_scope @@ -408,6 +409,21 @@ def _canonical_project_scope(values: Sequence[object]) -> tuple[str, ...]: return normalize_project_scope(values) +def _stored_open_loop_project_id(project_scope: Sequence[str]) -> str | None: + """Return a project id only when the single scope entry is a uuid. + + ``open_loops.project_id`` is a uuid column. A free-form name stays in + ``metadata_json.project_scope``. + """ + + if len(project_scope) != 1: + return None + try: + return str(UUID(str(project_scope[0]))) + except (ValueError, AttributeError, TypeError): + return None + + def _metadata_json(row: JsonObject) -> JsonObject: value = row.get("metadata_json") return value if isinstance(value, dict) else {} @@ -988,7 +1004,7 @@ def _create_candidate_open_loops( "status": "open", "priority": "normal", "source_id": source.get("id"), - "project_id": project_scope[0] if len(project_scope) == 1 else None, + "project_id": _stored_open_loop_project_id(project_scope), "domain": source.get("domain", "unknown"), "sensitivity": source.get("sensitivity", "unknown"), "metadata_json": { diff --git a/tests/integration/test_daily_brief_project_id_postgres.py b/tests/integration/test_daily_brief_project_id_postgres.py new file mode 100644 index 000000000..81c25dd2f --- /dev/null +++ b/tests/integration/test_daily_brief_project_id_postgres.py @@ -0,0 +1,74 @@ +"""A daily brief must not write a free-form project name into open_loops.project_id. + +On Postgres that column is uuid. v0.20.0 wrote the single project string, so a +source scoped ``Alice`` with a TODO line aborted the brief with +``invalid input syntax for type uuid``. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from uuid import uuid4 + +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_agent_control import ALL_SENSITIVITY +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from alicebot_api.vnext_store import PostgresVNextStore + + +def _source(store: PostgresVNextStore, *, projects: list[str], title: str, line: str) -> None: + store.create_source( + { + "source_type": "manual_text", + "title": title, + "content_hash": f"sha256:{uuid4().hex}", + "captured_at": datetime.now(UTC), + "source_created_at": datetime.now(UTC), + "domain": "project", + "sensitivity": "internal", + "metadata_json": { + "raw_text": line, + "project_scope": projects, + }, + }, + actor_type="user", + ) + + +def test_daily_brief_keeps_a_free_form_project_out_of_the_uuid_column(migrated_database_urls) -> None: + """A name stays in metadata. A single existing project uuid is stored. Two names leave the column empty. + + Mutation: in ``_stored_open_loop_project_id``, return ``project_scope[0]`` whenever the scope has one + entry. This test then fails while inserting the ``Alice`` loop. + """ + + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"brief-{uuid4().hex}@example.invalid", "Brief") + store = PostgresVNextStore(conn) + project = store.create_project( + { + "name": "Tracked", + "slug": f"tracked-{uuid4().hex[:8]}", + "domain": "project", + "sensitivity": "internal", + }, + actor_type="user", + ) + project_id = str(project["id"]) + _source(store, projects=["Alice"], title="Named project", line="TODO: publish the named note") + _source(store, projects=[project_id], title="Uuid project", line="TODO: publish the uuid note") + _source(store, projects=["Alice", "Bob"], title="Two projects", line="TODO: publish the pair") + VNextBrainService(store).generate_daily_brief( + BrainArtifactRequest(sensitivity_allowed=ALL_SENSITIVITY, discover_open_loops=True) + ) + loops = store.list_open_loops( + status="open", + sensitivity_allowed=list(ALL_SENSITIVITY), + limit=20, + ) + by_scope = {tuple(row["metadata_json"]["project_scope"]): str(row.get("project_id") or "") for row in loops} + assert by_scope[("Alice",)] == "" + assert by_scope[(project_id,)] == project_id + assert by_scope[("Alice", "Bob")] == "" diff --git a/tests/unit/test_daily_brief_project_id.py b/tests/unit/test_daily_brief_project_id.py new file mode 100644 index 000000000..451404160 --- /dev/null +++ b/tests/unit/test_daily_brief_project_id.py @@ -0,0 +1,47 @@ +"""Candidate open loops keep a free-form project name out of project_id.""" + +from __future__ import annotations + +from uuid import UUID + +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService, _stored_open_loop_project_id + + +class _Store: + def __init__(self) -> None: + self.payloads: list[dict] = [] + + def create_open_loop(self, payload: dict, *, actor_type: str = "system") -> dict: + self.payloads.append(payload) + return {"id": "loop", **payload} + + +def _source(projects: list[str]) -> dict: + return { + "id": "22222222-2222-2222-2222-222222222222", + "title": "Note", + "domain": "project", + "sensitivity": "internal", + "metadata_json": {"project_scope": projects}, + } + + +def test_a_free_form_name_is_not_a_stored_project_id() -> None: + """Mutation: return ``project_scope[0]`` for a one-item scope. ``Alice`` is then the project id.""" + + assert _stored_open_loop_project_id(("Alice",)) is None + assert _stored_open_loop_project_id(("prj_0123456789abcdef",)) is None + assert _stored_open_loop_project_id(("Alice", "Bob")) is None + canonical = "11111111-1111-1111-1111-111111111111" + assert _stored_open_loop_project_id((canonical.upper(),)) == canonical + assert _stored_open_loop_project_id((str(UUID(canonical)),)) == canonical + + store = _Store() + VNextBrainService(store)._create_candidate_open_loops( + BrainArtifactRequest(), + [("publish the note", _source(["Alice"]))], + workflow_digest="digest", + ) + payload = store.payloads[0] + assert payload["project_id"] is None + assert payload["metadata_json"]["project_scope"] == ["Alice"] From b96664b41932471dd2ecb97ba479cfd91afe93b7 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:49:32 +0200 Subject: [PATCH 056/270] Give reader guard fixtures real label and belief ancestry --- .../test_open_loop_references_read_fence.py | 7 +++++++ tests/unit/test_recall_framing.py | 17 ++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_open_loop_references_read_fence.py b/tests/unit/test_open_loop_references_read_fence.py index cb9f3610a..d2061a741 100644 --- a/tests/unit/test_open_loop_references_read_fence.py +++ b/tests/unit/test_open_loop_references_read_fence.py @@ -1379,6 +1379,13 @@ def get_sources_by_ids(self, ids: list[str]) -> list[dict[str, object]]: def get_memories_by_ids(self, ids: list[str]) -> list[dict[str, object]]: return [] + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + if kind == "source": + return self.get_sources_by_ids(ids) + if kind == "open_loop": + return [dict(row) for row in self.open_loops if str(row.get("id")) in ids] + return [] + def run(identity) -> dict[str, object]: # type: ignore[no-untyped-def] store = _Store() store.open_loops = [ diff --git a/tests/unit/test_recall_framing.py b/tests/unit/test_recall_framing.py index 768edac76..759b11fbd 100644 --- a/tests/unit/test_recall_framing.py +++ b/tests/unit/test_recall_framing.py @@ -372,11 +372,26 @@ def test_contradiction_quotes_are_framed_from_the_store_and_memory_text_stays( ) memory_id = committed["memory"]["id"] + # The schema binds each belief to an actual memory whose labels the + # effective-input guard can read. This original note contains the same + # claim used below, and remains unchanged across the context-pack read. + backing = _commit( + context, + title="Backing belief note", + canonical_text=belief_claim, + memory_type="belief", + domain="personal", + sensitivity="private", + confidence=0.95, + source_type="direct_user_instruction", + ) + backing_id = str(backing["memory"]["id"]) + def list_beliefs(self, **_kwargs: object) -> list[dict[str, object]]: return [ { "id": "belief-framing", - "memory_id": "belief-memory-framing", + "memory_id": backing_id, "claim": belief_claim, "status": "active", "memory_type": "belief", From 01b08b943aadbf79417966a34e6ab165d2ba285c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:52:44 +0200 Subject: [PATCH 057/270] test: pin workspace helper type annotation changes --- tests/unit/test_workspaces_router_split.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_workspaces_router_split.py b/tests/unit/test_workspaces_router_split.py index 20ec038c0..6b8422b91 100644 --- a/tests/unit/test_workspaces_router_split.py +++ b/tests/unit/test_workspaces_router_split.py @@ -79,13 +79,14 @@ } # Re-minted support/import receipts for effective-label workspace filtering. +# The later typing repair adds Sequence and annotates _workspace_rows. # Two local helpers admit rows and check event targets; the payload uses them # before returning lists/counts. Route bodies, mounts and middleware are unchanged. EXPECTED_ROUTE_AST_SHA256 = "fb8925ebcda058598b0c6d5e7eca83abe18606a0126bdb6cde0fd1c22444a795" -EXPECTED_SUPPORT_AST_SHA256 = "f5e8747a48d68e9916c7412c10bd2dea082bd4e34487842703b36cb2fba840b4" +EXPECTED_SUPPORT_AST_SHA256 = "0e5708e69de1dd1358ca91709d4a60effeddcd262fc8beddb19ba490cab3b0f1" EXPECTED_ROUTE_NAME_MANIFEST_SHA256 = "225c57c08bd8314156c56352dd1c53ffed3f556ce285c666dd6fca125115d0b4" EXPECTED_OPERATION_MANIFEST_SHA256 = "c320979b62d7ee8de244fe38bde5bf3761a4f9d76f76bf3cd8576c30fce9857e" -EXPECTED_IMPORT_MANIFEST_SHA256 = "6d6812ae6907dcbe9ee16ac9481c741241b256fac745133415ccd57813bc67bb" +EXPECTED_IMPORT_MANIFEST_SHA256 = "4a9df193d620b33578f1e42760b118a4267d3e1c8ff01ad92bee1f1bba1c1e9f" EXPECTED_CARRIER_NAMES_SHA256 = "2c109fc234a05dd8f44e4c34bee49e797fbb5e49e92413391541a7e504da328b" # Re-pinned 2026-10-02 (DB-005, legacy /v0 routes). One definition changed, # found by a per-definition AST diff against the previous pin: @@ -134,7 +135,7 @@ EXPECTED_SUPPORT_NODE_SHA256 = { "_vnext_status_counts": "0bf0ed228a14bd648a9d18fcd5f99ebf8c585bd29f4b5e81e1df17fe0201fd15", "_vnext_workspace_payload": "4b6e4a3d59d16538b0e859c425ede21207e9c80f11ab31c3af2ce4d604e14edf", - "_workspace_rows": "cc312a1300045147caf7340463b91d388c1932ba4edcf8f50fab22cf717999f6", + "_workspace_rows": "070bdfbd1eae10608bd8208b08367e1c0ea10e2064f03ad5a84121a190ed4cf0", "_workspace_event_visible": "8343c060909326a5cb69fa6f671ac62f160630ecf989f04d78e792ea74c0ea90", } EXPECTED_ROUTE_MANIFEST = [ @@ -482,7 +483,7 @@ def test_workspace_import_direction_pruning_timing_and_runtime_identities_are_ex main_definitions = _top_level_definitions(main_tree) router_imports = _import_manifest(router_tree) - assert len(router_imports) == 31 + assert len(router_imports) == 32 assert hashlib.sha256(json.dumps(router_imports, separators=(",", ":")).encode()).hexdigest() == ( EXPECTED_IMPORT_MANIFEST_SHA256 ) From 7d2b3af4d6de1ecbf0f7d04a68b376a688c68935 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 21:52:56 +0200 Subject: [PATCH 058/270] Model source and original-memory labels in guarded fixtures --- tests/unit/test_group_scope_consumers.py | 3 +++ tests/unit/test_vnext_main.py | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/tests/unit/test_group_scope_consumers.py b/tests/unit/test_group_scope_consumers.py index d5fbd7f0f..d5e5aad84 100644 --- a/tests/unit/test_group_scope_consumers.py +++ b/tests/unit/test_group_scope_consumers.py @@ -258,6 +258,9 @@ def list_memories_for_staleness_sweep( del reference_time, confirmation_before, review_memory_types, limit, projects return list(self.memories) + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + return [dict(row) for row in self.memories if str(row.get("id")) in ids] if kind == "memory" else [] + def update_memory(self, *, memory_id: str, patch: dict[str, object], actor_type: str = "system") -> dict[str, object]: del actor_type for memory in self.memories: diff --git a/tests/unit/test_vnext_main.py b/tests/unit/test_vnext_main.py index 997968e0c..377306289 100644 --- a/tests/unit/test_vnext_main.py +++ b/tests/unit/test_vnext_main.py @@ -2413,6 +2413,16 @@ def test_create_vnext_context_pack_endpoint_returns_structured_pack(monkeypatch) def test_create_vnext_context_pack_endpoint_keeps_uncorroborated_count_trace_only(monkeypatch) -> None: store = FakeVNextStore(None) for index, bike in enumerate(("commuter", "touring"), start=1): + # These copied notes have real original sources. Their labels settle, + # while no aggregate card corroborates the numeric answer. + store.sources[f"source-bike-{index}"] = { + "id": f"source-bike-{index}", + "source_type": "manual_text", + "title": "Maintenance note", + "domain": "personal", + "sensitivity": "private", + "metadata_json": {}, + } store.memories.append( { "id": f"memory-bike-{index}", From 9999f095394df15dc36bc48fede00e3a018822de Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 19:58:15 +0000 Subject: [PATCH 059/270] Blank an open loop for every spelling the saved-quote reader names. SQLite delete, prune, and markdown replacement now use one pass over the user's loops. v0.20.0 kept the loop text for every spelling other than the stored id or source:. --- CHANGELOG.md | 3 +- apps/api/src/alicebot_api/source_commands.py | 18 +- apps/api/src/alicebot_api/vnext_capture.py | 11 +- .../vnext_open_loop_references.py | 2 + .../vnext_stores/sqlite/graph_open_loops.py | 17 +- .../sqlite/open_loop_source_reference.py | 72 +++----- .../vnext_stores/sqlite/source_retirement.py | 97 ++++++++--- docs/alpha/known-limitations.md | 2 +- docs/integrations/importers.md | 18 +- .../unit/test_known_limitations_page_shape.py | 4 +- .../test_open_loop_references_read_fence.py | 11 +- .../unit/test_source_scrub_loop_references.py | 155 +++++++++++++----- .../unit/test_store_graph_open_loops_split.py | 11 +- 13 files changed, 256 insertions(+), 165 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f12d0d482..057daa113 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## Unreleased -- Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` now scrub an open loop that names the source only in its metadata: the id, or `source:`, as the text under `source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references` or `selected_source_ids` at any depth. That is the rule the open-loop lookup of a source already used. They blanked only loops whose `source_id` column held the id, so a loop with an empty column kept its title, description and metadata, and they reached a new export. The delete preview now counts the same loops as the receipt. The rule reads the id only as stored or as `source:`: a loop that names the source in any spelling other than those two keeps its text, although the memory rule reads several other spellings. No migration is required. +- Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. +- Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` now scrub an open loop that names the source only in its metadata: the id, or `source:`, as the text under `source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references` or `selected_source_ids` at any depth. That is the rule the open-loop lookup of a source already used. They blanked only loops whose `source_id` column held the id, so a loop with an empty column kept its title, description and metadata, and they reached a new export. The delete preview now counts the same loops as the receipt. Correction (2026-10-05): those other spellings are blanked too, by the same reader the saved-quote fence uses. No migration is required. - Unreleased (on main, not in v0.20.0): the SQLite derived-label repair now updates and records each row under the id it is stored with (SQLite keeps capitals, missing hyphens, braces and `urn:uuid:` as written) and matches rows by the normalised id only to read the graph. If two stored spellings of one id exist, each is compared with the label its own recorded inputs give it, so neither is left at its old label while the other is relabelled, and neither is lowered. An update that changes no row stops the repair with `DerivedDomainRepairError` before any event or the completion stamp is written, on open and on restore; migration `20261004_0095` refuses the same way. Before, a derived memory stored under such an id kept its label while its relabel event and the stamp were written, and the pair survived export and import. A PostgreSQL uuid column was not affected. A vault that the earlier repair already stamped keeps such a row at its old label until a restore repairs it again. No migration is required. - Unreleased (on main, not in v0.20.0): v0.20.0 could store a consolidation report with a sensitivity below the memories it prints. The report took its sensitivity from the near-duplicate clusters only, so a run whose proposals were roll-up cards had no cluster to take it from and was stored as `unknown`. A key whose ceiling is below those memories (a `trusted_local_agent` key for confidential ones, a `read_only_agent` key for private ones) could then read the card topic and the member ids through `GET /v0/vnext/artifacts/{id}`. The report now takes its domain and its sensitivity over every row it names: the cluster members, the members of every proposed roll-up group, the members of the groups that a skip line names by key (`topic:...`, `entity:...`, `semantic:cluster-`), the pending, accepted, expired or held roll-up cards it names by id, and the sources that the `source_refs` of its cluster members name, which the report still prints. The open-loop review is labelled over the sources whose ids it prints as well as over its loops. The run digest of both reports covers those sources, so a source that was reclassified makes a new report instead of returning the earlier one. The first run after the upgrade over loops that link a source, or over cluster members that cite one, makes one new report, and a run that names no source keeps the digest it had. A report whose inputs are all unrestricted is now stored with the label of those inputs, `internal` where it was `unknown`, and every permission profile reads the two the same way. Reports stored by v0.20.0 keep their labels, because the stored-row repair does not change sensitivity. The other report producers (daily brief, weekly synthesis, connection and contradiction reports, project updates, staleness reports) were checked and already label over every row they print. Memories and sources are counted apart when the label is taken, because an id is unique only within its own table: a source that shares a memory's id can no longer replace that memory's label. No migration is required. - Unreleased (on main, not in v0.20.0): v0.20.0 `alice-memory export` copied the vault into a private folder in the temporary directory and removed it only on a normal exit or an exception, so a SIGKILL or a power loss left a plaintext copy of the vault there until the system cleaned the directory. `sources list`, the `sources delete` and `sources prune` previews and `import-markdown --dry-run` on main use the same kind of folder, and `alice-memory import` in v0.20.0 does the same for a copy of the file it imports. Each folder now holds a small marker that names the process that made it (its PID and start time), and every command that makes one first removes the folders of processes that are gone, or whose PID now belongs to a process that started at a different time. The sweep only touches a real directory (never a symlink) directly under the temporary directory, with one of the three snapshot name prefixes, owned by the current user, mode 0700, holding a valid marker. It leaves everything else alone, keeps a folder whenever it cannot tell whether the process is running, and never fails the command. The `import-markdown --dry-run` copy of the vault and of `sleep_proposals.jsonl` is now created with mode 0600. No migration is required. diff --git a/apps/api/src/alicebot_api/source_commands.py b/apps/api/src/alicebot_api/source_commands.py index a16d28acb..d8bd92800 100644 --- a/apps/api/src/alicebot_api/source_commands.py +++ b/apps/api/src/alicebot_api/source_commands.py @@ -9,7 +9,7 @@ from alicebot_api.vault_sleep import SleepError from alicebot_api.vnext_stores.memory_lifecycle_common import is_redacted_memory from alicebot_api.vnext_stores.sqlite.source_retirement import ( - CandidateScrubRefused, citing_memories_by_source, optimize_scrub_indexes, source_open_loop_count) + CandidateScrubRefused, citing_memories_by_source, open_loops_naming_sources, optimize_scrub_indexes) RETAINED_DATA = ( "Source and import events keep prior titles, hashes and import folder paths. " @@ -40,8 +40,10 @@ def _targets(store, args): def _preview(store, rows): - # One pass over the memories answers for every source of the preview. - citing=citing_memories_by_source(store,[str(row['id']) for row in rows]) + # One pass over the memories, and one pass over the loops, answer for every source of the preview. + source_ids=[str(row['id']) for row in rows] + citing=citing_memories_by_source(store,source_ids) + loops=open_loops_naming_sources(store,source_ids) result = [] for row in rows: sid=str(row['id']) @@ -51,8 +53,7 @@ def _preview(store, rows): "(SELECT 1 FROM source_chunks c WHERE c.user_id=p.user_id AND c.id=p.source_chunk_id " "AND c.source_id=?))) AS provenance_quotes", (store.user_id,sid,store.user_id,sid,sid)) - # The loops the scrub blanks: the one rule of the source reverse lookup, not the column alone. - counts['open_loops']=source_open_loop_count(store,sid) + counts['open_loops']=len(loops[sid]) memories=[memory for memory in citing[sid] if not is_redacted_memory(memory)] counts['candidate_memories']=sum(memory['status'] in {'candidate','needs_review','rejected'} for memory in memories) counts['memories_citing_replaced']=[str(memory['id']) for memory in memories @@ -91,9 +92,12 @@ def run_sources(args): with store.savepoint(): rows=_targets(store,args) # One pass over the memories finds the citing memories of every source; each scrub reads those again. - citing=citing_memories_by_source(store,[str(row['id']) for row in rows]) + source_ids=[str(row['id']) for row in rows] + citing=citing_memories_by_source(store,source_ids) + loops=open_loops_naming_sources(store,source_ids) receipts=[{'id':str(row['id']),**store.scrub_source(str(row['id']), optimize=False, - citing_ids=[str(memory['id']) for memory in citing[str(row['id'])]])} for row in rows] + citing_ids=[str(memory['id']) for memory in citing[str(row['id'])]], + loop_ids=[str(loop['id']) for loop in loops[str(row['id'])]])} for row in rows] if receipts: optimize_scrub_indexes(store) print(json.dumps({'deleted_count':len(receipts),'deleted':receipts,'retained_data':RETAINED_DATA},sort_keys=True)) diff --git a/apps/api/src/alicebot_api/vnext_capture.py b/apps/api/src/alicebot_api/vnext_capture.py index 301341ccc..8b86ba37e 100644 --- a/apps/api/src/alicebot_api/vnext_capture.py +++ b/apps/api/src/alicebot_api/vnext_capture.py @@ -1346,11 +1346,14 @@ def capture_source(self, source_input: SourceCaptureInput) -> CaptureResult: return replace(result, kept_reason="matches_other_live_source") retired = [] citing = [] + cached = getattr(self, "_open_loop_names", None) for row in matches: if str(row['id']) == str(result.source_id): continue + loop_ids = None if cached is None else [str(loop["id"]) for loop in cached.get(str(row["id"]), [])] counts = getattr(self.store, "supersede_source")(str(row['id']), superseded_by=result.source_id, - allow_looser_classification=policy.allow_looser_classification, dry_run=policy.dry_run) + allow_looser_classification=policy.allow_looser_classification, dry_run=policy.dry_run, + loop_ids=loop_ids) retired.append({"id": str(row['id']), "title": printed_source_label(row.get('title'))}) citing.extend(counts['memories_citing_replaced']) return replace(result, superseded=tuple(retired), memories_citing_replaced=tuple(dict.fromkeys(citing))) @@ -1845,6 +1848,11 @@ class PreviewRollback(Exception): scan = getattr(self.store, "markdown_sources_by_path", None) with self.store.savepoint() if callable(scan) or dry_run else nullcontext(): self._markdown_path_index = scan() if callable(scan) else {} + self._open_loop_names = None + if policy.mode != "off": + from alicebot_api.vnext_stores.sqlite.source_retirement import open_loops_naming_sources + indexed = [str(row["id"]) for rows in self._markdown_path_index.values() for row in rows] + self._open_loop_names = open_loops_naming_sources(self.store, indexed) result = self._import_markdown_folder(folder, domain=domain, sensitivity=sensitivity, max_file_bytes=max_file_bytes, policy=policy) if dry_run: @@ -1853,6 +1861,7 @@ class PreviewRollback(Exception): return replace(result, dry_run=True) finally: self._markdown_path_index = None + self._open_loop_names = None return result def _import_markdown_folder( diff --git a/apps/api/src/alicebot_api/vnext_open_loop_references.py b/apps/api/src/alicebot_api/vnext_open_loop_references.py index 92b51c8ff..3a0ebd4ee 100644 --- a/apps/api/src/alicebot_api/vnext_open_loop_references.py +++ b/apps/api/src/alicebot_api/vnext_open_loop_references.py @@ -277,6 +277,8 @@ def _collect_ids(value: object, found: set[str], referenced: set[str], *, at_ref return if isinstance(value, str): ids = _ids_in_text(value) + if at_reference: + ids |= set(cited_source_ids(value).named) found.update(ids) if at_reference: referenced.update(ids) diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py index 47368f076..4b4b7f3c4 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py @@ -19,7 +19,7 @@ GRAPH_EDGE_COLUMNS, OPEN_LOOP_COLUMNS, ) -from alicebot_api.vnext_stores.sqlite.open_loop_source_reference import open_loop_source_reference_sql +from alicebot_api.vnext_stores.sqlite.source_retirement import open_loops_naming_sources from alicebot_api.vnext_stores.sqlite.primitives import ( _iso_or_none, _iso_or_now, @@ -664,22 +664,11 @@ def find_open_loop_by_automation_digest( ) def list_open_loops_referencing_source(self, *, source_id: str, limit: int = 500) -> list[VNextRow]: - """Bound open loops related to one source before LIMIT.""" + """Open loops that name one source, by the spellings ``cited_source_ids`` names, before LIMIT.""" if limit < 1: raise ValueError("limit must be positive") - reference, reference_params = open_loop_source_reference_sql(source_id) - return self._fetch_all( - f""" - SELECT {", ".join(OPEN_LOOP_COLUMNS)} - FROM open_loops - WHERE user_id = ? - AND {reference} - ORDER BY updated_at DESC, created_at DESC, id DESC - LIMIT ? - """, - (self.user_id, *reference_params, limit), - ) + return open_loops_naming_sources(self, [source_id]).get(str(source_id), [])[:limit] def list_open_loops( self, diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/open_loop_source_reference.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/open_loop_source_reference.py index 3e6e9466e..0f933c2d4 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/open_loop_source_reference.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/open_loop_source_reference.py @@ -1,57 +1,27 @@ """The one rule that says an open loop names a source, on SQLite. -Three places ask the question: the reverse lookup of a source (``list_open_loops_referencing_source``), the owner's -delete preview and receipt count, and the scrub that blanks a source's loops (``retire_dependents``, which both -``sources delete`` and ``import-markdown --supersede`` reach). They read this one statement, so a loop that the lookup -lists for a source is counted and blanked for it, and one the lookup does not list is not. - -A loop names a source when its ``source_id`` column holds the id, or when the text under one of the keys of -``SOURCE_REFERENCE_KEYS`` (``vnext_source_fence``) anywhere in its metadata is the id as stored or ``source:``. The -metadata is walked with ``json_tree``, so a key at any depth counts. The comparison is exact text, so any spelling other -than the id as stored or ``source:`` is not read (capitals, no hyphens, braces, a ``urn:uuid:`` or ``SOURCE:`` -prefix, a list, a JSON text, and every other form), although the shared reference reader of memories and saved quotes -(``cited_source_ids``) reads many of them. The scrub follows this rule and no further. +The reverse lookup of a source (``list_open_loops_referencing_source``), the owner's delete preview and receipt +count, the scrub that blanks a source's loops, and the reader that withholds ids all use ``cited_source_ids``. +A loop names a source when that reader names the id: the ``source_id`` column, or a reference under +``SOURCE_REFERENCE_KEYS`` at any depth, in any spelling ``cited_source_ids`` names (capitals, no hyphens, braces, +``urn:uuid:``, a list, JSON text). An id in prose under some other key is incidental and is not a name. """ from __future__ import annotations -# The keys are the literals of ``SOURCE_REFERENCE_KEYS``. A test reads this text and checks them against the set, so a -# key added to one and not to the other fails there. -OPEN_LOOP_SOURCE_REFERENCE_SQL = """( - open_loops.source_id = ? - OR EXISTS ( - SELECT 1 FROM json_tree(open_loops.metadata_json) AS ref - WHERE ref.key IN ( - 'source_id', 'source_ids', 'source_ref', 'source_refs', - 'source_references', 'selected_source_ids' - ) - AND CAST(ref.value AS TEXT) IN (?, ?) - ) - )""" - -# The two statements of the delete preview and the scrub are written out here, once, so neither caller builds SQL. -# Each is the fixed text around the constant above. Nothing a caller passes enters the text: the user id, the id of the -# source and the timestamps are bound parameters, in the order ``open_loop_source_reference_params`` documents. -# Bandit reads an f-string over SQL words as injection (B608) whatever it holds, so the two lines say why they are safe. -OPEN_LOOP_SOURCE_COUNT_SQL = f"""SELECT count(*) AS count FROM open_loops -WHERE user_id = ? AND {OPEN_LOOP_SOURCE_REFERENCE_SQL}""" # nosec B608 # fixed text and a module constant; every value is bound -OPEN_LOOP_SOURCE_BLANK_SQL = f"""UPDATE open_loops SET title = ?, description = ?, status = 'dismissed', -resolved_at = ?, closed_at = ?, resolution_note = ?, metadata_json = '{{}}', updated_at = ? -WHERE user_id = ? AND {OPEN_LOOP_SOURCE_REFERENCE_SQL}""" # nosec B608 # fixed text and a module constant; every value is bound - - -def open_loop_source_reference_params(source_id: object) -> tuple[str, str, str]: - """The three parameters of ``OPEN_LOOP_SOURCE_REFERENCE_SQL``, in order: the column test, the id, ``source:``.""" - - text = str(source_id) - return text, text, f"source:{text}" - - -def open_loop_source_reference_sql(source_id: object) -> tuple[str, tuple[str, str, str]]: - """``(condition, parameters)`` for the rows of ``open_loops`` that name ``source_id``. - - The condition reads the table by its own name, so it fits ``SELECT ... FROM open_loops`` and - ``UPDATE open_loops``. The caller adds its own ``user_id`` test. - """ - - return OPEN_LOOP_SOURCE_REFERENCE_SQL, open_loop_source_reference_params(source_id) +from collections.abc import Mapping + +from alicebot_api.vnext_source_fence import SOURCE_REFERENCE_KEYS, cited_source_ids + +# The lookup, the preview and the scrub read this set. It is the fence's set, not a second list of key names. +NAMED_REFERENCE_KEYS = SOURCE_REFERENCE_KEYS + + +def named_source_ids(row: Mapping[str, object]) -> frozenset[str]: + """The canonical source ids ``row`` names, and no incidental id from prose under another key.""" + + cited = cited_source_ids(row.get("metadata_json")) + column = row.get("source_id") + if column is not None and column != "": + cited = cited | cited_source_ids(column) + return cited.named diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/source_retirement.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/source_retirement.py index 12c1b2d0c..3d2112ead 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/source_retirement.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/source_retirement.py @@ -15,11 +15,8 @@ from alicebot_api.vnext_entities import ENTITY_MENTION_EDGE_TYPE from alicebot_api.vnext_project_scope import source_project_scope from alicebot_api.vnext_source_fence import memory_cited_source_ids -from alicebot_api.vnext_stores.sqlite.open_loop_source_reference import ( - OPEN_LOOP_SOURCE_BLANK_SQL, - OPEN_LOOP_SOURCE_COUNT_SQL, - open_loop_source_reference_params, -) +from alicebot_api.vnext_stores.sqlite.columns import OPEN_LOOP_COLUMNS +from alicebot_api.vnext_stores.sqlite.open_loop_source_reference import named_source_ids from alicebot_api.vnext_stores.sqlite.primitives import _utc_now_iso REMOVAL_MARKER = "[removed by the owner]" @@ -342,29 +339,77 @@ def close_mention_edges(self, from_type, from_id, now): return len(edges) -def source_open_loop_count(self, source_id): - """How many loops of the user, in any status, name ``source_id``, by the one rule of the reverse lookup of a source - (``open_loop_source_reference_sql``): the ``source_id`` column, or the id or ``source:`` under a reference key of - the loop's metadata. The delete preview and ``blank_open_loops`` count the same rows.""" +# One read of the user's loops. The column list is a module constant, and the user id is bound. +_USER_OPEN_LOOPS_SQL = ( + f"SELECT {', '.join(OPEN_LOOP_COLUMNS)} FROM open_loops WHERE user_id = ? " # nosec B608 + "ORDER BY updated_at DESC, created_at DESC, id DESC" +) +_BLANK_OPEN_LOOPS_SQL = """UPDATE open_loops SET title = ?, description = ?, status = 'dismissed', +resolved_at = ?, closed_at = ?, resolution_note = ?, metadata_json = '{}', updated_at = ? +WHERE user_id = ? AND id IN (SELECT value FROM json_each(?))""" - row = self._fetch_one( - 'count the open loops of a source', OPEN_LOOP_SOURCE_COUNT_SQL, - (self.user_id, *open_loop_source_reference_params(source_id))) - return int(row['count']) + +def _user_open_loops(self): + """Every open loop of the user, newest first. One call is one pass.""" + + return self._fetch_all(_USER_OPEN_LOOPS_SQL, (self.user_id,)) + + +def open_loops_naming_sources(self, source_ids): + """``{source id: [loop rows]}`` for the loops of the user that name each source. + + One pass over the user's loops answers for every source. A loop names a source when ``named_source_ids`` says so, + which is every spelling ``cited_source_ids`` names. The lists keep the read order. + """ + + ids = list(dict.fromkeys(str(source_id) for source_id in source_ids)) + found = {source_id: [] for source_id in ids} + if not ids: + return found + wanted: dict[str, list[str]] = {} + for source_id in ids: + try: + canonical = str(UUID(source_id)) + except ValueError: + canonical = source_id + wanted.setdefault(canonical, []).append(source_id) + for row in _user_open_loops(self): + for canonical in named_source_ids(row): + for source_id in wanted.get(canonical, ()): + found[source_id].append(row) + return found + + +def source_open_loop_count(self, source_id, *, named=None): + """How many loops of the user, in any status, name ``source_id``. + + ``named`` is the answer of ``open_loops_naming_sources`` for every source of one command, so a preview asks once. + """ + + if named is None: + named = open_loops_naming_sources(self, [source_id]) + return len(named[str(source_id)]) -def blank_open_loops(self, source_id, *, now): - """Dismiss every loop that names ``source_id``, whether it is open, resolved or dismissed, and blank its free text and - metadata, in the caller's transaction. Which loops is the one rule of ``open_loop_source_reference_sql``, so a loop - named only in its metadata is blanked as a loop named by its column is. Returns the number of loops.""" +def blank_open_loops(self, source_id, *, now, loop_ids=None): + """Dismiss every loop that names ``source_id`` and blank its free text and metadata. + ``loop_ids`` are the ids an earlier pass of this command found. Without them the loops are read here. An empty + list is an answer and does not read again. Returns the number of loops. + """ + + if loop_ids is None: + loop_ids = [str(row["id"]) for row in open_loops_naming_sources(self, [source_id])[str(source_id)]] + loop_ids = [str(loop_id) for loop_id in loop_ids] + if not loop_ids: + return 0 return self._execute( - OPEN_LOOP_SOURCE_BLANK_SQL, - (REMOVAL_MARKER, REMOVAL_MARKER, now, now, REMOVAL_MARKER, now, self.user_id, - *open_loop_source_reference_params(source_id))).rowcount + _BLANK_OPEN_LOOPS_SQL, + (REMOVAL_MARKER, REMOVAL_MARKER, now, now, REMOVAL_MARKER, now, self.user_id, json.dumps(loop_ids)), + ).rowcount -def retire_dependents(self, source_id, *, now, scrub_candidates=False, citing_ids=None): +def retire_dependents(self, source_id, *, now, scrub_candidates=False, citing_ids=None, loop_ids=None): """Retire what a source owns. ``citing_ids`` are the ids of the memories an earlier lookup of this transaction found for the source (``citing_memories_by_source``): a caller that retires many sources looks once for all of them, and the memories are read again here as they are stored now, so one that an earlier retirement redacted is left alone, as it @@ -390,12 +435,12 @@ def retire_dependents(self, source_id, *, now, scrub_candidates=False, citing_id self.update_memory(memory_id=mid, patch={'status': 'rejected'}, actor_type='user') close_mention_edges(self, 'memory', mid, now) edges = close_mention_edges(self, 'source', source_id, now) - loops = blank_open_loops(self, source_id, now=now) + loops = blank_open_loops(self, source_id, now=now, loop_ids=loop_ids) return {'candidate_memories': len(pending), 'mention_edges': edges, 'open_loops': loops, 'memories_citing_replaced': retained} -def supersede_source(self, source_id, *, superseded_by, allow_looser_classification=False, dry_run=False): +def supersede_source(self, source_id, *, superseded_by, allow_looser_classification=False, dry_run=False, loop_ids=None): with self.savepoint(): old = self.get_source(source_id) new = self.get_source(superseded_by) @@ -413,7 +458,7 @@ def supersede_source(self, source_id, *, superseded_by, allow_looser_classificat # The sidecar goes first. A later rollback may lose proposals, which # can be generated again, but cannot leave retired evidence in it. prune_sleep_rows(self, {source_id}, dry_run=dry_run) - counts = retire_dependents(self, source_id, now=now) + counts = retire_dependents(self, source_id, now=now, loop_ids=loop_ids) metadata = {**old['metadata_json'], 'superseded_by': superseded_by, 'superseded_at': now, 'supersede_reason': 'markdown_reimport'} self._execute("UPDATE sources SET deleted_at = ?, metadata_json = ? WHERE id = ? AND user_id = ?", @@ -476,7 +521,7 @@ def optimize_scrub_indexes(self): self._execute("INSERT INTO memories_fts(memories_fts) VALUES('optimize')") -def scrub_source(self, source_id, *, optimize=True, citing_ids=None): +def scrub_source(self, source_id, *, optimize=True, citing_ids=None, loop_ids=None): with self.savepoint(): self._execute("PRAGMA secure_delete=ON") rows = self.get_sources_by_ids([source_id], include_deleted=True) @@ -497,7 +542,7 @@ def scrub_source(self, source_id, *, optimize=True, citing_ids=None): OR EXISTS (SELECT 1 FROM source_chunks c WHERE c.user_id = provenance_links.user_id AND c.id = provenance_links.source_chunk_id AND c.source_id = ?))""", (REMOVAL_MARKER, self.user_id, source_id, source_id)).rowcount - counts = retire_dependents(self, source_id, now=now, scrub_candidates=True, citing_ids=citing_ids) + counts = retire_dependents(self, source_id, now=now, scrub_candidates=True, citing_ids=citing_ids, loop_ids=loop_ids) counts.update({'chunks':chunks, 'provenance_quotes':quotes, 'sleep_proposals':sleep_count}) self._append_mutation_event(event_type='source.deleted', target_type='source', target_id=source_id, actor_type='user', payload={'operation':'scrub', **counts}) diff --git a/docs/alpha/known-limitations.md b/docs/alpha/known-limitations.md index a53763d6a..e1330b7b7 100644 --- a/docs/alpha/known-limitations.md +++ b/docs/alpha/known-limitations.md @@ -74,7 +74,7 @@ Open in v0.20.0, with the detail in the [v0.20.0 release notes](../release/v0.20 - the Postgres HTTP commit routes take up to 20,000 characters of memory text, and `alice_memory_commit` on SQLite has no length limit (a 2,000,000-character memory was stored whole). Unreleased (on main, not in v0.20.0): `alice_memory_commit` refuses a memory whose text is over 20,000 characters, on SQLite and on any other store the tool runs on, with `invalid_request` and nothing saved; the legacy tools `alice_vnext_correct_memory` and `alice_vnext_propose_memory` (off unless `ALICE_MCP_LEGACY_TOOLS=1`) still take text of any length on SQLite - in v0.20.0 a key bound to one project can attach a source it cannot read through `alice_memory_commit` `source_refs`, the `provenance` of `alice_memory_correct` and, over HTTP on Postgres, `POST /v0/vnext/open-loops`, and a saved quote stays readable after its source is reclassified. Unreleased (on main, not in v0.20.0): those doors answer `not_found` (404 over HTTP) for a source or memory the caller may not read and the readers of a saved quote and of an open loop ask the reader's own fence again, but a link that passes the fence of an `admin_agent` writer still makes `alice_explain` of that memory fail for the keys of a project with a lower ceiling (see [Cited sources](mcp-tools.md#cited-sources) and [Saved quotes](mcp-tools.md#saved-quotes)) - the cited-source fence does not cover everything yet. Unreleased (on main, not in v0.20.0): memory proposals and the agent-output ingest store their `source_refs` as given, the owner dependency trace lists a memory by the first id of a multi-id ref only, `provenance_count` still counts a withheld link, and a memory or open loop saved before the fix keeps its link or id, so `alice_explain` still fails for such a memory; the rest is under [Cited sources](mcp-tools.md#cited-sources), [Saved quotes](mcp-tools.md#saved-quotes) and in the [CHANGELOG](../../CHANGELOG.md) -- Unreleased (on main, not in v0.20.0): on SQLite, `import-markdown --supersede` matches by resolved path (a moved folder is not matched), and `sources delete` and `prune` keep source events, hashes, memory text and backups, and blank an open loop only when its column or its metadata names the id as stored or `source:` (any other spelling keeps the loop's text); freed space keeps text until the [VACUUM step](../integrations/importers.md#list-delete-and-prune-sqlite-sources). +- Unreleased (on main, not in v0.20.0): on SQLite, `import-markdown --supersede` matches by resolved path (a moved folder is not matched), and `sources delete` and `prune` keep source events, hashes, memory text and backups, and blank an open loop that names the source in any spelling the saved-quote reader names, in one pass over the user's loops. In v0.20.0 any spelling other than the id as stored or `source:` kept the loop's text, and freed space keeps text until the [VACUUM step](../integrations/importers.md#list-delete-and-prune-sqlite-sources). - Unreleased (on main, not in v0.20.0): a summary of restricted inputs keeps their most frequent restricted label, not `unknown`, but one that combines project scopes may be readable through one of them, and a stored one whose inputs no longer resolve is not repaired. See [Derived row domains](mcp-tools.md#derived-row-domains) What v0.19.0 and v0.19.2 limited and v0.20.0 fixed or narrowed (request body size, the `Host` check, provider redirects, local-folder reads, the memory id fence, deep backup JSON, the lone surrogate turn, the query size bounds and the data directory variable) is recorded in the [v0.19.2 release notes](../release/v0.19.2-release-notes.md), the [v0.20.0 release notes](../release/v0.20.0-release-notes.md) and the [CHANGELOG](../../CHANGELOG.md). diff --git a/docs/integrations/importers.md b/docs/integrations/importers.md index 08469cf6a..0d93004b2 100644 --- a/docs/integrations/importers.md +++ b/docs/integrations/importers.md @@ -336,15 +336,15 @@ over the memories for their preview and one for their receipt, however many replaced sources a prune removes. Unreleased (on main, not in v0.20.0): an open loop belongs to a source when its -`source_id` column holds the id, or when the text under `source_id`, `source_ids`, -`source_ref`, `source_refs`, `source_references` or `selected_source_ids`, at any -depth of its metadata, is the id as stored or `source:`. This is the rule of -the lookup of the open loops that name a source. Replacement, `sources delete`, -`sources prune` and the delete preview all use it, so a loop whose column is -empty is closed and blanked too, and the preview counts the loops the receipt -reports. Unlike the memory rule above, it reads the id only as stored or as -`source:`, so a loop that names the source in any spelling other than the id -as stored or `source:` keeps its text. +`source_id` column or its metadata names that source in any spelling the +saved-quote reader names (capitals, no hyphens, braces, `urn:uuid:`, a list, or +JSON text) under `source_id`, `source_ids`, `source_ref`, `source_refs`, +`source_references` or `selected_source_ids`. Replacement, `sources delete`, +`sources prune` and the delete preview all use that rule, and the reader that +withholds ids uses it too. One pass over the user's loops answers for every +source of the command, and the preview counts the loops the receipt reports. +In v0.20.0 the rule read the id only as stored or as `source:`, so any other +spelling kept the loop's text. There is no restore command for sources. Import the old text with `--supersede` to make it live again. A pre-deletion export can conflict with rows in the same diff --git a/tests/unit/test_known_limitations_page_shape.py b/tests/unit/test_known_limitations_page_shape.py index e17102961..1de40d165 100644 --- a/tests/unit/test_known_limitations_page_shape.py +++ b/tests/unit/test_known_limitations_page_shape.py @@ -22,8 +22,8 @@ # eight sentences each. The caps sit just above the page as it is, so a limit can be reworded and a new one added, and # nothing grows back into a record without this test failing. A bullet is one or two sentences, and a closing # "See ..." pointer to the page that explains it does not count as one. -# Raised from 14,950 to 15,090 (2026-10-05) for the one clause on how far an open loop is scrubbed with its source. -MAX_PAGE_CHARS = 15_090 +# Raised from 15,090 to 15,167 (2026-10-05) for the open-loop spelling clause (page length 15,147 plus 20). +MAX_PAGE_CHARS = 15_167 MAX_BULLET_CHARS = 800 MAX_BULLET_SENTENCES = 2 diff --git a/tests/unit/test_open_loop_references_read_fence.py b/tests/unit/test_open_loop_references_read_fence.py index d94911e87..f38830daf 100644 --- a/tests/unit/test_open_loop_references_read_fence.py +++ b/tests/unit/test_open_loop_references_read_fence.py @@ -1113,15 +1113,10 @@ def test_the_reference_keys_include_every_key_the_reverse_lookup_of_a_source_rea """ from alicebot_api.vnext_open_loop_references import SOURCE_REFERENCE_KEYS - from alicebot_api.vnext_stores.sqlite.open_loop_source_reference import OPEN_LOOP_SOURCE_REFERENCE_SQL + from alicebot_api.vnext_stores.sqlite.open_loop_source_reference import NAMED_REFERENCE_KEYS, named_source_ids - # The rule the lookup, the delete preview and the scrub share lives in one statement (the lookup reads it from there). - text = OPEN_LOOP_SOURCE_REFERENCE_SQL - keys_in_sql = { - part.strip().strip("'") - for part in text[text.index("ref.key IN (") + len("ref.key IN (") : text.index(")", text.index("ref.key IN ("))].split(",") - } - assert keys_in_sql and keys_in_sql <= SOURCE_REFERENCE_KEYS, keys_in_sql + assert NAMED_REFERENCE_KEYS and NAMED_REFERENCE_KEYS <= SOURCE_REFERENCE_KEYS + assert "cited_source_ids" in named_source_ids.__code__.co_names # -- 5. every reader of a loop, classified ---------------------------------------------------------------------- diff --git a/tests/unit/test_source_scrub_loop_references.py b/tests/unit/test_source_scrub_loop_references.py index aafc10bf0..f09819f93 100644 --- a/tests/unit/test_source_scrub_loop_references.py +++ b/tests/unit/test_source_scrub_loop_references.py @@ -1,10 +1,8 @@ """An open loop that names a source only in its metadata is scrubbed with the source, on SQLite. -``list_open_loops_referencing_source`` (the reader) finds a loop by its ``source_id`` column or by a source id or a -``source:`` text under one of the keys of ``SOURCE_REFERENCE_KEYS`` anywhere in its metadata. The delete preview, the -scrub and the replacement path (``import-markdown --supersede``) once counted and blanked the column only, so a loop with -a NULL column and the id in its metadata kept its title, description and metadata, and they reached a new export. All -three now use the one rule of the reader (``open_loop_source_reference_sql``, in ``vnext_stores/sqlite/open_loop_source_reference.py``). +``list_open_loops_referencing_source`` (the reader) finds a loop by every spelling ``cited_source_ids`` names, in the +``source_id`` column or under one of the keys of ``SOURCE_REFERENCE_KEYS``. The delete preview, the scrub and the +replacement path (``import-markdown --supersede``) use that same rule, in one pass over the user's loops. Mutations, each alone, each named by the test that fails: @@ -15,8 +13,8 @@ * ``blank_open_loops`` drops the ``user_id`` filter: ``test_another_users_loop_that_names_the_source_is_left_alone``. * ``blank_open_loops`` stops clearing ``description``, ``resolution_note`` or ``metadata_json``: the delete and replacement tests fail on the blank row. -* the shared SQL drops a key (``selected_source_ids``), the ``source:`` spelling or the column test: the per-key cases, - ``test_every_case_is_a_loop_the_reader_names`` and ``test_the_shared_rule_reads_exactly_the_reference_keys`` fail. +* ``named_source_ids`` stops calling ``cited_source_ids``: ``test_each_spelling_is_blanked_on_delete_and_on_replace`` + and ``test_the_shared_rule_reads_exactly_the_reference_keys`` fail. * ``list_open_loops_referencing_source`` goes back to its own copy of the rule, and the copy drops a key: ``test_the_scrub_matches_exactly_what_the_reader_lists`` fails, and so does the one-rule test. * the preview count drops its ``user_id`` filter: ``test_another_users_loop_that_names_the_source_is_left_alone`` fails on @@ -25,7 +23,7 @@ dismissed loops, which name the source only in metadata (or, for one resolved loop, by the column). * the scrub keeps the old ``closed_at``, ``resolved_at`` or ``updated_at`` of a loop it blanks: the delete and replacement tests fail in ``_assert_scrubbed``, which reads each stamp before and after. -* a caller builds its own statement again (an f-string over the rule): ``test_the_lookup_the_preview_and_the_scrub_share_one_rule``. +* the preview or the receipt looks the loops up once per source: ``test_a_prune_reads_the_loops_once_for_the_preview_and_once_for_the_receipt``. """ from __future__ import annotations @@ -336,11 +334,8 @@ def test_another_users_loop_that_names_the_source_is_left_alone(tmp_path, capsys assert _read(db, "SELECT title FROM open_loops WHERE user_id = '%s'" % USER_ID) == [(REMOVAL_MARKER,)] -# Spellings other than the id as stored or ``source:``. The rule of the open-loop reader compares the text under a key -# to those two, exactly, so none of these is read by the reader or the scrub, whatever any other reader does with them -# (the shared reference reader of memories and saved quotes, ``cited_source_ids``, reads many). The scrub follows the -# reader and no further. -UNREAD_SPELLINGS = { +# Spellings ``cited_source_ids`` names. Each is the only reference on its loop. +NAMED_SPELLINGS = { "upper": lambda sid: {"source_id": sid.upper()}, "compact": lambda sid: {"source_id": sid.replace("-", "")}, "upper_prefix": lambda sid: {"source_id": f"SOURCE:{sid}"}, @@ -352,23 +347,117 @@ def test_another_users_loop_that_names_the_source_is_left_alone(tmp_path, capsys } +def _escaped_json(sid: str) -> dict: + """A JSON text whose first hex digit is a ``\\u`` escape, so a scan of the raw text misses the id.""" + + digit = sid[0] + escaped = "\\u00" + format(ord(digit), "02x") + sid[1:] + return {"source_refs": '{"source_id": "' + escaped + '"}'} + + +def test_each_spelling_is_blanked_on_delete_and_on_replace(tmp_path, capsys): + """Every spelling ``cited_source_ids`` names is listed, counted, and blanked. Preview and receipt counts match. + + A prose mention under another key is left. Mutation: ``named_source_ids`` returns only the id as stored and + ``source:`` (the compact, braced, urn and JSON-text loops keep their titles). + """ + + for path in ("delete", "replace"): + root = tmp_path / path + root.mkdir() + db = _vault(root) + folder = _folder(root, note="The cobalt door opens on Monday.") + sid = run_import(db, folder).source_ids[0] + spellings = {**NAMED_SPELLINGS, "escaped": _escaped_json} + with sqlite_user_connection(db, USER_ID) as conn: + store = SQLiteVNextStore(conn, USER_ID) + planted = { + name: _create_loop(store, f"zebra{name}", metadata=builder(sid)) + for name, builder in spellings.items() + } + prose = _create_loop(store, "zebraprose", metadata={"unrelated": f"mentions {sid} in prose"}) + listed = {str(row["id"]) for row in store.list_open_loops_referencing_source(source_id=sid)} + assert listed == set(planted.values()) + assert prose not in listed + if path == "delete": + assert _command(db, "delete", sid) == 2 + preview = json.loads(capsys.readouterr().out)["would_delete"][0]["open_loops"] + assert _command(db, "delete", sid, "--yes") == 0 + receipt = json.loads(capsys.readouterr().out)["deleted"][0]["open_loops"] + assert preview == receipt == len(planted) + else: + (folder / "note.md").write_text("The cobalt door opens on Friday.") + run_import(db, folder, supersede=True) + payload = json.loads( + _read(db, "SELECT payload_json FROM event_log WHERE event_type = 'source.superseded' ORDER BY rowid DESC")[0][0] + ) + assert payload["open_loops"] == len(planted) + blanked = {loop for loop, in _read(db, "SELECT id FROM open_loops WHERE title = '%s'" % REMOVAL_MARKER)} + assert blanked == set(planted.values()) + assert _read(db, "SELECT title FROM open_loops WHERE id = '%s'" % prose) == [("zebraprose",)] + + +def test_a_prune_reads_the_loops_once_for_the_preview_and_once_for_the_receipt(tmp_path, capsys, monkeypatch): + """Twelve replaced sources. The preview reads the user's loops once, and the receipt reads them once, not once per source. + + Mutation: ``_preview`` or ``run_sources`` calls ``open_loops_naming_sources`` inside the per-source loop. + """ + + from alicebot_api.vnext_stores.sqlite import source_retirement + + db = _vault(tmp_path) + folder = _folder(tmp_path, **{f"note{index}": f"The older amber statement {index}." for index in range(12)}) + run_import(db, folder) + for index in range(12): + (folder / f"note{index}.md").write_text(f"The current copper statement {index}.") + assert len(run_import(db, folder, supersede=True).superseded) == 12 + ids = [row[0] for row in _read(db, "SELECT id FROM sources WHERE deleted_at IS NOT NULL ORDER BY deleted_at, id")] + with sqlite_user_connection(db, USER_ID) as conn: + store = SQLiteVNextStore(conn, USER_ID) + for index, sid in enumerate(ids): + builder = list(NAMED_SPELLINGS.values())[index % len(NAMED_SPELLINGS)] + _create_loop(store, f"zebraprune{index}", metadata=builder(sid)) + for index in range(40): + _create_loop(store, f"zebrafiller{index}") + calls = [] + original = source_retirement._user_open_loops + + def _counting(store): + calls.append(store.user_id) + return original(store) + + monkeypatch.setattr(source_retirement, "_user_open_loops", _counting) + assert _command(db, "prune", "--superseded") == 2 + preview = json.loads(capsys.readouterr().out)["would_delete"] + assert len(calls) == 1 + assert sum(row["open_loops"] for row in preview) == 12 + calls.clear() + assert _command(db, "prune", "--superseded", "--yes") == 0 + receipt = json.loads(capsys.readouterr().out)["deleted"] + assert len(calls) == 1 + assert sum(row["open_loops"] for row in receipt) == sum(row["open_loops"] for row in preview) == 12 + + def test_the_scrub_matches_exactly_what_the_reader_lists(tmp_path, capsys): """One rule, not two: a loop is blanked by a delete if and only if the reader lists it for the source.""" db = _vault(tmp_path) sid = run_import(db, _folder(tmp_path, note="The cobalt door opens on Monday.")).source_ids[0] - ids = {} with sqlite_user_connection(db, USER_ID) as conn: store = SQLiteVNextStore(conn, USER_ID) - for name, metadata in UNREAD_SPELLINGS.items(): - ids[name] = _create_loop(store, f"zebra{name}", metadata=metadata(sid)) + planted = { + name: _create_loop(store, f"zebra{name}", metadata=builder(sid)) + for name, builder in NAMED_SPELLINGS.items() + } read = {key: _create_loop(store, f"zebrakey{index}", metadata={key: sid}) for index, key in enumerate(KEYS)} listed = {str(row["id"]) for row in store.list_open_loops_referencing_source(source_id=sid)} + assert _command(db, "delete", sid) == 2 + preview = json.loads(capsys.readouterr().out)["would_delete"][0]["open_loops"] assert _command(db, "delete", sid, "--yes") == 0 receipt = json.loads(capsys.readouterr().out)["deleted"][0] blanked = {loop for loop, in _read(db, "SELECT id FROM open_loops WHERE title = '%s'" % REMOVAL_MARKER)} - assert blanked == listed == set(read.values()) - assert receipt["open_loops"] == len(listed) == len(KEYS) + assert blanked == listed == set(planted.values()) | set(read.values()) + assert preview == receipt["open_loops"] == len(listed) def _code(function): @@ -407,29 +496,23 @@ def test_the_lookup_the_preview_and_the_scrub_share_one_rule(): """ from alicebot_api import source_commands - from alicebot_api.vnext_stores.sqlite import graph_open_loops, open_loop_source_reference as rule, source_retirement + from alicebot_api.vnext_stores.sqlite import graph_open_loops, source_retirement expected = { - graph_open_loops.list_open_loops_referencing_source: {"open_loop_source_reference_sql"}, - source_retirement.source_open_loop_count: {"OPEN_LOOP_SOURCE_COUNT_SQL", "open_loop_source_reference_params"}, - source_retirement.blank_open_loops: {"OPEN_LOOP_SOURCE_BLANK_SQL", "open_loop_source_reference_params"}, + graph_open_loops.list_open_loops_referencing_source: {"open_loops_naming_sources"}, + source_retirement.source_open_loop_count: {"open_loops_naming_sources"}, + source_retirement.blank_open_loops: {"open_loops_naming_sources"}, } for function, uses in expected.items(): names, strings, fstring = _code(function) assert uses <= names, (function.__name__, uses - names) text = "\n".join(strings) assert "json_tree" not in text and "source_id =" not in text, function.__name__ - if function is not graph_open_loops.list_open_loops_referencing_source: - # The reader keeps its column list in its own f-string; the preview and the scrub build no SQL at all. - assert not fstring and "open_loops" not in text, function.__name__ - # The two statements are the one condition inside fixed text, with the user test before it. - for statement in (rule.OPEN_LOOP_SOURCE_COUNT_SQL, rule.OPEN_LOOP_SOURCE_BLANK_SQL): - assert statement.count(rule.OPEN_LOOP_SOURCE_REFERENCE_SQL) == 1 - assert statement.index("user_id = ?") < statement.index(rule.OPEN_LOOP_SOURCE_REFERENCE_SQL) + assert not fstring, function.__name__ names, strings, _ = _code(source_retirement.retire_dependents) assert "blank_open_loops" in names and not any("UPDATE open_loops" in text for text in strings) names, strings, _ = _code(source_commands._preview) - assert "source_open_loop_count" in names and not any("FROM open_loops" in text for text in strings) + assert "open_loops_naming_sources" in names and not any("FROM open_loops" in text for text in strings) def test_the_shared_rule_reads_exactly_the_reference_keys(): @@ -438,13 +521,7 @@ def test_the_shared_rule_reads_exactly_the_reference_keys(): Mutation: drop a key from the statement, or add a seventh to it. """ - from alicebot_api.vnext_stores.sqlite.open_loop_source_reference import ( - OPEN_LOOP_SOURCE_REFERENCE_SQL, - open_loop_source_reference_sql, - ) + from alicebot_api.vnext_stores.sqlite.open_loop_source_reference import NAMED_REFERENCE_KEYS, named_source_ids - text = OPEN_LOOP_SOURCE_REFERENCE_SQL - inside = text[text.index("ref.key IN (") + len("ref.key IN (") : text.index(")", text.index("ref.key IN ("))] - assert {part.strip().strip("'") for part in inside.split(",")} == SOURCE_REFERENCE_KEYS - sql, params = open_loop_source_reference_sql(UUID(int=5)) - assert sql == text and params == (str(UUID(int=5)), str(UUID(int=5)), f"source:{UUID(int=5)}") + assert NAMED_REFERENCE_KEYS == SOURCE_REFERENCE_KEYS + assert "cited_source_ids" in named_source_ids.__code__.co_names diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index eae9d1a4d..48f3653c9 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -124,21 +124,20 @@ # to ``None`` ("not stated", which raises when the request holds the marker) and the single-scan reader takes the # domain filter and the sensitivity ceiling as required arguments (reviewed change, not drift). # Re-minted for the source scrub of open loops (2026-10-05): ``list_open_loops_referencing_source`` reads the rule - # "this loop names this source" from ``open_loop_source_reference_sql`` instead of holding its own copy, so the - # delete preview and the scrub count and blank the same loops. The rule text moved unchanged to - # ``vnext_stores/sqlite/open_loop_source_reference.py``; only the reader function and the receipt of the file + # "this loop names this source" from ``open_loops_naming_sources`` instead of holding its own copy, so the + # delete preview and the scrub count and blank the same loops. The reader function and the receipt of the file # change (reviewed change, not drift). - SQLITE_CARRIER_PATH: "9a2634bef621d32262b845c046820d8b19c64801ec9f9b462e978f364f16f643", + SQLITE_CARRIER_PATH: "bb2635700d902d26c6de3f26ebda4e1ba4997dcb2c9acc9aab918a1f6b242596", POSTGRES_COLUMNS_PATH: "5b0d972a55abf8590ce14394a37fd71b9b88ba7ab3de82d61efc1bddfc022b71", SQLITE_COLUMNS_PATH: "be81b8628d0831d3d02b280b5455fb02333db5740ebef8d85d58024384ae6556", } EXPECTED_METHOD_AST_MANIFESTS = { POSTGRES_CARRIER_PATH: "2558088459f1b9a565e1b366ffe0b7c4025c623a9e2ea78007d06a46793ce1b8", - SQLITE_CARRIER_PATH: "2850ba6057b1510759613aaa3798a226808a42470ee11cfb9c6e3afbf3e98e66", + SQLITE_CARRIER_PATH: "b371202b7256a6d002a74433cd1ce7cea667490416286ec4f1a60bc4641b34ef", } EXPECTED_METADATA_MANIFESTS = { POSTGRES_CARRIER_PATH: "6edb6a10e7a37dbbbbde97e5550422718a0112257666de8e23d49c60490fa13f", - SQLITE_CARRIER_PATH: "da4c86fd17190805004670b0bec8a40e03a4a9a29a262efa961d3bc62abea644", + SQLITE_CARRIER_PATH: "121ed7dcea3f8565c181e844e49e0bc229098f6dc6c9124d03bb12e6062ab562", } EXPECTED_COMMENT_MANIFESTS = { POSTGRES_CARRIER_PATH: (9, "bb34d175e716f5a929fa1ee5e7e30ba0e0b25be285cda3556a0c709719316c4e"), From cf5c90cb971c74cd784e3d2154b7b09aeebf6702 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 20:09:49 +0000 Subject: [PATCH 060/270] Accept a recorded project scope and floor on skip import. A repeat skip import replaces a memory or open loop domain, sensitivity, project scope, or project floor when an event recorded the file's previous value, and still refuses any other difference. --- apps/api/src/alicebot_api/onramp.py | 64 ++++++++++++++++ tests/unit/test_sqlite_label_repair_v3.py | 91 +++++++++++++++++++++++ 2 files changed, 155 insertions(+) diff --git a/apps/api/src/alicebot_api/onramp.py b/apps/api/src/alicebot_api/onramp.py index 6474ea1f3..8e14b979a 100644 --- a/apps/api/src/alicebot_api/onramp.py +++ b/apps/api/src/alicebot_api/onramp.py @@ -3689,6 +3689,32 @@ def _stored_row_matches( return False +def _decoded_object(value: object) -> dict[str, object]: + if isinstance(value, str): + try: + loaded = json.loads(value) + except json.JSONDecodeError: + return {} + return dict(loaded) if isinstance(loaded, dict) else {} + if isinstance(value, dict): + return dict(value) + return {} + + +def _scope_identity_token(row: Mapping[str, object]) -> str: + from alicebot_api.vnext_project_scope import resolve_project_scope + + return json.dumps(list(resolve_project_scope(row).identity), sort_keys=True) + + +def _floor_identity_token(metadata: Mapping[str, object]) -> str: + from alicebot_api.vnext_project_scope import project_scope_identity + + floor = metadata.get("project_floor") + values = floor if isinstance(floor, list) else [] + return json.dumps(list(project_scope_identity(values)), sort_keys=True) + + def _import_records( conn: sqlite3.Connection, store: SQLiteVNextStore, @@ -3801,6 +3827,44 @@ def _import_records( ): adjusted[index] = stored_value changed_dimension = True + if "metadata_json" in columns: + meta_index = columns.index("metadata_json") + file_meta = _decoded_object(adjusted[meta_index]) + stored_meta = _decoded_object(existing["metadata_json"]) + project_index = columns.index("project_id") if "project_id" in columns else None + file_project_id = adjusted[project_index] if project_index is not None else None + stored_project_id = existing["project_id"] if project_index is not None else None + file_row = {"project_id": file_project_id, "metadata_json": file_meta} + stored_row = {"project_id": stored_project_id, "metadata_json": stored_meta} + file_scope = _scope_identity_token(file_row) + stored_scope = _scope_identity_token(stored_row) + file_floor = _floor_identity_token(file_meta) + stored_floor = _floor_identity_token(stored_meta) + scope_known = file_scope == stored_scope or file_scope in label_repairs.get( + (table, row_id, "project_scope"), set() + ) + floor_known = file_floor == stored_floor or file_floor in label_repairs.get( + (table, row_id, "project_floor"), set() + ) + spelling_differs = ( + file_meta.get("project_scope") != stored_meta.get("project_scope") + or file_meta.get("project_floor") != stored_meta.get("project_floor") + or file_project_id != stored_project_id + ) + if scope_known and floor_known and spelling_differs: + new_meta = dict(file_meta) + if "project_scope" in stored_meta: + new_meta["project_scope"] = stored_meta["project_scope"] + else: + new_meta.pop("project_scope", None) + if "project_floor" in stored_meta: + new_meta["project_floor"] = stored_meta["project_floor"] + else: + new_meta.pop("project_floor", None) + adjusted[meta_index] = _encode_column_value("metadata_json", new_meta) + if project_index is not None: + adjusted[project_index] = stored_project_id + changed_dimension = True if changed_dimension: candidates.append(tuple(adjusted)) if not _stored_row_matches( diff --git a/tests/unit/test_sqlite_label_repair_v3.py b/tests/unit/test_sqlite_label_repair_v3.py index a8629ebdb..0d93a240e 100644 --- a/tests/unit/test_sqlite_label_repair_v3.py +++ b/tests/unit/test_sqlite_label_repair_v3.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import logging @@ -63,6 +64,96 @@ def test_open_raises_a_public_copy_of_a_confidential_source(tmp_path, monkeypatc assert SOURCE_TEXT not in json.dumps(payload) +def test_skip_accepts_a_recorded_project_scope_and_floor(tmp_path, monkeypatch, capsys) -> None: + """A file that still has the scope and floor from before the repair is skipped, and the stored labels stay. + + Mutation: drop the project_scope and project_floor rewrite in ``_import_records``. The second import then + refuses the row. + """ + + from alicebot_api.onramp import _export_line + from alicebot_api.onramp import main as onramp_main + + path = tmp_path / "vault.db" + with monkeypatch.context() as patch: + patch.setattr(sqlite_schema, "_relabel_derived_labels", lambda conn: None) + bootstrap_database(path, user_id=USER, user_email="local@alice") + with without_insert_floor(), sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source( + { + "source_type": "note", + "title": "Clinic note", + "content_hash": "sha256:clinic", + "domain": "project", + "sensitivity": "confidential", + "metadata_json": {"project_scope": ["beta"]}, + } + ) + memory = store.create_memory( + { + "memory_key": "copy", + "canonical_text": SOURCE_TEXT, + "status": "active", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"source_id": source["id"], "project_scope": ["alpha"], "project_floor": []}, + } + ) + memory_id = str(memory["id"]) + with sqlite_user_connection(path, USER) as conn: + row = conn.execute("SELECT sensitivity, metadata_json FROM memories WHERE id = ?", (memory_id,)).fetchone() + event = conn.execute( + "SELECT payload_json FROM event_log WHERE event_type = 'memory.labels_raised' AND target_id = ?", + (memory_id,), + ).fetchone() + stored_meta = json.loads(row["metadata_json"] if isinstance(row, dict) else row[1]) + payload = json.loads(event["payload_json"] if isinstance(event, dict) else event[0]) + previous = payload["previous"] + assert stored_meta.get("project_floor") != previous.get("project_floor") + export_path = tmp_path / "vault.jsonl" + assert onramp_main(["export", "--db", str(path), "--user-id", USER, "--out", str(export_path)]) == 0 + digest = hashlib.sha256() + lines = [] + for line in export_path.read_text(encoding="utf-8").splitlines(): + record = json.loads(line) + record_type = record.get("record_type") + body = record.get("record") + if record_type in {"export_header", "export_footer"}: + lines.append((record_type, body)) + continue + if record_type == "memory" and isinstance(body, dict) and body.get("id") == memory_id: + body["sensitivity"] = previous["sensitivity"] + metadata = body.get("metadata_json") or {} + metadata["project_scope"] = previous["project_scope"] + metadata["project_floor"] = previous["project_floor"] + body["metadata_json"] = metadata + canonical = _export_line(str(record_type), body) + "\n" + digest.update(canonical.encode("utf-8")) + lines.append((record_type, canonical)) + rewritten = [] + for record_type, body in lines: + if record_type == "export_footer" and isinstance(body, dict): + body["sha256"] = digest.hexdigest() + rewritten.append(_export_line(record_type, body) + "\n") + elif record_type == "export_header": + rewritten.append(_export_line(record_type, body) + "\n") + else: + rewritten.append(body) + export_path.write_text("".join(rewritten), encoding="utf-8") + capsys.readouterr() + code = onramp_main(["import", "--db", str(path), "--user-id", USER, "--in", str(export_path), "--mode", "skip"]) + captured = capsys.readouterr() + assert code == 0, captured.err + with sqlite_user_connection(path, USER) as conn: + row = conn.execute("SELECT sensitivity, metadata_json FROM memories WHERE id = ?", (memory_id,)).fetchone() + sensitivity = row["sensitivity"] if isinstance(row, dict) else row[0] + metadata = json.loads(row["metadata_json"] if isinstance(row, dict) else row[1]) + assert sensitivity == "confidential" + assert metadata.get("project_floor") == stored_meta.get("project_floor") + assert metadata.get("project_scope") == stored_meta.get("project_scope") + + def test_labels_check_names_the_rows_the_next_open_would_raise(tmp_path, monkeypatch, capsys) -> None: from alicebot_api.onramp import main as onramp_main From bc7738ab6e28e729d757e213231bb69bc654a467 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 20:09:49 +0000 Subject: [PATCH 061/270] Name project scope and floor in the skip-import sentence. The backup page now says a repeat skip accepts those dimensions when the file still holds the recorded previous value. --- docs/alpha/backup-and-restore.md | 6 +++--- tests/unit/test_derived_domain_docs.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/alpha/backup-and-restore.md b/docs/alpha/backup-and-restore.md index 769d5eedc..85ae5d99d 100644 --- a/docs/alpha/backup-and-restore.md +++ b/docs/alpha/backup-and-restore.md @@ -296,9 +296,9 @@ whether the destination is new or was already upgraded. A derived row whose recorded inputs include a restricted domain takes the most frequent restricted label among them, the highest sensitivity and every project, and each row the repair changes gets one audit event with its old and new labels. Repeating -`--mode skip` accepts a memory whose domain or sensitivity the repair changed -when the file still holds the value recorded before that change, and still -refuses any other field that differs. A vault that is upgraded without a restore +`--mode skip` accepts a memory or open loop whose domain, sensitivity, project +scope or project floor the repair changed when the file still holds the value +recorded before that change, and still refuses any other field that differs. A vault that is upgraded without a restore gets the same repair once, the next time it opens. That open pass never stops a vault from opening, and a row the owner had lowered on purpose is raised again once. A portable backup carries no generated artifacts and the SQLite schema has diff --git a/tests/unit/test_derived_domain_docs.py b/tests/unit/test_derived_domain_docs.py index b3876b8b0..2740feddb 100644 --- a/tests/unit/test_derived_domain_docs.py +++ b/tests/unit/test_derived_domain_docs.py @@ -178,6 +178,7 @@ def test_the_backup_guide_states_when_the_repair_runs_and_what_it_cannot_repair( assert "the highest sensitivity and every project" in restore assert "one audit event with its old and new labels" in restore assert "still refuses any other field that differs" in restore + assert "project scope or project floor" in restore assert "A vault that is upgraded without a restore gets the same repair once, the next time it opens" in restore assert "never stops a vault from opening" in restore assert "raised again once" in restore From 79a23317dab267e569b7d6d0117cb4eff43fd9da Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 20:09:49 +0000 Subject: [PATCH 062/270] Clamp a derived memory edit at its inputs and name the clamp. An owner edit that would lower a derived memory is stored at the higher label. The review answer sets label_floor_applied only when that happens, and the event cause is floor_clamped. --- .../alicebot_api/routers/vnext_memories.py | 3 + .../src/alicebot_api/vnext_label_writes.py | 84 +++++++++++++++++++ .../vnext_stores/postgres/memory_lifecycle.py | 4 + .../vnext_stores/sqlite/memory_lifecycle.py | 4 + tests/unit/test_label_floor_applied.py | 64 ++++++++++++++ 5 files changed, 159 insertions(+) create mode 100644 tests/unit/test_label_floor_applied.py diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index 4bc6667eb..b0156174c 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -1332,6 +1332,9 @@ def review_vnext_memory( actor_id=actor_id, ) review_payload: dict[str, object] = {"memory": updated} + if getattr(store, "_label_floor_applied", False): + review_payload["label_floor_applied"] = True + store._label_floor_applied = False if action == "reject": review_payload["rationale_withheld"] = rationale_withheld review_payload["text_withheld"] = text_withheld diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 9fd571284..015384ae2 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -412,6 +412,89 @@ def _label_fields(row: Mapping[str, object]) -> tuple[str, str, tuple[str, ...], ) +def clamp_owner_patch( + store: Any, *, kind: str, before: Mapping[str, object] | None, patch: Mapping[str, object] +) -> JsonObject: + """Keep a derived row at or above its inputs when an edit would lower it. + + The store writes the higher label, records ``labels_raised`` with cause + ``floor_clamped`` when the stored label changes, and sets + ``store._label_floor_applied`` so the review answer can name it. + """ + + store._label_floor_applied = False + proposed_patch = dict(patch) + if before is None or not is_derived(kind, before): + return proposed_patch + proposed = dict(before) + for key in ("domain", "sensitivity", "project_id"): + if key in proposed_patch and proposed_patch[key] is not None: + proposed[key] = proposed_patch[key] + if isinstance(proposed_patch.get("metadata_json"), dict): + stored_meta = before.get("metadata_json") + meta = dict(stored_meta) if isinstance(stored_meta, dict) else {} + meta.update(proposed_patch["metadata_json"]) + proposed["metadata_json"] = meta + proposed["kind"] = kind + nodes, exceeded = collect_label_rows(store, [proposed], max_nodes=PROPAGATION_BOUND) + if exceeded: + return proposed_patch + try: + label = settle_labels(nodes).by_stored(kind, str(before.get("id") or "")) + except KeyError: + return proposed_patch + if label.unverified: + return proposed_patch + requested = _label_fields(proposed) + settled = (label.domain, label.sensitivity, tuple(label.project_scope), tuple(label.project_floor)) + if ( + requested[0] == settled[0] + and requested[1] == settled[1] + and project_scope_identity(requested[2]) == project_scope_identity(settled[2]) + and project_scope_identity(requested[3]) == project_scope_identity(settled[3]) + ): + return proposed_patch + proposed_patch["domain"] = label.domain + proposed_patch["sensitivity"] = label.sensitivity + metadata = dict(proposed.get("metadata_json") or {}) + metadata["project_scope"] = list(label.project_scope) + metadata["project_floor"] = list(label.project_floor) + proposed_patch["metadata_json"] = metadata + stored = _label_fields(before) + if not ( + stored[0] == settled[0] + and stored[1] == settled[1] + and project_scope_identity(stored[2]) == project_scope_identity(settled[2]) + and project_scope_identity(stored[3]) == project_scope_identity(settled[3]) + ): + event = build_event_log_record( + event_type=f"{kind}.labels_raised", + actor_type="system", + target_type=kind, + target_id=str(before.get("id") or ""), + payload=labels_raised_payload( + cause="floor_clamped", + previous={ + "domain": stored[0], + "sensitivity": stored[1], + "project_scope": list(stored[2]), + "project_floor": list(stored[3]), + }, + new={ + "domain": label.domain, + "sensitivity": label.sensitivity, + "project_scope": list(label.project_scope), + "project_floor": list(label.project_floor), + }, + ), + ) + append = getattr(store, "append_event", None) + if callable(append): + append(event) + store._label_floor_applied = True + return proposed_patch + + def write_settled_label( store: Any, *, @@ -642,6 +725,7 @@ def remember_floor_event(store: Any, event: JsonObject | None, target_id: object "REFUSED_DETAIL", "RETRYABLE_DETAIL", "acquire_exclusive_label_lock", + "clamp_owner_patch", "count_rows_hidden_by_scope_move", "label_error_response", "propagate", diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py index 99be62c49..64294a5be 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py @@ -450,6 +450,10 @@ def update_memory( patch_metadata, label_write=label_write, ) + if before_label is not None: + from alicebot_api.vnext_label_writes import clamp_owner_patch + + patch = clamp_owner_patch(self, kind="memory", before=before_label, patch=patch) row = self._fetch_one( "update_memory", f""" diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py index d5b89e9fc..728f8f359 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py @@ -304,6 +304,10 @@ def update_memory( before_label = self.get_memory(str(memory_id)) refuse_updated_credential_activation(patch, lambda: self.get_memory(str(memory_id))) patch = _with_protected_metadata(self, memory_id, patch, label_write=label_write) + if before_label is not None: + from alicebot_api.vnext_label_writes import clamp_owner_patch + + patch = clamp_owner_patch(self, kind="memory", before=before_label, patch=patch) # One clock reading for the write: an archive sets ``updated_at`` and ``deleted_at`` together. now = _utc_now_iso() cursor = self._execute( diff --git a/tests/unit/test_label_floor_applied.py b/tests/unit/test_label_floor_applied.py new file mode 100644 index 000000000..69e507c00 --- /dev/null +++ b/tests/unit/test_label_floor_applied.py @@ -0,0 +1,64 @@ +"""An owner edit that would lower a derived memory is held at its inputs. + +The handoff names this ``label_floor_applied`` on the review answer, with a +``labels_raised`` event whose cause is ``floor_clamped``. +""" + +from __future__ import annotations + +import inspect +import json + +from alicebot_api.onramp import bootstrap_database +from alicebot_api.routers.vnext_memories import review_vnext_memory +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_label_writes import without_insert_floor +from tests.unit.test_derived_domain_fence import USER + + +def test_an_edit_below_the_inputs_is_clamped_and_named(tmp_path) -> None: + """Mutation: ``clamp_owner_patch`` returns the patch unchanged. The stored sensitivity stays ``public`` + and no ``floor_clamped`` event is written. + """ + + path = tmp_path / "vault.db" + bootstrap_database(path, user_id=USER, user_email="local@alice") + with without_insert_floor(), sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source( + { + "source_type": "note", + "title": "Clinic note", + "content_hash": "sha256:clinic", + "domain": "health", + "sensitivity": "confidential", + "metadata_json": {}, + } + ) + memory = store.create_memory( + { + "memory_key": "copy", + "canonical_text": "A confidential observation", + "status": "active", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"source_id": source["id"]}, + } + ) + updated = store.update_memory( + memory_id=str(memory["id"]), + patch={"sensitivity": "public", "domain": "project"}, + actor_type="user", + ) + event = conn.execute( + "SELECT payload_json FROM event_log WHERE event_type = 'memory.labels_raised' AND target_id = ?", + (str(memory["id"]),), + ).fetchone() + assert updated["sensitivity"] == "confidential" + assert store._label_floor_applied is True + raw = event["payload_json"] if isinstance(event, dict) else event[0] + payload = json.loads(raw) + assert payload["cause"] == "floor_clamped" + assert "title" not in json.dumps(payload) + assert "A confidential observation" not in json.dumps(payload) + assert 'review_payload["label_floor_applied"] = True' in inspect.getsource(review_vnext_memory) From 8884941411c641fd0b1958df857f16ef78013abd Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:17:45 +0200 Subject: [PATCH 063/270] Enforce live label lock order and complete scope move previews --- apps/api/src/alicebot_api/cli/capture.py | 3 + .../src/alicebot_api/vnext_artifact_review.py | 6 + .../src/alicebot_api/vnext_label_writes.py | 128 ++++++++++++++++-- apps/api/src/alicebot_api/vnext_projects.py | 6 + apps/api/src/alicebot_api/vnext_store.py | 33 +++++ .../vnext_stores/postgres/graph_open_loops.py | 13 ++ .../vnext_stores/postgres/memory_access.py | 2 + .../vnext_stores/postgres/memory_lifecycle.py | 6 +- .../vnext_stores/sqlite/graph_open_loops.py | 13 ++ tests/integration/conftest.py | 9 ++ .../test_label_lock_order_postgres.py | 63 +++++++++ tests/unit/test_label_lock_order.py | 86 ++++++++++++ tests/unit/test_label_lock_registry.py | 69 ++++++++++ tests/unit/test_label_writer_registry.py | 78 +++++++++++ tests/unit/test_source_move_label_preview.py | 53 ++++++++ .../test_sqlite_derived_labels_write_path.py | 2 +- 16 files changed, 554 insertions(+), 16 deletions(-) create mode 100644 tests/integration/test_label_lock_order_postgres.py create mode 100644 tests/unit/test_label_lock_order.py create mode 100644 tests/unit/test_label_lock_registry.py create mode 100644 tests/unit/test_label_writer_registry.py create mode 100644 tests/unit/test_source_move_label_preview.py diff --git a/apps/api/src/alicebot_api/cli/capture.py b/apps/api/src/alicebot_api/cli/capture.py index 241566da8..cce5ef0e6 100644 --- a/apps/api/src/alicebot_api/cli/capture.py +++ b/apps/api/src/alicebot_api/cli/capture.py @@ -41,6 +41,7 @@ from alicebot_api.vnext_embeddings import DeferredMemoryEmbedding from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_store import PostgresVNextStore +from alicebot_api.vnext_label_writes import takes_label_lock as _takes_label_lock from .constants import DEFAULT_VNEXT_DEMO_DATASET_PATH, DEMO_SECRET_MARKERS from .models import CLIContext from .arguments import _object_dict, _object_int, _object_list @@ -394,6 +395,7 @@ def _demo_tag(dataset_id: str) -> JsonObject: return {"demo": True, "demo_dataset_id": dataset_id} +@_takes_label_lock def _reset_vnext_demo_dataset(store: PostgresVNextStore, *, dataset_id: str) -> JsonObject: with store.conn.cursor() as cur: cur.execute( @@ -507,6 +509,7 @@ def _tag_demo_candidate_memories(store: PostgresVNextStore, *, dataset_id: str, return updated +@_takes_label_lock def _tag_demo_artifact(store: PostgresVNextStore, *, artifact_id: str, dataset_id: str) -> None: artifact = store.get_artifact(artifact_id) if artifact is None: diff --git a/apps/api/src/alicebot_api/vnext_artifact_review.py b/apps/api/src/alicebot_api/vnext_artifact_review.py index 3e965e244..b42178dbd 100644 --- a/apps/api/src/alicebot_api/vnext_artifact_review.py +++ b/apps/api/src/alicebot_api/vnext_artifact_review.py @@ -37,6 +37,12 @@ def dispatch_vnext_artifact_review( mutating it, so no caller can route from a stale or forged preloaded row. """ + lock_graph = getattr(store, "lock_graph_mutation", None) + if callable(lock_graph): + lock_graph() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) target = store.get_artifact_for_update(artifact_id) if target is None: raise VNextQueueNotFoundError(f"artifact {artifact_id} was not found") diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 015384ae2..c823feebd 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -25,6 +25,7 @@ generation_domain, identifier, is_derived, + input_admitted, labels_raised_payload, settle_labels, stored_scope, @@ -127,6 +128,62 @@ def _in_transaction(store: Any) -> bool: return True +def held_label_locks(store: Any) -> tuple[bool, bool, bool]: + """Read the live S, L and exclusive L grants, including savepoint rollback.""" + + with store.conn.cursor() as cur: + cur.execute(""" + SELECT + coalesce(bool_or(classid = (hashtext('vnext_supersession')::bigint & 4294967295)::oid), false) AS graph, + coalesce(bool_or(classid = (hashtext('vnext_labels')::bigint & 4294967295)::oid), false) AS labels, + coalesce(bool_or(classid = (hashtext('vnext_labels')::bigint & 4294967295)::oid + AND mode = 'ExclusiveLock'), false) AS exclusive + FROM pg_locks + WHERE locktype = 'advisory' AND pid = pg_backend_pid() AND granted + AND objsubid = 2 + AND objid = (hashtext(app.current_user_id()::text)::bigint & 4294967295)::oid + """) + row = cur.fetchone() + if isinstance(row, Mapping): + return bool(row["graph"]), bool(row["labels"]), bool(row["exclusive"]) + return bool(row[0]), bool(row[1]), bool(row[2]) + + +def before_graph_lock(store: Any) -> None: + """Strict tests refuse S after L using the current database grants.""" + + if STRICT_LOCK_ORDER: + graph, labels, _exclusive = held_label_locks(store) + if labels and not graph: + raise LabelLockOrderError("the graph lock must precede the label lock") + + +def require_exclusive_label_lock(store: Any) -> None: + """A changing hook must hold exclusive L before taking any row lock.""" + + if _sqlite(store): + store.lock_label_writes(exclusive=True) + return + _graph, _labels, exclusive = held_label_locks(store) + if exclusive: + return + if STRICT_LOCK_ORDER: + raise LabelLockOrderError("the label change requires the exclusive label lock before row locks") + acquire_exclusive_label_lock(store) + + +def prepare_label_patch( + store: Any, kind: str, before: Mapping[str, object] | None, patch: Mapping[str, object] +) -> JsonObject: + """Check a proposed label change before its UPDATE or FOR UPDATE statement.""" + + proposed = dict(before or {}) + proposed.update({key: value for key, value in patch.items() if value is not None}) + if before and _label_fields(before) != _label_fields(proposed): + require_exclusive_label_lock(store) + return dict(patch) + + def _label_tuple(payload: Mapping[str, object]) -> tuple[str, str, tuple[str, ...], tuple[str, ...]]: metadata = payload.get("metadata_json") meta = metadata if isinstance(metadata, Mapping) else {} @@ -467,6 +524,7 @@ def clamp_owner_patch( and project_scope_identity(stored[2]) == project_scope_identity(settled[2]) and project_scope_identity(stored[3]) == project_scope_identity(settled[3]) ): + require_exclusive_label_lock(store) event = build_event_log_record( event_type=f"{kind}.labels_raised", actor_type="system", @@ -510,7 +568,7 @@ def write_settled_label( """Label-only update. A statement that changes no row refuses the whole relabel.""" table = {"memory": "memories", "open_loop": "open_loops", "artifact": "generated_artifacts", "project": "projects"}[kind] - blob = json.dumps(dict(metadata)) + blob = json.dumps({key: metadata[key] for key in ("project_scope", "project_floor") if key in metadata}) if _sqlite(store): project_sql = ", project_id = ?" if table in {"memories", "open_loops"} else "" params: list[object] = [domain, sensitivity, blob] @@ -520,7 +578,7 @@ def write_settled_label( cursor = store._execute( f""" UPDATE {table} - SET domain = ?, sensitivity = ?, metadata_json = ?{project_sql} + SET domain = ?, sensitivity = ?, metadata_json = json_patch(metadata_json, ?){project_sql} WHERE id = ? AND user_id = ? AND domain = ? AND sensitivity = ? """, # nosec B608 # table comes from the closed kind map; values are bound tuple(params), @@ -532,22 +590,47 @@ def write_settled_label( if project_sql: params.append(project_id) params.extend([str(row_id), expected_domain, expected_sensitivity]) - store._fetch_one( - "write_settled_label", + row = store._fetch_optional_one( f""" UPDATE {table} - SET domain = %s, sensitivity = %s, metadata_json = %s::jsonb{project_sql} + SET domain = %s, sensitivity = %s, metadata_json = metadata_json || %s::jsonb{project_sql} WHERE id = %s::uuid AND domain = %s AND sensitivity = %s RETURNING id """, # nosec B608 # table comes from the closed kind map; values are bound tuple(params), ) + require_changed(int(row is not None), table, str(row_id)) + + +LABEL_TABLE_ORDER = ("generated_artifacts", "projects", "open_loops", "memories") +_LABEL_TABLES = {"artifact": "generated_artifacts", "project": "projects", "open_loop": "open_loops", "memory": "memories"} + + +def lock_settled_label_rows(store: Any, changes: Sequence[Mapping[str, object]]) -> None: + """Lock exactly the changed rows in a stable table and UUID order.""" + + if _sqlite(store): + return + grouped: dict[str, set[str]] = {} + for row in changes: + grouped.setdefault(_LABEL_TABLES[str(row["kind"])], set()).add(str(row["id"])) + with store.conn.cursor() as cur: + for table in LABEL_TABLE_ORDER: + ids = sorted(grouped.get(table, ())) + if ids: + cur.execute( + f"SELECT id FROM {table} WHERE id = ANY(%s::uuid[]) ORDER BY id FOR UPDATE", # nosec B608 # closed internal table map + (ids,), + ) + locked = cur.fetchall() + if len(locked) != len(ids): + raise DerivedDomainRepairError("a planned label row disappeared before it could be locked") def propagate(store: Any, changed: Sequence[tuple[str, str]], *, cause: str) -> int: """Recompute dependants of ``changed`` rows and write the ones that rise.""" - store.lock_label_writes(exclusive=True) + require_exclusive_label_lock(store) roots = [row_id for _kind, row_id in changed] affected = walk_dependants(store, roots) if not affected: @@ -561,7 +644,7 @@ def propagate(store: Any, changed: Sequence[tuple[str, str]], *, cause: str) -> if exceeded: raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") settled = settle_labels(nodes) - written = 0 + changes: list[tuple[Mapping[str, object], Any, tuple[str, str, tuple[str, ...], tuple[str, ...]]]] = [] for row in affected: label = settled.by_stored(str(row.get("kind")), str(row.get("id"))) if label.unverified: @@ -586,6 +669,10 @@ def propagate(store: Any, changed: Sequence[tuple[str, str]], *, cause: str) -> and project_scope_identity(previous[3]) == project_scope_identity(current[3]) ): continue + changes.append((row, label, previous)) + lock_settled_label_rows(store, [row for row, _label, _previous in changes]) + written = 0 + for row, label, previous in sorted(changes, key=lambda item: (LABEL_TABLE_ORDER.index(_LABEL_TABLES[str(item[0]["kind"])]), str(item[0]["id"]))): raw_metadata = row.get("metadata_json") metadata = dict(raw_metadata) if isinstance(raw_metadata, Mapping) else {} metadata["project_scope"] = list(label.project_scope) @@ -660,15 +747,21 @@ def count_rows_hidden_by_scope_move(store: Any, source: Mapping[str, object], ne metadata = dict(raw_metadata) if isinstance(raw_metadata, Mapping) else {} metadata["project_scope"] = list(new_scope) moved["metadata_json"] = metadata - before = settle_labels([current, *[dict(row) for row in affected]]) - after = settle_labels([moved, *[dict(row) for row in affected]]) + nodes, exceeded = collect_label_rows(store, [current, *[dict(row) for row in affected]], max_nodes=PROPAGATION_BOUND) + if exceeded: + raise LabelPropagationTooLarge("the source move preview exceeded the label propagation bound") + before = settle_labels(nodes, on_cycle="unverified") + moved_nodes = [moved if str(row.get("kind")) == "source" and str(row.get("id")) == source_id else row for row in nodes] + after = settle_labels(moved_nodes, on_cycle="unverified") hidden = 0 for row in affected: old = before.by_stored(str(row.get("kind")), str(row.get("id"))) new = after.by_stored(str(row.get("kind")), str(row.get("id"))) - old_ids = set(project_scope_identity(old.project_scope)) - new_ids = set(project_scope_identity(new.project_scope)) - if old_ids - new_ids: + if old.unverified or not old.project_scope: + continue + binding = project_scope_identity([*old.project_scope, *old.project_floor]) + new_row = {**row, "metadata_json": {"project_scope": list(new.project_scope), "project_floor": list(new.project_floor)}} + if new.unverified or not new.project_scope or not input_admitted(str(row["kind"]), new_row, binding): hidden += 1 return hidden @@ -695,10 +788,17 @@ def raise_source_to_replacement(store: Any, old: Mapping[str, object], replaceme def label_error_response(exc: BaseException) -> tuple[int, str, str | None] | None: """``(status, detail, retry_after)`` for a relabel failure, or None.""" - if isinstance(exc, (LabelPropagationTooLarge, DerivedDomainRepairError)): - return 409, REFUSED_DETAIL, None + if isinstance(exc, LabelPropagationTooLarge): + return 409, REFUSED_DETAIL + "; cause: propagation_bound", None + if isinstance(exc, DerivedDomainRepairError): + cause = "row_changed" if "changed no row" in str(exc) or "disappeared" in str(exc) else "dependency_cycle" + return 409, REFUSED_DETAIL + "; cause: " + cause, None + if isinstance(exc, LabelLockOrderError): + return 409, REFUSED_DETAIL + "; cause: lock_order", None if type(exc).__name__ in {"LockNotAvailable", "DeadlockDetected", "SerializationFailure"}: return 503, RETRYABLE_DETAIL, "2" + if type(exc).__module__.startswith("psycopg"): + return 409, REFUSED_DETAIL + "; cause: database_error", None return None diff --git a/apps/api/src/alicebot_api/vnext_projects.py b/apps/api/src/alicebot_api/vnext_projects.py index 01b39aabb..df7e58ef8 100644 --- a/apps/api/src/alicebot_api/vnext_projects.py +++ b/apps/api/src/alicebot_api/vnext_projects.py @@ -858,6 +858,12 @@ def review_project_update( ) -> JsonObject: if action not in PROJECT_UPDATE_ACTIONS: raise VNextProjectValidationError("project update action must be accept, edit, or reject") + lock_graph = getattr(self.store, "lock_graph_mutation", None) + if callable(lock_graph): + lock_graph() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(self.store) # The artifact is the review decision's serialization point. Every # accept/edit/reject path must inspect and transition the same locked # row so stale reviewers cannot split project, memory, and artifact diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index d7b1305e1..da66a578e 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -483,6 +483,10 @@ def lock_label_writes(self, *, exclusive: bool = False) -> None: cur.execute( f"SELECT {mode}(hashtext('vnext_labels'), hashtext(app.current_user_id()::text))" ) + from alicebot_api.vnext_label_writes import _in_transaction, LabelLockOrderError + + if not _in_transaction(self): + raise LabelLockOrderError("label writes require an open transaction") def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: """Narrow label rows for the insert floor. No text columns.""" @@ -1253,6 +1257,9 @@ def get_source_by_dedupe_key(self, dedupe_key: str) -> VNextRow | None: @takes_label_lock def update_source(self, *, source_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import prepare_label_patch + + patch = prepare_label_patch(self, "source", self.get_source(source_id), patch) with self.conn.cursor() as cur: cur.execute( f""" @@ -1726,6 +1733,9 @@ def search_sources( @takes_label_lock def create_project(self, project: JsonObject, *, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import apply_insert_floor, remember_floor_event + + project, floor_event = apply_insert_floor(self, "project", project) row = self._fetch_one( "create_project", f""" @@ -1774,6 +1784,7 @@ def create_project(self, project: JsonObject, *, actor_type: str = "system") -> target_id=row["id"], payload={"operation": "create", "fields": _sorted_field_names(project)}, ) + remember_floor_event(self, floor_event, row["id"]) return row def get_project(self, project_id: str) -> VNextRow | None: @@ -1846,6 +1857,26 @@ def list_projects( @takes_label_lock def update_project(self, *, project_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import ( + apply_insert_floor, merge_protected_metadata, prepare_label_patch, + propagate_after_write, remember_floor_event, + ) + + before = self.get_project(project_id) + patch = dict(patch) + metadata = patch.get("metadata_json") + floor_event = None + if isinstance(metadata, dict) and before is not None: + patch["metadata_json"] = merge_protected_metadata( + before.get("metadata_json") if isinstance(before.get("metadata_json"), dict) else {}, + metadata, + label_write="derived_from" in metadata, + ) + if "derived_from" in metadata: + floored, floor_event = apply_insert_floor(self, "project", {**before, **patch}) + for key in ("domain", "sensitivity", "metadata_json"): + patch[key] = floored[key] + patch = prepare_label_patch(self, "project", before, patch) row = self._fetch_one( "update_project", f""" @@ -1879,6 +1910,8 @@ def update_project(self, *, project_id: str, patch: JsonObject, actor_type: str target_id=row["id"], payload={"operation": "update", "changes": patch}, ) + remember_floor_event(self, floor_event, row["id"]) + propagate_after_write(self, kind="project", before=before, after=row) return row def create_person(self, person: JsonObject, *, actor_type: str = "system") -> VNextRow: diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py b/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py index 3c4faed08..d20c0a152 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py @@ -1206,6 +1206,18 @@ def list_open_loop_events( @takes_label_lock def update_open_loop(self, *, loop_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import clamp_owner_patch, merge_protected_metadata, prepare_label_patch, propagate_after_write + + before = self.get_open_loop(loop_id) + patch = dict(patch) + metadata = patch.get("metadata_json") + if before is not None and isinstance(metadata, dict): + patch["metadata_json"] = merge_protected_metadata( + before.get("metadata_json") if isinstance(before.get("metadata_json"), dict) else {}, + metadata, label_write=False, + ) + patch = prepare_label_patch(self, "open_loop", before, patch) + patch = clamp_owner_patch(self, kind="open_loop", before=before, patch=patch) row = self._fetch_one( "update_open_loop", f""" @@ -1243,6 +1255,7 @@ def update_open_loop(self, *, loop_id: str, patch: JsonObject, actor_type: str = target_id=row["id"], payload={"operation": "update", "changes": patch}, ) + propagate_after_write(self, kind="open_loop", before=before, after=row) return row @takes_label_lock diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py index d8bb10f57..9d4690d72 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py @@ -7,6 +7,7 @@ from typing import cast from alicebot_api.store import ContinuityStoreInvariantError +from alicebot_api.vnext_label_writes import takes_label_lock from alicebot_api.vnext_embeddings import ( EMBEDDING_SIGNATURE_METADATA_KEY, memory_embedding_signature_is_current, @@ -217,6 +218,7 @@ def list_memories_referencing_sources( return grouped +@takes_label_lock def list_pending_derived_candidates_for_member( self, *, diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py index 64294a5be..0287fe060 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py @@ -362,6 +362,9 @@ def lock_graph_mutation(self) -> None: correction, forgetting, and transitions. Released automatically at commit/rollback. """ + from alicebot_api.vnext_label_writes import before_graph_lock + + before_graph_lock(self) with self.conn.cursor() as cur: cur.execute( "SELECT pg_advisory_xact_lock(hashtext('vnext_supersession'), hashtext(app.current_user_id()::text))" @@ -451,8 +454,9 @@ def update_memory( label_write=label_write, ) if before_label is not None: - from alicebot_api.vnext_label_writes import clamp_owner_patch + from alicebot_api.vnext_label_writes import clamp_owner_patch, prepare_label_patch + patch = prepare_label_patch(self, "memory", before_label, patch) patch = clamp_owner_patch(self, kind="memory", before=before_label, patch=patch) row = self._fetch_one( "update_memory", diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py index 29d29ba62..9ed0b0dfd 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py @@ -967,6 +967,18 @@ def list_open_loop_events( @takes_label_lock def update_open_loop(self, *, loop_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import clamp_owner_patch, merge_protected_metadata, prepare_label_patch, propagate_after_write + + before = self.get_open_loop(loop_id) + patch = dict(patch) + metadata = patch.get("metadata_json") + if before is not None and isinstance(metadata, dict): + patch["metadata_json"] = merge_protected_metadata( + before.get("metadata_json") if isinstance(before.get("metadata_json"), dict) else {}, + metadata, label_write=False, + ) + patch = prepare_label_patch(self, "open_loop", before, patch) + patch = clamp_owner_patch(self, kind="open_loop", before=before, patch=patch) cursor = self._execute( """ UPDATE open_loops @@ -1010,6 +1022,7 @@ def update_open_loop(self, *, loop_id: str, patch: JsonObject, actor_type: str = target_id=row["id"], payload={"operation": "update", "changes": patch}, ) + propagate_after_write(self, kind="open_loop", before=before, after=row) return row @takes_label_lock diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0a8f8edf2..404554c6e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -20,6 +20,15 @@ TEMPLATE_MIGRATION_COUNT = 0 +@pytest.fixture(autouse=True) +def strict_label_lock_order(monkeypatch: pytest.MonkeyPatch) -> None: + """Every integration flow obeys S, L, then label-row locks.""" + + import alicebot_api.vnext_label_writes as label_writes + + monkeypatch.setattr(label_writes, "STRICT_LOCK_ORDER", True) + + def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption( "--require-executed-tests", diff --git a/tests/integration/test_label_lock_order_postgres.py b/tests/integration/test_label_lock_order_postgres.py new file mode 100644 index 000000000..db9215843 --- /dev/null +++ b/tests/integration/test_label_lock_order_postgres.py @@ -0,0 +1,63 @@ +"""Live advisory grants enforce S before L and survive savepoint rollback.""" +from uuid import uuid4 + +import pytest + +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_store import PostgresVNextStore +from alicebot_api.vnext_label_writes import LabelLockOrderError, held_label_locks + + +def _user(url, user): + with user_connection(url, user) as conn: + ContinuityStore(conn).create_user(user, f"locks-{user}@example.test", "Synthetic") + + +def test_real_pg_strict_s_after_l_is_refused(migrated_database_urls): + user = uuid4() + url = migrated_database_urls["app"] + _user(url, user) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + store.lock_label_writes() + assert held_label_locks(store) == (False, True, False) + with pytest.raises(LabelLockOrderError, match="graph lock must precede"): + store.lock_graph_mutation() + + +def test_real_pg_savepoint_rollback_releases_live_grants(migrated_database_urls): + user = uuid4() + url = migrated_database_urls["app"] + _user(url, user) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + with pytest.raises(ValueError, match="rollback"): + with conn.transaction(): + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + assert held_label_locks(store) == (True, True, True) + raise ValueError("rollback") + assert held_label_locks(store) == (False, False, False) + store.lock_graph_mutation() + store.lock_label_writes() + assert held_label_locks(store) == (True, True, False) + + +def test_real_pg_changing_hook_needs_exclusive_before_the_update(migrated_database_urls): + user = uuid4() + url = migrated_database_urls["app"] + _user(url, user) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + row = store.create_memory({"memory_key": "original", "canonical_text": "synthetic", "domain": "project", "sensitivity": "public"}) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + with pytest.raises(LabelLockOrderError, match="exclusive label lock"): + store.update_memory(memory_id=str(row["id"]), patch={"sensitivity": "confidential"}) + assert store.get_memory(str(row["id"]))["sensitivity"] == "public" + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + assert store.update_memory(memory_id=str(row["id"]), patch={"sensitivity": "confidential"})["sensitivity"] == "confidential" diff --git a/tests/unit/test_label_lock_order.py b/tests/unit/test_label_lock_order.py new file mode 100644 index 000000000..5ac522ddc --- /dev/null +++ b/tests/unit/test_label_lock_order.py @@ -0,0 +1,86 @@ +"""Strict lock checks read database grants rather than cached state.""" +from types import SimpleNamespace + +import pytest + +from alicebot_api import vnext_label_writes as writes +from alicebot_api.vnext_store import PostgresVNextStore + + +class Cursor: + def __init__(self, conn): + self.conn = conn + def __enter__(self): + return self + def __exit__(self, *args): + return None + def execute(self, query, params=()): + self.conn.queries.append(query) + if "pg_advisory_xact_lock_shared" in query: + self.conn.grants["labels"] = True + elif "pg_advisory_xact_lock(" in query: + self.conn.grants["graph"] = True + def fetchone(self): + return dict(self.conn.grants) + + +def _store(): + conn = SimpleNamespace(grants={"graph": False, "labels": False, "exclusive": False}, queries=[]) + conn.cursor = lambda: Cursor(conn) + return PostgresVNextStore(conn) + + +def test_strict_s_after_l_is_refused(monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", True) + store = _store() + store.lock_label_writes() + with pytest.raises(writes.LabelLockOrderError, match="graph lock must precede"): + store.lock_graph_mutation() + assert any("FROM pg_locks" in query for query in store.conn.queries) + + +def test_a_savepoint_rollback_cannot_leave_a_stale_lock_memo(monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", True) + store = _store() + store.lock_graph_mutation() + store.lock_label_writes() + store.conn.grants.update(graph=False, labels=False, exclusive=False) + store.lock_graph_mutation() + store.conn.grants.update(graph=False, labels=True) + with pytest.raises(writes.LabelLockOrderError): + store.lock_graph_mutation() + assert sum("FROM pg_locks" in query for query in store.conn.queries) == 3 + + +def test_strict_changing_hook_requires_exclusive_l(monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", True) + store = _store() + before = {"domain": "project", "sensitivity": "public"} + with pytest.raises(writes.LabelLockOrderError, match="exclusive label lock"): + writes.prepare_label_patch(store, "memory", before, {"sensitivity": "confidential"}) + store.conn.grants["exclusive"] = True + assert writes.prepare_label_patch(store, "memory", before, {"sensitivity": "confidential"}) == {"sensitivity": "confidential"} + + +def test_propagation_locks_tables_and_rows_in_order(): + store = _store() + statements = [] + class RowsCursor(Cursor): + def execute(self, query, params=()): + statements.append((query, params)) + self.ids = params[0] + def fetchall(self): + return [{"id": row_id} for row_id in self.ids] + store.conn.cursor = lambda: RowsCursor(store.conn) + changes = [{"kind": kind, "id": row_id} for kind, row_id in [("memory", "b"), ("artifact", "c"), ("open_loop", "d"), ("project", "e"), ("memory", "a")]] + writes.lock_settled_label_rows(store, changes) + assert [query.split("FROM ")[1].split()[0] for query, _params in statements] == list(writes.LABEL_TABLE_ORDER) + assert all("ORDER BY id FOR UPDATE" in query for query, _params in statements) + assert statements[-1][1] == (["a", "b"],) + + +def test_compare_and_set_miss_refuses_the_whole_label_write(): + from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError + store = SimpleNamespace(_fetch_optional_one=lambda *_: None) + with pytest.raises(DerivedDomainRepairError, match="changed no row"): + writes.write_settled_label(store, kind="memory", row_id="missing", domain="health", sensitivity="confidential", metadata={}, project_id=None, expected_domain="project", expected_sensitivity="public") diff --git a/tests/unit/test_label_lock_registry.py b/tests/unit/test_label_lock_registry.py new file mode 100644 index 000000000..8e73f72b8 --- /dev/null +++ b/tests/unit/test_label_lock_registry.py @@ -0,0 +1,69 @@ +"""Discover every store SQL write or row lock on a label table.""" +import ast +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] / "apps/api/src/alicebot_api" +TABLE = r"(?:memories|sources|open_loops|generated_artifacts|projects)" +WRITE = re.compile(rf"\b(?:INSERT\s+INTO|UPDATE|DELETE\s+FROM)\s+{TABLE}\b", re.I) +FROM = re.compile(rf"\b(?:FROM|JOIN)\s+{TABLE}\b", re.I) + + +def sql_text(node): + if isinstance(node, ast.JoinedStr): + return " ".join(item.value if isinstance(item, ast.Constant) and isinstance(item.value, str) else "{}" for item in node.values) + return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else "" + + +def label_sql_functions(tree): + for fn in ast.walk(tree): + if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for node in ast.walk(fn): + query = " ".join(sql_text(node).split()) + if WRITE.search(query) or ("FOR UPDATE" in query.upper() and FROM.search(query)): + yield fn, node + break + + +def test_all_discovered_label_writers_and_row_lockers_take_l(): + paths = [ROOT / "vnext_store.py", *sorted((ROOT / "vnext_stores/postgres").glob("*.py"))] + found = set() + missing = [] + for path in paths: + for fn, node in label_sql_functions(ast.parse(path.read_text())): + found.add((path.name, fn.name)) + if not any(isinstance(dec, ast.Name) and dec.id == "takes_label_lock" for dec in fn.decorator_list): + missing.append(f"{path.name}:{node.lineno} {fn.name}") + assert ("memory_access.py", "list_pending_derived_candidates_for_member") in found + assert ("memory_lifecycle.py", "lock_project_update_artifacts_for_redaction") in found + assert not missing, "label SQL without L: " + ", ".join(missing) + + +# Each application SQL exception has a dedicated transaction protocol. +DIRECT_SQL_PROTOCOLS = { + ("vnext_label_writes.py", "write_settled_label"): "exclusive L and label compare-and-set", + ("vnext_label_writes.py", "lock_settled_label_rows"): "exclusive L and deterministic table order", + ("vnext_label_repair.py", "relabel_labels_sqlite"): "SQLite immediate writer transaction", + ("vnext_derived_domain_backfill.py", "relabel_derived_rows_sqlite"): "frozen historical SQLite repair", + ("labels.py", "repair_labels_postgres"): "S, exclusive L, ordered rows and compare-and-set", + ("sqlite_schema.py", "_backfill_legacy_memory_project_scopes"): "schema bootstrap writer transaction", + ("sqlite_schema.py", "_backfill_source_dedupe_keys"): "schema bootstrap writer transaction", + ("sqlite_schema.py", "_repair_source_dedupe_identity"): "schema bootstrap writer transaction", + ("sqlite_schema.py", "_backfill_memory_agent_attribution"): "schema bootstrap writer transaction", + ("sqlite_schema.py", "_deduplicate_memory_lookup_values"): "schema bootstrap writer transaction", + ("sqlite_schema.py", "_repair_tombstone_lookup_value_holders"): "schema bootstrap writer transaction", +} + + +def test_application_code_cannot_write_label_tables_directly(): + missing = [] + for path in ROOT.rglob("*.py"): + if path.name in {"vnext_store.py", "sqlite_store.py", "store.py"} or "vnext_stores" in path.parts or "legacy_store" in path.parts: + continue + for fn, node in label_sql_functions(ast.parse(path.read_text())): + if any(isinstance(dec, ast.Name) and dec.id in {"takes_label_lock", "_takes_label_lock"} for dec in fn.decorator_list): + continue + if (path.name, fn.name) not in DIRECT_SQL_PROTOCOLS: + missing.append(f"{path.relative_to(ROOT)}:{node.lineno} {fn.name}") + assert not missing, "application label SQL without protocol: " + ", ".join(missing) diff --git a/tests/unit/test_label_writer_registry.py b/tests/unit/test_label_writer_registry.py new file mode 100644 index 000000000..e047e7e5a --- /dev/null +++ b/tests/unit/test_label_writer_registry.py @@ -0,0 +1,78 @@ +"""Discover label-changing SQL, update patches and the derived insert floors.""" +import ast +import re +from pathlib import Path + +from tests.unit.test_label_lock_registry import ROOT, label_sql_functions, sql_text + +LABEL_KEYS = {"domain", "sensitivity", "project_id", "project_scope", "project_floor", "derived_from"} +UPDATE_METHODS = {"update_memory", "update_source", "update_open_loop", "update_project"} +# Explicit function identities make every newly added caller reviewable. +LABEL_CALLERS = { + ("routers/vnext_memories.py", "review_vnext_memory"), + ("routers/vnext_memories.py", "review_vnext_source"), + ("vnext_projects.py", "review_project_update"), + ("vnext_label_writes.py", "raise_source_to_replacement"), + ("sqlite_store.py", "supersede_source"), + ("cli/smokes.py", "_run_vnext_smoke_operator_console"), +} + + +def keys_in(node): + return {item.value for item in ast.walk(node) if isinstance(item, ast.Constant) and isinstance(item.value, str)} & LABEL_KEYS + + +def test_every_label_changing_update_caller_is_registered(): + missing = [] + for path in ROOT.rglob("*.py"): + if "vnext_stores" in path.parts or path.name in {"sqlite_store.py", "vnext_store.py"}: + continue + tree = ast.parse(path.read_text()) + for fn in ast.walk(tree): + if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + patch_nodes = [node for node in ast.walk(fn) if isinstance(node, (ast.Assign, ast.AnnAssign)) and any(isinstance(item, ast.Name) and "patch" in item.id for item in ast.walk(node))] + for call in ast.walk(fn): + if not isinstance(call, ast.Call) or not isinstance(call.func, ast.Attribute) or call.func.attr not in UPDATE_METHODS: + continue + patch = next((kw.value for kw in call.keywords if kw.arg == "patch"), None) + if patch is not None and (keys_in(patch) or (isinstance(patch, ast.Name) and any(keys_in(item) for item in patch_nodes))): + key = (str(path.relative_to(ROOT)), fn.name) + if key not in LABEL_CALLERS: + missing.append(f"{key[0]}:{call.lineno} {fn.name}") + assert not missing, "unregistered label caller: " + ", ".join(missing) + + +def test_every_create_has_the_insert_floor_and_every_update_has_its_hook(): + floors = { + "vnext_store.py": {"create_artifact", "upsert_artifact_by_workflow_digest", "create_project", "update_project"}, + "vnext_stores/postgres/memory_lifecycle.py": {"create_memory"}, + "vnext_stores/postgres/graph_open_loops.py": {"create_open_loop"}, + "vnext_stores/sqlite/memory_lifecycle.py": {"create_memory"}, + "vnext_stores/sqlite/graph_open_loops.py": {"create_open_loop"}, + } + hooks = { + "vnext_store.py": {"update_source", "update_project"}, + "sqlite_store.py": {"update_source"}, + "vnext_stores/postgres/memory_lifecycle.py": {"update_memory"}, + "vnext_stores/postgres/graph_open_loops.py": {"update_open_loop"}, + "vnext_stores/sqlite/memory_lifecycle.py": {"update_memory"}, + "vnext_stores/sqlite/graph_open_loops.py": {"update_open_loop"}, + } + for registry, helper in ((floors, "apply_insert_floor"), (hooks, "propagate_after_write")): + for name, functions in registry.items(): + tree = ast.parse((ROOT / name).read_text()) + for function in functions: + fn = next(node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef) and node.name == function) + calls = {node.func.id for node in ast.walk(fn) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)} + assert helper in calls, f"{name}:{fn.lineno} {function} lacks {helper}" + + +def test_a_new_derived_insert_cannot_bypass_the_floor(): + for path in [ROOT / "vnext_store.py", ROOT / "sqlite_store.py", *sorted((ROOT / "vnext_stores").rglob("*.py"))]: + for fn, node in label_sql_functions(ast.parse(path.read_text())): + inserts = [sql_text(item) for item in ast.walk(fn)] + if not any(re.search(r"\bINSERT\s+INTO\s+(?:memories|open_loops|generated_artifacts|projects)\b", query, re.I) for query in inserts): + continue + calls = {item.func.id for item in ast.walk(fn) if isinstance(item, ast.Call) and isinstance(item.func, ast.Name)} + assert "apply_insert_floor" in calls, f"{path.name}:{node.lineno} {fn.name} inserts without a floor" diff --git a/tests/unit/test_source_move_label_preview.py b/tests/unit/test_source_move_label_preview.py new file mode 100644 index 000000000..cb6caa1ce --- /dev/null +++ b/tests/unit/test_source_move_label_preview.py @@ -0,0 +1,53 @@ +"""A source move previews loss of full project admission before any write.""" +from copy import deepcopy +from uuid import uuid4 + +import pytest + +from alicebot_api import vnext_label_writes as writes + + +def _source(scope): + return {"id": str(uuid4()), "kind": "source", "user_id": "synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": scope}} + + +def _report(sources, scope): + return {"id": str(uuid4()), "kind": "artifact", "user_id": "synthetic", "artifact_type": "daily_brief", "domain": "project", "sensitivity": "public", "metadata_json": {"source_ids": [row["id"] for row in sources], "project_scope": scope}} + + +class Store: + def __init__(self, rows): + self.rows = rows + self.reads = [] + def read_label_rows(self, kind, ids): + self.reads.append((kind, ids)) + return [row for row in self.rows if row["kind"] == kind and row["id"] in ids] + + +def test_stored_scope_unchanged_floor_move_still_counts_hidden_row(monkeypatch): + source = _source(["alpha"]) + report = _report([source], ["alpha"]) + store = Store([source, report]) + original = deepcopy(store.rows) + monkeypatch.setattr(writes, "walk_dependants", lambda *_: [report]) + assert writes.count_rows_hidden_by_scope_move(store, source, ["beta"]) == 1 + assert store.rows == original + + +def test_preview_reads_the_other_parent_and_its_ancestry(monkeypatch): + moved, other = _source(["alpha"]), _source(["alpha"]) + parent = _report([other], ["alpha"]) + report = _report([moved], ["alpha"]) + report["metadata_json"]["artifact_ids"] = [parent["id"]] + store = Store([moved, other, parent, report]) + monkeypatch.setattr(writes, "walk_dependants", lambda *_: [report]) + assert writes.count_rows_hidden_by_scope_move(store, moved, ["beta"]) == 1 + assert any(other["id"] in ids for _kind, ids in store.reads) + assert any(parent["id"] in ids for _kind, ids in store.reads) + + +def test_preview_does_not_count_a_row_already_unverified(monkeypatch): + source = _source(["alpha"]) + report = _report([source, _source(["alpha"])], ["alpha"]) + monkeypatch.setattr(writes, "walk_dependants", lambda *_: [report]) + assert writes.count_rows_hidden_by_scope_move(Store([source, report]), source, ["beta"]) == 0 diff --git a/tests/unit/test_sqlite_derived_labels_write_path.py b/tests/unit/test_sqlite_derived_labels_write_path.py index 3bfd61cca..ad231c68a 100644 --- a/tests/unit/test_sqlite_derived_labels_write_path.py +++ b/tests/unit/test_sqlite_derived_labels_write_path.py @@ -76,7 +76,7 @@ def test_an_update_that_omits_a_marker_keeps_it(tmp_path: Path) -> None: path = tmp_path / "vault.sqlite3" with _vault(path) as conn: store = SQLiteVNextStore(conn, USER) - health = add_memory(store, key="health", text="A restricted observation", domain="health") + health = add_memory(store, key="health", text="A restricted observation", domain="health", scope=(ALPHA,)) with without_insert_floor(): derived = store.create_memory( { From 5741c65c2fe0d2e7da0570e82f136f27929c6740 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:12:01 +0200 Subject: [PATCH 064/270] Repair stored labels atomically and report owner checks accurately --- .../20261005_0096_derived_label_floor.py | 19 +++-- apps/api/src/alicebot_api/cli/labels.py | 61 ++++++++------- apps/api/src/alicebot_api/db.py | 15 ++++ apps/api/src/alicebot_api/label_commands.py | 2 +- apps/api/src/alicebot_api/vault_doctor.py | 12 ++- apps/api/src/alicebot_api/vnext_doctor.py | 14 +++- .../src/alicebot_api/vnext_label_repair.py | 76 ++++++++++++++----- 7 files changed, 135 insertions(+), 64 deletions(-) diff --git a/apps/api/alembic/versions/20261005_0096_derived_label_floor.py b/apps/api/alembic/versions/20261005_0096_derived_label_floor.py index 94e52e144..1f8c128a8 100644 --- a/apps/api/alembic/versions/20261005_0096_derived_label_floor.py +++ b/apps/api/alembic/versions/20261005_0096_derived_label_floor.py @@ -14,7 +14,7 @@ from alicebot_api.vnext_derived_domain_backfill import require_changed from alicebot_api.vnext_derived_labels import labels_raised_payload from alicebot_api.vnext_event_log import build_event_log_record -from alicebot_api.vnext_label_repair import INPUT_SELECTS_V3, plan_label_repairs +from alicebot_api.vnext_label_repair import INPUT_SELECTS_V3, plan_label_repairs, label_project_id revision = "20261005_0096" down_revision = "20261004_0095" @@ -47,15 +47,17 @@ def upgrade() -> None: for table, statement in INPUT_SELECTS_V3.items(): tables[table] = list(connection.execute(text(statement)).mappings()) for table, user, row_id, previous, new, node in plan_label_repairs(tables): - metadata = dict(node.get("metadata_json") or {}) - metadata["project_scope"] = list(new["project_scope"]) - metadata["project_floor"] = list(new["project_floor"]) + metadata = {"project_scope": list(new["project_scope"]), "project_floor": list(new["project_floor"])} + project_sql = ", project_id = :project_id" if table in {"memories", "open_loops"} else "" + project_guard = " AND project_id IS NOT DISTINCT FROM :previous_project_id" if project_sql else "" require_changed( connection.execute( text( f"UPDATE {table} SET domain = :domain, sensitivity = :sensitivity, " - "metadata_json = CAST(:metadata AS jsonb) " - "WHERE user_id = CAST(:user AS uuid) AND id = CAST(:id AS uuid)" + f"metadata_json = metadata_json || CAST(:metadata AS jsonb){project_sql} " + "WHERE user_id = CAST(:user AS uuid) AND id = CAST(:id AS uuid) " + "AND domain = :previous_domain AND sensitivity = :previous_sensitivity " + f"AND metadata_json = CAST(:previous_metadata AS jsonb){project_guard}" ), { "domain": new["domain"], @@ -63,6 +65,11 @@ def upgrade() -> None: "metadata": json.dumps(metadata), "user": user, "id": row_id, + "previous_domain": previous["domain"], + "previous_sensitivity": previous["sensitivity"], + "previous_metadata": json.dumps(node.get("metadata_json") or {}), + "project_id": label_project_id(new["project_scope"]), + "previous_project_id": node.get("project_id"), }, ).rowcount, table, diff --git a/apps/api/src/alicebot_api/cli/labels.py b/apps/api/src/alicebot_api/cli/labels.py index 7f98472b5..fc3b92b37 100644 --- a/apps/api/src/alicebot_api/cli/labels.py +++ b/apps/api/src/alicebot_api/cli/labels.py @@ -3,29 +3,30 @@ from __future__ import annotations from alicebot_api.cli.shared import CLIContext, _vnext_store_context -from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError -from alicebot_api.vnext_label_repair import classify_stored_labels, format_label_check, plan_label_repairs +from alicebot_api.db import user_read_snapshot_connection +from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError, require_changed +from alicebot_api.vnext_label_repair import classify_stored_labels, format_label_check, plan_label_repairs, load_postgres_label_tables, label_project_id from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock -def _postgres_tables(conn) -> dict[str, list[dict[str, object]]]: - from alicebot_api.vnext_label_repair import INPUT_SELECTS_V3 +_postgres_tables = load_postgres_label_tables - tables: dict[str, list[dict[str, object]]] = {} - for table, statement in INPUT_SELECTS_V3.items(): - cursor = conn.execute(statement) - names = [column[0] for column in cursor.description] - tables[table] = [row if isinstance(row, dict) else dict(zip(names, row)) for row in cursor.fetchall()] - return tables + +def _lock_repair_rows(store, changes) -> None: + """Take exactly the planned row locks in the relabel table order.""" + for table in ("generated_artifacts", "projects", "open_loops", "memories"): + ids = [row_id for changed_table, _user, row_id, *_ in changes if changed_table == table] + if ids: + locked = store.conn.execute( + f"SELECT id FROM {table} WHERE id = ANY(%s::uuid[]) ORDER BY id FOR UPDATE", (ids,) + ).fetchall() + require_changed(int(len(locked) == len(ids)), table, "planned rows") def _run_vnext_labels_check(ctx: CLIContext, args: object) -> str: del args try: - with _vnext_store_context(ctx) as store: - conn = store.conn - if not conn.in_transaction: - conn.execute("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + with user_read_snapshot_connection(ctx.database_url, ctx.user_id) as conn: below, unverified = classify_stored_labels(_postgres_tables(conn)) except DerivedDomainRepairError as exc: print(f"labels check failed: {exc}") @@ -45,6 +46,7 @@ def _run_vnext_labels_repair(ctx: CLIContext, args: object) -> str: store.lock_graph_mutation() acquire_exclusive_label_lock(store) changes = plan_label_repairs(_postgres_tables(store.conn)) + _lock_repair_rows(store, changes) # The SQLite writer is not used here. Postgres repair applies the # same plan through label-only updates inside this transaction. applied = 0 @@ -54,25 +56,22 @@ def _run_vnext_labels_repair(ctx: CLIContext, args: object) -> str: from alicebot_api.vnext_derived_labels import labels_raised_payload from alicebot_api.vnext_event_log import build_event_log_record - metadata = dict(node.get("metadata_json") or {}) - metadata["project_scope"] = list(new["project_scope"]) - metadata["project_floor"] = list(new["project_floor"]) + metadata = {"project_scope": list(new["project_scope"]), "project_floor": list(new["project_floor"])} + project_sql = ", project_id = %s" if table in {"memories", "open_loops"} else "" + project_guard = " AND project_id IS NOT DISTINCT FROM %s" if project_sql else "" + params = [new["domain"], new["sensitivity"], json.dumps(metadata)] + if project_sql: + params.append(label_project_id(new["project_scope"])) + params.extend([user, row_id, previous["domain"], previous["sensitivity"], json.dumps(node.get("metadata_json") or {})]) + if project_sql: + params.append(node.get("project_id")) cursor = store.conn.execute( - f"UPDATE {table} SET domain = %s, sensitivity = %s, metadata_json = %s::jsonb " - "WHERE user_id = %s::uuid AND id = %s::uuid " - "AND domain = %s AND sensitivity = %s", - ( - new["domain"], - new["sensitivity"], - json.dumps(metadata), - user, - row_id, - previous["domain"], - previous["sensitivity"], - ), + f"UPDATE {table} SET domain = %s, sensitivity = %s, metadata_json = metadata_json || %s::jsonb " + f"{project_sql} WHERE user_id = %s::uuid AND id = %s::uuid " + f"AND domain = %s AND sensitivity = %s AND metadata_json = %s::jsonb{project_guard}", + tuple(params), ) - if cursor.rowcount != 1: - continue + require_changed(cursor.rowcount, table, row_id) target = { "memories": "memory", "open_loops": "open_loop", diff --git a/apps/api/src/alicebot_api/db.py b/apps/api/src/alicebot_api/db.py index 11c1c0953..e677a463c 100644 --- a/apps/api/src/alicebot_api/db.py +++ b/apps/api/src/alicebot_api/db.py @@ -68,6 +68,20 @@ def direct_user_connection(database_url: str, user_id: UUID) -> Iterator[UserCon yield conn +@contextmanager +def user_read_snapshot_connection(database_url: str, user_id: UUID) -> Iterator[UserConnection]: + """One repeatable read, read-only snapshot with transaction-local RLS. + + Set the transaction mode before the first query, including the identity + query, and clear that identity when the transaction ends. + """ + with psycopg.connect(database_url, row_factory=dict_row) as conn: + with conn.transaction(): + conn.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") + set_current_user(conn, user_id) + yield conn + + def _new_connection_pool(database_url: str) -> ConnectionPool[UserConnection]: pool = cast( ConnectionPool[UserConnection], @@ -143,6 +157,7 @@ def pooled_user_connection(pool: ConnectionPoolLike, user_id: UUID) -> Iterator[ "set_current_user", "set_current_user_account", "user_connection", + "user_read_snapshot_connection", ] diff --git a/apps/api/src/alicebot_api/label_commands.py b/apps/api/src/alicebot_api/label_commands.py index 708cad473..f6d523716 100644 --- a/apps/api/src/alicebot_api/label_commands.py +++ b/apps/api/src/alicebot_api/label_commands.py @@ -68,7 +68,7 @@ def run_labels(args) -> int: return 1 if below or unverified or raised_on_next_open else 0 try: with sqlite_user_connection(db, args.user_id) as conn: - relabel_labels_sqlite(conn) + relabel_labels_sqlite(conn, explicit=True) except DerivedDomainRepairError as exc: print(f"labels repair failed: {exc}") return 2 diff --git a/apps/api/src/alicebot_api/vault_doctor.py b/apps/api/src/alicebot_api/vault_doctor.py index 488bb0aba..6c3f61bd1 100644 --- a/apps/api/src/alicebot_api/vault_doctor.py +++ b/apps/api/src/alicebot_api/vault_doctor.py @@ -111,10 +111,16 @@ def compile_local_vault_doctor( (uid, CANDIDATE_STATUS), ) missing_vector_line = _missing_vector_line(store) - from alicebot_api.vnext_label_repair import label_gap_counts + from alicebot_api.vnext_label_repair import LabelCheckUnavailable, label_gap_counts - below, unverified = label_gap_counts(store) - label_line = f"derived labels: {below} below their inputs, {unverified} unverified" + try: + below, unverified = label_gap_counts(store) + label_line = f"derived labels: {below} below their inputs, {unverified} unverified" + labels_available = True + except LabelCheckUnavailable: + below, unverified = 0, 0 + labels_available = False + label_line = "derived labels: unavailable; run labels check" flagged_ids = _flagged_source_ids(store) superseded_count = count_prunable_sources(store) try: diff --git a/apps/api/src/alicebot_api/vnext_doctor.py b/apps/api/src/alicebot_api/vnext_doctor.py index 5ddd2543e..f40998d2e 100644 --- a/apps/api/src/alicebot_api/vnext_doctor.py +++ b/apps/api/src/alicebot_api/vnext_doctor.py @@ -288,14 +288,20 @@ def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: }, ) - from alicebot_api.vnext_label_repair import label_gap_counts + from alicebot_api.vnext_label_repair import LabelCheckUnavailable, label_gap_counts - below, unverified = label_gap_counts(self.store) - label_line = f"derived labels: {below} below their inputs, {unverified} unverified" + try: + below, unverified = label_gap_counts(self.store) + label_line = f"derived labels: {below} below their inputs, {unverified} unverified" + labels_available = True + except LabelCheckUnavailable: + below, unverified = 0, 0 + labels_available = False + label_line = "derived labels: unavailable; run labels check" self._check( checks, name="derived_labels", - ok=below == 0 and unverified == 0, + ok=labels_available and below == 0 and unverified == 0, severity="warning", message_ok=label_line, message_fail=label_line, diff --git a/apps/api/src/alicebot_api/vnext_label_repair.py b/apps/api/src/alicebot_api/vnext_label_repair.py index 8e4b6b910..2db1b4d59 100644 --- a/apps/api/src/alicebot_api/vnext_label_repair.py +++ b/apps/api/src/alicebot_api/vnext_label_repair.py @@ -40,9 +40,9 @@ "beliefs": "SELECT id, user_id, memory_id FROM beliefs", } _UPDATES = { - "memories": "UPDATE memories SET domain = ?, sensitivity = ?, metadata_json = ? WHERE user_id = ? AND id = ?", + "memories": "UPDATE memories SET domain = ?, sensitivity = ?, metadata_json = json_patch(metadata_json, ?), project_id = ? WHERE user_id = ? AND id = ? AND domain = ? AND sensitivity = ? AND metadata_json = ? AND project_id IS ?", "open_loops": ( - "UPDATE open_loops SET domain = ?, sensitivity = ?, metadata_json = ? WHERE user_id = ? AND id = ?" + "UPDATE open_loops SET domain = ?, sensitivity = ?, metadata_json = json_patch(metadata_json, ?), project_id = ? WHERE user_id = ? AND id = ? AND domain = ? AND sensitivity = ? AND metadata_json = ? AND project_id IS ?" ), } @@ -84,6 +84,7 @@ def plan_label_repairs( for row in rows: node = dict(row) node["kind"] = kind + node["_stored_metadata"] = row.get("metadata_json") node["metadata_json"] = _json_object(row.get("metadata_json")) if isinstance(row.get("value"), str): node["value"] = _json_object(row.get("value")) @@ -112,6 +113,18 @@ def plan_label_repairs( return changes +def label_project_id(scope: Sequence[object]) -> str | None: + """Mirror a single UUID scope; named and global scopes use metadata only.""" + from uuid import UUID + + if len(scope) != 1: + return None + try: + return str(UUID(str(scope[0]))) + except ValueError: + return None + + def _load_tables(conn) -> dict[str, list[dict[str, object]]]: available = { row[0] if not isinstance(row, dict) else row["name"] @@ -131,8 +144,8 @@ def _stamped(conn) -> bool: return conn.execute("SELECT value FROM alice_schema_state WHERE key = ?", (REPAIR_STATE_KEY,)).fetchone() is not None -def relabel_labels_sqlite(conn, *, restoring: bool = False) -> None: - """Raise stored derived labels once, or again on a restore. +def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = False) -> None: + """Raise stored derived labels once, or always for a restore or owner repair. When no transaction is open this begins one and reads the state key inside it. A caller that already has a transaction keeps it. @@ -143,18 +156,19 @@ def relabel_labels_sqlite(conn, *, restoring: bool = False) -> None: conn.execute("BEGIN IMMEDIATE") owns = True try: - if not restoring and _stamped(conn): + if not restoring and not explicit and _stamped(conn): if owns: conn.commit() return changes = plan_label_repairs(_load_tables(conn)) - for table, user, stored, _previous, new, node in changes: - metadata = dict(node.get("metadata_json") or {}) - metadata["project_scope"] = list(new["project_scope"]) - metadata["project_floor"] = list(new["project_floor"]) + for table, user, stored, previous, new, node in changes: + metadata = {"project_scope": list(new["project_scope"]), "project_floor": list(new["project_floor"])} + raw_metadata = node.get("_stored_metadata") changed = conn.execute( _UPDATES[table], - (new["domain"], new["sensitivity"], json.dumps(metadata), user, stored), + (new["domain"], new["sensitivity"], json.dumps(metadata), + label_project_id(new["project_scope"]), user, stored, + previous["domain"], previous["sensitivity"], raw_metadata, node.get("project_id")), ).rowcount require_changed(changed, table, stored) for table, user, stored, previous, new, _node in changes: @@ -205,6 +219,7 @@ def classify_stored_labels( for row in rows: node = dict(row) node["kind"] = kind + node["_stored_metadata"] = row.get("metadata_json") node["metadata_json"] = _json_object(row.get("metadata_json")) if isinstance(row.get("value"), str): node["value"] = _json_object(row.get("value")) @@ -258,23 +273,44 @@ def format_label_check( return "\n".join(lines) +class LabelCheckUnavailable(RuntimeError): + """The label planner could not read this store; no count is available.""" + + +def load_postgres_label_tables(conn) -> dict[str, list[dict[str, object]]]: + """Read every label input using PostgreSQL's current RLS identity.""" + + tables = {} + for table, statement in INPUT_SELECTS_V3.items(): + cursor = conn.execute(statement) + names = [column[0] for column in cursor.description] + tables[table] = [row if isinstance(row, dict) else dict(zip(names, row)) for row in cursor.fetchall()] + return tables + + def label_gap_counts(store: object) -> tuple[int, int]: - """How many derived rows are below their inputs, and how many are unverified. + """Counts from the real store, or an explicit unavailable result. - A store that cannot be read returns zeros so a doctor does not fail closed. + PostgreSQL reads use a savepoint so an unavailable table does not poison + the doctor's surrounding application transaction. """ conn = getattr(store, "conn", None) module = type(conn).__module__ if conn is not None else "" - if conn is None or not (module.startswith("sqlite3") or module.startswith("psycopg")): - return (0, 0) try: - tables = _load_tables(conn) + if module.startswith("sqlite3"): + tables = _load_tables(conn) + elif module.startswith("psycopg"): + with conn.transaction(): + tables = load_postgres_label_tables(conn) + else: + raise LabelCheckUnavailable("derived label counts are unavailable for this store") below, unverified = classify_stored_labels(tables) - except Exception: - return (0, 0) - unverified_count = sum(len(ids) for ids in unverified.values()) - return (len(below), unverified_count) + except LabelCheckUnavailable: + raise + except Exception as exc: + raise LabelCheckUnavailable("derived label counts could not be read") from exc + return len(below), sum(len(ids) for ids in unverified.values()) def recorded_sqlite_label_repairs(conn, user_id: str) -> dict[tuple[str, str, str], set[str]]: @@ -330,6 +366,8 @@ def recorded_sqlite_label_repairs(conn, user_id: str) -> dict[tuple[str, str, st __all__ = [ "INPUT_SELECTS_V3", + "LabelCheckUnavailable", + "load_postgres_label_tables", "REPAIR_STATE_KEY", "DerivedDomainRepairError", "classify_stored_labels", From 770047b1f4c4a491fe174d0a9839de6be81dd511 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:20:03 +0200 Subject: [PATCH 065/270] Validate incomplete legacy dependency counts --- .../src/alicebot_api/vnext_derived_labels.py | 10 ++++-- ...test_derived_labels_record_completeness.py | 36 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_derived_labels_record_completeness.py diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 63267ebcf..4f721b3fa 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -606,8 +606,10 @@ def _legacy_count_problem(kind: str, row: Mapping[str, object], meta: Mapping[st def _counts_against_lists(counts: object, lists: Mapping[str, object]) -> str: - if not isinstance(counts, Mapping): + if counts is None: return "" + if not isinstance(counts, Mapping): + return "malformed" present_lists: dict[str, list[object]] = {} for key, raw in lists.items(): if raw is None: @@ -617,11 +619,13 @@ def _counts_against_lists(counts: object, lists: Mapping[str, object]) -> str: present_lists[key] = list(raw) if any(len(values) != len(set(map(str, values))) for values in present_lists.values()): return "" - for key, values in present_lists.items(): + for key in lists: if key not in counts: continue expected = counts.get(key) - if isinstance(expected, int) and not isinstance(expected, bool) and expected != len(values): + if not isinstance(expected, int) or isinstance(expected, bool) or expected < 0: + return "malformed" + if expected != len(present_lists.get(key, [])): return "counts_disagree" return "" diff --git a/tests/unit/test_derived_labels_record_completeness.py b/tests/unit/test_derived_labels_record_completeness.py new file mode 100644 index 000000000..efac96c30 --- /dev/null +++ b/tests/unit/test_derived_labels_record_completeness.py @@ -0,0 +1,36 @@ +"""Missing legacy lists cannot pass a producer's positive completeness count.""" + +from __future__ import annotations + +import pytest + +from alicebot_api.vnext_derived_labels import settle_labels + + +def legacy_report(workflow, lists, counts): + metadata = {"workflow": workflow} + if workflow in {"daily_brief", "weekly_synthesis"}: + metadata["input_summary"] = {**lists, "counts": counts} + else: + metadata.update(lists) + metadata["input_counts"] = counts + return {"kind": "artifact", "id": "report", "domain": "project", "sensitivity": "public", "metadata_json": metadata} + + +@pytest.mark.parametrize("workflow", ("daily_brief", "weekly_synthesis", "connection_finder", "contradiction_finder")) +@pytest.mark.parametrize("lists,counts", (({}, {"sources": 1}), ({"source_ids": []}, {"sources": "1"}), + ({"source_ids": []}, {"sources": True}), ({"source_ids": []}, {"sources": -1}), + ({"source_ids": []}, []))) +def test_a_missing_or_malformed_legacy_completeness_record_is_unverified(workflow, lists, counts): + row = legacy_report(workflow, lists, counts) + label = settle_labels([row]).by_stored("artifact", "report") + assert label.unverified is True + assert label.reason == "legacy_counts" + + +@pytest.mark.parametrize("workflow", ("daily_brief", "weekly_synthesis", "connection_finder", "contradiction_finder")) +@pytest.mark.parametrize("lists,counts", (({}, {"sources": 0}), ({"source_ids": []}, {"sources": 0}), + ({"source_ids": []}, {}))) +def test_a_legitimate_empty_legacy_record_is_verified(workflow, lists, counts): + row = legacy_report(workflow, lists, counts) + assert settle_labels([row]).by_stored("artifact", "report").unverified is False From 366e56bac5605cffb3e7562b2bd1bf8ca2c7132b Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:20:03 +0200 Subject: [PATCH 066/270] Validate incomplete legacy dependency counts --- .../src/alicebot_api/vnext_derived_labels.py | 10 ++++-- ...test_derived_labels_record_completeness.py | 36 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_derived_labels_record_completeness.py diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index e602e9ede..3391fb4b9 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -605,8 +605,10 @@ def _legacy_count_problem(kind: str, row: Mapping[str, object], meta: Mapping[st def _counts_against_lists(counts: object, lists: Mapping[str, object]) -> str: - if not isinstance(counts, Mapping): + if counts is None: return "" + if not isinstance(counts, Mapping): + return "malformed" present_lists: dict[str, list[object]] = {} for key, raw in lists.items(): if raw is None: @@ -616,11 +618,13 @@ def _counts_against_lists(counts: object, lists: Mapping[str, object]) -> str: present_lists[key] = list(raw) if any(len(values) != len(set(map(str, values))) for values in present_lists.values()): return "" - for key, values in present_lists.items(): + for key in lists: if key not in counts: continue expected = counts.get(key) - if isinstance(expected, int) and not isinstance(expected, bool) and expected != len(values): + if not isinstance(expected, int) or isinstance(expected, bool) or expected < 0: + return "malformed" + if expected != len(present_lists.get(key, [])): return "counts_disagree" return "" diff --git a/tests/unit/test_derived_labels_record_completeness.py b/tests/unit/test_derived_labels_record_completeness.py new file mode 100644 index 000000000..efac96c30 --- /dev/null +++ b/tests/unit/test_derived_labels_record_completeness.py @@ -0,0 +1,36 @@ +"""Missing legacy lists cannot pass a producer's positive completeness count.""" + +from __future__ import annotations + +import pytest + +from alicebot_api.vnext_derived_labels import settle_labels + + +def legacy_report(workflow, lists, counts): + metadata = {"workflow": workflow} + if workflow in {"daily_brief", "weekly_synthesis"}: + metadata["input_summary"] = {**lists, "counts": counts} + else: + metadata.update(lists) + metadata["input_counts"] = counts + return {"kind": "artifact", "id": "report", "domain": "project", "sensitivity": "public", "metadata_json": metadata} + + +@pytest.mark.parametrize("workflow", ("daily_brief", "weekly_synthesis", "connection_finder", "contradiction_finder")) +@pytest.mark.parametrize("lists,counts", (({}, {"sources": 1}), ({"source_ids": []}, {"sources": "1"}), + ({"source_ids": []}, {"sources": True}), ({"source_ids": []}, {"sources": -1}), + ({"source_ids": []}, []))) +def test_a_missing_or_malformed_legacy_completeness_record_is_unverified(workflow, lists, counts): + row = legacy_report(workflow, lists, counts) + label = settle_labels([row]).by_stored("artifact", "report") + assert label.unverified is True + assert label.reason == "legacy_counts" + + +@pytest.mark.parametrize("workflow", ("daily_brief", "weekly_synthesis", "connection_finder", "contradiction_finder")) +@pytest.mark.parametrize("lists,counts", (({}, {"sources": 0}), ({"source_ids": []}, {"sources": 0}), + ({"source_ids": []}, {}))) +def test_a_legitimate_empty_legacy_record_is_verified(workflow, lists, counts): + row = legacy_report(workflow, lists, counts) + assert settle_labels([row]).by_stored("artifact", "report").unverified is False From 9fa365855440d83006eb348d19e7888ee8e484fc Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:27:25 +0200 Subject: [PATCH 067/270] Regenerate source candidates at current labels and lock review adapters first --- apps/api/src/alicebot_api/main.py | 1 + .../alicebot_api/mcp/evidence_artifacts.py | 4 + .../openapi_operation_contracts.py | 7 + .../alicebot_api/routers/vnext_memories.py | 32 ++++- .../src/alicebot_api/routers/vnext_review.py | 4 + .../src/alicebot_api/vnext_label_writes.py | 10 +- .../alicebot_api/vnext_source_regeneration.py | 73 ++++++++++ apps/api/src/alicebot_api/vnext_store.py | 17 ++- ...test_source_move_label_preview_postgres.py | 128 ++++++++++++++++++ 9 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 apps/api/src/alicebot_api/vnext_source_regeneration.py create mode 100644 tests/integration/test_source_move_label_preview_postgres.py diff --git a/apps/api/src/alicebot_api/main.py b/apps/api/src/alicebot_api/main.py index 2b2be310a..1378f61a4 100644 --- a/apps/api/src/alicebot_api/main.py +++ b/apps/api/src/alicebot_api/main.py @@ -779,6 +779,7 @@ async def receive() -> dict[str, object]: ("POST", "/v0/vnext/projects/update-candidates/{artifact_id}/review"), ("POST", "/v0/vnext/queue/process-next"), ("POST", "/v0/vnext/sources/{source_id}/review"), + ("POST", "/v0/vnext/sources/{source_id}/regenerate"), ("PUT", "/v0/vnext/settings/brain-charter"), } ) diff --git a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py index a44771f2f..994e08c04 100644 --- a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py +++ b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py @@ -812,6 +812,10 @@ def _handle_alice_vnext_artifact_review(context: MCPRuntimeContext, arguments: M actor_id: str | None = None trace_id: str | None = None with _vnext_store_context(context) as store: + store.lock_graph_mutation() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) _target, actor_type, actor_id, decision = _authorize_vnext_artifact_target( store, identity=identity, diff --git a/apps/api/src/alicebot_api/openapi_operation_contracts.py b/apps/api/src/alicebot_api/openapi_operation_contracts.py index edf9d9e61..e04734f5c 100644 --- a/apps/api/src/alicebot_api/openapi_operation_contracts.py +++ b/apps/api/src/alicebot_api/openapi_operation_contracts.py @@ -1148,6 +1148,10 @@ def _closed_source_schema( closed=True, ), ), + ("POST", "/v0/vnext/sources/{source_id}/regenerate"): ( + "RegenerateVnextSourceSuccessResponse", + _operation_schema("RegenerateVnextSourceSuccessResponse", ("source_id", "memory_ids", "open_loop_ids", "memory_count", "open_loop_count"), closed=True), + ), ("GET", "/v0/vnext/traces/sources/{source_id}"): ( "GetVnextSourceTraceSuccessResponse", _operation_schema( @@ -2082,6 +2086,9 @@ def _closed_source_schema( ("POST", "/v0/vnext/sources/{source_id}/review"): _typed_properties( objects=("source", "trace"), booleans=("archived",) ), + ("POST", "/v0/vnext/sources/{source_id}/regenerate"): _typed_properties( + strings=("source_id",), string_arrays=("memory_ids", "open_loop_ids"), integers=("memory_count", "open_loop_count"), + ), ("POST", "/v0/vnext/memories/{memory_id}/review"): _typed_properties( objects=("memory",), nullable_objects=("consolidation_acceptance",) ), diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index 8532fd847..98bc11a10 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -118,7 +118,11 @@ class VNextSourceReviewRequest(VNextAgentRequest): sensitivity: VNextSensitivity | None = None project_id: str | None = Field(default=None, min_length=1, max_length=120) review_note: str | None = Field(default=None, min_length=1, max_length=4000) - confirm_label_hide: bool = False + confirm_label_hide: bool = Field(default=False, description="Confirm a source project move after previewing the number of derived rows hidden from project-bound keys.") + + +class VNextSourceRegenerateRequest(VNextAgentRequest): + user_id: UUID = Field(description="Owner of the stored source whose candidate memories and open loops are regenerated. Earlier rows keep their labels and provenance.") class VNextConnectorSyncRequest(VNextAgentRequest): @@ -773,6 +777,32 @@ def get_vnext_source(source_id: UUID, user_id: UUID) -> JSONResponse: ) +@source_review_router.post("/v0/vnext/sources/{source_id}/regenerate", summary="Regenerate fresh candidates from a stored source", description="The local owner or an unbound admin can regenerate candidate memories and open loops from all stored chunks using the source's current labels. Existing sources and outputs remain unchanged. Rerun the report's generation route to rebuild a report.") +def regenerate_vnext_source(source_id: UUID, request: VNextSourceRegenerateRequest, authorization: str | None = Header(default=None)) -> JSONResponse: + from alicebot_api.vnext_label_writes import label_error_response + from alicebot_api.vnext_source_regeneration import regenerate_source_inputs + + settings = get_settings() + try: + with user_connection(settings.database_url, request.user_id) as conn: + store = PostgresVNextStore(conn) + identity = _vnext_authenticated_agent_identity(store, request, user_id=request.user_id, authorization=authorization) + if identity is not None and (identity.permission_profile != "admin_agent" or identity.project_scope_locked or identity.project_scope): + return _vnext_public_error_response(status_code=403, detail="source regeneration requires the owner or an unbound admin") + store.lock_label_writes() + source = store.get_source(str(source_id)) + if source is None: + return _vnext_public_error_response(status_code=404, detail="vNext source was not found") + payload = regenerate_source_inputs(store, source) + except Exception as exc: + mapped = label_error_response(exc) + if mapped is None: + raise + status, detail, retry_after = mapped + return JSONResponse(status_code=status, content={"detail": detail}, headers={"Retry-After": retry_after} if retry_after else None) + return JSONResponse(status_code=201, content=jsonable_encoder(payload)) + + @source_review_router.post("/v0/vnext/sources/{source_id}/review") def review_vnext_source(source_id: UUID, request: VNextSourceReviewRequest) -> JSONResponse: settings = get_settings() diff --git a/apps/api/src/alicebot_api/routers/vnext_review.py b/apps/api/src/alicebot_api/routers/vnext_review.py index 508c6babf..6805c0d9f 100644 --- a/apps/api/src/alicebot_api/routers/vnext_review.py +++ b/apps/api/src/alicebot_api/routers/vnext_review.py @@ -662,6 +662,10 @@ def review_vnext_artifact( identity = _vnext_authenticated_agent_identity( store, request, user_id=request.user_id, authorization=authorization ) + store.lock_graph_mutation() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) _artifact, decision = _vnext_authorized_artifact( store=store, identity=identity, diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index c823feebd..ebed03e80 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -487,10 +487,11 @@ def clamp_owner_patch( for key in ("domain", "sensitivity", "project_id"): if key in proposed_patch and proposed_patch[key] is not None: proposed[key] = proposed_patch[key] - if isinstance(proposed_patch.get("metadata_json"), dict): + patch_metadata = proposed_patch.get("metadata_json") + if isinstance(patch_metadata, dict): stored_meta = before.get("metadata_json") meta = dict(stored_meta) if isinstance(stored_meta, dict) else {} - meta.update(proposed_patch["metadata_json"]) + meta.update(patch_metadata) proposed["metadata_json"] = meta proposed["kind"] = kind nodes, exceeded = collect_label_rows(store, [proposed], max_nodes=PROPAGATION_BOUND) @@ -513,7 +514,8 @@ def clamp_owner_patch( return proposed_patch proposed_patch["domain"] = label.domain proposed_patch["sensitivity"] = label.sensitivity - metadata = dict(proposed.get("metadata_json") or {}) + proposed_metadata = proposed.get("metadata_json") + metadata = dict(proposed_metadata) if isinstance(proposed_metadata, Mapping) else {} metadata["project_scope"] = list(label.project_scope) metadata["project_floor"] = list(label.project_floor) proposed_patch["metadata_json"] = metadata @@ -644,7 +646,7 @@ def propagate(store: Any, changed: Sequence[tuple[str, str]], *, cause: str) -> if exceeded: raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") settled = settle_labels(nodes) - changes: list[tuple[Mapping[str, object], Any, tuple[str, str, tuple[str, ...], tuple[str, ...]]]] = [] + changes: list[tuple[dict[str, object], Any, tuple[str, str, tuple[str, ...], tuple[str, ...]]]] = [] for row in affected: label = settled.by_stored(str(row.get("kind")), str(row.get("id"))) if label.unverified: diff --git a/apps/api/src/alicebot_api/vnext_source_regeneration.py b/apps/api/src/alicebot_api/vnext_source_regeneration.py new file mode 100644 index 000000000..589d72010 --- /dev/null +++ b/apps/api/src/alicebot_api/vnext_source_regeneration.py @@ -0,0 +1,73 @@ +"""Regenerate fresh source copies without changing earlier rows or their provenance.""" +from __future__ import annotations + +from typing import Any +from uuid import UUID, uuid4 + +from alicebot_api.vnext_capture import extract_candidate_memories +from alicebot_api.vnext_derived_labels import with_derived_from +from alicebot_api.vnext_event_log import append_event +from alicebot_api.vnext_project_scope import source_project_scope +from alicebot_api.vnext_projects import _open_loop_candidates +from alicebot_api.vnext_repositories import JsonObject + + +def regenerate_source_inputs(store: Any, source: JsonObject) -> JsonObject: + """Create candidate memories and loops from all stored chunks at the current label.""" + + chunks = store.read_source_chunks_for_regeneration(str(source["id"])) + scope = source_project_scope(source) + project_id = None + if len(scope) == 1: + try: + project_id = str(UUID(scope[0])) + except ValueError: + pass + generation = str(uuid4()) + memories = [] + loops = [] + for index, candidate in enumerate(extract_candidate_memories(chunks)): + metadata = with_derived_from({ + "source_id": str(source["id"]), + "source_chunk_id": candidate.source_chunk_id, + "source_chunk_index": candidate.source_chunk_index, + "extraction_rule": candidate.extraction_rule, + "project_scope": list(scope), + "regeneration_id": generation, + **({"provenance_role": candidate.provenance_role, "assertion_class": candidate.assertion_class} if candidate.provenance_role is not None else {}), + }, {"sources": [source]}) + memory = store.create_memory({ + "memory_key": f"regenerated:{generation}:{index}", + "canonical_text": candidate.text, + "title": candidate.text[:120], + "summary": candidate.text[:280], + "value": {"text": candidate.text, "source_id": str(source["id"]), "source_chunk_id": candidate.source_chunk_id}, + "source_event_ids": [str(source["id"]), candidate.source_chunk_id], + "status": "candidate", "memory_type": candidate.memory_type, + "confidence": candidate.confidence, + "domain": source["domain"], "sensitivity": source["sensitivity"], + "project_id": project_id, "metadata_json": metadata, + }, actor_type="user") + store.create_provenance_link({ + "target_type": "memory", "target_id": str(memory["id"]), + "source_id": str(source["id"]), "source_chunk_id": candidate.source_chunk_id, + "quote": candidate.text, "evidence_role": "quoted_from", "confidence": candidate.confidence, + }, actor_type="user") + memories.append(str(memory["id"])) + source_metadata = source.get("metadata_json") + source_with_text = {**source, "metadata_json": {**(source_metadata if isinstance(source_metadata, dict) else {}), "raw_text": "\n".join(str(chunk["text"]) for chunk in chunks)}} + for loop_candidate in _open_loop_candidates(source_with_text): + loop_candidate["project_id"] = project_id + loop_metadata = loop_candidate.get("metadata_json") + loop_candidate["metadata_json"] = with_derived_from({ + **(loop_metadata if isinstance(loop_metadata, dict) else {}), + "source_id": str(source["id"]), "project_scope": list(scope), "regeneration_id": generation, + }, {"sources": [source]}) + loop = store.create_open_loop(loop_candidate, actor_type="user") + store.create_provenance_link({ + "target_type": "open_loop", "target_id": str(loop["id"]), + "source_id": str(source["id"]), "evidence_role": "quoted_from", + }, actor_type="user") + loops.append(str(loop["id"])) + append_event(store, event_type="source.inputs_regenerated", actor_type="user", target_type="source", target_id=str(source["id"]), payload={"memory_count": len(memories), "open_loop_count": len(loops)}) + return {"source_id": str(source["id"]), "memory_ids": memories, "open_loop_ids": loops, "memory_count": len(memories), "open_loop_count": len(loops)} diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index da66a578e..91d714d98 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -470,6 +470,7 @@ class PostgresVNextStore: def __init__(self, conn: UserConnection): self.conn = conn + self._label_floor_applied = False def lock_label_writes(self, *, exclusive: bool = False) -> None: """Shared label lock for a write, or the exclusive lock for a relabel. @@ -1455,6 +1456,19 @@ def list_source_chunks(self, source_id: str, *, limit: int = 500) -> list[VNextR (source_id, bounded_limit), ) + def read_source_chunks_for_regeneration(self, source_id: str) -> list[VNextRow]: + """Read the complete source, or refuse recovery before writing any output.""" + + from alicebot_api.vnext_derived_labels import PROPAGATION_BOUND, LabelPropagationTooLarge + + rows = self._fetch_all( + f"SELECT {SOURCE_CHUNK_COLUMNS} FROM source_chunks WHERE source_id = %s::uuid ORDER BY chunk_index, id LIMIT %s", + (source_id, PROPAGATION_BOUND + 1), + ) + if len(rows) > PROPAGATION_BOUND: + raise LabelPropagationTooLarge("source regeneration exceeded the source chunk bound") + return rows + def search_source_chunks( self, *, @@ -1867,8 +1881,9 @@ def update_project(self, *, project_id: str, patch: JsonObject, actor_type: str metadata = patch.get("metadata_json") floor_event = None if isinstance(metadata, dict) and before is not None: + before_metadata = before.get("metadata_json") patch["metadata_json"] = merge_protected_metadata( - before.get("metadata_json") if isinstance(before.get("metadata_json"), dict) else {}, + before_metadata if isinstance(before_metadata, dict) else {}, metadata, label_write="derived_from" in metadata, ) diff --git a/tests/integration/test_source_move_label_preview_postgres.py b/tests/integration/test_source_move_label_preview_postgres.py new file mode 100644 index 000000000..14056987b --- /dev/null +++ b/tests/integration/test_source_move_label_preview_postgres.py @@ -0,0 +1,128 @@ +"""Owner source moves preview real admission loss and regenerate fresh candidates.""" +import json +from copy import deepcopy +from uuid import UUID, uuid4 + +import anyio +import pytest + +import alicebot_api.main as main +from alicebot_api.config import Settings +from alicebot_api.db import user_connection +from alicebot_api.routers import vnext_memories as router +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_capture import VNextCaptureService +from alicebot_api.vnext_store import PostgresVNextStore + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +def _request(path, payload, raw_key): + messages = [] + body = json.dumps(payload).encode() + received = False + async def receive(): + nonlocal received + if received: + return {"type": "http.disconnect"} + received = True + return {"type": "http.request", "body": body, "more_body": False} + async def send(message): + messages.append(message) + scope = {"type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", "method": "POST", "scheme": "http", "path": path, "raw_path": path.encode(), "query_string": b"", "headers": [(b"host", b"127.0.0.1:8000"), (b"content-type", b"application/json"), (b"authorization", f"Bearer {raw_key}".encode())], "client": ("127.0.0.1", 50000), "server": ("testserver", 80), "root_path": ""} + anyio.run(main.app, scope, receive, send) + status = next(item["status"] for item in messages if item["type"] == "http.response.start") + return status, b"".join(item.get("body", b"") for item in messages if item["type"] == "http.response.body") + + +def _fixture(url): + user = uuid4() + with user_connection(url, user) as conn: + ContinuityStore(conn).create_user(user, f"move-{user}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + result = VNextCaptureService(store, defer_embeddings=True).capture_text("I prefer synthetic blue ink.\nTODO: Review the synthetic draft", domain="project", sensitivity="public", project_scope=[ALPHA]) + source = store.get_source(str(result.source_id)) + report = store.create_artifact({"artifact_type": "daily_brief", "title": "Synthetic report", "content_markdown": "Synthetic report", "domain": "project", "sensitivity": "public", "metadata_json": {"source_ids": [str(result.source_id)], "project_scope": [ALPHA]}}) + return user, source, report + + +def _snapshot(url, user, source): + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + rows = {table: store._fetch_all(f"SELECT * FROM {table} ORDER BY id") for table in ("sources", "memories", "open_loops", "generated_artifacts", "provenance_links", "event_log", "graph_edges")} + return rows + + +def test_source_move_preview_is_zero_write_then_confirmation_preserves_provenance(migrated_database_urls, monkeypatch): + url = migrated_database_urls["app"] + user, source, report = _fixture(url) + monkeypatch.setattr(router, "get_settings", lambda: Settings(database_url=url)) + before = _snapshot(url, user, source) + request = router.VNextSourceReviewRequest(user_id=user, action="assign_project", project_id=BETA) + preview = router.review_vnext_source(UUID(str(source["id"])), request) + payload = json.loads(preview.body) + assert preview.status_code == 200 and payload["preview"] is True + assert payload["derived_rows_hidden_from_project_keys"] == len(before["memories"]) + 1 + assert payload["confirm_required"] is True + assert _snapshot(url, user, source) == before + confirmed = router.review_vnext_source(UUID(str(source["id"])), request.model_copy(update={"confirm_label_hide": True})) + assert confirmed.status_code == 200 + after = _snapshot(url, user, source) + assert after["provenance_links"] == before["provenance_links"] + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + assert store.get_source(str(source["id"]))["metadata_json"]["project_scope"] == [BETA] + moved_report = store.get_artifact(str(report["id"])) + assert BETA in moved_report["metadata_json"]["project_floor"] + assert moved_report["metadata_json"]["source_ids"] == report["metadata_json"]["source_ids"] + + +def test_regeneration_uses_moved_source_without_lowering_or_rewriting_originals(migrated_database_urls, monkeypatch): + url = migrated_database_urls["app"] + user, source, report = _fixture(url) + monkeypatch.setattr(router, "get_settings", lambda: Settings(database_url=url)) + move = router.VNextSourceReviewRequest(user_id=user, action="assign_project", project_id=BETA, confirm_label_hide=True, sensitivity="confidential", domain="health") + assert router.review_vnext_source(UUID(str(source["id"])), move).status_code == 200 + before = _snapshot(url, user, source) + result = router.regenerate_vnext_source(UUID(str(source["id"])), router.VNextSourceRegenerateRequest(user_id=user), authorization=None) + payload = json.loads(result.body) + assert result.status_code == 201 + assert payload["memory_count"] >= 1 and payload["open_loop_count"] == 1 + after = _snapshot(url, user, source) + assert after["sources"] == before["sources"] + assert after["generated_artifacts"] == before["generated_artifacts"] + for table in ("memories", "open_loops", "provenance_links"): + by_id = {row["id"]: row for row in after[table]} + assert all(by_id[row["id"]] == row for row in before[table]) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + for kind, ids in (("memory", payload["memory_ids"]), ("open_loop", payload["open_loop_ids"])): + rows = store.read_label_rows(kind, ids) + assert len(rows) == len(ids) + for row in rows: + assert row["domain"] == "health" and row["sensitivity"] == "confidential" + assert row["metadata_json"]["project_scope"] == [BETA] + assert row["metadata_json"]["project_floor"] == [BETA] + links = store._fetch_all("SELECT source_id FROM provenance_links WHERE target_id = %s", (str(row["id"]),)) + assert {str(link["source_id"]) for link in links} == {str(source["id"])} + + +@pytest.mark.parametrize("profile,scope,expected", [("trusted_local_agent", None, 403), ("admin_agent", ALPHA, 403), ("admin_agent", None, 201)]) +def test_regeneration_authenticates_and_refuses_trusted_or_bound_keys(migrated_database_urls, monkeypatch, profile, scope, expected): + url = migrated_database_urls["app"] + user, source, _report = _fixture(url) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + _key, raw = create_agent_key(store, user_id=user, agent_id="synthetic-reader", permission_profile=profile, project_scope=scope) + settings = Settings(database_url=url, app_env="test") + monkeypatch.setattr(router, "get_settings", lambda: settings) + monkeypatch.setattr(main, "get_settings", lambda: settings) + before = _snapshot(url, user, source) + status, response = _request(f"/v0/vnext/sources/{source['id']}/regenerate", {"user_id": str(user)}, raw) + assert status == expected, response + if expected == 403: + after = _snapshot(url, user, source) + for table in ("sources", "memories", "open_loops", "generated_artifacts", "provenance_links"): + assert after[table] == before[table] From a47f772384084c038314242b5cb6d3e670e41c67 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:21:05 +0200 Subject: [PATCH 068/270] Exercise derived label concurrency and atomic propagation on PostgreSQL --- .../derived_labels_postgres_support.py | 216 ++++++++++ ...est_derived_labels_concurrency_postgres.py | 405 ++++++++++++++++++ ...t_derived_labels_main_behavior_postgres.py | 59 +++ ...est_derived_labels_propagation_postgres.py | 197 +++++++++ 4 files changed, 877 insertions(+) create mode 100644 tests/integration/derived_labels_postgres_support.py create mode 100644 tests/integration/test_derived_labels_concurrency_postgres.py create mode 100644 tests/integration/test_derived_labels_main_behavior_postgres.py create mode 100644 tests/integration/test_derived_labels_propagation_postgres.py diff --git a/tests/integration/derived_labels_postgres_support.py b/tests/integration/derived_labels_postgres_support.py new file mode 100644 index 000000000..c29f47aad --- /dev/null +++ b/tests/integration/derived_labels_postgres_support.py @@ -0,0 +1,216 @@ +"""Synthetic, role-separated fixtures for the label atomicity acceptance tests.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime +import json +import time +from urllib.parse import urlencode +from uuid import UUID, uuid4 + +import anyio +import psycopg +from psycopg.rows import dict_row +import pytest + +import alicebot_api.main as main_module +from alicebot_api.config import Settings +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_store import PostgresVNextStore + + +def invoke(method, path, *, user_id, payload=None, key=None): + """Invoke the mounted HTTP application, preserving response headers.""" + messages = [] + body = json.dumps(payload).encode() if payload is not None else b"" + received = False + + async def receive(): + nonlocal received + if received: + return {"type": "http.disconnect"} + received = True + return {"type": "http.request", "body": body, "more_body": False} + + async def send(message): + messages.append(message) + + headers = [(b"host", b"127.0.0.1:8000"), (b"content-type", b"application/json")] + if key: + headers.append((b"authorization", f"Bearer {key}".encode())) + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": method, + "scheme": "http", + "path": path, + "raw_path": path.encode(), + "query_string": urlencode({"user_id": str(user_id)}).encode(), + "headers": headers, + "client": ("127.0.0.1", 50000), + "server": ("testserver", 80), + "root_path": "", + } + anyio.run(main_module.app, scope, receive, send) + start = next(item for item in messages if item["type"] == "http.response.start") + raw = b"".join(item.get("body", b"") for item in messages if item["type"] == "http.response.body") + return int(start["status"]), json.loads(raw), dict(start.get("headers", [])) + + +@dataclass +class LabelHarness: + urls: dict[str, str] + user_id: UUID + + @contextmanager + def store(self) -> Iterator[PostgresVNextStore]: + with user_connection(self.urls["app"], self.user_id) as conn: + yield PostgresVNextStore(conn) + + def request(self, method, path, *, payload=None, key=None): + if payload is not None: + payload = {"user_id": str(self.user_id), **payload} + return invoke(method, path, user_id=self.user_id, payload=payload, key=key) + + def relabel(self, kind, row_id, **labels): + # Call the real owner handler independently of the middleware's key-provisioning gate. + from alicebot_api.routers import vnext_memories as router + + if kind == "source": + result = router.review_vnext_source( + UUID(str(row_id)), router.VNextSourceReviewRequest(user_id=self.user_id, action="update", **labels) + ) + else: + result = router.review_vnext_memory( + UUID(str(row_id)), + router.VNextMemoryReviewRequest(user_id=self.user_id, action="edit", **labels), + authorization=None, + ) + return ( + result.status_code, + json.loads(result.body), + {key.encode(): value.encode() for key, value in result.headers.items()}, + ) + + def key(self, profile, *, project=None): + with self.store() as store: + _record, raw = create_agent_key( + store, user_id=self.user_id, agent_id=str(uuid4()), permission_profile=profile, project_scope=project + ) + return raw + + def source(self, *, scope=(), text="TODO: Synthetic acceptance task", sensitivity="public"): + with self.store() as store: + return store.create_source( + { + "source_type": "note", + "title": "Synthetic acceptance", + "content_hash": str(uuid4()), + "domain": "project", + "sensitivity": sensitivity, + "metadata_json": {"project_scope": list(scope), "raw_text": text}, + } + ) + + def memory(self, *, source=None, parents=(), scope=()): + metadata = {"project_scope": list(scope)} + if source is not None: + metadata["source_id"] = str(source["id"]) + if parents: + metadata["consolidation"] = {"cluster_member_ids": [str(row["id"]) for row in parents]} + with self.store() as store: + return store.create_memory( + { + "memory_key": str(uuid4()), + "canonical_text": "Synthetic acceptance memory", + "status": "active", + "domain": "project", + "sensitivity": "public", + "metadata_json": metadata, + } + ) + + def snapshot(self): + with self.store() as store: + result = {} + with store.conn.cursor() as cur: + for table in ("sources", "memories", "open_loops", "generated_artifacts", "projects", "event_log"): + cur.execute(f"SELECT row_to_json(t) FROM {table} t ORDER BY id") # closed table names + result[table] = [row["row_to_json"] for row in cur.fetchall()] + return result + + def label_locks(self): + # Monitoring through a fresh app connection avoids disturbing the transaction under test. + with psycopg.connect(self.urls["app"], autocommit=True, row_factory=dict_row) as conn: + return conn.execute( + "SELECT pid, mode, granted FROM pg_locks WHERE locktype='advisory' AND classid=(hashtext('vnext_labels')::bigint & 4294967295) AND objid=(hashtext(%s)::bigint & 4294967295)", + (str(self.user_id),), + ).fetchall() + + def wait_exclusive(self, *, timeout=2.0): + end = time.monotonic() + timeout + while time.monotonic() < end: + rows = self.label_locks() + if any(row["mode"] == "ExclusiveLock" and not row["granted"] for row in rows): + return rows + time.sleep(0.01) + raise AssertionError(f"no waiting exclusive label lock in pg_locks: {self.label_locks()}") + + def wait_relabel(self, *, timeout=2.0): + """A review may serialise the relabel on S or L, in that order.""" + end = time.monotonic() + timeout + with psycopg.connect(self.urls["app"], autocommit=True, row_factory=dict_row) as conn: + while time.monotonic() < end: + rows = conn.execute( + "SELECT pid, mode, granted FROM pg_locks WHERE locktype='advisory' " + "AND classid IN ((hashtext('vnext_labels')::bigint & 4294967295), " + "(hashtext('vnext_supersession')::bigint & 4294967295)) " + "AND objid=(hashtext(%s)::bigint & 4294967295)", + (str(self.user_id),), + ).fetchall() + if any(row["mode"] == "ExclusiveLock" and not row["granted"] for row in rows): + return rows + time.sleep(0.01) + raise AssertionError(f"no waiting graph or label relabel lock: {rows}") + + +@pytest.fixture +def label_harness(migrated_database_urls, monkeypatch): + from alicebot_api import vnext_label_writes + from alicebot_api.routers import vnext_memories, vnext_projects, vnext_retrieval, vnext_review, workspaces + + settings = Settings(database_url=migrated_database_urls["app"]) + for module in (main_module, vnext_memories, vnext_projects, vnext_retrieval, vnext_review, workspaces): + monkeypatch.setattr(module, "get_settings", lambda: settings) + monkeypatch.setattr(vnext_label_writes, "STRICT_LOCK_ORDER", True) + monkeypatch.setenv("ALICE_LEGACY_SURFACES", "1") + monkeypatch.setenv("ALICE_MCP_LEGACY_TOOLS", "1") + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + harness = LabelHarness(migrated_database_urls, uuid4()) + with harness.store() as store: + ContinuityStore(store.conn).create_user( + harness.user_id, f"labels-{harness.user_id}@example.invalid", "Synthetic acceptance" + ) + return harness + + +def today(): + return datetime.now(UTC).date().isoformat() + + +def assert_raised(row, *, domain="health", sensitivity="confidential"): + assert row["domain"] == domain + assert row["sensitivity"] == sensitivity + + +def join_thread(thread, failures, *, timeout=6.0): + thread.join(timeout) + assert not thread.is_alive(), "worker did not finish within the acceptance deadline" + assert not failures, repr(failures) diff --git a/tests/integration/test_derived_labels_concurrency_postgres.py b/tests/integration/test_derived_labels_concurrency_postgres.py new file mode 100644 index 000000000..2249e4cd4 --- /dev/null +++ b/tests/integration/test_derived_labels_concurrency_postgres.py @@ -0,0 +1,405 @@ +"""Real PostgreSQL acceptance races for derived labels, always in strict mode.""" + +from __future__ import annotations + +import argparse +from copy import deepcopy +from datetime import UTC, datetime, timedelta +from threading import Barrier, Event, Thread +import time +from types import SimpleNamespace + +import pytest + +from alicebot_api.cli.automation import _run_vnext_artifact_review +from alicebot_api.cli.models import CLIContext +from alicebot_api.config import Settings +from alicebot_api.mcp_tools import MCPRuntimeContext, call_mcp_tool +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService +from alicebot_api.vnext_queue import VNextQueueService +from alicebot_api.vnext_scheduler import _StagedSchedulerStore, VNextSchedulerService +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.derived_labels_postgres_support import ( + assert_raised, + join_thread, + label_harness, + today, +) + + +def _thread(fn): + failures = [] + + def run(): + try: + fn() + except BaseException as exc: + failures.append(exc) + + thread = Thread(target=run, daemon=True) + thread.start() + return thread, failures + + +def test_a_report_built_from_stale_inputs_is_floored_at_insert(label_harness, monkeypatch): + h = label_harness + source = h.source() + reader = h.key("trusted_local_agent") + with h.store() as store: + start = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0) + stale = deepcopy( + VNextBrainService(store)._load_inputs( + BrainArtifactRequest(generated_for=today()), window_start=start, window_end=start + timedelta(days=1) + ) + ) + assert str(source["id"]) in [str(row["id"]) for row in stale[0]] + assert h.relabel("source", source["id"], domain="health", sensitivity="confidential")[0] == 200 + monkeypatch.setattr(VNextBrainService, "_load_inputs", lambda *_args, **_kwargs: deepcopy(stale)) + status, body, _ = h.request( + "POST", + "/v0/vnext/artifacts/generate/daily-brief", + payload={"options": {"generated_for": today(), "discover_open_loops": False}}, + key=reader, + ) + assert status == 201, body + assert "Synthetic acceptance" in body["content_markdown"] + # The 201 is the read made at selection; the durable row has the current floor. + with h.store() as store: + assert_raised(store.get_artifact(str(body["id"]))) + assert h.request("GET", f"/v0/vnext/artifacts/{body['id']}", key=reader)[0] == 403 + + +def test_a_relabel_waits_for_an_open_generation_and_then_labels_its_report(label_harness): + h = label_harness + source = h.source() + response = [] + with h.store() as store: + report = VNextBrainService(store).generate_daily_brief( + BrainArtifactRequest(generated_for=today(), discover_open_loops=False) + ) + thread, failures = _thread( + lambda: response.append(h.relabel("source", source["id"], domain="health", sensitivity="confidential")) + ) + try: + time.sleep(0.5) + assert thread.is_alive(), response + rows = h.wait_exclusive() + assert any(row["mode"] == "ShareLock" and row["granted"] for row in rows) + finally: + # Leaving the context commits the uncommitted report before joining A. + pass + join_thread(thread, failures) + assert response[0][0] == 200, response + with h.store() as store: + assert_raised(store.get_artifact(str(report["id"]))) + + +def test_a_relabel_behind_a_slow_provider_call_answers_retryable_and_changes_nothing(label_harness, monkeypatch): + h = label_harness + source = h.source() + h.memory(source=source) + before = h.snapshot() + provider_entered, release = Event(), Event() + generated = [] + + def provider(self, **kwargs): + provider_entered.set() + assert release.wait(8), "provider release deadline exceeded" + return SimpleNamespace( + content_markdown=kwargs["deterministic_markdown"], prompt_hash="synthetic", model_info={}, metadata={} + ) + + monkeypatch.setattr(VNextBrainService, "_model_backed_artifact", provider) + + def generate(): + with h.store() as store: + generated.append( + VNextBrainService(store).generate_daily_brief( + BrainArtifactRequest(generated_for=today(), generation_mode="model_backed") + ) + ) + + thread, failures = _thread(generate) + try: + assert provider_entered.wait(2) + assert any(row["mode"] == "ShareLock" and row["granted"] for row in h.label_locks()) + result = [] + started = time.monotonic() + relabel_thread, relabel_failures = _thread( + lambda: result.append(h.relabel("source", source["id"], domain="health", sensitivity="confidential")) + ) + relabel_thread.join(4.0) + assert not relabel_thread.is_alive(), "relabel exceeded 3 s bound plus 1 s" + assert not relabel_failures, relabel_failures + assert time.monotonic() - started < 4.0 + status, body, headers = result[0] + assert status == 503, body + assert headers[b"retry-after"] == b"2" + assert "nothing was changed" in body["detail"] + assert h.snapshot() == before + finally: + release.set() + join_thread(thread, failures) + assert h.relabel("source", source["id"], domain="health", sensitivity="confidential")[0] == 200 + with h.store() as store: + assert_raised(store.get_artifact(str(generated[0]["id"]))) + loops = store.list_open_loops( + status=None, sensitivity_allowed=["public", "private", "confidential", "regulated", "unknown"] + ) + assert loops, "model-backed generation must write its candidate before the provider" + for loop in loops: + assert_raised(loop) + + +def test_a_scheduler_plan_staged_before_a_relabel_is_floored_at_publish(label_harness): + h = label_harness + source = h.source() + with h.store() as store: + staged = _StagedSchedulerStore(store) + report = VNextBrainService(staged).generate_daily_brief( + BrainArtifactRequest(generated_for=today(), discover_open_loops=False) + ) + plan = staged.plan(report) + assert h.relabel("source", source["id"], domain="health", sensitivity="confidential")[0] == 200 + with h.store() as store: + published = plan.publish(store) + assert_raised(published) + assert any(row["mode"] == "ShareLock" and row["granted"] for row in h.label_locks()) + with h.store() as store: + assert_raised(store.get_artifact(str(published["id"]))) + + +def test_a_staged_staleness_mark_cannot_undo_a_relabel(label_harness): + h = label_harness + alpha, beta = "prj_" + "a" * 16, "prj_" + "b" * 16 + memory = h.memory(scope=(alpha,)) + with h.store() as store: + staged = _StagedSchedulerStore(store) + marked = VNextSchedulerService(staged)._mark_memory_stale( + memory, reason="synthetic", note="Synthetic mark", metadata={} + ) + plan = staged.plan(marked) + assert h.relabel("memory", memory["id"], domain="health", sensitivity="confidential")[0] == 200 + assert ( + h.request( + "POST", + f"/v0/vnext/memories/{memory['id']}/review", + payload={"action": "assign_project", "project_id": beta}, + )[0] + == 200 + ) + with h.store() as store: + before = store.get_memory(str(memory["id"])) + plan.publish(store) + after = store.get_memory(str(memory["id"])) + assert_raised(after) + assert after["metadata_json"]["project_scope"] == [beta] + assert after["metadata_json"].get("project_floor") == before["metadata_json"].get("project_floor") + assert "staleness" in after["metadata_json"] + + +def test_an_update_that_omits_a_marker_keeps_it(label_harness): + h = label_harness + original = h.memory() + derived = h.memory(parents=(original,)) + with h.store() as store: + before = store.get_memory(str(derived["id"])) + after = store.update_memory( + memory_id=str(derived["id"]), patch={"metadata_json": {"staleness": {"note": "synthetic"}}} + ) + for marker in ("consolidation", "derived_from"): + if marker in before["metadata_json"]: + assert after["metadata_json"][marker] == before["metadata_json"][marker] + assert "consolidation" in after["metadata_json"] + + +@pytest.mark.parametrize( + "entrypoint", ["http", "mcp", "cli", "project_accept", "project_edit", "project_reject", "direct_promote"] +) +def test_every_entry_point_that_locks_a_row_and_a_relabel_never_deadlock(label_harness, monkeypatch, entrypoint): + h = label_harness + original_fetch = PostgresVNextStore._fetch_optional_one + for round_no in range(25): + source = h.source(text=f"TODO: Synthetic acceptance task {entrypoint} {round_no}") + with h.store() as store: + if entrypoint.startswith("project_"): + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + project = store.create_project( + { + "name": f"Synthetic acceptance {round_no}", + "slug": f"synthetic-{round_no}", + "domain": "project", + "sensitivity": "public", + } + ) + # A real producer selects a source in the requested project. + store.update_source( + source_id=str(source["id"]), patch={"metadata_json": {"project_scope": [str(project["id"])]}} + ) + artifact = VNextProjectService(store).generate_project_update_candidate( + ProjectAutomationRequest(project_id=str(project["id"])) + ) + else: + artifact = VNextBrainService(store).generate_daily_brief( + BrainArtifactRequest(generated_for=today(), discover_open_loops=False) + ) + row_locked, release = Event(), Event() + + def paused_fetch(self, *args, **kwargs): + row = original_fetch(self, *args, **kwargs) + if ( + row + and str(row.get("id")) == str(artifact["id"]) + and "FOR UPDATE" in str(args).upper() + and not row_locked.is_set() + ): + row_locked.set() + assert release.wait(4), "review lock release deadline exceeded" + return row + + monkeypatch.setattr(PostgresVNextStore, "_fetch_optional_one", paused_fetch) + + def review(): + artifact_id = str(artifact["id"]) + if entrypoint == "http": + result = h.request("POST", f"/v0/vnext/artifacts/{artifact_id}/review", payload={"action": "promote"}) + assert result[0] == 200, result + elif entrypoint == "mcp": + call_mcp_tool( + MCPRuntimeContext(database_url=h.urls["app"], user_id=h.user_id), + name="alice_vnext_artifact_review", + arguments={"artifact_id": artifact_id, "action": "promote"}, + ) + elif entrypoint == "cli": + _run_vnext_artifact_review( + CLIContext(Settings(database_url=h.urls["app"]), h.urls["app"], h.user_id), + argparse.Namespace(artifact_id=artifact_id, action="promote"), + ) + else: + with h.store() as store: + if entrypoint.startswith("project_"): + VNextProjectService(store, defer_embeddings=True).review_project_update( + artifact_id=artifact_id, + action=entrypoint.split("_", 1)[1], + edited_current_state="Synthetic edited state" if entrypoint == "project_edit" else None, + ) + else: + VNextQueueService(store, defer_embeddings=True)._promote_artifact( + artifact_id=artifact_id, + actor_type="user", + actor_id=str(h.user_id), + trace_id=None, + run_id=None, + ) + + reviewer, review_failures = _thread(review) + result = [] + relabeller = None + try: + assert row_locked.wait(2), (entrypoint, round_no, review_failures) + relabeller, relabel_failures = _thread( + lambda: result.append(h.relabel("source", source["id"], domain="health", sensitivity="confidential")) + ) + h.wait_relabel() + finally: + release.set() + join_thread(reviewer, review_failures) + if relabeller is not None: + join_thread(relabeller, relabel_failures) + monkeypatch.setattr(PostgresVNextStore, "_fetch_optional_one", original_fetch) + assert result[0][0] == 200, (entrypoint, round_no, result) + with h.store() as store: + assert_raised(store.get_artifact(str(artifact["id"]))) + from alicebot_api.vnext_label_writes import walk_dependants + + for row in walk_dependants(store, [str(source["id"])]): + assert_raised(row) + + +def test_two_relabels_with_a_shared_dependant_do_not_deadlock(label_harness): + h = label_harness + first, second = h.memory(), h.memory() + shared = h.memory(parents=(first, second)) + barrier = Barrier(2) + results = [] + + def change(memory): + barrier.wait(2) + results.append(h.relabel("memory", memory["id"], domain="health", sensitivity="confidential")) + + a, af = _thread(lambda: change(first)) + b, bf = _thread(lambda: change(second)) + join_thread(a, af) + join_thread(b, bf) + assert [row[0] for row in results] == [200, 200], results + with h.store() as store: + assert_raised(store.get_memory(str(shared["id"]))) + + +@pytest.mark.parametrize("method", ["get_artifact_for_update", "get_memory_for_update", "get_project_for_update"]) +def test_each_row_locker_holds_the_shared_label_lock(label_harness, method): + h = label_harness + memory = h.memory() + with h.store() as store: + project = store.create_project({"name": "Synthetic locker", "slug": "synthetic-locker"}) + artifact = VNextBrainService(store).generate_daily_brief( + BrainArtifactRequest(generated_for=today(), discover_open_loops=False) + ) + row_id = { + "get_artifact_for_update": artifact["id"], + "get_memory_for_update": memory["id"], + "get_project_for_update": project["id"], + }[method] + with h.store() as store: + getattr(store, method)(str(row_id)) + assert any(row["mode"] == "ShareLock" and row["granted"] for row in h.label_locks()), method + + +@pytest.mark.parametrize( + "method", ["create_memory", "create_open_loop", "create_artifact", "upsert_artifact_by_workflow_digest"] +) +def test_each_postgres_creator_floors_a_stale_source_copy(label_harness, method): + from uuid import uuid4 + + h = label_harness + source = h.source() + assert h.relabel("source", source["id"], domain="health", sensitivity="confidential")[0] == 200 + with h.store() as store: + metadata = {"source_id": str(source["id"])} + if method == "create_memory": + row = store.create_memory( + { + "memory_key": str(uuid4()), + "canonical_text": "Synthetic copy", + "domain": "project", + "sensitivity": "public", + "metadata_json": metadata, + } + ) + elif method == "create_open_loop": + row = store.create_open_loop( + { + "title": "Synthetic copy", + "source_id": str(source["id"]), + "domain": "project", + "sensitivity": "public", + "metadata_json": {**metadata, "discovered_by": "vnext_daily_brief"}, + } + ) + else: + payload = { + "artifact_type": "daily_brief", + "title": "Synthetic copy", + "content_markdown": "Synthetic copy", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"workflow": "daily_brief", "source_refs": [str(source["id"])]}, + } + row = ( + store.create_artifact(payload) + if method == "create_artifact" + else store.upsert_artifact_by_workflow_digest(payload, workflow="daily_brief", digest=str(uuid4())) + ) + assert_raised(row) diff --git a/tests/integration/test_derived_labels_main_behavior_postgres.py b/tests/integration/test_derived_labels_main_behavior_postgres.py new file mode 100644 index 000000000..c76fbe109 --- /dev/null +++ b/tests/integration/test_derived_labels_main_behavior_postgres.py @@ -0,0 +1,59 @@ +"""Execute the original stale-label behaviors even before the new kernel exists.""" + +from uuid import uuid4 + +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_store import PostgresVNextStore + + +def test_source_relabel_reaches_an_existing_report_on_main(migrated_database_urls): + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user( + user_id, f"main-behavior-{user_id}@example.invalid", "Synthetic main behavior" + ) + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + source = store.create_source( + {"source_type": "note", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public"} + ) + report = store.create_artifact( + { + "artifact_type": "daily_brief", + "title": "Synthetic report", + "content_markdown": "Synthetic report", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"workflow": "daily_brief", "source_refs": [str(source["id"])]}, + } + ) + lock = getattr(store, "lock_label_writes", None) + if callable(lock): + lock(exclusive=True) + store.update_source(source_id=str(source["id"]), patch={"domain": "health", "sensitivity": "confidential"}) + updated = store.get_artifact(str(report["id"])) + assert updated["domain"] == "health" + assert updated["sensitivity"] == "confidential" + + +def test_insert_floor_reads_current_source_labels_on_main(migrated_database_urls): + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"main-floor-{user_id}@example.invalid", "Synthetic main floor") + store = PostgresVNextStore(conn) + source = store.create_source( + {"source_type": "note", "content_hash": str(uuid4()), "domain": "health", "sensitivity": "confidential"} + ) + report = store.create_artifact( + { + "artifact_type": "daily_brief", + "title": "Synthetic report", + "content_markdown": "Synthetic report", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"workflow": "daily_brief", "source_refs": [str(source["id"])]}, + } + ) + assert report["domain"] == "health" + assert report["sensitivity"] == "confidential" diff --git a/tests/integration/test_derived_labels_propagation_postgres.py b/tests/integration/test_derived_labels_propagation_postgres.py new file mode 100644 index 000000000..8bd144bcf --- /dev/null +++ b/tests/integration/test_derived_labels_propagation_postgres.py @@ -0,0 +1,197 @@ +"""Propagation must finish with its original, over actual generated chains.""" + +from __future__ import annotations + +import json + +import psycopg +import pytest + +from alicebot_api import vnext_label_writes +from alicebot_api.store import ContinuityStoreInvariantError +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from alicebot_api.vnext_capture import VNextCaptureService +from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService +from alicebot_api.vnext_queue import VNextQueueService +from tests.integration.derived_labels_postgres_support import assert_raised, label_harness, today + + +@pytest.mark.parametrize("failure_kind", ["database", "writer"]) +def test_a_failed_propagation_rolls_the_original_back_with_it(label_harness, monkeypatch, failure_kind): + h = label_harness + source = h.source() + copies = [h.memory(source=source) for _ in range(4)] + h.memory(parents=tuple(copies)) + before = h.snapshot() + original = vnext_label_writes.write_settled_label + attempts = [] + + def fail_third(store, **kwargs): + attempts.append(kwargs["row_id"]) + if len(attempts) == 3: + if failure_kind == "writer": + raise ContinuityStoreInvariantError("synthetic third dependant failure") + # A genuine database exception must abort the transaction after two writes/events. + store.conn.execute("SELECT 1 / 0") + return original(store, **kwargs) + + monkeypatch.setattr(vnext_label_writes, "write_settled_label", fail_third) + try: + status, body, _ = h.relabel("source", source["id"], domain="health", sensitivity="confidential") + except psycopg.errors.DivisionByZero: + # Baseline exposes the error; atomic rollback is still asserted, separately from mapping. + status = None + assert len(attempts) == 3 + assert h.snapshot() == before + assert status == 409, "all database propagation failures must return the fixed whole-refusal answer" + assert "nothing was changed" in body["detail"] + + +def _build_chain(h, *, source=None, project=None, label=("project", "public")): + with h.store() as store: + # Taking S first also verifies capture's real strict-order path when it reacquires S. + store.lock_graph_mutation() + if project is None: + project = store.create_project( + {"name": "SyntheticChain", "slug": "synthetic-chain", "domain": "project", "sensitivity": "public"} + ) + if source is None: + capture = VNextCaptureService(store, defer_embeddings=True).capture_text( + "Fact: SyntheticChain acceptance is prepared.\nTODO: SyntheticChain followup", + title="SyntheticChain acceptance", + domain=label[0], + sensitivity=label[1], + project_scope=(str(project["id"]),), + ) + assert capture.status == "imported", capture + assert capture.candidate_memory_count == 2 + source = store.get_source(str(capture.source_id)) + copies = store.list_memories_referencing_source(source_id=str(source["id"])) + assert len(copies) >= 2 + brain = VNextBrainService(store) + request = BrainArtifactRequest( + generated_for=today(), + projects=(str(project["id"]),), + sensitivity_allowed=("public", "internal", "private", "confidential", "regulated", "unknown"), + ) + daily = brain.generate_daily_brief(request) + weekly = brain.generate_weekly_synthesis(request) + assert str(daily["id"]) in weekly["metadata_json"]["derived_from"]["artifacts"] + promoted = VNextQueueService(store, defer_embeddings=True)._promote_artifact( + artifact_id=str(weekly["id"]), actor_type="user", actor_id=str(h.user_id), trace_id=None, run_id=None + ) + update = VNextProjectService(store, defer_embeddings=True).generate_project_update_candidate( + ProjectAutomationRequest(project_id=str(project["id"]), sensitivity_allowed=request.sensitivity_allowed) + ) + VNextProjectService(store, defer_embeddings=True).review_project_update( + artifact_id=str(update["id"]), action="accept", actor_type="user" + ) + rows = [ + *(("memory", str(row["id"])) for row in copies), + ("artifact", str(daily["id"])), + ("artifact", str(weekly["id"])), + ("memory", str(promoted["promoted_memory_id"])), + ("artifact", str(update["id"])), + ("memory", str(update["metadata_json"]["candidate_memory_id"])), + ("project", str(project["id"])), + ] + loops = store.list_open_loops(status=None, sensitivity_allowed=list(request.sensitivity_allowed)) + loops = [row for row in loops if str(row.get("source_id")) == str(source["id"])] + assert loops, "daily producer must create a candidate loop" + rows.extend(("open_loop", str(row["id"])) for row in loops) + assert store.get_project(str(project["id"]))["metadata_json"]["derived_from"] + return source, project, rows + + +def _read_row(store, kind, row_id): + return { + "memory": store.get_memory, + "artifact": store.get_artifact, + "open_loop": store.get_open_loop, + "project": store.get_project, + }[kind](row_id) + + +def _assert_chain(h, rows, *, domain, sensitivity): + with h.store() as store: + for kind, row_id in rows: + row = _read_row(store, kind, row_id) + assert_raised(row, domain=domain, sensitivity=sensitivity) + + +def test_a_source_relabel_reaches_the_extracted_memories_the_loop_every_report_and_the_project_state(label_harness): + h = label_harness + source, project, rows = _build_chain(h) + assert h.relabel("source", source["id"], domain="health")[0] == 200 + _assert_chain(h, rows, domain="health", sensitivity="public") + assert h.relabel("source", source["id"], sensitivity="confidential")[0] == 200 + _assert_chain(h, rows, domain="health", sensitivity="confidential") + # All original roles in the chain must be recreated at the raised floor. + with h.store() as store: + updated_source = store.get_source(str(source["id"])) + _control_source, _control_project, control = _build_chain(h, source=updated_source, project=project) + _assert_chain(h, control, domain="health", sensitivity="confidential") + + withheld = {row_id for _kind, row_id in rows} + for profile in ("read_only_agent", "trusted_local_agent"): + key = h.key(profile) + for kind, row_id in rows: + if kind == "artifact": + exact = f"/v0/vnext/artifacts/{row_id}" + elif kind == "memory": + exact = f"/v0/vnext/memories/{row_id}/audit" + elif kind == "open_loop": + # The coupled review door performs the exact target admission before mutation. + status, body, _ = h.request( + "POST", f"/v0/vnext/open-loops/{row_id}/review", payload={"action": "close"}, key=key + ) + assert status == 403, (profile, kind, body) + continue + else: + exact = f"/v0/vnext/projects/{row_id}/dashboard" + status, body, _ = h.request("GET", exact, key=key) + if kind == "project": + # O12 operator screens redact the response rather than requiring an exact policy refusal. + assert row_id not in json.dumps(body), (profile, kind, status, body) + else: + assert status == 403, (profile, kind, status, body) + for path in ("/v0/vnext/artifacts", "/v0/vnext/projects", "/v0/vnext/workspace", "/v0/vnext/context-tree"): + status, body, _ = h.request("GET", path, key=key) + # Read-only keys cannot use operator routes. Their refusal must carry no row data. + assert status == (403 if profile == "read_only_agent" else 200), (path, body) + encoded = json.dumps(body) + assert not withheld.intersection({row_id for row_id in withheld if row_id in encoded}), ( + profile, + path, + body, + ) + status, body, _ = h.request( + "POST", + "/v0/vnext/context-packs", + payload={"query": "SyntheticChain", "scope": {}, "options": {"include_sources": True}}, + key=key, + ) + assert status == 201, body + assert all(row_id not in json.dumps(body) for row_id in withheld) + + # A project-bound key loses the entire moved chain; provenance and a union floor survive. + bound = h.key("admin_agent", project=str(project["id"])) + beta = "prj_" + "b" * 16 + from alicebot_api.routers import vnext_memories as router + from uuid import UUID + + result = router.review_vnext_source( + UUID(str(source["id"])), + router.VNextSourceReviewRequest( + user_id=h.user_id, action="assign_project", project_id=beta, confirm_label_hide=True + ), + ) + move_status, move = result.status_code, json.loads(result.body) + assert move_status == 200, move + with h.store() as store: + for kind, row_id in rows: + row = _read_row(store, kind, row_id) + assert_raised(row) + assert beta in row["metadata_json"]["project_floor"] + if kind == "artifact": + assert h.request("GET", f"/v0/vnext/artifacts/{row_id}", key=bound)[0] == 403 From 3d532154510276967e8522a62ea14a8004fe0ba1 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:27:44 +0200 Subject: [PATCH 069/270] Use ordered repair locks and retain the CLI compatibility contract --- apps/api/src/alicebot_api/cli/__init__.py | 3 + apps/api/src/alicebot_api/cli/labels.py | 59 ++++++++++++------ apps/api/src/alicebot_api/vault_doctor.py | 4 +- apps/api/src/alicebot_api/vnext_doctor.py | 29 ++++----- .../src/alicebot_api/vnext_label_repair.py | 61 ++++++++++++------- 5 files changed, 97 insertions(+), 59 deletions(-) diff --git a/apps/api/src/alicebot_api/cli/__init__.py b/apps/api/src/alicebot_api/cli/__init__.py index b75de90b8..a8c313536 100644 --- a/apps/api/src/alicebot_api/cli/__init__.py +++ b/apps/api/src/alicebot_api/cli/__init__.py @@ -50,6 +50,9 @@ for _carrier in _CARRIER_MODULES: globals().pop(_carrier.__name__.rsplit(".", maxsplit=1)[-1], None) +# The owner label commands do not extend the compatibility facade. +globals().pop("labels", None) + _PUBLIC_NAME_ORDER = "annotations argparse Iterator Sequence contextmanager redirect_stderr dataclass UTC datetime StringIO json logging os Path sys tempfile time TypedDict cast URLError Request urlopen UUID uuid4 psycopg format_artifact_detail_output format_capture_output format_continuity_brief_output format_contradiction_case_detail_output format_contradiction_case_list_output format_contradiction_sync_output format_explain_output format_lifecycle_detail_output format_lifecycle_list_output format_memory_operation_candidates_output format_memory_operation_commit_output format_memory_operations_output format_open_loops_output format_recall_output format_resume_output format_review_apply_output format_review_detail_output format_review_queue_output format_status_output format_task_brief_comparison_output format_task_brief_output format_temporal_explain_output format_temporal_state_output format_temporal_timeline_output format_trust_signals_output format_trusted_fact_pattern_explain_output format_trusted_fact_pattern_list_output format_trusted_fact_playbook_explain_output format_trusted_fact_playbook_list_output Settings get_runtime_settings get_settings ContinuityCaptureValidationError capture_continuity_input ContinuityBriefValidationError compile_continuity_brief ContinuityEvidenceNotFoundError build_continuity_explain get_continuity_artifact_detail ContinuityContradictionNotFoundError ContinuityContradictionValidationError get_contradiction_case list_contradiction_cases resolve_contradiction_case sync_contradictions MemoryMutationValidationError commit_memory_operations generate_memory_operation_candidates list_memory_operation_candidates list_memory_operations default_continuity_promotable default_continuity_searchable ContinuityLifecycleNotFoundError ContinuityLifecycleValidationError get_continuity_lifecycle_state list_continuity_lifecycle_state ContinuityOpenLoopValidationError compile_continuity_open_loop_dashboard get_thread_health_dashboard ContinuityRecallValidationError query_continuity_recall ContinuityResumptionValidationError compile_continuity_resumption_brief ContinuityReviewNotFoundError ContinuityReviewValidationError apply_continuity_correction get_continuity_review_detail list_continuity_review_queue list_trust_signals get_memory_hygiene_dashboard_summary CONTINUITY_CAPTURE_EXPLICIT_SIGNALS CONTINUITY_CORRECTION_ACTIONS CONTRADICTION_RESOLUTION_ACTIONS CONTINUITY_BRIEF_TYPE_ORDER DEFAULT_CONTINUITY_CAPTURE_LIMIT DEFAULT_CONTINUITY_BRIEF_CONFLICT_LIMIT DEFAULT_CONTINUITY_BRIEF_RELEVANT_FACT_LIMIT DEFAULT_CONTINUITY_BRIEF_TIMELINE_LIMIT DEFAULT_CONTINUITY_LIFECYCLE_LIMIT DEFAULT_CONTINUITY_OPEN_LOOP_LIMIT DEFAULT_CONTINUITY_RECALL_LIMIT DEFAULT_CONTINUITY_RESUMPTION_OPEN_LOOP_LIMIT DEFAULT_CONTINUITY_RESUMPTION_RECENT_CHANGES_LIMIT DEFAULT_CONTINUITY_REVIEW_LIMIT DEFAULT_TEMPORAL_TIMELINE_LIMIT DEFAULT_TASK_BRIEF_TOKEN_BUDGET DEFAULT_TRUSTED_FACT_PROMOTION_LIMIT MAX_CONTINUITY_REVIEW_LIMIT MAX_CONTINUITY_OPEN_LOOP_LIMIT MAX_CONTINUITY_RECALL_LIMIT MAX_CONTINUITY_BRIEF_CONFLICT_LIMIT MAX_CONTINUITY_BRIEF_RELEVANT_FACT_LIMIT MAX_CONTINUITY_BRIEF_TIMELINE_LIMIT MAX_CONTINUITY_LIFECYCLE_LIMIT MAX_CONTINUITY_RESUMPTION_OPEN_LOOP_LIMIT MAX_CONTINUITY_RESUMPTION_RECENT_CHANGES_LIMIT MAX_TASK_BRIEF_TOKEN_BUDGET MAX_TEMPORAL_TIMELINE_LIMIT MAX_TRUSTED_FACT_PROMOTION_LIMIT ContradictionCaseListQueryInput ContradictionResolveInput ContradictionSyncInput ContinuityCaptureCreateInput ContinuityBriefRequestInput ContinuityCorrectionInput ContinuityLifecycleQueryInput ContinuityOpenLoopDashboardQueryInput ContinuityRecallQueryInput ContinuityResumptionBriefRequestInput ContinuityReviewQueueQueryInput MemoryOperationCommitInput MemoryOperationGenerateInput MemoryOperationListInput TaskBriefCompileRequestInput TemporalExplainQueryInput TemporalStateAtQueryInput TemporalTimelineQueryInput TrustSignalListQueryInput TrustedFactPatternListQueryInput TrustedFactPlaybookListQueryInput TaskBriefNotFoundError TaskBriefValidationError compare_task_briefs compile_and_persist_task_brief get_persisted_task_brief ping_database user_connection get_public_eval_run list_public_eval_runs list_public_eval_suites run_public_evals write_public_eval_report get_retrieval_evaluation_summary ContinuityStore ContinuityJsonObject legacy_surfaces_enabled TemporalStateValidationError get_temporal_explain get_temporal_state_at get_temporal_timeline TrustedFactPromotionNotFoundError get_trusted_fact_pattern get_trusted_fact_playbook list_trusted_fact_patterns list_trusted_fact_playbooks PERMISSION_PROFILES AgentIdentity PolicyDecision agent_metadata append_policy_events ensure_policy_allowed evaluate_agent_policy summarize_agent_policy_telemetry AgentKeyValidationError create_agent_key dispatch_vnext_artifact_review VNextCaptureService VNextCaptureValidationError BrainArtifactRequest VNextBrainService VNextBrainValidationError ConnectionFinderRequest VNextConnectionService VNextConnectionValidationError VNextConnectorService VNextConnectorValidationError list_connector_definitions load_connector_items_from_file scan_local_folder ContextTreeRequest VNextContextTreeService VNextContextTreeStore ContradictionFinderRequest VNextContradictionService VNextContradictionValidationError VNextDogfoodingService LOCAL_VNEXT_FRONTEND_ORIGINS VNextDoctorService local_live_cors_status VNEXT_EVAL_SUITE_ORDER run_vnext_evals write_vnext_benchmark_corpus write_vnext_eval_report ProjectAutomationRequest VNextProjectService VNextProjectValidationError QueueTaskRequest VNextQueueService VNextQueueValidationError JsonObject BUDGET_STRATEGIES CONTEXT_DEPTHS VNextRetrievalRequest VNextRetrievalService VNextRetrievalStore VNextRetrievalValidationError SchedulerRunRequest VNextSchedulerService VNextSchedulerStore VNextSchedulerValidationError WORKFLOW_TYPES default_schedule DEFAULT_LOG_FILE DEFAULT_PID_FILE DEFAULT_STATUS_FILE SchedulerRuntimeConfig daemon_status run_due_workflows_durable run_foreground_daemon run_now_durable start_background_daemon stop_daemon DeferredMemoryEmbedding EMBEDDINGS_API_KEY_ENV EMBEDDINGS_BASE_URL_ENV EMBEDDINGS_MODEL_ENV EMBEDDING_SIGNATURE_VERSION MAX_EMBEDDINGS_BATCH_SIZE endpoint_fingerprint get_embedding_provider memory_embedding_text persist_deferred_memory_embeddings_best_effort append_event json_safe redact_memory_flow VNextMemoryCommitService VNextMemoryCommitValidationError memory_commit_request_from_payload InMemorySecretProvider PostgresVNextStore DEFAULT_CLI_USER_ID DEFAULT_VNEXT_SENSITIVITY_ALLOWED MAINTENANCE_REPORT_PATH_ENV DEFAULT_MAINTENANCE_REPORT_PATH DEFAULT_VNEXT_DEMO_DATASET_PATH REVIEW_STATUS_CHOICES DEMO_SECRET_MARKERS logger EvalGateFailure EmbeddingBackfillFailure PartialCommandFailure CLIContext ModelGenerationKwargs build_parser main".split() _public_values = {name: globals().pop(name) for name in _PUBLIC_NAME_ORDER} globals().update(_public_values) diff --git a/apps/api/src/alicebot_api/cli/labels.py b/apps/api/src/alicebot_api/cli/labels.py index fc3b92b37..93f290062 100644 --- a/apps/api/src/alicebot_api/cli/labels.py +++ b/apps/api/src/alicebot_api/cli/labels.py @@ -2,27 +2,24 @@ from __future__ import annotations +from psycopg import sql + from alicebot_api.cli.shared import CLIContext, _vnext_store_context from alicebot_api.db import user_read_snapshot_connection from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError, require_changed -from alicebot_api.vnext_label_repair import classify_stored_labels, format_label_check, plan_label_repairs, load_postgres_label_tables, label_project_id -from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock +from alicebot_api.vnext_label_repair import ( + classify_stored_labels, + format_label_check, + plan_label_repairs, + load_postgres_label_tables, + label_project_id, +) +from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock, lock_settled_label_rows _postgres_tables = load_postgres_label_tables -def _lock_repair_rows(store, changes) -> None: - """Take exactly the planned row locks in the relabel table order.""" - for table in ("generated_artifacts", "projects", "open_loops", "memories"): - ids = [row_id for changed_table, _user, row_id, *_ in changes if changed_table == table] - if ids: - locked = store.conn.execute( - f"SELECT id FROM {table} WHERE id = ANY(%s::uuid[]) ORDER BY id FOR UPDATE", (ids,) - ).fetchall() - require_changed(int(len(locked) == len(ids)), table, "planned rows") - - def _run_vnext_labels_check(ctx: CLIContext, args: object) -> str: del args try: @@ -46,7 +43,21 @@ def _run_vnext_labels_repair(ctx: CLIContext, args: object) -> str: store.lock_graph_mutation() acquire_exclusive_label_lock(store) changes = plan_label_repairs(_postgres_tables(store.conn)) - _lock_repair_rows(store, changes) + lock_settled_label_rows( + store, + [ + { + "kind": { + "generated_artifacts": "artifact", + "projects": "project", + "open_loops": "open_loop", + "memories": "memory", + }[table], + "id": row_id, + } + for table, _user, row_id, *_ in changes + ], + ) # The SQLite writer is not used here. Postgres repair applies the # same plan through label-only updates inside this transaction. applied = 0 @@ -59,16 +70,26 @@ def _run_vnext_labels_repair(ctx: CLIContext, args: object) -> str: metadata = {"project_scope": list(new["project_scope"]), "project_floor": list(new["project_floor"])} project_sql = ", project_id = %s" if table in {"memories", "open_loops"} else "" project_guard = " AND project_id IS NOT DISTINCT FROM %s" if project_sql else "" - params = [new["domain"], new["sensitivity"], json.dumps(metadata)] + params: list[object] = [new["domain"], new["sensitivity"], json.dumps(metadata)] if project_sql: params.append(label_project_id(new["project_scope"])) - params.extend([user, row_id, previous["domain"], previous["sensitivity"], json.dumps(node.get("metadata_json") or {})]) + params.extend( + [ + user, + row_id, + previous["domain"], + previous["sensitivity"], + json.dumps(node.get("metadata_json") or {}), + ] + ) if project_sql: params.append(node.get("project_id")) cursor = store.conn.execute( - f"UPDATE {table} SET domain = %s, sensitivity = %s, metadata_json = metadata_json || %s::jsonb " - f"{project_sql} WHERE user_id = %s::uuid AND id = %s::uuid " - f"AND domain = %s AND sensitivity = %s AND metadata_json = %s::jsonb{project_guard}", + sql.SQL( + "UPDATE {} SET domain = %s, sensitivity = %s, metadata_json = metadata_json || %s::jsonb " + "{} WHERE user_id = %s::uuid AND id = %s::uuid " + "AND domain = %s AND sensitivity = %s AND metadata_json = %s::jsonb{}" + ).format(sql.Identifier(table), sql.SQL(project_sql), sql.SQL(project_guard)), tuple(params), ) require_changed(cursor.rowcount, table, row_id) diff --git a/apps/api/src/alicebot_api/vault_doctor.py b/apps/api/src/alicebot_api/vault_doctor.py index 6c3f61bd1..9442b562a 100644 --- a/apps/api/src/alicebot_api/vault_doctor.py +++ b/apps/api/src/alicebot_api/vault_doctor.py @@ -90,9 +90,7 @@ def compile_local_vault_doctor( """Render the vault census for the acting local user.""" if COMMITTED_MEMORY_STATUSES != ("active", "accepted"): - raise RuntimeError( - "committed-fact COUNT SQL is written for active and accepted only" - ) + raise RuntimeError("committed-fact COUNT SQL is written for active and accepted only") resolved = Path(db_path).expanduser().resolve() with sqlite_user_connection(resolved, user_id) as connection: diff --git a/apps/api/src/alicebot_api/vnext_doctor.py b/apps/api/src/alicebot_api/vnext_doctor.py index f40998d2e..7081eda06 100644 --- a/apps/api/src/alicebot_api/vnext_doctor.py +++ b/apps/api/src/alicebot_api/vnext_doctor.py @@ -22,9 +22,7 @@ # The workspace dashboard runs this doctor on every load. Stop and say so # rather than scanning the rest of a large source table in that request. _FLAGGED_SOURCE_SCAN_LIMIT = 10_000 -LOCAL_VNEXT_CORS_RECOMMENDED_FIX = ( - "CORS_ALLOWED_ORIGINS=http://127.0.0.1:3000,http://localhost:3000" -) +LOCAL_VNEXT_CORS_RECOMMENDED_FIX = "CORS_ALLOWED_ORIGINS=http://127.0.0.1:3000,http://localhost:3000" PGVECTOR_MINIMUM_VERSION = (0, 8, 0) PGVECTOR_MINIMUM_VERSION_TEXT = ".".join(str(part) for part in PGVECTOR_MINIMUM_VERSION) @@ -159,10 +157,7 @@ def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: f"pgvector {pgvector_version} satisfies the required " f">= {PGVECTOR_MINIMUM_VERSION_TEXT} runtime contract." ), - message_fail=( - "pgvector is missing, unparseable, or older than " - f"{PGVECTOR_MINIMUM_VERSION_TEXT}." - ), + message_fail=(f"pgvector is missing, unparseable, or older than {PGVECTOR_MINIMUM_VERSION_TEXT}."), recommended_fix=( "Install pgvector >= " f"{PGVECTOR_MINIMUM_VERSION_TEXT}, run ALTER EXTENSION vector UPDATE, " @@ -220,7 +215,9 @@ def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: ) scheduler = daemon_status() - scheduler_known = not bool(scheduler.get("stopped")) or "pid file" not in str(scheduler.get("message", "")).casefold() + scheduler_known = ( + not bool(scheduler.get("stopped")) or "pid file" not in str(scheduler.get("message", "")).casefold() + ) self._check( checks, name="scheduler_daemon", @@ -232,7 +229,9 @@ def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: details=cast(JsonObject, scheduler), ) - health = VNextConnectorService(cast(Any, self.store), secret_provider=self.secret_provider).connector_health_all() + health = VNextConnectorService( + cast(Any, self.store), secret_provider=self.secret_provider + ).connector_health_all() failing_connectors = [] for item in cast(list[JsonObject], health.get("items", [])): failed_value = item.get("items_failed", 0) @@ -266,9 +265,7 @@ def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: flagged_ids, stopped_early = _flagged_source_scan(self.store) remedy = _flagged_source_remedy(self.store) if flagged_ids: - message = ( - f"{len(flagged_ids)} stored sources carry credential material. {remedy}" - ) + message = f"{len(flagged_ids)} stored sources carry credential material. {remedy}" else: message = "No stored source carries credential material." if stopped_early: @@ -318,7 +315,9 @@ def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: "warning_count": len(warnings), "checks": [check.to_record() for check in checks], "recommended_fixes": [ - check.recommended_fix for check in checks if check.status == "fail" and check.recommended_fix is not None + check.recommended_fix + for check in checks + if check.status == "fail" and check.recommended_fix is not None ], "migration_status": migration_status, "connector_health": health, @@ -400,9 +399,7 @@ def local_live_cors_status( ) -> JsonObject: merged_env = _merged_local_env(os.environ if env is None else env, cwd or Path.cwd()) frontend_api_base_url = ( - merged_env.get("NEXT_PUBLIC_ALICEBOT_API_BASE_URL") - or merged_env.get("ALICEBOT_API_BASE_URL") - or "" + merged_env.get("NEXT_PUBLIC_ALICEBOT_API_BASE_URL") or merged_env.get("ALICEBOT_API_BASE_URL") or "" ).strip() frontend_user_id = ( merged_env.get("NEXT_PUBLIC_ALICEBOT_USER_ID") or merged_env.get("ALICEBOT_USER_ID") or "" diff --git a/apps/api/src/alicebot_api/vnext_label_repair.py b/apps/api/src/alicebot_api/vnext_label_repair.py index 2db1b4d59..3dad92c6d 100644 --- a/apps/api/src/alicebot_api/vnext_label_repair.py +++ b/apps/api/src/alicebot_api/vnext_label_repair.py @@ -8,12 +8,23 @@ import json from collections.abc import Mapping, Sequence +from typing import TypedDict from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError, require_changed from alicebot_api.vnext_derived_labels import labels_raised_payload, settle_labels from alicebot_api.vnext_event_log import build_event_log_record from alicebot_api.vnext_project_scope import project_scope_identity + +class LabelParts(TypedDict): + domain: str + sensitivity: str + project_scope: list[str] + project_floor: list[str] + + +LabelRepair = tuple[str, str, str, LabelParts, LabelParts, dict[str, object]] + REPAIR_STATE_KEY = "derived_labels_v3" _TABLE_KIND = { "sources": "source", @@ -26,12 +37,9 @@ _WRITABLE = frozenset({"memories", "open_loops", "generated_artifacts", "projects"}) INPUT_SELECTS_V3 = { "sources": "SELECT id, user_id, domain, sensitivity, metadata_json, deleted_at FROM sources", - "memories": ( - "SELECT id, user_id, domain, sensitivity, metadata_json, value, project_id, deleted_at FROM memories" - ), + "memories": ("SELECT id, user_id, domain, sensitivity, metadata_json, value, project_id, deleted_at FROM memories"), "open_loops": ( - "SELECT id, user_id, domain, sensitivity, metadata_json, project_id, source_id, memory_id " - "FROM open_loops" + "SELECT id, user_id, domain, sensitivity, metadata_json, project_id, source_id, memory_id FROM open_loops" ), "generated_artifacts": ( "SELECT id, user_id, domain, sensitivity, metadata_json, artifact_type FROM generated_artifacts" @@ -70,7 +78,7 @@ def _same(previous: Mapping[str, object], new: Mapping[str, object]) -> bool: def plan_label_repairs( tables: Mapping[str, Sequence[Mapping[str, object]]], -) -> list[tuple[str, str, str, dict[str, object], dict[str, object], dict[str, object]]]: +) -> list[LabelRepair]: """Label changes for derived rows whose stored label is below the inputs. A malformed record makes its row unverified and is not rewritten. A cycle @@ -91,17 +99,17 @@ def plan_label_repairs( nodes.append(node) index.append((table, node)) settled = settle_labels(nodes, on_cycle="raise") - changes = [] + changes: list[LabelRepair] = [] for (table, node), label in zip(index, settled.rows, strict=True): if not label.derived or label.unverified or table not in _WRITABLE: continue - previous = { + previous: LabelParts = { "domain": str(node.get("domain") or "unknown"), "sensitivity": str(node.get("sensitivity") or "unknown"), "project_scope": list(label.stored_scope), "project_floor": list(label.stored_floor), } - new = { + new: LabelParts = { "domain": label.domain, "sensitivity": label.sensitivity, "project_scope": list(label.project_scope), @@ -141,7 +149,9 @@ def _load_tables(conn) -> dict[str, list[dict[str, object]]]: def _stamped(conn) -> bool: - return conn.execute("SELECT value FROM alice_schema_state WHERE key = ?", (REPAIR_STATE_KEY,)).fetchone() is not None + return ( + conn.execute("SELECT value FROM alice_schema_state WHERE key = ?", (REPAIR_STATE_KEY,)).fetchone() is not None + ) def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = False) -> None: @@ -166,9 +176,18 @@ def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = Fal raw_metadata = node.get("_stored_metadata") changed = conn.execute( _UPDATES[table], - (new["domain"], new["sensitivity"], json.dumps(metadata), - label_project_id(new["project_scope"]), user, stored, - previous["domain"], previous["sensitivity"], raw_metadata, node.get("project_id")), + ( + new["domain"], + new["sensitivity"], + json.dumps(metadata), + label_project_id(new["project_scope"]), + user, + stored, + previous["domain"], + previous["sensitivity"], + raw_metadata, + node.get("project_id"), + ), ).rowcount require_changed(changed, table, stored) for table, user, stored, previous, new, _node in changes: @@ -206,7 +225,7 @@ def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = Fal def classify_stored_labels( tables: Mapping[str, Sequence[Mapping[str, object]]], -) -> tuple[list[tuple[str, str, str, dict[str, object], dict[str, object], dict[str, object]]], dict[str, list[str]]]: +) -> tuple[list[LabelRepair], dict[str, list[str]]]: """Rows below their inputs, and unverified ids grouped by reason. A cycle is reported as unverified instead of raising. ``labels check`` uses this. @@ -226,7 +245,7 @@ def classify_stored_labels( nodes.append(node) index.append((table, node)) settled = settle_labels(nodes, on_cycle="unverified") - below = [] + below: list[LabelRepair] = [] unverified: dict[str, list[str]] = {} for (table, node), label in zip(index, settled.rows, strict=True): if not label.derived: @@ -237,13 +256,13 @@ def classify_stored_labels( continue if table not in _WRITABLE: continue - previous = { + previous: LabelParts = { "domain": str(node.get("domain") or "unknown"), "sensitivity": str(node.get("sensitivity") or "unknown"), "project_scope": list(label.stored_scope), "project_floor": list(label.stored_floor), } - new = { + new: LabelParts = { "domain": label.domain, "sensitivity": label.sensitivity, "project_scope": list(label.project_scope), @@ -296,7 +315,9 @@ def label_gap_counts(store: object) -> tuple[int, int]: """ conn = getattr(store, "conn", None) - module = type(conn).__module__ if conn is not None else "" + if conn is None: + raise LabelCheckUnavailable("derived label counts are unavailable for this store") + module = type(conn).__module__ try: if module.startswith("sqlite3"): tables = _load_tables(conn) @@ -350,9 +371,7 @@ def recorded_sqlite_label_repairs(conn, user_id: str) -> dict[tuple[str, str, st if not isinstance(previous, Mapping): continue repairs.setdefault((table, row_id, "domain"), set()).add(str(previous.get("domain") or "unknown")) - repairs.setdefault((table, row_id, "sensitivity"), set()).add( - str(previous.get("sensitivity") or "unknown") - ) + repairs.setdefault((table, row_id, "sensitivity"), set()).add(str(previous.get("sensitivity") or "unknown")) scope = previous.get("project_scope") floor = previous.get("project_floor") repairs.setdefault((table, row_id, "project_scope"), set()).add( From 0d623567538f11fa55eebe068f03a95cabfd650a Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:27:44 +0200 Subject: [PATCH 070/270] Prove label upgrade restore and interruption safety through real readers --- .../test_derived_labels_migration_postgres.py | 456 ++++++++++++++++++ tests/unit/test_cli_package_split.py | 8 +- tests/unit/test_derived_domain_mutations.py | 6 + tests/unit/test_sqlite_derived_labels_v3.py | 429 ++++++++++++++++ 4 files changed, 897 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_derived_labels_migration_postgres.py create mode 100644 tests/unit/test_sqlite_derived_labels_v3.py diff --git a/tests/integration/test_derived_labels_migration_postgres.py b/tests/integration/test_derived_labels_migration_postgres.py new file mode 100644 index 000000000..0c9152a5e --- /dev/null +++ b/tests/integration/test_derived_labels_migration_postgres.py @@ -0,0 +1,456 @@ +"""Owner migration and application repair acceptance with real restricted readers.""" + +from __future__ import annotations + +from contextlib import contextmanager +import json +import threading +from types import SimpleNamespace +from uuid import uuid4 + +from alembic import command, op +import psycopg +import pytest + +import alicebot_api.main as main_module +from alicebot_api import vnext_label_repair as repair +from alicebot_api.cli import labels +from alicebot_api.config import Settings +from alicebot_api.db import close_connection_pools, user_connection, user_read_snapshot_connection +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError +from alicebot_api.migrations import make_alembic_config +from alicebot_api.routers import vnext_retrieval as retrieval_router +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_doctor import VNextDoctorService +from alicebot_api.vnext_derived_labels import with_derived_from +from alicebot_api.vnext_label_guard import LabelGuard +from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.test_vnext_omitted_domains_api import invoke_request +from tests.integration.conftest import _create_role_separated_database, _drop_database +from urllib.parse import urlsplit + +TABLES = ("sources", "memories", "open_loops", "generated_artifacts", "beliefs", "event_log", "projects") +SENTINEL = "Violet private migration sentinel" + + +@contextmanager +def owner_bracket(url): + with psycopg.connect(url) as conn: + for table in TABLES: + conn.execute(f"ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY") + yield conn + for table in TABLES: + conn.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + + +def assert_roles_and_force(urls): + for url in (urls["admin"], urls["app"]): + with psycopg.connect(url) as conn: + assert conn.execute( + "SELECT rolsuper, rolbypassrls FROM pg_roles WHERE rolname=current_user" + ).fetchone() == (False, False) + with psycopg.connect(urls["admin"]) as conn: + assert all( + row[0] + for row in conn.execute("SELECT relforcerowsecurity FROM pg_class WHERE relname=ANY(%s)", (list(TABLES),)) + ) + + +def seed_stale(urls, user=None): + user = user or uuid4() + with without_insert_floor(), user_connection(urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"fixture-{user}@example.invalid", "Synthetic fixture") + store = PostgresVNextStore(conn) + project = store.create_project({"slug": "alpha", "name": "Alpha", "domain": "project", "sensitivity": "public"}) + source = store.create_source( + { + "source_type": "note", + "title": "Synthetic private input", + "content_hash": "sha256:" + uuid4().hex, + "domain": "health", + "sensitivity": "confidential", + "metadata_json": {}, + } + ) + copies = [] + for i in range(3): + copies.append( + store.create_memory( + { + "memory_key": f"derived.{i}", + "canonical_text": SENTINEL, + "domain": "unknown", + "sensitivity": "public", + "status": "active", + "project_id": str(project["id"]), + "metadata_json": {"source_id": str(source["id"]), "project_scope": [str(project["id"])]}, + } + ) + ) + loop = store.create_open_loop( + { + "title": SENTINEL, + "domain": "unknown", + "sensitivity": "public", + "source_id": str(source["id"]), + "project_id": str(project["id"]), + "metadata_json": { + "discovered_by": "vnext_daily_brief", + "source_id": str(source["id"]), + "project_scope": [str(project["id"])], + }, + }, + actor_type="user", + ) + belief = store.create_belief( + {"memory_id": str(copies[0]["id"]), "claim": "Synthetic belief", "status": "active", "confidence": 0.8}, + actor_type="user", + ) + reports = [] + for key, metadata in ( + ("source", {"input_summary": {"source_ids": [str(source["id"])]}}), + ("memory", {"input_summary": {"memory_ids": [str(copies[0]["id"])]}}), + ("loop", {"input_summary": {"open_loop_ids": [str(loop["id"])]}}), + ("belief", {"belief_ids": [str(belief["id"])]}), + ): + reports.append( + store.create_artifact( + { + "artifact_type": "contradiction_report" if key == "belief" else "daily_brief", + "title": f"Synthetic {key} report", + "content_markdown": SENTINEL, + "status": "needs_review", + "domain": "unknown", + "sensitivity": "public", + "metadata_json": {**metadata, "project_scope": [str(project["id"])]}, + } + ) + ) + # Raw state matches a pre-floor accepted project update while keeping its real recorded parents. + conn.execute( + "UPDATE projects SET metadata_json=%s::jsonb WHERE id=%s", + (json.dumps(with_derived_from({}, {"memories": [copies[0]], "artifacts": [reports[0]]})), project["id"]), + ) + return user, { + "memories": [str(r["id"]) for r in copies], + "open_loops": [str(loop["id"])], + "generated_artifacts": [str(r["id"]) for r in reports], + "projects": [str(project["id"])], + } + + +def stored(urls, user, targets): + with user_connection(urls["app"], user) as conn: + rows = { + table: conn.execute( + f"SELECT domain, sensitivity, metadata_json{', project_id' if table in {'memories', 'open_loops'} else ''} FROM {table} WHERE id=ANY(%s::uuid[]) ORDER BY id", + (ids,), + ).fetchall() + for table, ids in targets.items() + } + events = conn.execute( + "SELECT target_id, payload_json FROM event_log WHERE event_type LIKE '%%.labels_raised' ORDER BY id" + ).fetchall() + return rows, events + + +def assert_repaired(urls, user, targets): + rows, events = stored(urls, user, targets) + for table, values in rows.items(): + assert len(values) == len(targets[table]) + for row in values: + assert (row["domain"], row["sensitivity"]) == ("health", "confidential"), (table, row) + assert row["metadata_json"]["project_scope"] == [], (table, row) + assert row["metadata_json"]["project_floor"] == [], (table, row) + if "project_id" in row: + assert row["project_id"] is None + assert len(events) == sum(len(ids) for ids in targets.values()) + assert SENTINEL not in json.dumps(events, default=str) + return rows, events + + +def restricted_reads(urls, user, targets, monkeypatch, *, guard_off=False): + with user_connection(urls["app"], user) as conn: + _, raw = create_agent_key( + PostgresVNextStore(conn), user_id=user, agent_id="migration-reader", permission_profile="read_only_agent" + ) + with monkeypatch.context() as patch: + for module in (main_module, retrieval_router): + patch.setattr(module, "get_settings", lambda: Settings(database_url=urls["app"])) + patch.setenv("ALICE_AGENT_API_KEY", raw) + patch.setenv("ALICE_MCP_FULL_TOOLS", "1") + patch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + if guard_off: + patch.setattr(LabelGuard, "effective_row", lambda self, kind, row: row) + status, pack = invoke_request( + "POST", + "/v0/vnext/context-packs", + authorization=f"Bearer {raw}", + payload={"user_id": str(user), "query": "Violet", "scope": {}, "options": {"include_sources": True}}, + ) + assert status == 201, pack + assert SENTINEL not in json.dumps(pack), (guard_off, pack) + context = MCPRuntimeContext(database_url=urls["app"], user_id=user) + result = call_mcp_tool(context, name="alice_recall", arguments={"query": "Violet"}) + assert SENTINEL not in json.dumps(result, default=str) + for row_id in targets["memories"]: + with pytest.raises(MCPToolError, match="requested explanation is unavailable"): + call_mcp_tool(context, name="alice_explain", arguments={"memory_id": row_id}) + + +def cli_context(urls, user): + return SimpleNamespace(database_url=urls["app"], user_id=user) + + +@pytest.mark.parametrize("revision", ("20260721_0094", "20261004_0095")) +def test_upgrade_repairs_all_labels_as_nobypassrls_owner(database_urls, monkeypatch, revision): + command.upgrade(make_alembic_config(database_urls["admin"]), revision) + user, targets = seed_stale(database_urls) + command.upgrade(make_alembic_config(database_urls["admin"]), "head") + assert_roles_and_force(database_urls) + assert_repaired(database_urls, user, targets) + restricted_reads(database_urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(database_urls, user, targets, monkeypatch) + + +def test_head_restored_rows_are_guarded_then_fixed_by_owner_command(migrated_database_urls, monkeypatch): + urls = migrated_database_urls + user, targets = seed_stale(urls) + assert stored(urls, user, targets)[0]["memories"][0]["sensitivity"] == "public" + restricted_reads(urls, user, targets, monkeypatch) + assert labels._run_vnext_labels_repair(cli_context(urls, user), None).startswith("labels repair updated ") + assert_repaired(urls, user, targets) + restricted_reads(urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(urls, user, targets, monkeypatch) + assert labels._run_vnext_labels_check(cli_context(urls, user), None) == "below_inputs 0" + + +def test_labels_check_is_one_read_only_repeatable_snapshot(migrated_database_urls, monkeypatch): + urls = migrated_database_urls + user, targets = seed_stale(urls) + original = labels._postgres_tables + + def inspect_snapshot(conn): + assert conn.execute("SHOW transaction_isolation").fetchone()["transaction_isolation"] == "repeatable read" + assert conn.execute("SHOW transaction_read_only").fetchone()["transaction_read_only"] == "on" + before = conn.execute("SELECT count(*) AS count FROM memories").fetchone()["count"] + with user_connection(urls["app"], user) as other: + other.execute("UPDATE memories SET sensitivity='regulated' WHERE id=%s", (targets["memories"][0],)) + rows = original(conn) + assert len(rows["memories"]) == before + assert all(row["sensitivity"] == "public" for row in rows["memories"]) + with pytest.raises(psycopg.errors.ReadOnlySqlTransaction): + with conn.transaction(): + conn.execute("UPDATE memories SET sensitivity='public'") + return rows + + monkeypatch.setattr(labels, "_postgres_tables", inspect_snapshot) + with pytest.raises(SystemExit) as caught: + labels._run_vnext_labels_check(cli_context(urls, user), None) + assert caught.value.code == 1 + with user_read_snapshot_connection(urls["app"], user) as conn: + assert conn.execute("SELECT app.current_user_id() AS id").fetchone()["id"] == user + with psycopg.connect(urls["app"]) as conn: + assert conn.execute("SELECT app.current_user_id()").fetchone()[0] is None + + +def test_repair_compare_and_set_failure_after_writes_rolls_everything_back(migrated_database_urls, monkeypatch): + urls = migrated_database_urls + user, targets = seed_stale(urls) + before = stored(urls, user, targets) + original = labels.require_changed + writes = [] + + def fail_third(count, table, row_id): + if row_id != "planned rows": + writes.append((count, table, row_id)) + if len(writes) == 3: + count = 0 + return original(count, table, row_id) + + with monkeypatch.context() as patch: + patch.setattr(labels, "require_changed", fail_third) + with pytest.raises(SystemExit) as caught: + labels._run_vnext_labels_repair(cli_context(urls, user), None) + assert caught.value.code == 2 and len(writes) == 3 + assert stored(urls, user, targets) == before + labels._run_vnext_labels_repair(cli_context(urls, user), None) + assert_repaired(urls, user, targets) + restricted_reads(urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(urls, user, targets, monkeypatch) + + +def test_doctor_counts_real_postgres_rows_and_failure_stays_warning(migrated_database_urls, monkeypatch): + urls = migrated_database_urls + user, targets = seed_stale(urls) + with user_connection(urls["app"], user) as conn: + store = PostgresVNextStore(conn) + below, unverified = repair.label_gap_counts(store) + assert below == sum(map(len, targets.values())) and unverified == 0 + + def broken_loader(conn): + conn.execute("SELECT * FROM synthetic_missing_label_table") + + monkeypatch.setattr(repair, "load_postgres_label_tables", broken_loader) + with pytest.raises(repair.LabelCheckUnavailable): + repair.label_gap_counts(store) + assert conn.execute("SELECT 1 AS ok").fetchone()["ok"] == 1 + result = VNextDoctorService(store).run(ci=True) + check = next(row for row in result["checks"] if row["name"] == "derived_labels") + assert check["severity"] == "warning" and check["status"] == "fail" + assert "unavailable" in check["message"] and "0 below" not in check["message"] + assert conn.execute("SELECT 1 AS ok").fetchone()["ok"] == 1 + + +def test_interrupted_migration_rolls_back_and_restored_backup_repeats(database_urls, monkeypatch): + urls = database_urls + config = make_alembic_config(urls["admin"]) + command.upgrade(config, "20261004_0095") + user, targets = seed_stale(urls) + before = stored(urls, user, targets) + backup_name = "alicebot_backup_" + uuid4().hex[:12] + restored_name = "alicebot_restored_" + uuid4().hex[:12] + close_connection_pools() + _create_role_separated_database(backup_name, template=urlsplit(urls["admin"]).path.lstrip("/")) + try: + original = repair.plan_label_repairs + + def interrupted_plan(tables): + # Alembic iterates this sequence and really writes its first three rows + # before the iterator raises; labels, events and FORCE stay atomic. + changes = original(tables) + for index, change in enumerate(changes): + if index == 3: + raise RuntimeError("injected after three migration writes") + yield change + + with monkeypatch.context() as patch: + patch.setattr(repair, "plan_label_repairs", interrupted_plan) + with pytest.raises(RuntimeError, match="three migration writes"): + command.upgrade(config, "head") + assert stored(urls, user, targets) == before + assert_roles_and_force(urls) + with psycopg.connect(urls["admin"]) as conn: + assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == "20261004_0095" + command.upgrade(config, "head") + assert_repaired(urls, user, targets) + restricted_reads(urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(urls, user, targets, monkeypatch) + close_connection_pools() + restored_urls = _create_role_separated_database(restored_name, template=backup_name) + try: + assert stored(restored_urls, user, targets) == before + command.upgrade(make_alembic_config(restored_urls["admin"]), "head") + assert_repaired(restored_urls, user, targets) + assert_roles_and_force(restored_urls) + restricted_reads(restored_urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(restored_urls, user, targets, monkeypatch) + finally: + close_connection_pools() + _drop_database(restored_name) + finally: + _drop_database(backup_name) + + +@pytest.mark.parametrize("table", TABLES) +def test_each_force_bracket_guard_is_behaviorally_required(database_urls, monkeypatch, table): + urls = database_urls + command.upgrade(make_alembic_config(urls["admin"]), "20261004_0095") + user, targets = seed_stale(urls) + original = op.execute + + def omit_one(statement, *args, **kwargs): + if str(statement) == f"ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY": + return None + return original(statement, *args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(op, "execute", omit_one) + try: + command.upgrade(make_alembic_config(urls["admin"]), "head") + except Exception as exc: + assert table == "event_log", (table, exc) + else: + with pytest.raises(AssertionError): + assert_repaired(urls, user, targets) + # Return every intentional mutant to a safe settled state and a real reader. + command.downgrade(make_alembic_config(urls["admin"]), "20261004_0095") + command.upgrade(make_alembic_config(urls["admin"]), "head") + assert_repaired(urls, user, targets) + assert_roles_and_force(urls) + restricted_reads(urls, user, targets, monkeypatch) + + +def test_downgrade_to_0095_and_upgrade_is_idempotent(database_urls, monkeypatch): + urls = database_urls + config = make_alembic_config(urls["admin"]) + command.upgrade(config, "20261004_0095") + user, targets = seed_stale(urls) + command.upgrade(config, "head") + before = assert_repaired(urls, user, targets) + command.downgrade(config, "20261004_0095") + command.upgrade(config, "head") + assert stored(urls, user, targets) == before + restricted_reads(urls, user, targets, monkeypatch) + + +def test_labels_repair_cannot_overwrite_a_relabel(migrated_database_urls, monkeypatch): + from concurrent.futures import ThreadPoolExecutor + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock, held_label_locks + + urls = migrated_database_urls + user, targets = seed_stale(urls) + planned, release, relabel_started = threading.Event(), threading.Event(), threading.Event() + original = labels._postgres_tables + + def pause_snapshot(conn): + assert held_label_locks(PostgresVNextStore(conn)) == (True, True, True) + rows = original(conn) + planned.set() + assert release.wait(10) + return rows + + monkeypatch.setattr(labels, "_postgres_tables", pause_snapshot) + + def raise_source(): + with user_connection(urls["app"], user) as conn: + store = PostgresVNextStore(conn) + source_id = str(conn.execute("SELECT id FROM sources").fetchone()["id"]) + relabel_started.set() + store.lock_graph_mutation() + acquire_exclusive_label_lock(store) + store.update_source(source_id=source_id, patch={"sensitivity": "regulated"}, actor_type="user") + + with ThreadPoolExecutor(max_workers=2) as executor: + repaired = executor.submit(labels._run_vnext_labels_repair, cli_context(urls, user), None) + assert planned.wait(10) + relabelled = executor.submit(raise_source) + assert relabel_started.wait(10) + try: + # The second transaction has really reached its lock request. + import time + + deadline = time.monotonic() + 5 + waiting = False + while time.monotonic() < deadline: + with psycopg.connect(urls["admin"]) as conn: + waiting = bool( + conn.execute( + "SELECT count(*) FROM pg_locks WHERE locktype='advisory' AND NOT granted AND database=(SELECT oid FROM pg_database WHERE datname=current_database())" + ).fetchone()[0] + ) + if waiting: + break + time.sleep(0.01) + assert waiting and not relabelled.done() + finally: + release.set() + repaired.result(timeout=10) + relabelled.result(timeout=10) + rows, _events = stored(urls, user, targets) + assert all(row["sensitivity"] == "regulated" for values in rows.values() for row in values) + restricted_reads(urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(urls, user, targets, monkeypatch) diff --git a/tests/unit/test_cli_package_split.py b/tests/unit/test_cli_package_split.py index 14875781a..2cc8a95d3 100644 --- a/tests/unit/test_cli_package_split.py +++ b/tests/unit/test_cli_package_split.py @@ -35,6 +35,7 @@ "errors.py", "evals.py", "memories.py", + "labels.py", "models.py", "parser.py", "runner.py", @@ -49,11 +50,13 @@ # and again after adding it to `vnext sources capture-file`, `vnext connectors # browser-clipper capture` and `vnext agents ingest-output` (three more # actions, no new command). +# The owner label check and repair commands add three parsers, four actions, +# two leaves and two handlers under both flag states. # Both keys are updated together: leaving one at its old value is how the # other half silently rots. EXPECTED_PARSER_RECEIPTS = { - False: (159, 735, 122, 118), - True: (163, 768, 125, 121), + False: (162, 739, 124, 120), + True: (166, 772, 127, 123), } EXPECTED_PUBLIC_NAME_COUNT = 270 EXPECTED_PUBLIC_NAMES_SHA256 = "8d97ffb088d5d8dea239c81589e9c109b81f7dc50b916d7bfb593ce13acae5fa" @@ -68,6 +71,7 @@ "errors", "evals", "memories", + "labels", "models", "parser", "runner", diff --git a/tests/unit/test_derived_domain_mutations.py b/tests/unit/test_derived_domain_mutations.py index aa28d4819..0e73a7ae5 100644 --- a/tests/unit/test_derived_domain_mutations.py +++ b/tests/unit/test_derived_domain_mutations.py @@ -78,6 +78,12 @@ def fresh(check): def restore(directory, patch): + # This harness protects the frozen v2 repair. Its v3 successor must not + # mask a removed v2 guard; v3 has its own staged-restore mutation cases. + from alicebot_api import sqlite_schema, vnext_label_repair + patch.setattr(sqlite_schema, "_relabel_derived_labels", lambda conn: None) + patch.setattr(vnext_label_repair, "relabel_labels_sqlite", lambda conn, **kwargs: None) + patch.setattr(vnext_label_repair, "recorded_sqlite_label_repairs", lambda conn, user: {}) review.test_restore_repairs_derived_rows_before_publication(directory, patch, True) diff --git a/tests/unit/test_sqlite_derived_labels_v3.py b/tests/unit/test_sqlite_derived_labels_v3.py new file mode 100644 index 000000000..b07162b93 --- /dev/null +++ b/tests/unit/test_sqlite_derived_labels_v3.py @@ -0,0 +1,429 @@ +"""Stored labels, staged restore and crash recovery through real key readers.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import select +import shutil +import sqlite3 +import subprocess +import sys +from uuid import UUID + +import pytest + +from alicebot_api import sqlite_schema, vnext_label_repair as repair +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError +from alicebot_api.onramp import bootstrap_database, main as onramp_main, sqlite_url_for_path +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_label_guard import LabelGuard +from alicebot_api.vnext_label_writes import without_insert_floor +from tests.unit.test_derived_domain_fence import USER + +SENTINEL = "Violet confidential recovery sentinel" + + +def old_vault(path, monkeypatch, *, copies=1, cross_project=False, domain="health"): + with monkeypatch.context() as patch: + patch.setattr(sqlite_schema, "_relabel_derived_domains", lambda conn: None) + patch.setattr(sqlite_schema, "_relabel_derived_labels", lambda conn: None) + bootstrap_database(path, user_id=USER, user_email="fixture@example.invalid") + with without_insert_floor(), sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source( + { + "source_type": "note", + "title": "Synthetic input", + "content_hash": "sha256:violet", + "domain": domain, + "sensitivity": "internal" if cross_project else "confidential", + "metadata_json": {} if cross_project else {"project_scope": ["beta"]}, + } + ) + ids = [] + for index in range(copies): + row = store.create_memory( + { + "memory_key": f"copy.{index}", + "canonical_text": SENTINEL, + "status": "active", + "domain": "project" if cross_project else "unknown", + "sensitivity": "public", + "project_id": "alpha", + "metadata_json": {"source_id": source["id"], "project_scope": ["alpha"]}, + } + ) + ids.append(str(row["id"])) + if cross_project: + a = store.create_memory( + { + "memory_key": "alpha.input", + "canonical_text": "Violet alpha input", + "status": "active", + "domain": "project", + "sensitivity": "internal", + "metadata_json": {"project_scope": ["alpha"]}, + } + ) + b = store.create_memory( + { + "memory_key": "beta.input", + "canonical_text": "Violet beta input", + "status": "active", + "domain": "project", + "sensitivity": "internal", + "metadata_json": {"project_scope": ["beta"]}, + } + ) + for key, marker in ( + ("rollup", {"rollup": {"member_ids": [a["id"], b["id"]]}}), + ("consolidation", {"consolidation": {"cluster_member_ids": [a["id"], b["id"]]}}), + ): + row = store.create_memory( + { + "memory_key": key, + "canonical_text": SENTINEL, + "status": "active", + "domain": "project", + "sensitivity": "internal", + "value": {"rollup": {"member_ids": [a["id"], b["id"]]}} if key == "rollup" else {}, + "metadata_json": { + **marker, + "candidate_kind": "memory_rollup" if key == "rollup" else "memory_consolidation", + "project_scope": ["alpha", "beta"], + }, + } + ) + ids.append(str(row["id"])) + return ids + + +def export_old(path, backup, monkeypatch): + with monkeypatch.context() as patch: + patch.setattr(sqlite_schema, "_relabel_derived_domains", lambda conn: None) + patch.setattr(sqlite_schema, "_relabel_derived_labels", lambda conn: None) + assert onramp_main(["export", "--db", str(path), "--user-id", USER, "--out", str(backup)]) == 0 + + +def read_stored_columns(path, ids): + with sqlite3.connect(path) as conn: + rows = [ + conn.execute( + "SELECT domain, sensitivity, project_id, metadata_json FROM memories WHERE id=?", (row_id,) + ).fetchone() + for row_id in ids + ] + events = conn.execute("SELECT payload_json FROM event_log WHERE event_type='memory.labels_raised'").fetchall() + state = conn.execute("SELECT value FROM alice_schema_state WHERE key=?", (repair.REPAIR_STATE_KEY,)).fetchone() + return [(row[0], row[1], row[2], json.loads(row[3])) for row in rows], events, state + + +def restricted_reads(path, ids, monkeypatch, *, guard_off=False, project=None): + with sqlite_user_connection(path, USER) as conn: + _, raw = create_agent_key( + SQLiteVNextStore(conn, USER), + user_id=USER, + agent_id="recovery-reader", + permission_profile="read_only_agent", + project_scope=project, + ) + with monkeypatch.context() as patch: + patch.setenv("ALICE_AGENT_API_KEY", raw) + patch.setenv("ALICE_MCP_FULL_TOOLS", "1") + patch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + if guard_off: + patch.setattr(LabelGuard, "effective_row", lambda self, kind, row: row) + context = MCPRuntimeContext(database_url=sqlite_url_for_path(path), user_id=UUID(USER)) + for name, args in (("alice_recall", {"query": "Violet"}), ("alice_context_pack", {"query": "Violet"})): + result = call_mcp_tool(context, name=name, arguments=args) + assert SENTINEL not in json.dumps(result, default=str), (name, guard_off, result) + assert not any(row_id in json.dumps(result, default=str) for row_id in ids) + for row_id in ids: + with pytest.raises(MCPToolError, match="requested explanation is unavailable"): + call_mcp_tool(context, name="alice_explain", arguments={"memory_id": row_id}) + + +@pytest.mark.parametrize("upgraded", (False, True), ids=("fresh", "already_v3")) +def test_old_backup_is_repaired_before_publication(tmp_path, monkeypatch, upgraded): + old, target, backup = tmp_path / "old.db", tmp_path / "target.db", tmp_path / "old.jsonl" + ids = old_vault(old, monkeypatch) + export_old(old, backup, monkeypatch) + assert read_stored_columns(old, ids)[0][0][1] == "public" + if upgraded: + bootstrap_database(target, user_id=USER, user_email="fixture@example.invalid") + assert read_stored_columns(target, [])[2] is not None + assert onramp_main(["import", "--db", str(target), "--user-id", USER, "--in", str(backup)]) == 0 + rows, events, state = read_stored_columns(target, ids) + assert rows[0][:3] == ("health", "confidential", None) + assert rows[0][3]["project_scope"] == [] + assert rows[0][3]["project_floor"] == ["beta"] + assert len(events) == 1 and state is not None + assert SENTINEL not in json.dumps(events) + restricted_reads(target, ids, monkeypatch, guard_off=True) + restricted_reads(target, ids, monkeypatch) + + +def test_old_backup_with_cross_project_aggregates(tmp_path, monkeypatch): + old, target, backup = tmp_path / "old.db", tmp_path / "target.db", tmp_path / "old.jsonl" + ids = old_vault(old, monkeypatch, cross_project=True, domain="project") + export_old(old, backup, monkeypatch) + assert onramp_main(["import", "--db", str(target), "--user-id", USER, "--in", str(backup)]) == 0 + rows, events, _ = read_stored_columns(target, ids) + assert all(row[2] is None and row[3]["project_scope"] == [] for row in rows) + assert rows[0][3]["project_floor"] == [] + assert all(row[3]["project_floor"] == ["alpha", "beta"] for row in rows[1:]) + assert len(events) == 3 + restricted_reads(target, ids, monkeypatch, guard_off=True, project="alpha") + restricted_reads(target, ids, monkeypatch, project="alpha") + + +def test_explicit_repair_ignores_completion_stamp(tmp_path, monkeypatch): + path = tmp_path / "stamped.db" + ids = old_vault(path, monkeypatch, domain="project") + with sqlite3.connect(path) as conn: + conn.execute("INSERT OR REPLACE INTO alice_schema_state VALUES (?, '1')", (repair.REPAIR_STATE_KEY,)) + assert onramp_main(["labels", "repair", "--db", str(path), "--user-id", USER]) == 0 + rows, events, _ = read_stored_columns(path, ids) + assert rows[0][1] == "confidential", "explicit repair must revisit a stamped vault" + assert len(events) == 1 + restricted_reads(path, ids, monkeypatch, guard_off=True) + restricted_reads(path, ids, monkeypatch) + + +def test_physical_copy_is_repaired_on_first_open(tmp_path, monkeypatch): + old, target = tmp_path / "old.db", tmp_path / "copied.db" + ids = old_vault(old, monkeypatch, domain="project") + shutil.copy2(old, target) + assert read_stored_columns(target, ids)[0][0][1] == "public" + with sqlite_user_connection(target, USER): + pass + assert read_stored_columns(target, ids)[0][0][1] == "confidential" + restricted_reads(target, ids, monkeypatch, guard_off=True) + restricted_reads(target, ids, monkeypatch) + + +def snapshot(path): + with sqlite3.connect(path) as conn: + return { + table: conn.execute(f"SELECT * FROM {table} ORDER BY 1").fetchall() + for table in ("memories", "sources", "event_log", "alice_schema_state") + } + + +def assert_recovered(path, ids, monkeypatch): + with sqlite_user_connection(path, USER): + pass + rows, _, state = read_stored_columns(path, ids) + assert all(row[1] == "confidential" for row in rows) and state is not None + restricted_reads(path, ids, monkeypatch, guard_off=True) + restricted_reads(path, ids, monkeypatch) + + +def test_interrupted_sqlite_repair_rolls_back_and_backup_repeats(tmp_path, monkeypatch): + path, backup = tmp_path / "interrupted.db", tmp_path / "preupgrade.db" + ids = old_vault(path, monkeypatch, copies=3, domain="project") + shutil.copy2(path, backup) + before = snapshot(path) + original = repair.require_changed + calls = [] + + def fail_after_writes(*args): + original(*args) + calls.append(args) + if len(calls) == 2: + raise RuntimeError("injected after second label write") + + with sqlite3.connect(path) as conn, monkeypatch.context() as patch: + patch.setattr(repair, "require_changed", fail_after_writes) + with pytest.raises(RuntimeError, match="after second"): + repair.relabel_labels_sqlite(conn, restoring=True) + assert len(calls) == 2 and snapshot(path) == before + assert_recovered(path, ids, monkeypatch) + repeated = tmp_path / "restored_preupgrade.db" + shutil.copy2(backup, repeated) + assert snapshot(repeated) == before + assert_recovered(repeated, ids, monkeypatch) + + +def test_killed_sqlite_repair_rolls_back_and_backup_repeats(tmp_path, monkeypatch): + path, backup = tmp_path / "killed.db", tmp_path / "preupgrade.db" + ids = old_vault(path, monkeypatch, copies=3, domain="project") + shutil.copy2(path, backup) + before = snapshot(path) + program = """ +import sqlite3, sys, time +from alicebot_api import vnext_label_repair as repair +original = repair.require_changed +calls = 0 +def pause(*args): + global calls + original(*args) + calls += 1 + if calls == 2: + print('second write reached', flush=True) + time.sleep(30) +repair.require_changed = pause +with sqlite3.connect(sys.argv[1]) as conn: + repair.relabel_labels_sqlite(conn, restoring=True) +""" + process = subprocess.Popen( + [sys.executable, "-c", program, str(path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=os.environ.copy(), + ) + try: + assert select.select([process.stdout], [], [], 10)[0], "child never reached its writes" + assert process.stdout.readline().strip() == "second write reached" + process.kill() + process.wait(timeout=10) + assert snapshot(path) == before + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + assert_recovered(path, ids, monkeypatch) + repeated = tmp_path / "restored_preupgrade.db" + shutil.copy2(backup, repeated) + assert snapshot(repeated) == before + assert_recovered(repeated, ids, monkeypatch) + + +def test_repeating_skip_composes_v2_v3_legacy_scope_and_open_loop(tmp_path, monkeypatch): + old, target, backup = tmp_path / "old.db", tmp_path / "target.db", tmp_path / "old.jsonl" + ids = old_vault(old, monkeypatch) + with monkeypatch.context() as patch: + patch.setattr(sqlite_schema, "_relabel_derived_domains", lambda conn: None) + patch.setattr(sqlite_schema, "_relabel_derived_labels", lambda conn: None) + with without_insert_floor(), sqlite_user_connection(old, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source_id = conn.execute("SELECT id FROM sources").fetchone()["id"] + legacy = store.create_memory( + { + "memory_key": "legacy.project", + "canonical_text": SENTINEL, + "domain": "unknown", + "sensitivity": "public", + "status": "active", + "project_id": "alpha", + "metadata_json": {"source_id": source_id}, + } + ) + loop = store.create_open_loop( + { + "title": SENTINEL, + "domain": "unknown", + "sensitivity": "public", + "source_id": source_id, + "metadata_json": {"source_id": source_id, "discovered_by": "vnext_daily_brief"}, + } + ) + ids.append(str(legacy["id"])) + v2_input = store.create_memory( + { + "memory_key": "v2.input", + "canonical_text": "Violet private original", + "domain": "health", + "sensitivity": "confidential", + "status": "active", + } + ) + aggregate = store.create_memory( + { + "memory_key": "v2.aggregate", + "canonical_text": SENTINEL, + "domain": "unknown", + "sensitivity": "public", + "status": "active", + "metadata_json": {"consolidation": {"cluster_member_ids": [v2_input["id"]]}}, + } + ) + ids.append(str(aggregate["id"])) + export_old(old, backup, monkeypatch) + argv = ["import", "--db", str(target), "--user-id", USER, "--in", str(backup)] + assert onramp_main(argv) == 0 + before = snapshot(target) + with sqlite3.connect(target) as conn: + assert conn.execute("SELECT domain,sensitivity FROM open_loops WHERE id=?", (loop["id"],)).fetchone() == ( + "health", + "confidential", + ) + assert ( + conn.execute("SELECT count(*) FROM event_log WHERE event_type='memory.domain_relabelled'").fetchone()[0] + >= 1 + ) + assert onramp_main([*argv, "--mode", "skip"]) == 0 + assert snapshot(target) == before, "repeat skip must change no stored row, event, or stamp" + restricted_reads(target, ids, monkeypatch, guard_off=True) + restricted_reads(target, ids, monkeypatch) + + +def test_repeating_skip_accepts_a_later_supersede_label_raise(tmp_path, monkeypatch): + old, target, backup = tmp_path / "old.db", tmp_path / "target.db", tmp_path / "memory_backup.jsonl" + ids = old_vault(old, monkeypatch, domain="project") + with monkeypatch.context() as patch: + patch.setattr(sqlite_schema, "_relabel_derived_domains", lambda conn: None) + patch.setattr(sqlite_schema, "_relabel_derived_labels", lambda conn: None) + with sqlite_user_connection(old, USER) as conn: + source = dict(conn.execute("SELECT * FROM sources").fetchone()) + source["source_type"], source["connector_name"], source["raw_path"] = ( + "markdown", + "markdown_folder", + "fixture.md", + ) + conn.execute( + "UPDATE sources SET source_type=?, connector_name=?, raw_path=? WHERE id=?", + ("markdown", "markdown_folder", "fixture.md", source["id"]), + ) + export_old(old, backup, monkeypatch) + # Old unversioned memory-only backups are valid. The source already lives + # in this destination and is intentionally outside this incremental file. + records = [json.loads(line) for line in backup.read_text().splitlines()] + backup.write_text("\n".join(json.dumps(row) for row in records if row["record_type"] == "memory") + "\n") + bootstrap_database(target, user_id=USER, user_email="fixture@example.invalid") + with sqlite_user_connection(target, USER) as conn: + source["metadata_json"] = json.loads(source["metadata_json"]) + SQLiteVNextStore(conn, USER).create_source(source) + argv = ["import", "--db", str(target), "--user-id", USER, "--in", str(backup)] + assert onramp_main(argv) == 0 + with sqlite_user_connection(target, USER) as conn: + store = SQLiteVNextStore(conn, USER) + replacement = store.create_source( + {**source, "id": None, "dedupe_key": None, "content_hash": "sha256:replacement", "sensitivity": "regulated"} + ) + store.supersede_source(str(source["id"]), superseded_by=str(replacement["id"])) + assert store.get_memory(ids[0])["sensitivity"] == "regulated" + payloads = [ + json.loads(row["payload_json"]) + for row in conn.execute( + "SELECT payload_json FROM event_log WHERE event_type='memory.labels_raised' AND target_id=?", (ids[0],) + ) + ] + assert any(row["cause"] == "input_relabelled" for row in payloads) + with sqlite_user_connection(target, USER): + pass + before = snapshot(target) + assert onramp_main([*argv, "--mode", "skip"]) == 0 + assert snapshot(target) == before + restricted_reads(target, ids, monkeypatch, guard_off=True) + restricted_reads(target, ids, monkeypatch) + + +def test_unrepairable_open_keeps_guarded_read_available(tmp_path, monkeypatch, caplog): + path = tmp_path / "unrepairable.db" + ids = old_vault(path, monkeypatch, domain="project") + + def fail(conn, **kwargs): + raise repair.DerivedDomainRepairError("synthetic cycle did not settle") + + monkeypatch.setattr(repair, "relabel_labels_sqlite", fail) + with sqlite_user_connection(path, USER): + pass + assert read_stored_columns(path, ids)[2] is None + assert "alice-memory: label repair did not run" in caplog.text + restricted_reads(path, ids, monkeypatch) From 7bf56d059268a7ce47858a202f189de2a71a6986 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:27:20 +0200 Subject: [PATCH 071/270] Enforce canonical dependency record completeness --- apps/api/src/alicebot_api/vnext_derived_labels.py | 7 +------ tests/unit/test_derived_labels_record_completeness.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 4f721b3fa..e9dae28b8 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -535,16 +535,13 @@ def _derived_from_deps(record: object) -> tuple[str, set[tuple[str, str]]]: return "malformed", set() found: set[tuple[str, str]] = set() lists: dict[str, list[str]] = {} - repeated = False for key, kind in _DERIVED_FROM_KIND.items(): if key not in record: lists[key] = [] continue raw = record.get(key) - if not isinstance(raw, list) or any(not isinstance(item, str) for item in raw): + if not isinstance(raw, list) or any(not isinstance(item, str) or not item.strip() for item in raw): return "malformed", set() - if len(raw) != len(set(raw)): - repeated = True ids = _strings(raw) lists[key] = ids _add_ids(found, kind, ids) @@ -555,8 +552,6 @@ def _derived_from_deps(record: object) -> tuple[str, set[tuple[str, str]]]: return "", found if not isinstance(counts, Mapping): return "malformed", found - if repeated: - return "", found for key, ids in lists.items(): if key not in counts: if ids: diff --git a/tests/unit/test_derived_labels_record_completeness.py b/tests/unit/test_derived_labels_record_completeness.py index efac96c30..ff9d6b679 100644 --- a/tests/unit/test_derived_labels_record_completeness.py +++ b/tests/unit/test_derived_labels_record_completeness.py @@ -34,3 +34,13 @@ def test_a_missing_or_malformed_legacy_completeness_record_is_unverified(workflo def test_a_legitimate_empty_legacy_record_is_verified(workflow, lists, counts): row = legacy_report(workflow, lists, counts) assert settle_labels([row]).by_stored("artifact", "report").unverified is False + + +@pytest.mark.parametrize("ids,count", ((["source", "source"], 1), ([""], 1), ([" "], 1))) +def test_canonical_counts_and_identifiers_do_not_use_the_legacy_duplicate_exception(ids, count): + source = {"kind": "source", "id": "source", "domain": "project", "sensitivity": "public", "metadata_json": {}} + row = {"kind": "artifact", "id": "report", "metadata_json": {"derived_from": { + "v": 1, "sources": ids, "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], + "counts": {"sources": count, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, + }}} + assert settle_labels([source, row]).by_stored("artifact", "report").unverified is True From dd2d3871a90eac26632e41651bdd6dd37ca780c8 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:20:03 +0200 Subject: [PATCH 072/270] Validate incomplete legacy dependency counts --- .../src/alicebot_api/vnext_derived_labels.py | 10 ++++-- ...test_derived_labels_record_completeness.py | 36 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_derived_labels_record_completeness.py diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index e602e9ede..3391fb4b9 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -605,8 +605,10 @@ def _legacy_count_problem(kind: str, row: Mapping[str, object], meta: Mapping[st def _counts_against_lists(counts: object, lists: Mapping[str, object]) -> str: - if not isinstance(counts, Mapping): + if counts is None: return "" + if not isinstance(counts, Mapping): + return "malformed" present_lists: dict[str, list[object]] = {} for key, raw in lists.items(): if raw is None: @@ -616,11 +618,13 @@ def _counts_against_lists(counts: object, lists: Mapping[str, object]) -> str: present_lists[key] = list(raw) if any(len(values) != len(set(map(str, values))) for values in present_lists.values()): return "" - for key, values in present_lists.items(): + for key in lists: if key not in counts: continue expected = counts.get(key) - if isinstance(expected, int) and not isinstance(expected, bool) and expected != len(values): + if not isinstance(expected, int) or isinstance(expected, bool) or expected < 0: + return "malformed" + if expected != len(present_lists.get(key, [])): return "counts_disagree" return "" diff --git a/tests/unit/test_derived_labels_record_completeness.py b/tests/unit/test_derived_labels_record_completeness.py new file mode 100644 index 000000000..efac96c30 --- /dev/null +++ b/tests/unit/test_derived_labels_record_completeness.py @@ -0,0 +1,36 @@ +"""Missing legacy lists cannot pass a producer's positive completeness count.""" + +from __future__ import annotations + +import pytest + +from alicebot_api.vnext_derived_labels import settle_labels + + +def legacy_report(workflow, lists, counts): + metadata = {"workflow": workflow} + if workflow in {"daily_brief", "weekly_synthesis"}: + metadata["input_summary"] = {**lists, "counts": counts} + else: + metadata.update(lists) + metadata["input_counts"] = counts + return {"kind": "artifact", "id": "report", "domain": "project", "sensitivity": "public", "metadata_json": metadata} + + +@pytest.mark.parametrize("workflow", ("daily_brief", "weekly_synthesis", "connection_finder", "contradiction_finder")) +@pytest.mark.parametrize("lists,counts", (({}, {"sources": 1}), ({"source_ids": []}, {"sources": "1"}), + ({"source_ids": []}, {"sources": True}), ({"source_ids": []}, {"sources": -1}), + ({"source_ids": []}, []))) +def test_a_missing_or_malformed_legacy_completeness_record_is_unverified(workflow, lists, counts): + row = legacy_report(workflow, lists, counts) + label = settle_labels([row]).by_stored("artifact", "report") + assert label.unverified is True + assert label.reason == "legacy_counts" + + +@pytest.mark.parametrize("workflow", ("daily_brief", "weekly_synthesis", "connection_finder", "contradiction_finder")) +@pytest.mark.parametrize("lists,counts", (({}, {"sources": 0}), ({"source_ids": []}, {"sources": 0}), + ({"source_ids": []}, {}))) +def test_a_legitimate_empty_legacy_record_is_verified(workflow, lists, counts): + row = legacy_report(workflow, lists, counts) + assert settle_labels([row]).by_stored("artifact", "report").unverified is False From 8e6f426d58cb187858313858b3ea82c092e202a5 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:27:20 +0200 Subject: [PATCH 073/270] Enforce canonical dependency record completeness --- apps/api/src/alicebot_api/vnext_derived_labels.py | 7 +------ tests/unit/test_derived_labels_record_completeness.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 3391fb4b9..97c4b3bc9 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -534,16 +534,13 @@ def _derived_from_deps(record: object) -> tuple[str, set[tuple[str, str]]]: return "malformed", set() found: set[tuple[str, str]] = set() lists: dict[str, list[str]] = {} - repeated = False for key, kind in _DERIVED_FROM_KIND.items(): if key not in record: lists[key] = [] continue raw = record.get(key) - if not isinstance(raw, list) or any(not isinstance(item, str) for item in raw): + if not isinstance(raw, list) or any(not isinstance(item, str) or not item.strip() for item in raw): return "malformed", set() - if len(raw) != len(set(raw)): - repeated = True ids = _strings(raw) lists[key] = ids _add_ids(found, kind, ids) @@ -554,8 +551,6 @@ def _derived_from_deps(record: object) -> tuple[str, set[tuple[str, str]]]: return "", found if not isinstance(counts, Mapping): return "malformed", found - if repeated: - return "", found for key, ids in lists.items(): if key not in counts: if ids: diff --git a/tests/unit/test_derived_labels_record_completeness.py b/tests/unit/test_derived_labels_record_completeness.py index efac96c30..ff9d6b679 100644 --- a/tests/unit/test_derived_labels_record_completeness.py +++ b/tests/unit/test_derived_labels_record_completeness.py @@ -34,3 +34,13 @@ def test_a_missing_or_malformed_legacy_completeness_record_is_unverified(workflo def test_a_legitimate_empty_legacy_record_is_verified(workflow, lists, counts): row = legacy_report(workflow, lists, counts) assert settle_labels([row]).by_stored("artifact", "report").unverified is False + + +@pytest.mark.parametrize("ids,count", ((["source", "source"], 1), ([""], 1), ([" "], 1))) +def test_canonical_counts_and_identifiers_do_not_use_the_legacy_duplicate_exception(ids, count): + source = {"kind": "source", "id": "source", "domain": "project", "sensitivity": "public", "metadata_json": {}} + row = {"kind": "artifact", "id": "report", "metadata_json": {"derived_from": { + "v": 1, "sources": ids, "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], + "counts": {"sources": count, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, + }}} + assert settle_labels([source, row]).by_stored("artifact", "report").unverified is True From 658560c73ca0604cc0ba9078fc092f7c87faa61c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:17:45 +0200 Subject: [PATCH 074/270] Enforce live label lock order and complete scope move previews --- apps/api/src/alicebot_api/cli/capture.py | 3 + .../src/alicebot_api/vnext_artifact_review.py | 6 + .../src/alicebot_api/vnext_label_writes.py | 128 ++++++++++++++++-- apps/api/src/alicebot_api/vnext_projects.py | 6 + apps/api/src/alicebot_api/vnext_store.py | 33 +++++ .../vnext_stores/postgres/graph_open_loops.py | 13 ++ .../vnext_stores/postgres/memory_access.py | 2 + .../vnext_stores/postgres/memory_lifecycle.py | 6 +- .../vnext_stores/sqlite/graph_open_loops.py | 13 ++ tests/integration/conftest.py | 9 ++ .../test_label_lock_order_postgres.py | 63 +++++++++ tests/unit/test_label_lock_order.py | 86 ++++++++++++ tests/unit/test_label_lock_registry.py | 69 ++++++++++ tests/unit/test_label_writer_registry.py | 78 +++++++++++ tests/unit/test_source_move_label_preview.py | 53 ++++++++ .../test_sqlite_derived_labels_write_path.py | 2 +- 16 files changed, 554 insertions(+), 16 deletions(-) create mode 100644 tests/integration/test_label_lock_order_postgres.py create mode 100644 tests/unit/test_label_lock_order.py create mode 100644 tests/unit/test_label_lock_registry.py create mode 100644 tests/unit/test_label_writer_registry.py create mode 100644 tests/unit/test_source_move_label_preview.py diff --git a/apps/api/src/alicebot_api/cli/capture.py b/apps/api/src/alicebot_api/cli/capture.py index 241566da8..cce5ef0e6 100644 --- a/apps/api/src/alicebot_api/cli/capture.py +++ b/apps/api/src/alicebot_api/cli/capture.py @@ -41,6 +41,7 @@ from alicebot_api.vnext_embeddings import DeferredMemoryEmbedding from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_store import PostgresVNextStore +from alicebot_api.vnext_label_writes import takes_label_lock as _takes_label_lock from .constants import DEFAULT_VNEXT_DEMO_DATASET_PATH, DEMO_SECRET_MARKERS from .models import CLIContext from .arguments import _object_dict, _object_int, _object_list @@ -394,6 +395,7 @@ def _demo_tag(dataset_id: str) -> JsonObject: return {"demo": True, "demo_dataset_id": dataset_id} +@_takes_label_lock def _reset_vnext_demo_dataset(store: PostgresVNextStore, *, dataset_id: str) -> JsonObject: with store.conn.cursor() as cur: cur.execute( @@ -507,6 +509,7 @@ def _tag_demo_candidate_memories(store: PostgresVNextStore, *, dataset_id: str, return updated +@_takes_label_lock def _tag_demo_artifact(store: PostgresVNextStore, *, artifact_id: str, dataset_id: str) -> None: artifact = store.get_artifact(artifact_id) if artifact is None: diff --git a/apps/api/src/alicebot_api/vnext_artifact_review.py b/apps/api/src/alicebot_api/vnext_artifact_review.py index 3e965e244..b42178dbd 100644 --- a/apps/api/src/alicebot_api/vnext_artifact_review.py +++ b/apps/api/src/alicebot_api/vnext_artifact_review.py @@ -37,6 +37,12 @@ def dispatch_vnext_artifact_review( mutating it, so no caller can route from a stale or forged preloaded row. """ + lock_graph = getattr(store, "lock_graph_mutation", None) + if callable(lock_graph): + lock_graph() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) target = store.get_artifact_for_update(artifact_id) if target is None: raise VNextQueueNotFoundError(f"artifact {artifact_id} was not found") diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 015384ae2..c823feebd 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -25,6 +25,7 @@ generation_domain, identifier, is_derived, + input_admitted, labels_raised_payload, settle_labels, stored_scope, @@ -127,6 +128,62 @@ def _in_transaction(store: Any) -> bool: return True +def held_label_locks(store: Any) -> tuple[bool, bool, bool]: + """Read the live S, L and exclusive L grants, including savepoint rollback.""" + + with store.conn.cursor() as cur: + cur.execute(""" + SELECT + coalesce(bool_or(classid = (hashtext('vnext_supersession')::bigint & 4294967295)::oid), false) AS graph, + coalesce(bool_or(classid = (hashtext('vnext_labels')::bigint & 4294967295)::oid), false) AS labels, + coalesce(bool_or(classid = (hashtext('vnext_labels')::bigint & 4294967295)::oid + AND mode = 'ExclusiveLock'), false) AS exclusive + FROM pg_locks + WHERE locktype = 'advisory' AND pid = pg_backend_pid() AND granted + AND objsubid = 2 + AND objid = (hashtext(app.current_user_id()::text)::bigint & 4294967295)::oid + """) + row = cur.fetchone() + if isinstance(row, Mapping): + return bool(row["graph"]), bool(row["labels"]), bool(row["exclusive"]) + return bool(row[0]), bool(row[1]), bool(row[2]) + + +def before_graph_lock(store: Any) -> None: + """Strict tests refuse S after L using the current database grants.""" + + if STRICT_LOCK_ORDER: + graph, labels, _exclusive = held_label_locks(store) + if labels and not graph: + raise LabelLockOrderError("the graph lock must precede the label lock") + + +def require_exclusive_label_lock(store: Any) -> None: + """A changing hook must hold exclusive L before taking any row lock.""" + + if _sqlite(store): + store.lock_label_writes(exclusive=True) + return + _graph, _labels, exclusive = held_label_locks(store) + if exclusive: + return + if STRICT_LOCK_ORDER: + raise LabelLockOrderError("the label change requires the exclusive label lock before row locks") + acquire_exclusive_label_lock(store) + + +def prepare_label_patch( + store: Any, kind: str, before: Mapping[str, object] | None, patch: Mapping[str, object] +) -> JsonObject: + """Check a proposed label change before its UPDATE or FOR UPDATE statement.""" + + proposed = dict(before or {}) + proposed.update({key: value for key, value in patch.items() if value is not None}) + if before and _label_fields(before) != _label_fields(proposed): + require_exclusive_label_lock(store) + return dict(patch) + + def _label_tuple(payload: Mapping[str, object]) -> tuple[str, str, tuple[str, ...], tuple[str, ...]]: metadata = payload.get("metadata_json") meta = metadata if isinstance(metadata, Mapping) else {} @@ -467,6 +524,7 @@ def clamp_owner_patch( and project_scope_identity(stored[2]) == project_scope_identity(settled[2]) and project_scope_identity(stored[3]) == project_scope_identity(settled[3]) ): + require_exclusive_label_lock(store) event = build_event_log_record( event_type=f"{kind}.labels_raised", actor_type="system", @@ -510,7 +568,7 @@ def write_settled_label( """Label-only update. A statement that changes no row refuses the whole relabel.""" table = {"memory": "memories", "open_loop": "open_loops", "artifact": "generated_artifacts", "project": "projects"}[kind] - blob = json.dumps(dict(metadata)) + blob = json.dumps({key: metadata[key] for key in ("project_scope", "project_floor") if key in metadata}) if _sqlite(store): project_sql = ", project_id = ?" if table in {"memories", "open_loops"} else "" params: list[object] = [domain, sensitivity, blob] @@ -520,7 +578,7 @@ def write_settled_label( cursor = store._execute( f""" UPDATE {table} - SET domain = ?, sensitivity = ?, metadata_json = ?{project_sql} + SET domain = ?, sensitivity = ?, metadata_json = json_patch(metadata_json, ?){project_sql} WHERE id = ? AND user_id = ? AND domain = ? AND sensitivity = ? """, # nosec B608 # table comes from the closed kind map; values are bound tuple(params), @@ -532,22 +590,47 @@ def write_settled_label( if project_sql: params.append(project_id) params.extend([str(row_id), expected_domain, expected_sensitivity]) - store._fetch_one( - "write_settled_label", + row = store._fetch_optional_one( f""" UPDATE {table} - SET domain = %s, sensitivity = %s, metadata_json = %s::jsonb{project_sql} + SET domain = %s, sensitivity = %s, metadata_json = metadata_json || %s::jsonb{project_sql} WHERE id = %s::uuid AND domain = %s AND sensitivity = %s RETURNING id """, # nosec B608 # table comes from the closed kind map; values are bound tuple(params), ) + require_changed(int(row is not None), table, str(row_id)) + + +LABEL_TABLE_ORDER = ("generated_artifacts", "projects", "open_loops", "memories") +_LABEL_TABLES = {"artifact": "generated_artifacts", "project": "projects", "open_loop": "open_loops", "memory": "memories"} + + +def lock_settled_label_rows(store: Any, changes: Sequence[Mapping[str, object]]) -> None: + """Lock exactly the changed rows in a stable table and UUID order.""" + + if _sqlite(store): + return + grouped: dict[str, set[str]] = {} + for row in changes: + grouped.setdefault(_LABEL_TABLES[str(row["kind"])], set()).add(str(row["id"])) + with store.conn.cursor() as cur: + for table in LABEL_TABLE_ORDER: + ids = sorted(grouped.get(table, ())) + if ids: + cur.execute( + f"SELECT id FROM {table} WHERE id = ANY(%s::uuid[]) ORDER BY id FOR UPDATE", # nosec B608 # closed internal table map + (ids,), + ) + locked = cur.fetchall() + if len(locked) != len(ids): + raise DerivedDomainRepairError("a planned label row disappeared before it could be locked") def propagate(store: Any, changed: Sequence[tuple[str, str]], *, cause: str) -> int: """Recompute dependants of ``changed`` rows and write the ones that rise.""" - store.lock_label_writes(exclusive=True) + require_exclusive_label_lock(store) roots = [row_id for _kind, row_id in changed] affected = walk_dependants(store, roots) if not affected: @@ -561,7 +644,7 @@ def propagate(store: Any, changed: Sequence[tuple[str, str]], *, cause: str) -> if exceeded: raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") settled = settle_labels(nodes) - written = 0 + changes: list[tuple[Mapping[str, object], Any, tuple[str, str, tuple[str, ...], tuple[str, ...]]]] = [] for row in affected: label = settled.by_stored(str(row.get("kind")), str(row.get("id"))) if label.unverified: @@ -586,6 +669,10 @@ def propagate(store: Any, changed: Sequence[tuple[str, str]], *, cause: str) -> and project_scope_identity(previous[3]) == project_scope_identity(current[3]) ): continue + changes.append((row, label, previous)) + lock_settled_label_rows(store, [row for row, _label, _previous in changes]) + written = 0 + for row, label, previous in sorted(changes, key=lambda item: (LABEL_TABLE_ORDER.index(_LABEL_TABLES[str(item[0]["kind"])]), str(item[0]["id"]))): raw_metadata = row.get("metadata_json") metadata = dict(raw_metadata) if isinstance(raw_metadata, Mapping) else {} metadata["project_scope"] = list(label.project_scope) @@ -660,15 +747,21 @@ def count_rows_hidden_by_scope_move(store: Any, source: Mapping[str, object], ne metadata = dict(raw_metadata) if isinstance(raw_metadata, Mapping) else {} metadata["project_scope"] = list(new_scope) moved["metadata_json"] = metadata - before = settle_labels([current, *[dict(row) for row in affected]]) - after = settle_labels([moved, *[dict(row) for row in affected]]) + nodes, exceeded = collect_label_rows(store, [current, *[dict(row) for row in affected]], max_nodes=PROPAGATION_BOUND) + if exceeded: + raise LabelPropagationTooLarge("the source move preview exceeded the label propagation bound") + before = settle_labels(nodes, on_cycle="unverified") + moved_nodes = [moved if str(row.get("kind")) == "source" and str(row.get("id")) == source_id else row for row in nodes] + after = settle_labels(moved_nodes, on_cycle="unverified") hidden = 0 for row in affected: old = before.by_stored(str(row.get("kind")), str(row.get("id"))) new = after.by_stored(str(row.get("kind")), str(row.get("id"))) - old_ids = set(project_scope_identity(old.project_scope)) - new_ids = set(project_scope_identity(new.project_scope)) - if old_ids - new_ids: + if old.unverified or not old.project_scope: + continue + binding = project_scope_identity([*old.project_scope, *old.project_floor]) + new_row = {**row, "metadata_json": {"project_scope": list(new.project_scope), "project_floor": list(new.project_floor)}} + if new.unverified or not new.project_scope or not input_admitted(str(row["kind"]), new_row, binding): hidden += 1 return hidden @@ -695,10 +788,17 @@ def raise_source_to_replacement(store: Any, old: Mapping[str, object], replaceme def label_error_response(exc: BaseException) -> tuple[int, str, str | None] | None: """``(status, detail, retry_after)`` for a relabel failure, or None.""" - if isinstance(exc, (LabelPropagationTooLarge, DerivedDomainRepairError)): - return 409, REFUSED_DETAIL, None + if isinstance(exc, LabelPropagationTooLarge): + return 409, REFUSED_DETAIL + "; cause: propagation_bound", None + if isinstance(exc, DerivedDomainRepairError): + cause = "row_changed" if "changed no row" in str(exc) or "disappeared" in str(exc) else "dependency_cycle" + return 409, REFUSED_DETAIL + "; cause: " + cause, None + if isinstance(exc, LabelLockOrderError): + return 409, REFUSED_DETAIL + "; cause: lock_order", None if type(exc).__name__ in {"LockNotAvailable", "DeadlockDetected", "SerializationFailure"}: return 503, RETRYABLE_DETAIL, "2" + if type(exc).__module__.startswith("psycopg"): + return 409, REFUSED_DETAIL + "; cause: database_error", None return None diff --git a/apps/api/src/alicebot_api/vnext_projects.py b/apps/api/src/alicebot_api/vnext_projects.py index b34b1a88b..8f0785c92 100644 --- a/apps/api/src/alicebot_api/vnext_projects.py +++ b/apps/api/src/alicebot_api/vnext_projects.py @@ -824,6 +824,12 @@ def review_project_update( ) -> JsonObject: if action not in PROJECT_UPDATE_ACTIONS: raise VNextProjectValidationError("project update action must be accept, edit, or reject") + lock_graph = getattr(self.store, "lock_graph_mutation", None) + if callable(lock_graph): + lock_graph() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(self.store) # The artifact is the review decision's serialization point. Every # accept/edit/reject path must inspect and transition the same locked # row so stale reviewers cannot split project, memory, and artifact diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index b8d7572ca..788b172e1 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -480,6 +480,10 @@ def lock_label_writes(self, *, exclusive: bool = False) -> None: cur.execute( f"SELECT {mode}(hashtext('vnext_labels'), hashtext(app.current_user_id()::text))" ) + from alicebot_api.vnext_label_writes import _in_transaction, LabelLockOrderError + + if not _in_transaction(self): + raise LabelLockOrderError("label writes require an open transaction") def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: """Narrow label rows for the insert floor. No text columns.""" @@ -1250,6 +1254,9 @@ def get_source_by_dedupe_key(self, dedupe_key: str) -> VNextRow | None: @takes_label_lock def update_source(self, *, source_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import prepare_label_patch + + patch = prepare_label_patch(self, "source", self.get_source(source_id), patch) with self.conn.cursor() as cur: cur.execute( f""" @@ -1723,6 +1730,9 @@ def search_sources( @takes_label_lock def create_project(self, project: JsonObject, *, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import apply_insert_floor, remember_floor_event + + project, floor_event = apply_insert_floor(self, "project", project) row = self._fetch_one( "create_project", f""" @@ -1771,6 +1781,7 @@ def create_project(self, project: JsonObject, *, actor_type: str = "system") -> target_id=row["id"], payload={"operation": "create", "fields": _sorted_field_names(project)}, ) + remember_floor_event(self, floor_event, row["id"]) return row def get_project(self, project_id: str) -> VNextRow | None: @@ -1843,6 +1854,26 @@ def list_projects( @takes_label_lock def update_project(self, *, project_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import ( + apply_insert_floor, merge_protected_metadata, prepare_label_patch, + propagate_after_write, remember_floor_event, + ) + + before = self.get_project(project_id) + patch = dict(patch) + metadata = patch.get("metadata_json") + floor_event = None + if isinstance(metadata, dict) and before is not None: + patch["metadata_json"] = merge_protected_metadata( + before.get("metadata_json") if isinstance(before.get("metadata_json"), dict) else {}, + metadata, + label_write="derived_from" in metadata, + ) + if "derived_from" in metadata: + floored, floor_event = apply_insert_floor(self, "project", {**before, **patch}) + for key in ("domain", "sensitivity", "metadata_json"): + patch[key] = floored[key] + patch = prepare_label_patch(self, "project", before, patch) row = self._fetch_one( "update_project", f""" @@ -1876,6 +1907,8 @@ def update_project(self, *, project_id: str, patch: JsonObject, actor_type: str target_id=row["id"], payload={"operation": "update", "changes": patch}, ) + remember_floor_event(self, floor_event, row["id"]) + propagate_after_write(self, kind="project", before=before, after=row) return row def create_person(self, person: JsonObject, *, actor_type: str = "system") -> VNextRow: diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py b/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py index 3c4faed08..d20c0a152 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py @@ -1206,6 +1206,18 @@ def list_open_loop_events( @takes_label_lock def update_open_loop(self, *, loop_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import clamp_owner_patch, merge_protected_metadata, prepare_label_patch, propagate_after_write + + before = self.get_open_loop(loop_id) + patch = dict(patch) + metadata = patch.get("metadata_json") + if before is not None and isinstance(metadata, dict): + patch["metadata_json"] = merge_protected_metadata( + before.get("metadata_json") if isinstance(before.get("metadata_json"), dict) else {}, + metadata, label_write=False, + ) + patch = prepare_label_patch(self, "open_loop", before, patch) + patch = clamp_owner_patch(self, kind="open_loop", before=before, patch=patch) row = self._fetch_one( "update_open_loop", f""" @@ -1243,6 +1255,7 @@ def update_open_loop(self, *, loop_id: str, patch: JsonObject, actor_type: str = target_id=row["id"], payload={"operation": "update", "changes": patch}, ) + propagate_after_write(self, kind="open_loop", before=before, after=row) return row @takes_label_lock diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py index ac34bb114..f2d6c1df5 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py @@ -7,6 +7,7 @@ from typing import cast from alicebot_api.store import ContinuityStoreInvariantError +from alicebot_api.vnext_label_writes import takes_label_lock from alicebot_api.vnext_embeddings import ( EMBEDDING_SIGNATURE_METADATA_KEY, memory_embedding_signature_is_current, @@ -216,6 +217,7 @@ def list_memories_referencing_sources( return grouped +@takes_label_lock def list_pending_derived_candidates_for_member( self, *, diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py index 64294a5be..0287fe060 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py @@ -362,6 +362,9 @@ def lock_graph_mutation(self) -> None: correction, forgetting, and transitions. Released automatically at commit/rollback. """ + from alicebot_api.vnext_label_writes import before_graph_lock + + before_graph_lock(self) with self.conn.cursor() as cur: cur.execute( "SELECT pg_advisory_xact_lock(hashtext('vnext_supersession'), hashtext(app.current_user_id()::text))" @@ -451,8 +454,9 @@ def update_memory( label_write=label_write, ) if before_label is not None: - from alicebot_api.vnext_label_writes import clamp_owner_patch + from alicebot_api.vnext_label_writes import clamp_owner_patch, prepare_label_patch + patch = prepare_label_patch(self, "memory", before_label, patch) patch = clamp_owner_patch(self, kind="memory", before=before_label, patch=patch) row = self._fetch_one( "update_memory", diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py index cc43135e1..25514f3de 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py @@ -977,6 +977,18 @@ def list_open_loop_events( @takes_label_lock def update_open_loop(self, *, loop_id: str, patch: JsonObject, actor_type: str = "system") -> VNextRow: + from alicebot_api.vnext_label_writes import clamp_owner_patch, merge_protected_metadata, prepare_label_patch, propagate_after_write + + before = self.get_open_loop(loop_id) + patch = dict(patch) + metadata = patch.get("metadata_json") + if before is not None and isinstance(metadata, dict): + patch["metadata_json"] = merge_protected_metadata( + before.get("metadata_json") if isinstance(before.get("metadata_json"), dict) else {}, + metadata, label_write=False, + ) + patch = prepare_label_patch(self, "open_loop", before, patch) + patch = clamp_owner_patch(self, kind="open_loop", before=before, patch=patch) cursor = self._execute( """ UPDATE open_loops @@ -1020,6 +1032,7 @@ def update_open_loop(self, *, loop_id: str, patch: JsonObject, actor_type: str = target_id=row["id"], payload={"operation": "update", "changes": patch}, ) + propagate_after_write(self, kind="open_loop", before=before, after=row) return row @takes_label_lock diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0a8f8edf2..404554c6e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -20,6 +20,15 @@ TEMPLATE_MIGRATION_COUNT = 0 +@pytest.fixture(autouse=True) +def strict_label_lock_order(monkeypatch: pytest.MonkeyPatch) -> None: + """Every integration flow obeys S, L, then label-row locks.""" + + import alicebot_api.vnext_label_writes as label_writes + + monkeypatch.setattr(label_writes, "STRICT_LOCK_ORDER", True) + + def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption( "--require-executed-tests", diff --git a/tests/integration/test_label_lock_order_postgres.py b/tests/integration/test_label_lock_order_postgres.py new file mode 100644 index 000000000..db9215843 --- /dev/null +++ b/tests/integration/test_label_lock_order_postgres.py @@ -0,0 +1,63 @@ +"""Live advisory grants enforce S before L and survive savepoint rollback.""" +from uuid import uuid4 + +import pytest + +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_store import PostgresVNextStore +from alicebot_api.vnext_label_writes import LabelLockOrderError, held_label_locks + + +def _user(url, user): + with user_connection(url, user) as conn: + ContinuityStore(conn).create_user(user, f"locks-{user}@example.test", "Synthetic") + + +def test_real_pg_strict_s_after_l_is_refused(migrated_database_urls): + user = uuid4() + url = migrated_database_urls["app"] + _user(url, user) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + store.lock_label_writes() + assert held_label_locks(store) == (False, True, False) + with pytest.raises(LabelLockOrderError, match="graph lock must precede"): + store.lock_graph_mutation() + + +def test_real_pg_savepoint_rollback_releases_live_grants(migrated_database_urls): + user = uuid4() + url = migrated_database_urls["app"] + _user(url, user) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + with pytest.raises(ValueError, match="rollback"): + with conn.transaction(): + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + assert held_label_locks(store) == (True, True, True) + raise ValueError("rollback") + assert held_label_locks(store) == (False, False, False) + store.lock_graph_mutation() + store.lock_label_writes() + assert held_label_locks(store) == (True, True, False) + + +def test_real_pg_changing_hook_needs_exclusive_before_the_update(migrated_database_urls): + user = uuid4() + url = migrated_database_urls["app"] + _user(url, user) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + row = store.create_memory({"memory_key": "original", "canonical_text": "synthetic", "domain": "project", "sensitivity": "public"}) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + with pytest.raises(LabelLockOrderError, match="exclusive label lock"): + store.update_memory(memory_id=str(row["id"]), patch={"sensitivity": "confidential"}) + assert store.get_memory(str(row["id"]))["sensitivity"] == "public" + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + assert store.update_memory(memory_id=str(row["id"]), patch={"sensitivity": "confidential"})["sensitivity"] == "confidential" diff --git a/tests/unit/test_label_lock_order.py b/tests/unit/test_label_lock_order.py new file mode 100644 index 000000000..5ac522ddc --- /dev/null +++ b/tests/unit/test_label_lock_order.py @@ -0,0 +1,86 @@ +"""Strict lock checks read database grants rather than cached state.""" +from types import SimpleNamespace + +import pytest + +from alicebot_api import vnext_label_writes as writes +from alicebot_api.vnext_store import PostgresVNextStore + + +class Cursor: + def __init__(self, conn): + self.conn = conn + def __enter__(self): + return self + def __exit__(self, *args): + return None + def execute(self, query, params=()): + self.conn.queries.append(query) + if "pg_advisory_xact_lock_shared" in query: + self.conn.grants["labels"] = True + elif "pg_advisory_xact_lock(" in query: + self.conn.grants["graph"] = True + def fetchone(self): + return dict(self.conn.grants) + + +def _store(): + conn = SimpleNamespace(grants={"graph": False, "labels": False, "exclusive": False}, queries=[]) + conn.cursor = lambda: Cursor(conn) + return PostgresVNextStore(conn) + + +def test_strict_s_after_l_is_refused(monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", True) + store = _store() + store.lock_label_writes() + with pytest.raises(writes.LabelLockOrderError, match="graph lock must precede"): + store.lock_graph_mutation() + assert any("FROM pg_locks" in query for query in store.conn.queries) + + +def test_a_savepoint_rollback_cannot_leave_a_stale_lock_memo(monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", True) + store = _store() + store.lock_graph_mutation() + store.lock_label_writes() + store.conn.grants.update(graph=False, labels=False, exclusive=False) + store.lock_graph_mutation() + store.conn.grants.update(graph=False, labels=True) + with pytest.raises(writes.LabelLockOrderError): + store.lock_graph_mutation() + assert sum("FROM pg_locks" in query for query in store.conn.queries) == 3 + + +def test_strict_changing_hook_requires_exclusive_l(monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", True) + store = _store() + before = {"domain": "project", "sensitivity": "public"} + with pytest.raises(writes.LabelLockOrderError, match="exclusive label lock"): + writes.prepare_label_patch(store, "memory", before, {"sensitivity": "confidential"}) + store.conn.grants["exclusive"] = True + assert writes.prepare_label_patch(store, "memory", before, {"sensitivity": "confidential"}) == {"sensitivity": "confidential"} + + +def test_propagation_locks_tables_and_rows_in_order(): + store = _store() + statements = [] + class RowsCursor(Cursor): + def execute(self, query, params=()): + statements.append((query, params)) + self.ids = params[0] + def fetchall(self): + return [{"id": row_id} for row_id in self.ids] + store.conn.cursor = lambda: RowsCursor(store.conn) + changes = [{"kind": kind, "id": row_id} for kind, row_id in [("memory", "b"), ("artifact", "c"), ("open_loop", "d"), ("project", "e"), ("memory", "a")]] + writes.lock_settled_label_rows(store, changes) + assert [query.split("FROM ")[1].split()[0] for query, _params in statements] == list(writes.LABEL_TABLE_ORDER) + assert all("ORDER BY id FOR UPDATE" in query for query, _params in statements) + assert statements[-1][1] == (["a", "b"],) + + +def test_compare_and_set_miss_refuses_the_whole_label_write(): + from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError + store = SimpleNamespace(_fetch_optional_one=lambda *_: None) + with pytest.raises(DerivedDomainRepairError, match="changed no row"): + writes.write_settled_label(store, kind="memory", row_id="missing", domain="health", sensitivity="confidential", metadata={}, project_id=None, expected_domain="project", expected_sensitivity="public") diff --git a/tests/unit/test_label_lock_registry.py b/tests/unit/test_label_lock_registry.py new file mode 100644 index 000000000..8e73f72b8 --- /dev/null +++ b/tests/unit/test_label_lock_registry.py @@ -0,0 +1,69 @@ +"""Discover every store SQL write or row lock on a label table.""" +import ast +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] / "apps/api/src/alicebot_api" +TABLE = r"(?:memories|sources|open_loops|generated_artifacts|projects)" +WRITE = re.compile(rf"\b(?:INSERT\s+INTO|UPDATE|DELETE\s+FROM)\s+{TABLE}\b", re.I) +FROM = re.compile(rf"\b(?:FROM|JOIN)\s+{TABLE}\b", re.I) + + +def sql_text(node): + if isinstance(node, ast.JoinedStr): + return " ".join(item.value if isinstance(item, ast.Constant) and isinstance(item.value, str) else "{}" for item in node.values) + return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else "" + + +def label_sql_functions(tree): + for fn in ast.walk(tree): + if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for node in ast.walk(fn): + query = " ".join(sql_text(node).split()) + if WRITE.search(query) or ("FOR UPDATE" in query.upper() and FROM.search(query)): + yield fn, node + break + + +def test_all_discovered_label_writers_and_row_lockers_take_l(): + paths = [ROOT / "vnext_store.py", *sorted((ROOT / "vnext_stores/postgres").glob("*.py"))] + found = set() + missing = [] + for path in paths: + for fn, node in label_sql_functions(ast.parse(path.read_text())): + found.add((path.name, fn.name)) + if not any(isinstance(dec, ast.Name) and dec.id == "takes_label_lock" for dec in fn.decorator_list): + missing.append(f"{path.name}:{node.lineno} {fn.name}") + assert ("memory_access.py", "list_pending_derived_candidates_for_member") in found + assert ("memory_lifecycle.py", "lock_project_update_artifacts_for_redaction") in found + assert not missing, "label SQL without L: " + ", ".join(missing) + + +# Each application SQL exception has a dedicated transaction protocol. +DIRECT_SQL_PROTOCOLS = { + ("vnext_label_writes.py", "write_settled_label"): "exclusive L and label compare-and-set", + ("vnext_label_writes.py", "lock_settled_label_rows"): "exclusive L and deterministic table order", + ("vnext_label_repair.py", "relabel_labels_sqlite"): "SQLite immediate writer transaction", + ("vnext_derived_domain_backfill.py", "relabel_derived_rows_sqlite"): "frozen historical SQLite repair", + ("labels.py", "repair_labels_postgres"): "S, exclusive L, ordered rows and compare-and-set", + ("sqlite_schema.py", "_backfill_legacy_memory_project_scopes"): "schema bootstrap writer transaction", + ("sqlite_schema.py", "_backfill_source_dedupe_keys"): "schema bootstrap writer transaction", + ("sqlite_schema.py", "_repair_source_dedupe_identity"): "schema bootstrap writer transaction", + ("sqlite_schema.py", "_backfill_memory_agent_attribution"): "schema bootstrap writer transaction", + ("sqlite_schema.py", "_deduplicate_memory_lookup_values"): "schema bootstrap writer transaction", + ("sqlite_schema.py", "_repair_tombstone_lookup_value_holders"): "schema bootstrap writer transaction", +} + + +def test_application_code_cannot_write_label_tables_directly(): + missing = [] + for path in ROOT.rglob("*.py"): + if path.name in {"vnext_store.py", "sqlite_store.py", "store.py"} or "vnext_stores" in path.parts or "legacy_store" in path.parts: + continue + for fn, node in label_sql_functions(ast.parse(path.read_text())): + if any(isinstance(dec, ast.Name) and dec.id in {"takes_label_lock", "_takes_label_lock"} for dec in fn.decorator_list): + continue + if (path.name, fn.name) not in DIRECT_SQL_PROTOCOLS: + missing.append(f"{path.relative_to(ROOT)}:{node.lineno} {fn.name}") + assert not missing, "application label SQL without protocol: " + ", ".join(missing) diff --git a/tests/unit/test_label_writer_registry.py b/tests/unit/test_label_writer_registry.py new file mode 100644 index 000000000..e047e7e5a --- /dev/null +++ b/tests/unit/test_label_writer_registry.py @@ -0,0 +1,78 @@ +"""Discover label-changing SQL, update patches and the derived insert floors.""" +import ast +import re +from pathlib import Path + +from tests.unit.test_label_lock_registry import ROOT, label_sql_functions, sql_text + +LABEL_KEYS = {"domain", "sensitivity", "project_id", "project_scope", "project_floor", "derived_from"} +UPDATE_METHODS = {"update_memory", "update_source", "update_open_loop", "update_project"} +# Explicit function identities make every newly added caller reviewable. +LABEL_CALLERS = { + ("routers/vnext_memories.py", "review_vnext_memory"), + ("routers/vnext_memories.py", "review_vnext_source"), + ("vnext_projects.py", "review_project_update"), + ("vnext_label_writes.py", "raise_source_to_replacement"), + ("sqlite_store.py", "supersede_source"), + ("cli/smokes.py", "_run_vnext_smoke_operator_console"), +} + + +def keys_in(node): + return {item.value for item in ast.walk(node) if isinstance(item, ast.Constant) and isinstance(item.value, str)} & LABEL_KEYS + + +def test_every_label_changing_update_caller_is_registered(): + missing = [] + for path in ROOT.rglob("*.py"): + if "vnext_stores" in path.parts or path.name in {"sqlite_store.py", "vnext_store.py"}: + continue + tree = ast.parse(path.read_text()) + for fn in ast.walk(tree): + if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + patch_nodes = [node for node in ast.walk(fn) if isinstance(node, (ast.Assign, ast.AnnAssign)) and any(isinstance(item, ast.Name) and "patch" in item.id for item in ast.walk(node))] + for call in ast.walk(fn): + if not isinstance(call, ast.Call) or not isinstance(call.func, ast.Attribute) or call.func.attr not in UPDATE_METHODS: + continue + patch = next((kw.value for kw in call.keywords if kw.arg == "patch"), None) + if patch is not None and (keys_in(patch) or (isinstance(patch, ast.Name) and any(keys_in(item) for item in patch_nodes))): + key = (str(path.relative_to(ROOT)), fn.name) + if key not in LABEL_CALLERS: + missing.append(f"{key[0]}:{call.lineno} {fn.name}") + assert not missing, "unregistered label caller: " + ", ".join(missing) + + +def test_every_create_has_the_insert_floor_and_every_update_has_its_hook(): + floors = { + "vnext_store.py": {"create_artifact", "upsert_artifact_by_workflow_digest", "create_project", "update_project"}, + "vnext_stores/postgres/memory_lifecycle.py": {"create_memory"}, + "vnext_stores/postgres/graph_open_loops.py": {"create_open_loop"}, + "vnext_stores/sqlite/memory_lifecycle.py": {"create_memory"}, + "vnext_stores/sqlite/graph_open_loops.py": {"create_open_loop"}, + } + hooks = { + "vnext_store.py": {"update_source", "update_project"}, + "sqlite_store.py": {"update_source"}, + "vnext_stores/postgres/memory_lifecycle.py": {"update_memory"}, + "vnext_stores/postgres/graph_open_loops.py": {"update_open_loop"}, + "vnext_stores/sqlite/memory_lifecycle.py": {"update_memory"}, + "vnext_stores/sqlite/graph_open_loops.py": {"update_open_loop"}, + } + for registry, helper in ((floors, "apply_insert_floor"), (hooks, "propagate_after_write")): + for name, functions in registry.items(): + tree = ast.parse((ROOT / name).read_text()) + for function in functions: + fn = next(node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef) and node.name == function) + calls = {node.func.id for node in ast.walk(fn) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)} + assert helper in calls, f"{name}:{fn.lineno} {function} lacks {helper}" + + +def test_a_new_derived_insert_cannot_bypass_the_floor(): + for path in [ROOT / "vnext_store.py", ROOT / "sqlite_store.py", *sorted((ROOT / "vnext_stores").rglob("*.py"))]: + for fn, node in label_sql_functions(ast.parse(path.read_text())): + inserts = [sql_text(item) for item in ast.walk(fn)] + if not any(re.search(r"\bINSERT\s+INTO\s+(?:memories|open_loops|generated_artifacts|projects)\b", query, re.I) for query in inserts): + continue + calls = {item.func.id for item in ast.walk(fn) if isinstance(item, ast.Call) and isinstance(item.func, ast.Name)} + assert "apply_insert_floor" in calls, f"{path.name}:{node.lineno} {fn.name} inserts without a floor" diff --git a/tests/unit/test_source_move_label_preview.py b/tests/unit/test_source_move_label_preview.py new file mode 100644 index 000000000..cb6caa1ce --- /dev/null +++ b/tests/unit/test_source_move_label_preview.py @@ -0,0 +1,53 @@ +"""A source move previews loss of full project admission before any write.""" +from copy import deepcopy +from uuid import uuid4 + +import pytest + +from alicebot_api import vnext_label_writes as writes + + +def _source(scope): + return {"id": str(uuid4()), "kind": "source", "user_id": "synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": scope}} + + +def _report(sources, scope): + return {"id": str(uuid4()), "kind": "artifact", "user_id": "synthetic", "artifact_type": "daily_brief", "domain": "project", "sensitivity": "public", "metadata_json": {"source_ids": [row["id"] for row in sources], "project_scope": scope}} + + +class Store: + def __init__(self, rows): + self.rows = rows + self.reads = [] + def read_label_rows(self, kind, ids): + self.reads.append((kind, ids)) + return [row for row in self.rows if row["kind"] == kind and row["id"] in ids] + + +def test_stored_scope_unchanged_floor_move_still_counts_hidden_row(monkeypatch): + source = _source(["alpha"]) + report = _report([source], ["alpha"]) + store = Store([source, report]) + original = deepcopy(store.rows) + monkeypatch.setattr(writes, "walk_dependants", lambda *_: [report]) + assert writes.count_rows_hidden_by_scope_move(store, source, ["beta"]) == 1 + assert store.rows == original + + +def test_preview_reads_the_other_parent_and_its_ancestry(monkeypatch): + moved, other = _source(["alpha"]), _source(["alpha"]) + parent = _report([other], ["alpha"]) + report = _report([moved], ["alpha"]) + report["metadata_json"]["artifact_ids"] = [parent["id"]] + store = Store([moved, other, parent, report]) + monkeypatch.setattr(writes, "walk_dependants", lambda *_: [report]) + assert writes.count_rows_hidden_by_scope_move(store, moved, ["beta"]) == 1 + assert any(other["id"] in ids for _kind, ids in store.reads) + assert any(parent["id"] in ids for _kind, ids in store.reads) + + +def test_preview_does_not_count_a_row_already_unverified(monkeypatch): + source = _source(["alpha"]) + report = _report([source, _source(["alpha"])], ["alpha"]) + monkeypatch.setattr(writes, "walk_dependants", lambda *_: [report]) + assert writes.count_rows_hidden_by_scope_move(Store([source, report]), source, ["beta"]) == 0 diff --git a/tests/unit/test_sqlite_derived_labels_write_path.py b/tests/unit/test_sqlite_derived_labels_write_path.py index 3bfd61cca..ad231c68a 100644 --- a/tests/unit/test_sqlite_derived_labels_write_path.py +++ b/tests/unit/test_sqlite_derived_labels_write_path.py @@ -76,7 +76,7 @@ def test_an_update_that_omits_a_marker_keeps_it(tmp_path: Path) -> None: path = tmp_path / "vault.sqlite3" with _vault(path) as conn: store = SQLiteVNextStore(conn, USER) - health = add_memory(store, key="health", text="A restricted observation", domain="health") + health = add_memory(store, key="health", text="A restricted observation", domain="health", scope=(ALPHA,)) with without_insert_floor(): derived = store.create_memory( { From 6bf053b3267f88fd82898ea412557415fc04eb57 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:27:25 +0200 Subject: [PATCH 075/270] Regenerate source candidates at current labels and lock review adapters first --- apps/api/src/alicebot_api/main.py | 1 + .../alicebot_api/mcp/evidence_artifacts.py | 4 + .../openapi_operation_contracts.py | 7 + .../alicebot_api/routers/vnext_memories.py | 32 ++++- .../src/alicebot_api/routers/vnext_review.py | 4 + .../src/alicebot_api/vnext_label_writes.py | 10 +- .../alicebot_api/vnext_source_regeneration.py | 73 ++++++++++ apps/api/src/alicebot_api/vnext_store.py | 17 ++- ...test_source_move_label_preview_postgres.py | 128 ++++++++++++++++++ 9 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 apps/api/src/alicebot_api/vnext_source_regeneration.py create mode 100644 tests/integration/test_source_move_label_preview_postgres.py diff --git a/apps/api/src/alicebot_api/main.py b/apps/api/src/alicebot_api/main.py index 2b2be310a..1378f61a4 100644 --- a/apps/api/src/alicebot_api/main.py +++ b/apps/api/src/alicebot_api/main.py @@ -779,6 +779,7 @@ async def receive() -> dict[str, object]: ("POST", "/v0/vnext/projects/update-candidates/{artifact_id}/review"), ("POST", "/v0/vnext/queue/process-next"), ("POST", "/v0/vnext/sources/{source_id}/review"), + ("POST", "/v0/vnext/sources/{source_id}/regenerate"), ("PUT", "/v0/vnext/settings/brain-charter"), } ) diff --git a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py index 0a384563a..c6efae11f 100644 --- a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py +++ b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py @@ -791,6 +791,10 @@ def _handle_alice_vnext_artifact_review(context: MCPRuntimeContext, arguments: M actor_id: str | None = None trace_id: str | None = None with _vnext_store_context(context) as store: + store.lock_graph_mutation() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) _target, actor_type, actor_id, decision = _authorize_vnext_artifact_target( store, identity=identity, diff --git a/apps/api/src/alicebot_api/openapi_operation_contracts.py b/apps/api/src/alicebot_api/openapi_operation_contracts.py index edf9d9e61..e04734f5c 100644 --- a/apps/api/src/alicebot_api/openapi_operation_contracts.py +++ b/apps/api/src/alicebot_api/openapi_operation_contracts.py @@ -1148,6 +1148,10 @@ def _closed_source_schema( closed=True, ), ), + ("POST", "/v0/vnext/sources/{source_id}/regenerate"): ( + "RegenerateVnextSourceSuccessResponse", + _operation_schema("RegenerateVnextSourceSuccessResponse", ("source_id", "memory_ids", "open_loop_ids", "memory_count", "open_loop_count"), closed=True), + ), ("GET", "/v0/vnext/traces/sources/{source_id}"): ( "GetVnextSourceTraceSuccessResponse", _operation_schema( @@ -2082,6 +2086,9 @@ def _closed_source_schema( ("POST", "/v0/vnext/sources/{source_id}/review"): _typed_properties( objects=("source", "trace"), booleans=("archived",) ), + ("POST", "/v0/vnext/sources/{source_id}/regenerate"): _typed_properties( + strings=("source_id",), string_arrays=("memory_ids", "open_loop_ids"), integers=("memory_count", "open_loop_count"), + ), ("POST", "/v0/vnext/memories/{memory_id}/review"): _typed_properties( objects=("memory",), nullable_objects=("consolidation_acceptance",) ), diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index b0156174c..c58712977 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -116,7 +116,11 @@ class VNextSourceReviewRequest(VNextAgentRequest): sensitivity: VNextSensitivity | None = None project_id: str | None = Field(default=None, min_length=1, max_length=120) review_note: str | None = Field(default=None, min_length=1, max_length=4000) - confirm_label_hide: bool = False + confirm_label_hide: bool = Field(default=False, description="Confirm a source project move after previewing the number of derived rows hidden from project-bound keys.") + + +class VNextSourceRegenerateRequest(VNextAgentRequest): + user_id: UUID = Field(description="Owner of the stored source whose candidate memories and open loops are regenerated. Earlier rows keep their labels and provenance.") class VNextConnectorSyncRequest(VNextAgentRequest): @@ -756,6 +760,32 @@ def get_vnext_source(source_id: UUID, user_id: UUID) -> JSONResponse: ) +@source_review_router.post("/v0/vnext/sources/{source_id}/regenerate", summary="Regenerate fresh candidates from a stored source", description="The local owner or an unbound admin can regenerate candidate memories and open loops from all stored chunks using the source's current labels. Existing sources and outputs remain unchanged. Rerun the report's generation route to rebuild a report.") +def regenerate_vnext_source(source_id: UUID, request: VNextSourceRegenerateRequest, authorization: str | None = Header(default=None)) -> JSONResponse: + from alicebot_api.vnext_label_writes import label_error_response + from alicebot_api.vnext_source_regeneration import regenerate_source_inputs + + settings = get_settings() + try: + with user_connection(settings.database_url, request.user_id) as conn: + store = PostgresVNextStore(conn) + identity = _vnext_authenticated_agent_identity(store, request, user_id=request.user_id, authorization=authorization) + if identity is not None and (identity.permission_profile != "admin_agent" or identity.project_scope_locked or identity.project_scope): + return _vnext_public_error_response(status_code=403, detail="source regeneration requires the owner or an unbound admin") + store.lock_label_writes() + source = store.get_source(str(source_id)) + if source is None: + return _vnext_public_error_response(status_code=404, detail="vNext source was not found") + payload = regenerate_source_inputs(store, source) + except Exception as exc: + mapped = label_error_response(exc) + if mapped is None: + raise + status, detail, retry_after = mapped + return JSONResponse(status_code=status, content={"detail": detail}, headers={"Retry-After": retry_after} if retry_after else None) + return JSONResponse(status_code=201, content=jsonable_encoder(payload)) + + @source_review_router.post("/v0/vnext/sources/{source_id}/review") def review_vnext_source(source_id: UUID, request: VNextSourceReviewRequest) -> JSONResponse: settings = get_settings() diff --git a/apps/api/src/alicebot_api/routers/vnext_review.py b/apps/api/src/alicebot_api/routers/vnext_review.py index cd0a9adb1..6ebc447e0 100644 --- a/apps/api/src/alicebot_api/routers/vnext_review.py +++ b/apps/api/src/alicebot_api/routers/vnext_review.py @@ -634,6 +634,10 @@ def review_vnext_artifact( identity = _vnext_authenticated_agent_identity( store, request, user_id=request.user_id, authorization=authorization ) + store.lock_graph_mutation() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) _artifact, decision = _vnext_authorized_artifact( store=store, identity=identity, diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index c823feebd..ebed03e80 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -487,10 +487,11 @@ def clamp_owner_patch( for key in ("domain", "sensitivity", "project_id"): if key in proposed_patch and proposed_patch[key] is not None: proposed[key] = proposed_patch[key] - if isinstance(proposed_patch.get("metadata_json"), dict): + patch_metadata = proposed_patch.get("metadata_json") + if isinstance(patch_metadata, dict): stored_meta = before.get("metadata_json") meta = dict(stored_meta) if isinstance(stored_meta, dict) else {} - meta.update(proposed_patch["metadata_json"]) + meta.update(patch_metadata) proposed["metadata_json"] = meta proposed["kind"] = kind nodes, exceeded = collect_label_rows(store, [proposed], max_nodes=PROPAGATION_BOUND) @@ -513,7 +514,8 @@ def clamp_owner_patch( return proposed_patch proposed_patch["domain"] = label.domain proposed_patch["sensitivity"] = label.sensitivity - metadata = dict(proposed.get("metadata_json") or {}) + proposed_metadata = proposed.get("metadata_json") + metadata = dict(proposed_metadata) if isinstance(proposed_metadata, Mapping) else {} metadata["project_scope"] = list(label.project_scope) metadata["project_floor"] = list(label.project_floor) proposed_patch["metadata_json"] = metadata @@ -644,7 +646,7 @@ def propagate(store: Any, changed: Sequence[tuple[str, str]], *, cause: str) -> if exceeded: raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") settled = settle_labels(nodes) - changes: list[tuple[Mapping[str, object], Any, tuple[str, str, tuple[str, ...], tuple[str, ...]]]] = [] + changes: list[tuple[dict[str, object], Any, tuple[str, str, tuple[str, ...], tuple[str, ...]]]] = [] for row in affected: label = settled.by_stored(str(row.get("kind")), str(row.get("id"))) if label.unverified: diff --git a/apps/api/src/alicebot_api/vnext_source_regeneration.py b/apps/api/src/alicebot_api/vnext_source_regeneration.py new file mode 100644 index 000000000..589d72010 --- /dev/null +++ b/apps/api/src/alicebot_api/vnext_source_regeneration.py @@ -0,0 +1,73 @@ +"""Regenerate fresh source copies without changing earlier rows or their provenance.""" +from __future__ import annotations + +from typing import Any +from uuid import UUID, uuid4 + +from alicebot_api.vnext_capture import extract_candidate_memories +from alicebot_api.vnext_derived_labels import with_derived_from +from alicebot_api.vnext_event_log import append_event +from alicebot_api.vnext_project_scope import source_project_scope +from alicebot_api.vnext_projects import _open_loop_candidates +from alicebot_api.vnext_repositories import JsonObject + + +def regenerate_source_inputs(store: Any, source: JsonObject) -> JsonObject: + """Create candidate memories and loops from all stored chunks at the current label.""" + + chunks = store.read_source_chunks_for_regeneration(str(source["id"])) + scope = source_project_scope(source) + project_id = None + if len(scope) == 1: + try: + project_id = str(UUID(scope[0])) + except ValueError: + pass + generation = str(uuid4()) + memories = [] + loops = [] + for index, candidate in enumerate(extract_candidate_memories(chunks)): + metadata = with_derived_from({ + "source_id": str(source["id"]), + "source_chunk_id": candidate.source_chunk_id, + "source_chunk_index": candidate.source_chunk_index, + "extraction_rule": candidate.extraction_rule, + "project_scope": list(scope), + "regeneration_id": generation, + **({"provenance_role": candidate.provenance_role, "assertion_class": candidate.assertion_class} if candidate.provenance_role is not None else {}), + }, {"sources": [source]}) + memory = store.create_memory({ + "memory_key": f"regenerated:{generation}:{index}", + "canonical_text": candidate.text, + "title": candidate.text[:120], + "summary": candidate.text[:280], + "value": {"text": candidate.text, "source_id": str(source["id"]), "source_chunk_id": candidate.source_chunk_id}, + "source_event_ids": [str(source["id"]), candidate.source_chunk_id], + "status": "candidate", "memory_type": candidate.memory_type, + "confidence": candidate.confidence, + "domain": source["domain"], "sensitivity": source["sensitivity"], + "project_id": project_id, "metadata_json": metadata, + }, actor_type="user") + store.create_provenance_link({ + "target_type": "memory", "target_id": str(memory["id"]), + "source_id": str(source["id"]), "source_chunk_id": candidate.source_chunk_id, + "quote": candidate.text, "evidence_role": "quoted_from", "confidence": candidate.confidence, + }, actor_type="user") + memories.append(str(memory["id"])) + source_metadata = source.get("metadata_json") + source_with_text = {**source, "metadata_json": {**(source_metadata if isinstance(source_metadata, dict) else {}), "raw_text": "\n".join(str(chunk["text"]) for chunk in chunks)}} + for loop_candidate in _open_loop_candidates(source_with_text): + loop_candidate["project_id"] = project_id + loop_metadata = loop_candidate.get("metadata_json") + loop_candidate["metadata_json"] = with_derived_from({ + **(loop_metadata if isinstance(loop_metadata, dict) else {}), + "source_id": str(source["id"]), "project_scope": list(scope), "regeneration_id": generation, + }, {"sources": [source]}) + loop = store.create_open_loop(loop_candidate, actor_type="user") + store.create_provenance_link({ + "target_type": "open_loop", "target_id": str(loop["id"]), + "source_id": str(source["id"]), "evidence_role": "quoted_from", + }, actor_type="user") + loops.append(str(loop["id"])) + append_event(store, event_type="source.inputs_regenerated", actor_type="user", target_type="source", target_id=str(source["id"]), payload={"memory_count": len(memories), "open_loop_count": len(loops)}) + return {"source_id": str(source["id"]), "memory_ids": memories, "open_loop_ids": loops, "memory_count": len(memories), "open_loop_count": len(loops)} diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 788b172e1..59aeee2f9 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -467,6 +467,7 @@ class PostgresVNextStore: def __init__(self, conn: UserConnection): self.conn = conn + self._label_floor_applied = False def lock_label_writes(self, *, exclusive: bool = False) -> None: """Shared label lock for a write, or the exclusive lock for a relabel. @@ -1452,6 +1453,19 @@ def list_source_chunks(self, source_id: str, *, limit: int = 500) -> list[VNextR (source_id, bounded_limit), ) + def read_source_chunks_for_regeneration(self, source_id: str) -> list[VNextRow]: + """Read the complete source, or refuse recovery before writing any output.""" + + from alicebot_api.vnext_derived_labels import PROPAGATION_BOUND, LabelPropagationTooLarge + + rows = self._fetch_all( + f"SELECT {SOURCE_CHUNK_COLUMNS} FROM source_chunks WHERE source_id = %s::uuid ORDER BY chunk_index, id LIMIT %s", + (source_id, PROPAGATION_BOUND + 1), + ) + if len(rows) > PROPAGATION_BOUND: + raise LabelPropagationTooLarge("source regeneration exceeded the source chunk bound") + return rows + def search_source_chunks( self, *, @@ -1864,8 +1878,9 @@ def update_project(self, *, project_id: str, patch: JsonObject, actor_type: str metadata = patch.get("metadata_json") floor_event = None if isinstance(metadata, dict) and before is not None: + before_metadata = before.get("metadata_json") patch["metadata_json"] = merge_protected_metadata( - before.get("metadata_json") if isinstance(before.get("metadata_json"), dict) else {}, + before_metadata if isinstance(before_metadata, dict) else {}, metadata, label_write="derived_from" in metadata, ) diff --git a/tests/integration/test_source_move_label_preview_postgres.py b/tests/integration/test_source_move_label_preview_postgres.py new file mode 100644 index 000000000..14056987b --- /dev/null +++ b/tests/integration/test_source_move_label_preview_postgres.py @@ -0,0 +1,128 @@ +"""Owner source moves preview real admission loss and regenerate fresh candidates.""" +import json +from copy import deepcopy +from uuid import UUID, uuid4 + +import anyio +import pytest + +import alicebot_api.main as main +from alicebot_api.config import Settings +from alicebot_api.db import user_connection +from alicebot_api.routers import vnext_memories as router +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_capture import VNextCaptureService +from alicebot_api.vnext_store import PostgresVNextStore + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +def _request(path, payload, raw_key): + messages = [] + body = json.dumps(payload).encode() + received = False + async def receive(): + nonlocal received + if received: + return {"type": "http.disconnect"} + received = True + return {"type": "http.request", "body": body, "more_body": False} + async def send(message): + messages.append(message) + scope = {"type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", "method": "POST", "scheme": "http", "path": path, "raw_path": path.encode(), "query_string": b"", "headers": [(b"host", b"127.0.0.1:8000"), (b"content-type", b"application/json"), (b"authorization", f"Bearer {raw_key}".encode())], "client": ("127.0.0.1", 50000), "server": ("testserver", 80), "root_path": ""} + anyio.run(main.app, scope, receive, send) + status = next(item["status"] for item in messages if item["type"] == "http.response.start") + return status, b"".join(item.get("body", b"") for item in messages if item["type"] == "http.response.body") + + +def _fixture(url): + user = uuid4() + with user_connection(url, user) as conn: + ContinuityStore(conn).create_user(user, f"move-{user}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + result = VNextCaptureService(store, defer_embeddings=True).capture_text("I prefer synthetic blue ink.\nTODO: Review the synthetic draft", domain="project", sensitivity="public", project_scope=[ALPHA]) + source = store.get_source(str(result.source_id)) + report = store.create_artifact({"artifact_type": "daily_brief", "title": "Synthetic report", "content_markdown": "Synthetic report", "domain": "project", "sensitivity": "public", "metadata_json": {"source_ids": [str(result.source_id)], "project_scope": [ALPHA]}}) + return user, source, report + + +def _snapshot(url, user, source): + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + rows = {table: store._fetch_all(f"SELECT * FROM {table} ORDER BY id") for table in ("sources", "memories", "open_loops", "generated_artifacts", "provenance_links", "event_log", "graph_edges")} + return rows + + +def test_source_move_preview_is_zero_write_then_confirmation_preserves_provenance(migrated_database_urls, monkeypatch): + url = migrated_database_urls["app"] + user, source, report = _fixture(url) + monkeypatch.setattr(router, "get_settings", lambda: Settings(database_url=url)) + before = _snapshot(url, user, source) + request = router.VNextSourceReviewRequest(user_id=user, action="assign_project", project_id=BETA) + preview = router.review_vnext_source(UUID(str(source["id"])), request) + payload = json.loads(preview.body) + assert preview.status_code == 200 and payload["preview"] is True + assert payload["derived_rows_hidden_from_project_keys"] == len(before["memories"]) + 1 + assert payload["confirm_required"] is True + assert _snapshot(url, user, source) == before + confirmed = router.review_vnext_source(UUID(str(source["id"])), request.model_copy(update={"confirm_label_hide": True})) + assert confirmed.status_code == 200 + after = _snapshot(url, user, source) + assert after["provenance_links"] == before["provenance_links"] + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + assert store.get_source(str(source["id"]))["metadata_json"]["project_scope"] == [BETA] + moved_report = store.get_artifact(str(report["id"])) + assert BETA in moved_report["metadata_json"]["project_floor"] + assert moved_report["metadata_json"]["source_ids"] == report["metadata_json"]["source_ids"] + + +def test_regeneration_uses_moved_source_without_lowering_or_rewriting_originals(migrated_database_urls, monkeypatch): + url = migrated_database_urls["app"] + user, source, report = _fixture(url) + monkeypatch.setattr(router, "get_settings", lambda: Settings(database_url=url)) + move = router.VNextSourceReviewRequest(user_id=user, action="assign_project", project_id=BETA, confirm_label_hide=True, sensitivity="confidential", domain="health") + assert router.review_vnext_source(UUID(str(source["id"])), move).status_code == 200 + before = _snapshot(url, user, source) + result = router.regenerate_vnext_source(UUID(str(source["id"])), router.VNextSourceRegenerateRequest(user_id=user), authorization=None) + payload = json.loads(result.body) + assert result.status_code == 201 + assert payload["memory_count"] >= 1 and payload["open_loop_count"] == 1 + after = _snapshot(url, user, source) + assert after["sources"] == before["sources"] + assert after["generated_artifacts"] == before["generated_artifacts"] + for table in ("memories", "open_loops", "provenance_links"): + by_id = {row["id"]: row for row in after[table]} + assert all(by_id[row["id"]] == row for row in before[table]) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + for kind, ids in (("memory", payload["memory_ids"]), ("open_loop", payload["open_loop_ids"])): + rows = store.read_label_rows(kind, ids) + assert len(rows) == len(ids) + for row in rows: + assert row["domain"] == "health" and row["sensitivity"] == "confidential" + assert row["metadata_json"]["project_scope"] == [BETA] + assert row["metadata_json"]["project_floor"] == [BETA] + links = store._fetch_all("SELECT source_id FROM provenance_links WHERE target_id = %s", (str(row["id"]),)) + assert {str(link["source_id"]) for link in links} == {str(source["id"])} + + +@pytest.mark.parametrize("profile,scope,expected", [("trusted_local_agent", None, 403), ("admin_agent", ALPHA, 403), ("admin_agent", None, 201)]) +def test_regeneration_authenticates_and_refuses_trusted_or_bound_keys(migrated_database_urls, monkeypatch, profile, scope, expected): + url = migrated_database_urls["app"] + user, source, _report = _fixture(url) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + _key, raw = create_agent_key(store, user_id=user, agent_id="synthetic-reader", permission_profile=profile, project_scope=scope) + settings = Settings(database_url=url, app_env="test") + monkeypatch.setattr(router, "get_settings", lambda: settings) + monkeypatch.setattr(main, "get_settings", lambda: settings) + before = _snapshot(url, user, source) + status, response = _request(f"/v0/vnext/sources/{source['id']}/regenerate", {"user_id": str(user)}, raw) + assert status == expected, response + if expected == 403: + after = _snapshot(url, user, source) + for table in ("sources", "memories", "open_loops", "generated_artifacts", "provenance_links"): + assert after[table] == before[table] From ea00c272f7b9fd4d8b5c03a77eb452d4aaf12eff Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:27:32 +0200 Subject: [PATCH 076/270] Pin the scheduler publication first writer lock --- ...est_derived_labels_concurrency_postgres.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/integration/test_derived_labels_concurrency_postgres.py b/tests/integration/test_derived_labels_concurrency_postgres.py index 2249e4cd4..793e87f57 100644 --- a/tests/integration/test_derived_labels_concurrency_postgres.py +++ b/tests/integration/test_derived_labels_concurrency_postgres.py @@ -170,6 +170,28 @@ def test_a_scheduler_plan_staged_before_a_relabel_is_floored_at_publish(label_ha assert_raised(store.get_artifact(str(published["id"]))) +def test_scheduler_direct_create_takes_the_shared_publish_lock(label_harness): + h = label_harness + source = h.source() + with h.store() as store: + staged = _StagedSchedulerStore(store) + artifact = staged.create_artifact( + { + "artifact_type": "daily_brief", + "title": "Synthetic staged report", + "content_markdown": "Synthetic staged report", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"workflow": "daily_brief", "source_refs": [str(source["id"])]}, + } + ) + plan = staged.plan(artifact) + assert h.relabel("source", source["id"], domain="health", sensitivity="confidential")[0] == 200 + with h.store() as store: + assert_raised(plan.publish(store)) + assert any(row["mode"] == "ShareLock" and row["granted"] for row in h.label_locks()) + + def test_a_staged_staleness_mark_cannot_undo_a_relabel(label_harness): h = label_harness alpha, beta = "prj_" + "a" * 16, "prj_" + "b" * 16 From 898d3251a3202a8317f1fc2092f5c0b21f0a1f1b Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:28:08 +0200 Subject: [PATCH 077/270] Inject the real compare and set refusal on the third dependant --- tests/integration/test_derived_labels_propagation_postgres.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_derived_labels_propagation_postgres.py b/tests/integration/test_derived_labels_propagation_postgres.py index 8bd144bcf..6f0cf8df1 100644 --- a/tests/integration/test_derived_labels_propagation_postgres.py +++ b/tests/integration/test_derived_labels_propagation_postgres.py @@ -8,7 +8,7 @@ import pytest from alicebot_api import vnext_label_writes -from alicebot_api.store import ContinuityStoreInvariantError +from alicebot_api.vnext_derived_domain_backfill import require_changed from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService from alicebot_api.vnext_capture import VNextCaptureService from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService @@ -30,7 +30,7 @@ def fail_third(store, **kwargs): attempts.append(kwargs["row_id"]) if len(attempts) == 3: if failure_kind == "writer": - raise ContinuityStoreInvariantError("synthetic third dependant failure") + require_changed(0, "memories", kwargs["row_id"]) # A genuine database exception must abort the transaction after two writes/events. store.conn.execute("SELECT 1 / 0") return original(store, **kwargs) From 7604d2d6ab88bd00dc014c87c98a06576845608c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:28:59 +0200 Subject: [PATCH 078/270] Withhold encoded source references in open-loop reads --- CHANGELOG.md | 2 +- .../vnext_open_loop_references.py | 12 ++- .../test_open_loop_references_read_fence.py | 52 +++++++++++ .../unit/test_source_scrub_loop_references.py | 91 +++++++++++++++++-- 4 files changed, 148 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4c7840a8..89cea6093 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. +- Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings, including JSON escapes and repeated keys. Admitted references and surrounding values stay, and the owner receives live references unchanged. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. - Unreleased (on main, not in v0.20.0): the derived-row pages state the three-part label, who reads an unverified row, and that the five operator screens apply the caller's sensitivity ceiling. The v0.20.0 notes omitted both limits. No migration is required. - Unreleased (on main, not in v0.20.0): PostgreSQL migration `20261005_0096` raises a stored derived row to the labels of its inputs, and `alicebot vnext labels` and `alice-memory labels` check and repair the same rows. The doctors print how many derived rows are below their inputs or unverified, as a warning. v0.20.0 left the stored label where it was written. - Unreleased (on main, not in v0.20.0): opening a SQLite vault raises a derived row whose stored label is below its inputs, and a restore does the same before it publishes. A repair that cannot finish leaves the vault open and the completion key unstamped. v0.20.0 left the stored label where it was written. diff --git a/apps/api/src/alicebot_api/vnext_open_loop_references.py b/apps/api/src/alicebot_api/vnext_open_loop_references.py index 3a0ebd4ee..80100c097 100644 --- a/apps/api/src/alicebot_api/vnext_open_loop_references.py +++ b/apps/api/src/alicebot_api/vnext_open_loop_references.py @@ -67,11 +67,12 @@ from __future__ import annotations import inspect +import json import re from collections.abc import Callable, Iterable, Mapping, Sequence from uuid import UUID -from alicebot_api.vnext_source_fence import SOURCE_REFERENCE_KEYS, SourceReadFence, cited_source_ids +from alicebot_api.vnext_source_fence import SOURCE_REFERENCE_KEYS, SourceReadFence, _json_container, cited_source_ids JsonObject = dict[str, object] @@ -131,6 +132,9 @@ def withhold_unreadable_references( if memory is not None: memory_ids.add(memory) referenced.add(memory) + named = cited_source_ids(row.get("metadata_json")).named + metadata_ids.update(named) + referenced.update(named) _collect_ids(row.get("metadata_json"), metadata_ids, referenced, at_reference=False, depth=0) source_rows = _rows_by_id(store, sorted(source_ids | metadata_ids), bulk="get_sources_by_ids", single="get_source") memory_rows = _rows_by_id(store, sorted(memory_ids | metadata_ids), bulk="get_memories_by_ids", single="get_memory") @@ -366,6 +370,12 @@ def _scrub(value: object, *, depth: int, withheld: frozenset[str]) -> object: if depth > _METADATA_MAX_DEPTH: return _DROPPED if isinstance(value, str): + decoded = _json_container(value) + if decoded is not None: + checked = _scrub(decoded, depth=depth + 1, withheld=withheld) + if checked is _DROPPED: + return _DROPPED + return value if checked == decoded else json.dumps(checked) return _scrub_text(value, withheld=withheld) if isinstance(value, Mapping): output: dict[object, object] = {} diff --git a/tests/unit/test_open_loop_references_read_fence.py b/tests/unit/test_open_loop_references_read_fence.py index 801fdf54b..e77423303 100644 --- a/tests/unit/test_open_loop_references_read_fence.py +++ b/tests/unit/test_open_loop_references_read_fence.py @@ -313,6 +313,45 @@ def test_the_open_loop_list_withholds_what_the_reader_may_not_read(world: _World world.check_payload(reader, items, where="list") +@pytest.mark.parametrize("reference_key", ("source_refs", "sources")) +@pytest.mark.parametrize("layout", ("object", "duplicate_key", "nested_text")) +def test_encoded_json_references_are_withheld_on_the_real_key_list(world: _World, reference_key: str, layout: str) -> None: + """A real key list removes decoded refused ids and keeps the admitted id and nearby values. + + The owner receives the stored text unchanged. Mutations: stop decoding JSON in ``_scrub``, or stop adding the + canonical metadata names before lookup (the ``sources`` alias then leaves the refused id). + """ + + hidden, admitted = world.sources["beta"], world.sources["own"] + encode = lambda value: "".join("\\u%04x" % ord(char) for char in value) + refs = '["' + encode(hidden) + '", "' + encode(admitted) + '"]' + if layout == "object": + text = '{"source_ids": ' + refs + ', "kept": "cobalt"}' + elif layout == "duplicate_key": + text = '{"source_id": "' + encode(hidden) + '", "source_id": "' + encode(admitted) + '", "kept": "cobalt"}' + else: + text = json.dumps({"source_refs": '{"source_ids": ' + refs + ', "kept": "cobalt"}', "outer": "amber"}) + metadata = {"project_scope": ["alpha"], reference_key: text, "kept": "control"} + world._plant("encoded", metadata=metadata) + loop_id = world.loops["encoded"] + # This is the only loop returned, so another loop's column cannot supply the encoded id to the shared lookup. + world.vault.sql("DELETE FROM open_loops WHERE id != ?", (loop_id,)) + owner = world.vault.wire("alice_open_loops", {"status": "all", "limit": 100}, key=None) + assert owner["is_error"] is False + assert len(owner["payload"]["items"]) == 1 + stored = next(item for item in owner["payload"]["items"] if str(item["id"]) == loop_id) + assert stored["metadata_json"] == metadata + item = next(item for item in world.list_items("alpha_project") if str(item["id"]) == loop_id) + checked = item["metadata_json"] + from alicebot_api.vnext_source_fence import cited_source_ids + + assert cited_source_ids(checked).named == {admitted} + assert checked["kept"] == "control" + assert "cobalt" in str(checked[reference_key]) + if layout == "nested_text": + assert "amber" in str(checked[reference_key]) + + @pytest.mark.parametrize("reader", _UPDATERS) @pytest.mark.parametrize("name", sorted(("own", "health", "confidential", "old_beta", "old_global", "old_deleted", "old_ghost", "old_upper", "old_metadata"))) def test_the_open_loop_update_actions_withhold_what_the_reader_may_not_read(world: _World, reader: str, name: str) -> None: @@ -1035,6 +1074,19 @@ def test_metadata_nested_deeper_than_the_scan_reads_is_dropped_and_does_not_rais assert depth < 100 and "bottom" not in json.dumps(metadata) +def test_json_text_at_the_metadata_depth_limit_is_dropped_without_raising() -> None: + """Decoded JSON obeys the same depth bound. Mutation: serialize ``_DROPPED`` instead of propagating it.""" + + from alicebot_api.vnext_open_loop_references import _METADATA_MAX_DEPTH + + nested: object = json.dumps({"source_refs": [str(uuid4())], "note": "cobalt"}) + for _ in range(_METADATA_MAX_DEPTH - 1): + nested = {"deep": nested} + out = withhold_unreadable_references(_Rows(), [_loop(metadata_json={"kept": "control", "deep": nested})], fence=_FENCE) + assert out[0]["metadata_json"]["kept"] == "control" + assert "cobalt" not in json.dumps(out) + + def test_the_owners_fence_admits_a_live_row_of_any_project_and_refuses_a_deleted_one() -> None: """``SourceReadFence.unfenced()`` is the owner's fence. It admits another project's row and a global one, and refuses a deleted one, so the owner is shown every live reference and no reference to a forgotten row. diff --git a/tests/unit/test_source_scrub_loop_references.py b/tests/unit/test_source_scrub_loop_references.py index f09819f93..a249ca654 100644 --- a/tests/unit/test_source_scrub_loop_references.py +++ b/tests/unit/test_source_scrub_loop_references.py @@ -334,6 +334,33 @@ def test_another_users_loop_that_names_the_source_is_left_alone(tmp_path, capsys assert _read(db, "SELECT title FROM open_loops WHERE user_id = '%s'" % USER_ID) == [(REMOVAL_MARKER,)] +def test_cached_loop_ids_cannot_blank_another_users_loop(tmp_path): + """The scrub checks the user even when its cached ids include another user's row. + + Mutation: remove the ``user_id`` condition from ``_BLANK_OPEN_LOOPS_SQL``. + """ + + from alicebot_api.vnext_stores.sqlite.source_retirement import blank_open_loops + + db = _vault(tmp_path) + sid = run_import(db, _folder(tmp_path, note="The cobalt door opens on Monday.")).source_ids[0] + other, neighbour = str(uuid4()), str(uuid4()) + with closing(sqlite3.connect(db)) as conn: + conn.execute("INSERT INTO users (id, email) VALUES (?, ?)", (other, "other@example.invalid")) + conn.execute( + "INSERT INTO open_loops (id, user_id, title, status, metadata_json) VALUES (?, ?, ?, 'open', ?)", + (neighbour, other, "zebraneighbour", json.dumps({"source_id": sid})), + ) + conn.commit() + before = _read(db, "SELECT * FROM open_loops WHERE id = '%s'" % neighbour) + with sqlite_user_connection(db, USER_ID) as conn: + store = SQLiteVNextStore(conn, USER_ID) + mine = _create_loop(store, "zebramine", metadata={"source_id": sid}) + assert blank_open_loops(store, sid, now="2026-10-05T00:00:00Z", loop_ids=[mine, neighbour]) == 1 + assert _read(db, "SELECT title FROM open_loops WHERE id = '%s'" % mine) == [(REMOVAL_MARKER,)] + assert _read(db, "SELECT * FROM open_loops WHERE id = '%s'" % neighbour) == before + + # Spellings ``cited_source_ids`` names. Each is the only reference on its loop. NAMED_SPELLINGS = { "upper": lambda sid: {"source_id": sid.upper()}, @@ -344,6 +371,17 @@ def test_another_users_loop_that_names_the_source_is_left_alone(tmp_path, capsys "braced": lambda sid: {"source_id": "{" + sid + "}"}, "urn": lambda sid: {"source_id": f"urn:uuid:{sid}"}, "spaced": lambda sid: {"source_id": f" {sid} "}, + "repeated_prefix": lambda sid: {"source_id": f"source: SOURCE:{sid}"}, + "chunk_suffix": lambda sid: {"source_ref": f"source:{sid}#chunk-1"}, + "alice_url": lambda sid: {"source_refs": f"alice://sources/{sid}#chunk-1"}, + "id_object": lambda sid: {"source_refs": [{"id": sid}]}, + "ref_object": lambda sid: {"source_refs": {"ref": sid}}, + "token_list": lambda sid: {"source_ids": f"{sid}; source:{sid}"}, + "source_sentence": lambda sid: {"source_refs": f"See source:{sid} for the note"}, + "sources_alias": lambda sid: {"sources": [sid]}, + "case_key": lambda sid: {"SOURCE_REFS": [sid]}, + "escaped_all": lambda sid: {"source_refs": '{"source_id": "' + "".join("\\u%04x" % ord(c) for c in sid) + '"}'}, + "duplicate_key": lambda sid: {"source_refs": '{"source_id": "' + sid + '", "source_id": "' + sid + '"}'}, } @@ -398,7 +436,7 @@ def test_each_spelling_is_blanked_on_delete_and_on_replace(tmp_path, capsys): def test_a_prune_reads_the_loops_once_for_the_preview_and_once_for_the_receipt(tmp_path, capsys, monkeypatch): - """Twelve replaced sources. The preview reads the user's loops once, and the receipt reads them once, not once per source. + """Every spelling on its own replaced source. The preview and receipt each read the user's loops once. Mutation: ``_preview`` or ``run_sources`` calls ``open_loops_naming_sources`` inside the per-source loop. """ @@ -406,16 +444,18 @@ def test_a_prune_reads_the_loops_once_for_the_preview_and_once_for_the_receipt(t from alicebot_api.vnext_stores.sqlite import source_retirement db = _vault(tmp_path) - folder = _folder(tmp_path, **{f"note{index}": f"The older amber statement {index}." for index in range(12)}) + spellings = {**NAMED_SPELLINGS, "escaped": _escaped_json} + count = max(12, len(spellings)) + folder = _folder(tmp_path, **{f"note{index}": f"The older amber statement {index}." for index in range(count)}) run_import(db, folder) - for index in range(12): + for index in range(count): (folder / f"note{index}.md").write_text(f"The current copper statement {index}.") - assert len(run_import(db, folder, supersede=True).superseded) == 12 + assert len(run_import(db, folder, supersede=True).superseded) == count ids = [row[0] for row in _read(db, "SELECT id FROM sources WHERE deleted_at IS NOT NULL ORDER BY deleted_at, id")] with sqlite_user_connection(db, USER_ID) as conn: store = SQLiteVNextStore(conn, USER_ID) for index, sid in enumerate(ids): - builder = list(NAMED_SPELLINGS.values())[index % len(NAMED_SPELLINGS)] + builder = list(spellings.values())[index % len(spellings)] _create_loop(store, f"zebraprune{index}", metadata=builder(sid)) for index in range(40): _create_loop(store, f"zebrafiller{index}") @@ -430,12 +470,49 @@ def _counting(store): assert _command(db, "prune", "--superseded") == 2 preview = json.loads(capsys.readouterr().out)["would_delete"] assert len(calls) == 1 - assert sum(row["open_loops"] for row in preview) == 12 + assert sum(row["open_loops"] for row in preview) == count calls.clear() assert _command(db, "prune", "--superseded", "--yes") == 0 receipt = json.loads(capsys.readouterr().out)["deleted"] assert len(calls) == 1 - assert sum(row["open_loops"] for row in receipt) == sum(row["open_loops"] for row in preview) == 12 + assert sum(row["open_loops"] for row in receipt) == sum(row["open_loops"] for row in preview) == count + assert _read(db, "SELECT count(*) FROM open_loops WHERE title LIKE 'zebraprune%'") == [(0,)] + assert _read(db, "SELECT count(*) FROM open_loops WHERE title LIKE 'zebrafiller%'") == [(40,)] + + +def test_replacement_reads_all_source_loops_once(tmp_path, monkeypatch): + """A multi-file replacement blanks every spelling using one loop pass for the whole import. + + Mutation: stop passing cached ``loop_ids`` to ``supersede_source`` in the capture service. + """ + + from alicebot_api.vnext_stores.sqlite import source_retirement + + db = _vault(tmp_path) + spellings = {**NAMED_SPELLINGS, "escaped": _escaped_json} + folder = _folder(tmp_path, **{f"note{index}": f"The older amber statement {index}." for index in range(len(spellings))}) + source_ids = run_import(db, folder).source_ids + with sqlite_user_connection(db, USER_ID) as conn: + store = SQLiteVNextStore(conn, USER_ID) + for index, (sid, builder) in enumerate(zip(source_ids, spellings.values(), strict=True)): + _create_loop(store, f"zebrareplaced{index}", metadata=builder(sid)) + for index in range(len(spellings)): + (folder / f"note{index}.md").write_text(f"The current copper statement {index}.") + calls = [] + original = source_retirement._user_open_loops + + def _counting(store): + calls.append(store.user_id) + return original(store) + + monkeypatch.setattr(source_retirement, "_user_open_loops", _counting) + result = run_import(db, folder, supersede=True) + assert len(result.superseded) == len(spellings) + assert len(calls) == 1 + assert _read(db, "SELECT count(*) FROM open_loops WHERE title LIKE 'zebrareplaced%'") == [(0,)] + events = _read(db, "SELECT payload_json FROM event_log WHERE event_type = 'source.superseded'") + assert len(events) == len(spellings) + assert all(json.loads(payload)["open_loops"] == 1 for payload, in events) def test_the_scrub_matches_exactly_what_the_reader_lists(tmp_path, capsys): From a9230c57a6955ae26045df658f98e162e929be20 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:29:41 +0200 Subject: [PATCH 079/270] Use the shared label reader in upgrade evidence --- scripts/run_phase5_ops_evidence.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/scripts/run_phase5_ops_evidence.py b/scripts/run_phase5_ops_evidence.py index bf0d9f01f..843cb968d 100755 --- a/scripts/run_phase5_ops_evidence.py +++ b/scripts/run_phase5_ops_evidence.py @@ -1095,19 +1095,10 @@ def _verify_derived_labels(admin_url: str) -> None: """The migrations must leave no derived row below its inputs or unverified.""" from alicebot_api.db import direct_user_connection - from alicebot_api.vnext_label_repair import classify_stored_labels + from alicebot_api.vnext_label_repair import classify_stored_labels, load_postgres_label_tables with direct_user_connection(admin_url, USER_ID) as conn: - from alicebot_api.vnext_label_repair import INPUT_SELECTS_V3 - - tables: dict[str, list[dict[str, object]]] = {} - for table, statement in INPUT_SELECTS_V3.items(): - cursor = conn.execute(statement) - names = [column[0] for column in cursor.description] - tables[table] = [ - row if isinstance(row, dict) else dict(zip(names, row)) for row in cursor.fetchall() - ] - below, unverified = classify_stored_labels(tables) + below, unverified = classify_stored_labels(load_postgres_label_tables(conn)) if below or any(unverified.values()): raise EvidenceError("derived_labels_not_repaired") From b73f2f418e578f04178823da590017ebb8c1ce83 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:33:00 +0200 Subject: [PATCH 080/270] Fix effective project scope and exact audit authorization --- apps/api/src/alicebot_api/mcp/retrieval.py | 4 +- .../alicebot_api/routers/vnext_memories.py | 29 ++++++- apps/api/src/alicebot_api/sqlite_store.py | 2 +- .../api/src/alicebot_api/vnext_label_guard.py | 6 ++ apps/api/src/alicebot_api/vnext_store.py | 2 +- tests/unit/test_derived_labels_real_keys.py | 82 +++++++++++++++++++ 6 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 tests/unit/test_derived_labels_real_keys.py diff --git a/apps/api/src/alicebot_api/mcp/retrieval.py b/apps/api/src/alicebot_api/mcp/retrieval.py index 7290d2a09..ee9db7cb8 100644 --- a/apps/api/src/alicebot_api/mcp/retrieval.py +++ b/apps/api/src/alicebot_api/mcp/retrieval.py @@ -809,6 +809,7 @@ def _resume_event_honours_policy_fence( # row it points at is the thing to test (the event queries leave them out in SQL # too, which keeps a held-back event from using up a place). exclude_global_domains: frozenset[str], + effective_project_scope: tuple[str, ...] = (), ) -> bool: target_type = event.get("target_type") target_id = event.get("target_id") @@ -836,7 +837,7 @@ def _resume_event_honours_policy_fence( rows=[row], domains=effective_domains, sensitivity_allowed=effective_sensitivity_allowed, - projects=(), + projects=effective_project_scope, ): return False return _resource_matches_domains(row, effective_domains) and _resource_matches_sensitivity( @@ -1226,6 +1227,7 @@ def read_events(scope: tuple[str, ...], excluded: frozenset[str], count: int) -> effective_domains=effective_domains, effective_sensitivity_allowed=effective_sensitivity_allowed, exclude_global_domains=held_back, + effective_project_scope=effective_project_scope, ) ] event_rows.sort(key=_event_recency, reverse=True) diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index 98bc11a10..037c16fa6 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Mapping from datetime import UTC, datetime from typing import Literal from uuid import UUID @@ -42,6 +43,7 @@ _vnext_agent_identity, _vnext_agent_record, _vnext_authenticated_agent_identity, + _vnext_exact_resource_policy, _vnext_load_source_trace, _vnext_metadata, _vnext_permission_response, @@ -1872,12 +1874,35 @@ def list_vnext_recent_memory_commits(user_id: UUID, limit: int = Query(default=2 @memory_router.get("/v0/vnext/memories/{memory_id}/audit") -def get_vnext_memory_audit(memory_id: UUID, user_id: UUID) -> JSONResponse: +def get_vnext_memory_audit( + memory_id: UUID, + user_id: UUID, + authorization: str | None = Header(default=None), +) -> JSONResponse: + from alicebot_api.vnext_label_guard import apply_unverified_rule, effective_row_for_fence + settings = get_settings() try: with user_connection(settings.database_url, user_id) as conn: store = PostgresVNextStore(conn) - payload = VNextMemoryCommitService(store).audit(memory_id=str(memory_id)) + identity = resolve_protected_agent_identity( + store, user_id=user_id, raw_key=agent_key_from_authorization(authorization), payload={}, + ) + + def authorize_memory(memory: Mapping[str, object]) -> None: + effective = effective_row_for_fence(store, identity, "memory", memory) + decision = _vnext_exact_resource_policy(identity=identity, action="memory.audit", resource=dict(effective)) + decision = apply_unverified_rule(decision, effective, identity) + append_policy_events(store, identity=identity, decision=decision, target_type="memory", target_id=str(memory["id"])) + if decision.decision == "blocked": + raise AgentPolicyBlockedError(decision) + + try: + payload = VNextMemoryCommitService(store).audit(memory_id=str(memory_id), authorize_memory=authorize_memory) + except AgentPolicyBlockedError as exc: + return _vnext_permission_response(exc.decision) + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) except VNextMemoryCommitValidationError as exc: return public_exception_response(exc, status_code=404) return JSONResponse(status_code=200, content=jsonable_encoder(payload)) diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index 4808aceb8..05d3be249 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -419,7 +419,7 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: return [] extra = "" if table == "memories": - extra = ", value, project_id" + extra = ", value, project_id, source_event_ids, deleted_at, status" elif table == "open_loops": extra = ", project_id, source_id, memory_id" from uuid import UUID diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index 8122354d0..ec57e7ebe 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -116,6 +116,12 @@ def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[ metadata["project_floor"] = list(label.project_floor) copy["unverified"] = False copy["metadata_json"] = metadata + # Store records expose scope and floor at the top level as well. The + # resolver reads those first, so both representations must agree. + copy["project_scope"] = list(metadata["project_scope"]) + copy["project_floor"] = list(metadata["project_floor"]) + if not copy["project_scope"]: + copy["project_id"] = None return copy def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 91d714d98..94292aa91 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -514,7 +514,7 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: ) extra = "" if table == "memories": - extra = ", value, project_id" + extra = ", value, project_id, source_event_ids, deleted_at, status" elif table == "open_loops": extra = ", project_id, source_id, memory_id" elif table == "beliefs": diff --git a/tests/unit/test_derived_labels_real_keys.py b/tests/unit/test_derived_labels_real_keys.py new file mode 100644 index 000000000..1e10d640c --- /dev/null +++ b/tests/unit/test_derived_labels_real_keys.py @@ -0,0 +1,82 @@ +"""Core read doors authenticate actual SQLite keys before effective admission.""" + +from __future__ import annotations + +import json +from uuid import UUID, uuid4 + +import pytest + +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError +from alicebot_api.onramp import bootstrap_database, sqlite_url_for_path +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_agent_control import ALL_SENSITIVITY +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_event_log import append_event +from alicebot_api.vnext_label_writes import without_insert_floor + + +READERS = ("owner", "admin", "trusted", "read_only", "bound_admin", "bound_trusted", "bound_read_only") +ALPHA = "prj_" + "a" * 16 + + +def seed_read_rows(store): + source = store.create_source({"source_type": "note", "title": "Restricted parent", "content_hash": str(uuid4()), + "domain": "health", "sensitivity": "confidential", "metadata_json": {"project_scope": [ALPHA]}}) + rows = {} + for state in ("verified_public", "verified_confidential", "unverified"): + metadata = {"project_scope": [ALPHA]} + if state != "verified_public": + metadata["source_id"] = str(source["id"]) if state == "verified_confidential" else str(uuid4()) + with without_insert_floor(): + rows[state] = store.create_memory({"memory_key": state, "canonical_text": f"Cedar sentinel {state}", + "title": f"Hidden title {state}", "memory_type": "decision", "status": "active", + "domain": "project", "sensitivity": "public", "metadata_json": metadata}) + append_event(store, event_type="memory.labels_raised", actor_type="system", + target_type="memory", target_id=str(rows[state]["id"]), payload={"cause": "repair_v3"}) + return rows + + +def real_reader_key(store, user_id, reader): + if reader == "owner": + return None + profile = {"admin": "admin_agent", "trusted": "trusted_local_agent", "read_only": "read_only_agent"}[reader.removeprefix("bound_")] + _record, raw = create_agent_key(store, user_id=user_id, agent_id=reader, permission_profile=profile, + project_scope=ALPHA if reader.startswith("bound_") else None) + return raw + + +def expected_read(reader, state): + return state == "verified_public" or reader in {"owner", "admin"} or (reader == "bound_admin" and state == "verified_confidential") + + +@pytest.mark.parametrize("reader", READERS) +def test_sqlite_real_key_core_doors(tmp_path, monkeypatch, reader): + user_id = uuid4() + path = tmp_path / "reads.sqlite3" + bootstrap_database(path, user_id=str(user_id), user_email="synthetic@example.invalid") + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.setenv("ALICE_PROJECT_SCOPING", "off") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + with sqlite_user_connection(path, user_id) as conn: + store = SQLiteVNextStore(conn, user_id) + rows = seed_read_rows(store) + key = real_reader_key(store, user_id, reader) + if key: + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + context = MCPRuntimeContext(database_url=sqlite_url_for_path(path), user_id=UUID(str(user_id))) + for state, row in rows.items(): + admitted = expected_read(reader, state) + for tool in ("alice_recall", "alice_context_pack", "alice_recent_decisions", "alice_explain", "alice_resume"): + arguments = {"memory_id": str(row["id"])} if tool == "alice_explain" else {"query": str(row["canonical_text"]), "sensitivity_allowed": list(ALL_SENSITIVITY)} + try: + result = call_mcp_tool(context, name=tool, arguments=arguments) + except MCPToolError: + assert not admitted, (reader, state, tool) + continue + rendered = json.dumps(result, default=str) + assert (str(row["id"]) in rendered) is admitted, (reader, state, tool, rendered) + if not admitted: + assert str(row["title"]) not in rendered From c779009eba95d8031c1cf4f51832d00a0f945c06 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:33:32 +0200 Subject: [PATCH 081/270] Count complete readable populations and register read boundaries --- .../src/alicebot_api/routers/_vnext_shared.py | 49 +++--- .../src/alicebot_api/routers/workspaces.py | 51 ++---- apps/api/src/alicebot_api/sqlite_store.py | 44 +++++ apps/api/src/alicebot_api/vnext_dogfooding.py | 12 +- .../api/src/alicebot_api/vnext_label_guard.py | 97 ++++++----- apps/api/src/alicebot_api/vnext_store.py | 67 ++++++++ ...derived_labels_read_acceptance_postgres.py | 132 ++++++++++++++ tests/unit/test_complete_readable_counts.py | 162 ++++++++++++++++++ tests/unit/test_label_door_registry.py | 92 +++++++++- 9 files changed, 592 insertions(+), 114 deletions(-) create mode 100644 tests/integration/test_derived_labels_read_acceptance_postgres.py create mode 100644 tests/unit/test_complete_readable_counts.py diff --git a/apps/api/src/alicebot_api/routers/_vnext_shared.py b/apps/api/src/alicebot_api/routers/_vnext_shared.py index f0f0a8bfa..99fddf907 100644 --- a/apps/api/src/alicebot_api/routers/_vnext_shared.py +++ b/apps/api/src/alicebot_api/routers/_vnext_shared.py @@ -280,6 +280,21 @@ def _vnext_source_trace( } +def _vnext_readable_trace_rows(store, kind, fetch, identity, *, admit=None): + """Deepen the prefix until readable truncation can be answered.""" + + from alicebot_api.vnext_label_guard import apply_sensitivity_ceiling + + limit = _VNEXT_SOURCE_TRACE_COLLECTION_LIMIT + prefix = limit + 1 + while True: + fetched = list(fetch(prefix)) + admitted = list(admit(fetched)) if admit is not None else apply_sensitivity_ceiling(store, kind=kind, rows=fetched, identity=identity) + if len(admitted) > limit or len(fetched) < prefix: + return _vnext_bounded_trace_rows(admitted) + prefix *= 2 + + def _vnext_load_source_trace( *, store: PostgresVNextStore, @@ -299,39 +314,27 @@ def _vnext_load_source_trace( if not apply_sensitivity_ceiling(store, kind="source", rows=[source], identity=caller): return None source_id = str(source["id"]) - memories, memories_complete = _vnext_bounded_trace_rows( - store.list_memories_referencing_source( - source_id=source_id, - limit=_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT + 1, - ) + memories, memories_complete = _vnext_readable_trace_rows( + store, "memory", lambda limit: store.list_memories_referencing_source(source_id=source_id, limit=limit), caller ) - artifacts, artifacts_complete = _vnext_bounded_trace_rows( - store.list_artifacts_referencing_source( - source_id=source_id, - limit=_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT + 1, - ) + artifacts, artifacts_complete = _vnext_readable_trace_rows( + store, "artifact", lambda limit: store.list_artifacts_referencing_source(source_id=source_id, limit=limit), caller ) - open_loops, open_loops_complete = _vnext_bounded_trace_rows( - store.list_open_loops_referencing_source( - source_id=source_id, - limit=_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT + 1, - ) + open_loops, open_loops_complete = _vnext_readable_trace_rows( + store, "open_loop", lambda limit: store.list_open_loops_referencing_source(source_id=source_id, limit=limit), caller ) - memories = apply_sensitivity_ceiling(store, kind="memory", rows=memories, identity=caller) - artifacts = apply_sensitivity_ceiling(store, kind="artifact", rows=artifacts, identity=caller) - open_loops = apply_sensitivity_ceiling(store, kind="open_loop", rows=open_loops, identity=caller) kept_ids = {str(row.get("id")) for row in (*memories, *artifacts, *open_loops)} kept_ids.add(source_id) - events, direct_events_complete = _vnext_bounded_trace_rows( - store.list_events_for_source_trace( + events, direct_events_complete = _vnext_readable_trace_rows( + store, "event", lambda limit: store.list_events_for_source_trace( source_id=source_id, memory_ids=[str(memory["id"]) for memory in memories], artifact_ids=[str(artifact["id"]) for artifact in artifacts], open_loop_ids=[str(open_loop["id"]) for open_loop in open_loops], - limit=_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT + 1, - ) + limit=limit, + ), caller, + admit=lambda rows: [event for event in rows if str(event.get("target_id") or "") in kept_ids], ) - events = [event for event in events if str(event.get("target_id") or "") in kept_ids] events_complete = direct_events_complete and memories_complete and artifacts_complete and open_loops_complete return _vnext_source_trace( store=store, diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index 1ae7c314a..c378393e5 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -85,13 +85,14 @@ def _workspace_rows(store: PostgresVNextStore, kind: str, rows: Sequence[Mapping def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: - from alicebot_api.vnext_label_guard import LabelGuard, readable_count, readable_status_counts + from alicebot_api.vnext_label_guard import LabelGuard sensitivity_allowed = ["public", "internal", "private", "unknown"] + guard = LabelGuard.for_filters(store, (), sensitivity_allowed, ()) review_statuses = ["candidate", "needs_review", "private_only", "accepted", "rejected"] fetched_sources = store.list_sources(sensitivity_allowed=sensitivity_allowed, limit=20) sources = _workspace_rows(store, "source", fetched_sources, sensitivity_allowed) - source_count = readable_count(store.count_sources(), len(fetched_sources), len(sources)) + source_count = sum(guard.readable_status_counts("source").values()) list_memories_by_statuses = getattr(store, "list_memories_by_statuses", None) if callable(list_memories_by_statuses): fetched_memories = list_memories_by_statuses( @@ -104,53 +105,31 @@ def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: memory for memory in store.list_memories(status=None) if str(memory.get("status")) in set(review_statuses) ][:30] review_memories = _workspace_rows(store, "memory", fetched_memories, sensitivity_allowed) - count_memories_by_status = getattr(store, "count_memories_by_status", None) - memory_status_counts = ( - count_memories_by_status(sensitivity_allowed=sensitivity_allowed) - if callable(count_memories_by_status) - else _vnext_status_counts(fetched_memories) - ) - memory_status_counts = readable_status_counts(memory_status_counts, fetched_memories, review_memories) + memory_status_counts = guard.readable_status_counts("memory") review_memory_total = sum(memory_status_counts.get(status, 0) for status in review_statuses) fetched_artifacts = store.list_artifacts(sensitivity_allowed=sensitivity_allowed, limit=30) artifacts = _workspace_rows(store, "artifact", fetched_artifacts, sensitivity_allowed) - artifact_count = readable_count(store.count_artifacts(), len(fetched_artifacts), len(artifacts)) - artifact_status_counts = readable_status_counts( - store.count_artifacts_by_status(), - fetched_artifacts, - artifacts, + artifact_status_counts = guard.readable_status_counts("artifact") + artifact_count = sum(artifact_status_counts.values()) + quality_evals = guard.admit_related_rows(store.list_artifact_quality_ratings(limit=50), kind="artifact", field="artifact_id") + quality_eval_count = sum( + len(guard.admit_related_rows(batch, kind="artifact", field="artifact_id")) + for batch in store.iter_label_ratings() ) - quality_evals = store.list_artifact_quality_ratings(limit=50) - quality_eval_count = store.count_artifact_quality_ratings() fetched_projects = store.list_projects(status=None, sensitivity_allowed=sensitivity_allowed, limit=20) projects = _workspace_rows(store, "project", fetched_projects, sensitivity_allowed) - project_count = readable_count(store.count_projects(), len(fetched_projects), len(projects)) + project_count = sum(guard.readable_status_counts("project").values()) fetched_loops = store.list_open_loops(status=None, sensitivity_allowed=sensitivity_allowed, limit=30) open_loops = _workspace_rows(store, "open_loop", fetched_loops, sensitivity_allowed) - open_loop_status_counts = readable_status_counts( - store.count_open_loops_by_status(), - fetched_loops, - open_loops, - ) - stored_open_loop_count = store.count_open_loops(status="open") - open_loop_count = ( - stored_open_loop_count - if len(fetched_loops) == len(open_loops) - else int(open_loop_status_counts.get("open", stored_open_loop_count)) - ) + open_loop_status_counts = guard.readable_status_counts("open_loop") + open_loop_count = open_loop_status_counts.get("open", 0) people = store.list_people(sensitivity_allowed=sensitivity_allowed, limit=12) fetched_beliefs = store.list_beliefs(status=None, sensitivity_allowed=sensitivity_allowed, limit=12) beliefs = LabelGuard.for_filters(store, (), sensitivity_allowed, ()).admit_beliefs(fetched_beliefs) tasks = store.list_tasks(status=None, limit=12) fetched_events = store.list_events(limit=20) - recent_events = [ - event - for event in fetched_events - if _workspace_event_visible(store, event, sensitivity_allowed) - ] - count_events = getattr(store, "count_events", None) - stored_event_count = count_events() if callable(count_events) else len(fetched_events) - event_count = readable_count(stored_event_count, len(fetched_events), len(recent_events)) + recent_events = guard.admit_events(fetched_events) + event_count = guard.readable_event_count() agent_identities = store.list_agent_identities(limit=20) agent_count = store.count_agent_identities() agent_events = store.list_agent_events(limit=50) diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index 05d3be249..b43d456aa 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -443,6 +443,50 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: (self.user_id, *wanted, *canonical), ) + def iter_label_rows(self, kind: str, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + """Complete counted population, in narrow tenant-bound keyset batches.""" + + if batch_size < 1: + raise ValueError("batch_size must be positive") + table = {"source": "sources", "memory": "memories", "open_loop": "open_loops"}.get(kind) + if table is None: + raise ValueError("unsupported label kind") + extra = "" + if kind == "memory": + extra = ", status, value, project_id, source_event_ids, deleted_at" + elif kind == "open_loop": + extra = ", status, project_id, source_id, memory_id" + live = " AND deleted_at IS NULL" if kind in {"source", "memory"} else "" + after = "" + while True: + rows = self._fetch_all( + f"""SELECT id, user_id, domain, sensitivity, metadata_json{extra} + FROM {table} WHERE user_id = ? AND id > ?{live} + ORDER BY id LIMIT ?""", + (self.user_id, after, batch_size), + ) + if not rows: + return + yield rows + after = str(rows[-1]["id"]) + + def iter_label_events(self, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + """Complete event targets for readable counts, without event payloads.""" + + if batch_size < 1: + raise ValueError("batch_size must be positive") + after = "" + while True: + rows = self._fetch_all( + """SELECT id, target_type, target_id, event_type FROM event_log + WHERE user_id = ? AND id > ? ORDER BY id LIMIT ?""", + (self.user_id, after, batch_size), + ) + if not rows: + return + yield rows + after = str(rows[-1]["id"]) + # -- fetch helpers (mirror PostgresVNextStore conventions) ------------ def _execute(self, query: str, params: tuple[object, ...] = ()) -> sqlite3.Cursor: diff --git a/apps/api/src/alicebot_api/vnext_dogfooding.py b/apps/api/src/alicebot_api/vnext_dogfooding.py index 8620af6e9..5a14906ed 100644 --- a/apps/api/src/alicebot_api/vnext_dogfooding.py +++ b/apps/api/src/alicebot_api/vnext_dogfooding.py @@ -172,24 +172,20 @@ def __init__(self, store: VNextDogfoodingStore) -> None: def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None) -> JsonObject: from alicebot_api.vnext_agent_control import ALL_SENSITIVITY - from alicebot_api.vnext_label_guard import admit_loaded, readable_status_counts + from alicebot_api.vnext_label_guard import LabelGuard, admit_loaded sources = self.store.list_sources(limit=500) try: memories = self.store.list_memories(status=None, limit=500) except TypeError: # Compatibility for external/test stores on the old protocol. memories = self.store.list_memories(status=None)[:500] - count_memories_by_status = getattr(self.store, "count_memories_by_status", None) - memory_status_counts = ( - count_memories_by_status() if callable(count_memories_by_status) else _status_counts(memories) - ) artifacts = self.store.list_artifacts(limit=500) ratings = self.store.list_artifact_quality_ratings(limit=500) open_loops = self.store.list_open_loops(status=None, limit=500) # None is the owner and an admin key: every sensitivity, so the guard # reads nothing and the lists stay as the store returned them. ceiling = sensitivity_allowed if sensitivity_allowed is not None else ALL_SENSITIVITY - fetched_memories = memories + guard = LabelGuard.for_filters(self.store, (), ceiling, ()) sources = admit_loaded(self.store, kind="source", rows=sources, domains=(), sensitivity_allowed=ceiling, projects=()) memories = admit_loaded( self.store, kind="memory", rows=memories, domains=(), sensitivity_allowed=ceiling, projects=() @@ -200,11 +196,13 @@ def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None) -> Js open_loops = admit_loaded( self.store, kind="open_loop", rows=open_loops, domains=(), sensitivity_allowed=ceiling, projects=() ) - memory_status_counts = readable_status_counts(memory_status_counts, fetched_memories, memories) + memory_status_counts = guard.readable_status_counts("memory") try: events = self.store.list_events(limit=5_000) except TypeError: # Compatibility for external/test stores on the old protocol. events = self.store.list_events()[:5_000] + events = guard.admit_events(events) + ratings = guard.admit_related_rows(ratings, kind="artifact", field="artifact_id") scheduler_runs = self.store.list_scheduler_runs(limit=20) now = datetime.now(UTC) today_cutoff = now.replace(hour=0, minute=0, second=0, microsecond=0) diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index ec57e7ebe..a9e4de7db 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -129,10 +129,6 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: if not self.active: return [row for row in rows if isinstance(row, Mapping)] - # Compatibility stores without an ancestry reader may retain SQL-filtered - # rows only when no locked all-of binding needs verification. - if not callable(getattr(self.store, "read_label_rows", None)): - return [] if self.all_of is not None else [row for row in rows if isinstance(row, Mapping)] kept: list[_Row] = [] for row in rows: if not isinstance(row, Mapping): @@ -142,6 +138,61 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: kept.append(row) return kept + def readable_status_counts(self, kind: str) -> dict[str, int]: + """Count the complete population through the same effective admission. + + Display pages and stored-label SQL counts cannot establish this total. + Stores must expose the complete narrow population rather than guessing + a restricted total from a sample. + """ + + iterator = getattr(self.store, "iter_label_rows", None) + if not callable(iterator): + raise TypeError("readable counts require complete label enumeration") + counts: dict[str, int] = {} + for batch in iterator(kind): + for row in self.admit_rows(kind, batch): + status = str(row.get("status", "unknown")) + counts[status] = counts.get(status, 0) + 1 + return counts + + def admit_related_rows(self, rows: Sequence[_Row], *, kind: str, field: str) -> list[_Row]: + """Admit an event or rating by its target's current effective label.""" + + if not self.active: + return [row for row in rows if isinstance(row, Mapping)] + reader = getattr(self.store, "read_label_rows", None) + if not callable(reader): + return [] + ids = list(dict.fromkeys(str(row.get(field)) for row in rows if row.get(field))) + found = list(reader(kind, ids)) if ids else [] + admitted = {str(row.get("id")) for row in ( + self.admit_beliefs(found) if kind == "belief" else self.admit_rows(kind, found) + )} + return [row for row in rows if str(row.get(field) or "") in admitted] + + def admit_events(self, rows: Sequence[_Row]) -> list[_Row]: + """Known label targets are admitted before an event exposes their IDs.""" + + if not self.active: + return [row for row in rows if isinstance(row, Mapping)] + admitted: set[int] = set() + kinds = {"source", "memory", "open_loop", "artifact", "project", "belief"} + for kind in kinds: + targets = [row for row in rows if str(row.get("target_type")) == kind] + admitted.update(id(row) for row in self.admit_related_rows(targets, kind=kind, field="target_id")) + return [row for row in rows if id(row) in admitted or ( + str(row.get("target_type")) not in kinds and not str(row.get("event_type", "")).endswith(".labels_raised") + )] + + def readable_event_count(self) -> int: + """Complete event count after current target admission.""" + + iterator = getattr(self.store, "iter_label_events", None) + if not callable(iterator): + raise TypeError("readable counts require complete event enumeration") + return sum(len(self.admit_events(batch)) for batch in iterator()) + def admit_beliefs(self, beliefs: Sequence[_Row]) -> list[_Row]: """Beliefs whose backing memory the filters admit. One batched read.""" @@ -279,44 +330,6 @@ def apply_sensitivity_ceiling( ) -def readable_count(sql_count: int, fetched: int, admitted: int) -> int: - """A stored count, reduced only by rows this page's guard dropped. - - When the guard drops nothing the stored count is returned unchanged. - When the fetched page is the whole set, the count is the admitted length. - """ - - dropped = fetched - admitted - if dropped <= 0: - return sql_count - if sql_count <= fetched: - return admitted - return max(0, sql_count - dropped) - - -def readable_status_counts( - counts: Mapping[str, int], - fetched: Sequence[Mapping[str, object]], - admitted: Sequence[Mapping[str, object]], - *, - field: str = "status", -) -> dict[str, int]: - """Status counts with one taken off for each row the guard dropped.""" - - if len(fetched) == len(admitted): - return dict(counts) - kept = {id(row) for row in admitted} - updated = dict(counts) - for row in fetched: - if id(row) in kept: - continue - status = str(row.get(field, "unknown")) - current = int(updated.get(status, 0)) - if current > 0: - updated[status] = current - 1 - return updated - - def apply_unverified_rule( decision: PolicyDecision, row: Mapping[str, object] | None, diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 94292aa91..654dadaa9 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -530,6 +530,73 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: (wanted,), ) + def iter_label_rows(self, kind: str, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + """Complete counted population, in narrow keyset batches under tenant RLS.""" + + if batch_size < 1: + raise ValueError("batch_size must be positive") + table = {"source": "sources", "memory": "memories", "open_loop": "open_loops", + "artifact": "generated_artifacts", "project": "projects"}.get(kind) + if table is None: + raise ValueError("unsupported label kind") + extra = "" + if kind == "memory": + extra = ", status, value, project_id, source_event_ids, deleted_at" + elif kind == "open_loop": + extra = ", status, project_id, source_id, memory_id" + elif kind == "artifact": + extra = ", status, artifact_type" + elif kind == "project": + extra = ", status" + live = " AND deleted_at IS NULL" if kind in {"source", "memory"} else "" + after: str | None = None + while True: + rows = self._fetch_all( + f"""SELECT id, user_id, domain, sensitivity, metadata_json{extra} + FROM {table} + WHERE (%s::uuid IS NULL OR id > %s::uuid){live} + ORDER BY id LIMIT %s""", + (after, after, batch_size), + ) + if not rows: + return + yield rows + after = str(rows[-1]["id"]) + + def iter_label_events(self, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + """Complete event targets for readable counts, without event payloads.""" + + if batch_size < 1: + raise ValueError("batch_size must be positive") + after: str | None = None + while True: + rows = self._fetch_all( + """SELECT id, target_type, target_id, event_type FROM event_log + WHERE (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s""", + (after, after, batch_size), + ) + if not rows: + return + yield rows + after = str(rows[-1]["id"]) + + def iter_label_ratings(self, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + """Complete rating targets for counts, without feedback text.""" + + if batch_size < 1: + raise ValueError("batch_size must be positive") + after: str | None = None + while True: + rows = self._fetch_all( + """SELECT id, artifact_id FROM artifact_quality_ratings + WHERE (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s""", + (after, after, batch_size), + ) + if not rows: + return + yield rows + after = str(rows[-1]["id"]) + def _fetch_one( self, operation_name: str, diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py new file mode 100644 index 000000000..b52c7074d --- /dev/null +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -0,0 +1,132 @@ +"""Real keys, operator screens and complete readable counts on PostgreSQL.""" + +from __future__ import annotations + +import json +from uuid import UUID, uuid4 + +import pytest + +from alicebot_api.config import Settings +from alicebot_api.db import user_connection +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError +from alicebot_api.routers import vnext_review, vnext_retrieval, vnext_projects, vnext_memories, workspaces +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_agent_control import ALL_SENSITIVITY +from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_store import PostgresVNextStore +from tests.unit.test_derived_labels_real_keys import READERS, expected_read, real_reader_key, seed_read_rows + + +def _user(app_url): + user_id = uuid4() + with user_connection(app_url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, "synthetic@example.invalid", "Synthetic") + return user_id + + +@pytest.mark.parametrize("reader", READERS) +def test_postgres_real_key_core_doors(migrated_database_urls, monkeypatch, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + monkeypatch.setattr(vnext_memories, "get_settings", lambda: Settings(database_url=app_url)) + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + rows = seed_read_rows(store) + key = real_reader_key(store, user_id, reader) + if key: + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + context = MCPRuntimeContext(database_url=app_url, user_id=user_id) + for state, row in rows.items(): + admitted = expected_read(reader, state) + audit = vnext_memories.get_vnext_memory_audit(UUID(str(row["id"])), user_id, authorization=f"Bearer {key}" if key else None) + assert audit.status_code == (200 if admitted else 403), (reader, state, audit.body) + if not admitted: + assert str(row["id"]) not in audit.body.decode() + for tool in ("alice_recall", "alice_context_pack", "alice_recent_decisions", "alice_explain", "alice_resume"): + arguments = {"memory_id": str(row["id"])} if tool == "alice_explain" else {"query": str(row["canonical_text"]), "sensitivity_allowed": list(ALL_SENSITIVITY)} + try: + result = call_mcp_tool(context, name=tool, arguments=arguments) + except MCPToolError: + assert not admitted, (reader, state, tool) + continue + rendered = json.dumps(result, default=str) + assert (str(row["id"]) in rendered) is admitted, (reader, state, tool, rendered) + if not admitted: + assert str(row["title"]) not in rendered + + +@pytest.mark.parametrize("reader", ("owner", "admin", "trusted")) +def test_all_five_operator_screens_with_real_keys(migrated_database_urls, monkeypatch, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + for module in (vnext_review, vnext_retrieval, vnext_projects): + monkeypatch.setattr(module, "get_settings", lambda: Settings(database_url=app_url)) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Cedar hidden source", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential"}) + public_source = store.create_source({"source_type": "note", "title": "Public source", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public"}) + memory = store.create_memory({"memory_key": "belief", "canonical_text": "Cedar hidden belief", "status": "active", "domain": "project", "sensitivity": "confidential"}) + artifact = store.create_artifact({"artifact_type": "daily_brief", "title": "Cedar hidden artifact", "content_markdown": "Cedar hidden artifact", "domain": "project", "sensitivity": "confidential", "metadata_json": {"derived_from": {"v": 1, "sources": [str(public_source["id"])], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}}, "source_refs": [str(public_source["id"])]}}) + project = store.create_project({"name": "Cedar hidden project", "slug": "cedar-hidden", "current_state": "Cedar hidden state", "domain": "project", "sensitivity": "confidential"}) + belief_id = uuid4() + conn.execute("INSERT INTO beliefs(id,user_id,memory_id,claim) VALUES (%s,%s,%s,%s)", (belief_id, user_id, memory["id"], "Cedar hidden belief")) + key = real_reader_key(store, user_id, reader) + auth = f"Bearer {key}" if key else None + responses = ( + (vnext_review.list_vnext_artifacts(user_id, authorization=auth), str(artifact["id"])), + (vnext_retrieval.get_vnext_source_trace(UUID(str(source["id"])), user_id, authorization=auth), str(source["id"])), + (vnext_projects.list_vnext_projects(user_id, authorization=auth), str(project["id"])), + (vnext_projects.get_vnext_project_dashboard(str(project["id"]), user_id, authorization=auth), str(project["id"])), + (vnext_review.get_vnext_belief_state(str(belief_id), user_id, authorization=auth), str(belief_id)), + ) + for response, identifier in responses: + body = response.body.decode() + if reader == "trusted": + assert identifier not in body + assert "Cedar hidden" not in body + assert response.status_code in {200, 404} + if response.status_code == 200: + assert json.loads(body)["count"] == 0 + else: + assert response.status_code == 200, body + assert identifier in body + # A visible source trace must not carry a hidden artifact or its count. + trace = vnext_retrieval.get_vnext_source_trace(UUID(str(public_source["id"])), user_id, authorization=auth) + body = json.loads(trace.body) + assert body["summary"]["artifact_count"] == (0 if reader == "trusted" else 1) + assert (str(artifact["id"]) in trace.body.decode()) is (reader != "trusted") + + +def test_workspace_counts_full_population_with_sql_hidden_and_stale_rows(migrated_database_urls, monkeypatch): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + # The doctor has its own store-specific repair acceptance test. Keep this + # count probe independent of that ancillary diagnostic implementation. + monkeypatch.setattr(workspaces.VNextDoctorService, "run", lambda self, **kwargs: {}) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + secret = store.create_source({"source_type": "note", "title": "Cedar hidden", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential"}) + for index in range(205): + store.create_source({"source_type": "note", "title": "Public source", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public"}) + for index in range(35): + store.create_memory({"memory_key": f"visible-{index}", "canonical_text": "Public fact", "status": "candidate", "domain": "project", "sensitivity": "public"}) + with without_insert_floor(): + for index in range(40): + store.create_memory({"memory_key": f"hidden-{index}", "canonical_text": "Cedar hidden", "status": "candidate", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(secret["id"])}}) + store.create_artifact({"artifact_type": "daily_brief", "title": "Cedar hidden", "content_markdown": "Cedar hidden", "domain": "project", "sensitivity": "confidential", "metadata_json": {"derived_from": {"v": 1, "sources": [], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": 0, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}}}}) + store.create_project({"name": "Cedar hidden", "slug": "hidden", "domain": "project", "sensitivity": "confidential"}) + body = workspaces._vnext_workspace_payload(store) + assert body["summary"]["source_count"] == 205 + assert body["summary"]["candidate_memory_count"] == 35 + assert body["summary"]["artifact_count"] == 0 + assert body["summary"]["project_count"] == 0 + assert body["samples"]["sources"]["has_more"] is True + assert "Cedar hidden" not in json.dumps(body, default=str) + batches = list(store.iter_label_rows("source", batch_size=100)) + assert [len(batch) for batch in batches] == [100, 100, 6] + assert all("content_markdown" not in row and "title" not in row for batch in batches for row in batch) diff --git a/tests/unit/test_complete_readable_counts.py b/tests/unit/test_complete_readable_counts.py new file mode 100644 index 000000000..97eb1bbea --- /dev/null +++ b/tests/unit/test_complete_readable_counts.py @@ -0,0 +1,162 @@ +"""Totals and trace completeness use the full effectively readable population.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from alicebot_api.routers import _vnext_shared, workspaces +from alicebot_api.vnext_agent_control import ALL_SENSITIVITY, AgentIdentity +from alicebot_api.vnext_dogfooding import VNextDogfoodingService +from alicebot_api.vnext_label_guard import LabelGuard + + +class PopulationStore: + def __init__(self): + self.rows = {kind: [] for kind in ("source", "memory", "artifact", "project", "open_loop")} + self.events = [] + + def read_label_rows(self, kind, ids): + return [row for row in self.rows[kind] if row["id"] in ids] + + def iter_label_rows(self, kind, *, batch_size=200): + for start in range(0, len(self.rows[kind]), batch_size): + yield self.rows[kind][start:start + batch_size] + + def iter_label_events(self, *, batch_size=200): + for start in range(0, len(self.events), batch_size): + yield self.events[start:start + batch_size] + + def iter_label_ratings(self): + return iter(()) + + def list_memories(self, *, status=None, limit=None, **kwargs): + return self.rows["memory"][:limit] + + def list_memories_by_statuses(self, *, statuses, sensitivity_allowed, limit): + return [row for row in self.rows["memory"] if row["status"] in statuses and row["sensitivity"] in sensitivity_allowed][:limit] + + def count_memories_by_status(self, **kwargs): + return {"candidate": len(self.rows["memory"])} + + def list_events(self, **kwargs): + return self.events[:kwargs.get("limit")] + + def __getattr__(self, name): + kinds = {"sources": "source", "artifacts": "artifact", "projects": "project", "open_loops": "open_loop"} + suffix = name.removeprefix("list_") + if suffix in kinds: + def listed(**kwargs): + rows = self.rows[kinds[suffix]] + sensitivities = kwargs.get("sensitivity_allowed") + if sensitivities: + rows = [row for row in rows if row["sensitivity"] in sensitivities] + return rows[:kwargs.get("limit")] + return listed + if name.startswith("list_"): + return lambda *args, **kwargs: [] + if name.startswith("count_"): + return lambda **kwargs: len(self.rows[kinds[name.removeprefix("count_")]]) if name.removeprefix("count_") in kinds else 0 + if name == "get_brain_charter": + return lambda: {} + raise AttributeError(name) + + +def _row(identifier, sensitivity="public", **kwargs): + return {"id": identifier, "domain": "project", "sensitivity": sensitivity, "status": "candidate", "metadata_json": {}, **kwargs} + + +def _quiet_services(monkeypatch): + for name in ("VNextSchedulerService", "VNextConnectorService", "VNextDoctorService", "VNextProjectService", "VNextMemoryCommitService"): + monkeypatch.setattr(workspaces, name, lambda store: SimpleNamespace( + status=lambda: {}, connector_health_all=lambda: [], run=lambda **kwargs: {}, + project_dashboard=lambda **kwargs: {}, recent_commits=lambda **kwargs: {"recent_commits": []}, + inline_confirmations=lambda **kwargs: [])) + monkeypatch.setattr(workspaces, "daemon_status", lambda: {}) + monkeypatch.setattr("alicebot_api.vnext_dogfooding.VNextConnectorService.connector_health_all", lambda self: []) + + +def test_workspace_counts_sql_hidden_and_beyond_display_page(monkeypatch): + _quiet_services(monkeypatch) + store = PopulationStore() + for kind in store.rows: + store.rows[kind] = [_row(f"{kind}-{index}") for index in range(35)] + store.rows[kind] += [_row(f"{kind}-hidden-{index}", "confidential") for index in range(205)] + store.rows["open_loop"] = [{**row, "status": "open"} for row in store.rows["open_loop"]] + store.events = [{"id": str(index), "target_type": "memory", "target_id": row["id"], "event_type": "memory.labels_raised"} for index, row in enumerate(store.rows["memory"])] + body = workspaces._vnext_workspace_payload(store) + summary = body["summary"] + for field in ("source_count", "artifact_count", "project_count", "open_loop_count", "event_count", "candidate_memory_count"): + assert summary[field] == 35, (field, summary[field]) + assert summary["memory_status_counts"] == {"candidate": 35} + assert summary["artifact_status_counts"] == {"candidate": 35} + assert summary["open_loop_status_counts"] == {"open": 35} + assert body["samples"]["sources"]["has_more"] is True + assert "hidden" not in str(body) + + +def test_all_sql_prefiltered_rows_leave_zero_totals(monkeypatch): + _quiet_services(monkeypatch) + store = PopulationStore() + for kind in store.rows: + store.rows[kind] = [_row(f"{kind}-hidden", "confidential")] + body = workspaces._vnext_workspace_payload(store) + for field in ("source_count", "artifact_count", "project_count", "open_loop_count", "candidate_memory_count"): + assert body["summary"][field] == 0 + assert all(not sample["has_more"] for sample in body["samples"].values()) + + +def test_dogfooding_counts_hidden_rows_beyond_500(monkeypatch): + _quiet_services(monkeypatch) + store = PopulationStore() + store.rows["memory"] = [_row(str(index)) for index in range(500)] + [_row("hidden", "confidential")] + trusted = VNextDogfoodingService(store).dashboard(sensitivity_allowed=("public", "internal", "private", "unknown")) + owner = VNextDogfoodingService(store).dashboard() + assert trusted["memory_status_counts"] == {"candidate": 500} + assert trusted["sample_scope"]["memories"]["total_count"] == 500 + assert owner["memory_status_counts"] == {"candidate": 501} + + +def test_count_rejects_a_store_without_complete_enumeration(): + with pytest.raises(TypeError, match="complete label enumeration"): + LabelGuard.for_filters(object(), (), ("public",)).readable_status_counts("memory") + + +def test_derived_totals_use_effective_labels_and_keep_owner_control(): + store = PopulationStore() + store.rows["source"] = [_row("11111111-1111-4111-8111-111111111111", "confidential")] + store.rows["memory"] = [_row("22222222-2222-4222-8222-222222222222", metadata_json={"source_id": store.rows["source"][0]["id"]})] + trusted = LabelGuard.for_filters(store, (), ("public", "internal", "private", "unknown")) + owner = LabelGuard.for_filters(store, (), ALL_SENSITIVITY) + assert trusted.readable_status_counts("memory") == {} + assert owner.readable_status_counts("memory") == {"candidate": 1} + + +def test_trace_completeness_does_not_reveal_hidden_501st_row(monkeypatch): + monkeypatch.setattr(_vnext_shared, "_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT", 2) + store = PopulationStore() + rows = [_row(str(index)) for index in range(2)] + [_row("hidden", "confidential")] + fetches = [] + def fetch(limit): + fetches.append(limit) + return rows[:limit] + trusted = AgentIdentity(agent_id="reader", permission_profile="trusted_local_agent", agent_type="unknown") + admitted, complete = _vnext_shared._vnext_readable_trace_rows(store, "memory", fetch, trusted) + assert [row["id"] for row in admitted] == ["0", "1"] + assert complete is True + assert fetches == [3, 6] + owner, complete = _vnext_shared._vnext_readable_trace_rows(store, "memory", fetch, None) + assert len(owner) == 2 + assert complete is False + + +def test_trace_event_completeness_uses_admitted_targets(monkeypatch): + monkeypatch.setattr(_vnext_shared, "_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT", 2) + events = [{"id": str(index), "target_id": "visible"} for index in range(2)] + [{"id": "hidden-event", "target_id": "hidden"}] + admitted, complete = _vnext_shared._vnext_readable_trace_rows( + PopulationStore(), "event", lambda limit: events[:limit], None, + admit=lambda rows: [row for row in rows if row["target_id"] == "visible"], + ) + assert [row["id"] for row in admitted] == ["0", "1"] + assert complete is True diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py index f04a8be44..e82773f3e 100644 --- a/tests/unit/test_label_door_registry.py +++ b/tests/unit/test_label_door_registry.py @@ -9,6 +9,7 @@ SRC = ROOT / "apps/api/src" READS = { + "get_source", "list_sources", "get_sources_by_ids", "get_memory", "get_memory_for_update", "get_memories_by_ids", @@ -31,6 +32,10 @@ "list_projects", "get_project", "get_project_for_update", + "list_open_loops_referencing_source", "list_events", "list_memory_events", "list_open_loop_events", + "list_events_for_source_trace", "list_recent_agentic_commits", "list_pending_inline_confirmations", + "count_sources", "count_artifacts", "count_artifacts_by_status", "count_projects", "count_memories_by_status", + "count_open_loops", "count_open_loops_by_status", "count_events", "iter_label_rows", "iter_label_events", "iter_label_ratings", } GUARD_CALLS = { @@ -43,6 +48,7 @@ "admit_loaded", "apply_sensitivity_ceiling", "sensitivity_ceiling", + "admit_events", "admit_related_rows", "readable_status_counts", "readable_event_count", } # function -> helper that holds the guard call, or None when the function calls it @@ -57,6 +63,7 @@ "mcp/review.py:_vnext_memory_review": None, "mcp/review.py:_vnext_memory_correct": None, "routers/vnext_memories.py:review_vnext_memory": None, + "routers/vnext_memories.py:get_vnext_memory_audit": None, "mcp/memories.py:redact_memory_flow": None, "routers/vnext_projects.py:review_vnext_open_loop": None, "mcp/retrieval.py:_handle_alice_open_loops": None, @@ -91,6 +98,18 @@ "vnext_retrieval.py:VNextRetrievalService._contradicting_evidence": None, "vnext_retrieval.py:VNextRetrievalService._recent_changes": None, "vnext_retrieval.py:VNextRetrievalService.memory_visibility": None, + "vnext_brain.py:VNextBrainService._load_inputs": None, + "vnext_connections.py:VNextConnectionService.generate_connection_report": None, + "vnext_contradictions.py:VNextContradictionService.generate_contradiction_report": None, + "vnext_consolidation.py:VNextConsolidationService._cluster_memories": None, + "vnext_consolidation.py:VNextConsolidationService.generate_memory_consolidation": None, + "vnext_rollups.py:VNextRollupService._collect_rows": None, + "vnext_rollups.py:VNextRollupService._existing_rollup_state": None, + "vnext_scheduler.py:VNextSchedulerService._run_staleness_sweep": None, + "vnext_scheduler.py:VNextSchedulerService._generate_open_loop_review_artifact": None, + "vnext_context_tree.py:_tree_event_visible": None, + "routers/vnext_retrieval.py:get_vnext_source_trace": "routers/_vnext_shared.py:_vnext_load_source_trace", + "routers/vnext_retrieval.py:get_vnext_artifact_trace": "routers/_vnext_shared.py:_vnext_authorized_artifact", } NOT_A_DOOR = { @@ -115,9 +134,40 @@ "vnext_queue.py:VNextQueueService.review_artifact": "the HTTP review route authorizes through _vnext_authorized_artifact first", "vnext_queue.py:VNextQueueService._promote_artifact": "the HTTP review route authorizes through _vnext_authorized_artifact first", "vnext_queue.py:VNextQueueService.export_artifact_markdown": "the HTTP export route authorizes through _vnext_authorized_artifact first", + "mcp/evidence_artifacts.py:_authorize_memory_audit_provenance": "original source pointers use SourceReadFence.admits before disclosure", + "routers/vnext_memories.py:get_vnext_source": "original source operator route; existing domain and project exemptions are preserved", + "routers/vnext_memories.py:get_vnext_connector_status": "operator connector telemetry; original-source labels retain existing behavior", + "routers/vnext_memories.py:review_vnext_source": "write path over an original source; existing exact policy applies", + "routers/vnext_memories.py:delete_vnext_source": "owner mutation of an original source", + "vnext_memory_commit.py:VNextMemoryCommitService.auto_promoted_by_agent": "write sweep; every target is authorized by expire before mutation", + "vnext_memory_commit.py:VNextMemoryCommitService._transition_memory": "writer checks source validity; no new read response", + "vnext_source_fence.py:_rows_by_id": "narrow loader; SavedProvenanceReader applies effective labels before presenting", + "vnext_source_fence.py:source_rows_including_archived": "original-source loader for provenance and producer label computation", + "vnext_source_fence.py:SavedProvenanceReader._row_for": "private loader; SavedProvenanceReader._admits settles each row", + "vnext_retrieval.py:_current_memory_id": "pointer loader; caller memory_visibility settles before exposing the pointer", + "vnext_retrieval.py:_memories_referencing_sources": "internal loader; expand_provenance_once applies effective admission", + "vnext_retrieval.py:VNextRetrievalService._sources_by_ids": "original-source lookup; SourceReadFence admits before formatting", + "vnext_retrieval.py:VNextRetrievalService._query_embedding": "store capability discovery only; does not execute a row reader", + "vnext_retrieval.py:VNextRetrievalService._source_stage_lists": "original-source stage; existing source admission remains", + "vnext_retrieval.py:VNextRetrievalService._supersession_context": "pointer metadata is fenced through memory_visibility at output", + "session_briefing.py:_merge_recent_change_targets": "private loader checks _event_target_honours_fence before adding a target", + "session_briefing.py:_resolve_excerpt_query": "original source lookup; source fence and withheld event targets constrain excerpts", + "vnext_context_tree.py:VNextContextTreeService._scoped_events": "build_tree filters every resulting event through _tree_event_visible", + "vnext_dogfooding.py:VNextDogfoodingService.record_insight_feedback": "write route authorizes artifact through _vnext_authorized_artifact", + "vnext_contradictions.py:_project_scoped_beliefs": "internal loader; generate_contradiction_report admits through backing memories", + "vnext_consolidation.py:_list_memories_bounded": "private loader; _cluster_memories applies effective admission", + "vnext_consolidation.py:_existing_cluster_candidates": "owner acceptance/idempotency lookup; group-scope consumers retain existing behavior", + "vnext_scheduler.py:_StagedSchedulerStore.update_memory": "staged write replay; returned row is not a disclosure door", + "vnext_scheduler.py:VNextSchedulerService.status": "scheduler telemetry; events are constrained to scheduler targets", + "vnext_scheduler.py:VNextSchedulerService._generate_project_update_scan_artifact": "internal project scan; generate_project_update_candidate applies effective admission", + "vnext_connectors.py:VNextConnectorService.get_cursor": "connector cursor events only; no labelled targets", + "vnext_connectors.py:VNextConnectorService.get_config": "connector configuration events only; no labelled targets", + "vnext_connectors.py:VNextConnectorService.connector_health": "connector state telemetry only; no labels_raised events", + "vnext_artifact_review.py:dispatch_vnext_artifact_review": "writer entry; calling route or MCP authorizes the artifact before dispatch", + "vnext_memory_commit.py:VNextMemoryCommitService._guard_supersession_acyclic": "write validation traverses pointers without exposing their content", } -SCAN_MODULES = ( +SCAN_MODULES = tuple(sorted({ "alicebot_api/routers/_vnext_shared.py", "alicebot_api/mcp/evidence_artifacts.py", "alicebot_api/mcp/review.py", @@ -130,7 +180,13 @@ "alicebot_api/vnext_open_loop_references.py", "alicebot_api/vnext_queue.py", "alicebot_api/mcp/retrieval.py", -) + "alicebot_api/vnext_retrieval.py", "alicebot_api/session_briefing.py", + "alicebot_api/routers/workspaces.py", "alicebot_api/vnext_context_tree.py", "alicebot_api/vnext_dogfooding.py", + "alicebot_api/vnext_brain.py", "alicebot_api/vnext_connections.py", "alicebot_api/vnext_contradictions.py", + "alicebot_api/vnext_consolidation.py", "alicebot_api/vnext_rollups.py", "alicebot_api/vnext_scheduler.py", + "alicebot_api/vnext_connectors.py", "alicebot_api/vnext_artifact_review.py", + *(str(path.relative_to(SRC)) for directory in ("routers", "mcp") for path in (SRC / "alicebot_api" / directory).glob("*.py")), +})) def _functions(tree: ast.AST) -> list[tuple[str, ast.FunctionDef]]: @@ -157,6 +213,19 @@ def _called_names(node: ast.AST) -> set[str]: return names +def _reader_names(node: ast.AST) -> set[str]: + """Actual AST calls, bound-method callbacks, and dynamic reader lookup.""" + + names = _called_names(node) & READS + for child in ast.walk(node): + if isinstance(child, ast.Attribute) and child.attr in READS: + names.add(child.attr) + elif isinstance(child, ast.Call) and isinstance(child.func, ast.Name) and child.func.id == "getattr": + if len(child.args) > 1 and isinstance(child.args[1], ast.Constant) and child.args[1].value in READS: + names.add(child.args[1].value) + return names + + def _has_guard(node: ast.AST) -> bool: return bool(_called_names(node) & GUARD_CALLS) @@ -172,9 +241,13 @@ def test_every_exact_door_calls_the_guard() -> None: if helper is None: assert _has_guard(functions[name]), key else: - assert helper in _called_names(functions[name]), key - assert helper in functions, key - assert _has_guard(functions[helper]), helper + helper_path, helper_name = helper.split(":", 1) if ":" in helper else (path, helper) + assert helper_name in _called_names(functions[name]), key + if helper_path not in modules: + modules[helper_path] = ast.parse((SRC / "alicebot_api" / helper_path).read_text(encoding="utf-8")) + helper_functions = dict(_functions(modules[helper_path])) + assert helper_name in helper_functions, key + assert _has_guard(helper_functions[helper_name]), helper def test_every_scanned_reader_is_classified() -> None: @@ -184,8 +257,15 @@ def test_every_scanned_reader_is_classified() -> None: tree = ast.parse((SRC / relative).read_text(encoding="utf-8")) short = relative.removeprefix("alicebot_api/") for name, node in _functions(tree): - if _called_names(node) & READS: + if _reader_names(node): key = f"{short}:{name}" if key not in classified: missing.append(key) assert missing == [] + + +def test_discovery_catches_new_direct_and_dynamic_readers() -> None: + for source in ("def added(store): return store.list_events()", "def added(store): return getattr(store, 'list_memories')()", "def added(store): return invoke(store.list_beliefs)"): + node = ast.parse(source).body[0] + assert _reader_names(node) + assert "added" not in DOORS and "added" not in NOT_A_DOOR From 8c78c4fe6ac4eebd86ef565ab4367adb8769fc4c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:33:16 +0200 Subject: [PATCH 082/270] Include generated weekly candidate memories in chain assertions --- tests/integration/test_derived_labels_propagation_postgres.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_derived_labels_propagation_postgres.py b/tests/integration/test_derived_labels_propagation_postgres.py index 6f0cf8df1..767bd4698 100644 --- a/tests/integration/test_derived_labels_propagation_postgres.py +++ b/tests/integration/test_derived_labels_propagation_postgres.py @@ -95,6 +95,7 @@ def _build_chain(h, *, source=None, project=None, label=("project", "public")): ("memory", str(update["metadata_json"]["candidate_memory_id"])), ("project", str(project["id"])), ] + rows.extend(("memory", str(row_id)) for row_id in weekly["metadata_json"]["candidate_memory_ids"]) loops = store.list_open_loops(status=None, sensitivity_allowed=list(request.sensitivity_allowed)) loops = [row for row in loops if str(row.get("source_id")) == str(source["id"])] assert loops, "daily producer must create a candidate loop" From 1ab457aa7c44fb9c990edbce2ba473391d0ac0b0 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:33:15 +0200 Subject: [PATCH 083/270] docs: clarify complete counts and derived label recovery --- CHANGELOG.md | 10 ++- docs/alpha/backup-and-restore.md | 11 ++- docs/alpha/doctor.md | 7 ++ docs/alpha/mcp-tools.md | 8 +-- tests/unit/test_derived_domain_docs.py | 4 +- tests/unit/test_derived_labels_docs.py | 92 +++++++++++++++++++++++++- 6 files changed, 117 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89cea6093..60fca387c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,12 @@ - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings, including JSON escapes and repeated keys. Admitted references and surrounding values stay, and the owner receives live references unchanged. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. - Unreleased (on main, not in v0.20.0): the derived-row pages state the three-part label, who reads an unverified row, and that the five operator screens apply the caller's sensitivity ceiling. The v0.20.0 notes omitted both limits. No migration is required. -- Unreleased (on main, not in v0.20.0): PostgreSQL migration `20261005_0096` raises a stored derived row to the labels of its inputs, and `alicebot vnext labels` and `alice-memory labels` check and repair the same rows. The doctors print how many derived rows are below their inputs or unverified, as a warning. v0.20.0 left the stored label where it was written. -- Unreleased (on main, not in v0.20.0): opening a SQLite vault raises a derived row whose stored label is below its inputs, and a restore does the same before it publishes. A repair that cannot finish leaves the vault open and the completion key unstamped. v0.20.0 left the stored label where it was written. -- Unreleased (on main, not in v0.20.0): recall, context packs, resume, recent decisions, the session brief, the review queue, open-loop lists, and the workspace, dashboard, context tree, and dogfooding lists drop a loaded row whose effective label is outside the request. The artifact list, the source trace, the project list, the dashboard project row, and the belief state route apply the caller's sensitivity ceiling, including titles, ids, and counts. Resume, recent decisions, the session brief, audit, and explain apply the target row's current effective label, so an excluded key does not learn of a label change. v0.20.0 returned those rows from the label stored on them. No migration is required. -- Unreleased (on main, not in v0.20.0): a daily brief, connection report, contradiction report, consolidation, roll-up, project update, staleness sweep, and open-loop review drop an input whose effective label is outside the request. v0.20.0 kept a public copy of a confidential input in the report text. No migration is required. +- Unreleased (on main, not in v0.20.0): PostgreSQL migration `20261005_0096` raises a stored derived row to the labels of its inputs, and SQLite does the same on the next open and before a restore is published. The open pass can leave a vault open with its completion key unstamped after a failed repair; explicit repair and restore refuse the whole change on failure. `alicebot vnext labels` and `alice-memory labels` check and repair the same rows, including rows added after an earlier repair. The doctors print how many derived rows are below their inputs or unverified, as a warning, and report failed checks as unavailable. v0.20.0 left the stored label where it was written. Migration `20261005_0096` must run before serving requests as the NOSUPERUSER NOBYPASSRLS table owner. +- Unreleased (on main, not in v0.20.0): recall, context packs, resume, recent decisions, the session brief, the review queue, open-loop lists, and the workspace, dashboard, context tree, and dogfooding lists drop a loaded row whose effective label is outside the request. The artifact list, the source trace, the project list, the dashboard project row, and the belief state route apply the caller's sensitivity ceiling, including titles, ids, and counts. Resume, recent decisions, the session brief, audit, and explain apply the target row's current effective label, so an excluded key does not learn of a label change. The producer input readers also drop an input whose effective label is outside the request before it appears in report text. v0.20.0 returned rows by their stored labels and could copy a public row with confidential inputs into a new report. No migration is required. - Unreleased (on main, not in v0.20.0): an exact read of a derived artifact, a write that cites a derived memory, explain, memory review, redaction, and an open-loop update use the labels of that row's inputs. v0.20.0 used the label stored on the row, so a public copy of a confidential input could be read. A row whose inputs cannot be checked is refused to a locked key. The owner and an unbound admin still read the stored row. No migration is required. - Unreleased (on main, not in v0.20.0): a locked key's consolidation, roll-up, staleness sweep and open-loop review keep only inputs whose scope and floor are both inside the binding. The overlap check stays, and the exact test runs after it. Consolidation and roll-ups compare the group scope (scope united with floor), so a card stored with an empty scope and a floor of two projects is still found and can be accepted. The operator artifact list, given a project, matches that floor as well as the scope. Each new report and aggregate records `derived_from` for the rows it used. A project view that also asks for global rows keeps a row with no Alice project id only when every Alice project id in its floor is in that view. v0.20.0 selected by overlap of the scope alone and wrote no `derived_from`. No migration is required. -- Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. Raising a source or a memory walks the rows derived from it and raises those too. A candidate open loop and a generated artifact take the same insert floor. Replacing a SQLite source with a stricter one raises the memories that cite it before they are retired. Moving a source to another project answers with how many derived rows a project-bound key would lose, and waits for confirm_label_hide before it writes. A relabel that cannot finish answers 409, and one that waits more than 3 seconds for the label lock answers 503. No migration is required. -- Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. The function is what the write path calls. A read that does not go through that path still behaves as it does in v0.20.0. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. +- Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. Raising a source or a memory walks the rows derived from it and raises those too. A candidate open loop and a generated artifact take the same insert floor. Replacing a SQLite source with a stricter one raises the memories that cite it before they are retired. Moving a source to another project previews how many derived rows a project-bound key would lose and requires `confirm_label_hide` when that count is nonzero. The owner or an unbound admin recovers fresh candidates through `POST /v0/vnext/sources/{source_id}/regenerate`, then reruns the normal report generation route; existing provenance is kept. A relabel that cannot finish answers 409 with a named cause, and one that waits more than 3 seconds for the label lock answers 503 with `Retry-After: 2` and nothing changed. No migration is required. +- Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. This pure kernel is shared by the later write, read and repair paths; it does not rewrite stored rows on its own. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. - Unreleased (on main, not in v0.20.0): a daily brief that discovers an open loop writes `project_id` only when the single project is a UUID, and keeps a free-form name in `metadata_json.project_scope`. v0.20.0 wrote that name into the UUID column, so a brief with a TODO line and a project such as `Alice` failed on Postgres. No migration is required. - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` now scrub an open loop that names the source only in its metadata: the id, or `source:`, as the text under `source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references` or `selected_source_ids` at any depth. That is the rule the open-loop lookup of a source already used. They blanked only loops whose `source_id` column held the id, so a loop with an empty column kept its title, description and metadata, and they reached a new export. The delete preview now counts the same loops as the receipt. Correction (2026-10-05): those other spellings are blanked too, by the same reader the saved-quote fence uses. No migration is required. - Unreleased (on main, not in v0.20.0): the SQLite derived-label repair now updates and records each row under the id it is stored with (SQLite keeps capitals, missing hyphens, braces and `urn:uuid:` as written) and matches rows by the normalised id only to read the graph. If two stored spellings of one id exist, each is compared with the label its own recorded inputs give it, so neither is left at its old label while the other is relabelled, and neither is lowered. An update that changes no row stops the repair with `DerivedDomainRepairError` before any event or the completion stamp is written, on open and on restore; migration `20261004_0095` refuses the same way. Before, a derived memory stored under such an id kept its label while its relabel event and the stamp were written, and the pair survived export and import. A PostgreSQL uuid column was not affected. Correction (2026-10-05): the v3 pass raises such a row on the next open, including a vault the earlier repair already stamped. No migration is required. diff --git a/docs/alpha/backup-and-restore.md b/docs/alpha/backup-and-restore.md index 85ae5d99d..4e93d5e41 100644 --- a/docs/alpha/backup-and-restore.md +++ b/docs/alpha/backup-and-restore.md @@ -313,7 +313,11 @@ counts as a recorded input is under [Derived row domains](mcp-tools.md#derived-r Unreleased (on main, not in v0.20.0): `alice-memory labels check` prints how many derived rows are below their inputs or unverified, and `alice-memory labels repair` -raises the rows that are below their inputs. Run check after a restore. +raises the rows that are below their inputs. Check reads a private snapshot and +does not upgrade the live vault. Explicit repair checks every time, including +after the open pass has stamped completion, under `BEGIN IMMEDIATE`; a failure +rolls back the whole repair. The open pass remains a one-time upgrade, and a +restore always repairs its staged copy before publication. Run check after a restore. This command restores a SQLite database. It is not a PostgreSQL import. @@ -466,7 +470,10 @@ inside its own transaction and turns it back on before it commits. A failure, including derived rows in a cycle whose labels do not settle within a bounded number of changes or an update that changes no row, rolls the relabels, their audit events and the FORCE change back together. The downgrade keeps the repaired -labels. +labels. `alicebot vnext labels check` uses one `REPEATABLE READ READ ONLY` +snapshot, set before the acting user's row-security identity. Explicit repair +takes the supersession lock, then the exclusive label lock, then ordered row +locks; an update that changes no row rolls back the whole repair. ## Upgrade checkpoint diff --git a/docs/alpha/doctor.md b/docs/alpha/doctor.md index 85dd6a853..fd254873f 100644 --- a/docs/alpha/doctor.md +++ b/docs/alpha/doctor.md @@ -23,6 +23,13 @@ Expected success: - warnings include a recommended fix - no secret value appears in output +Unreleased (on main, not in v0.20.0): the doctors report `derived labels: N below +their inputs, M unverified` as a warning. If the label check cannot read the +store, they report `derived labels: unavailable; run labels check`, also as a +warning. A failed read is never reported as zero rows. Run the store's `labels +check` command to inspect the cause, then `labels repair` for stale labels; +missing inputs need restoring or regeneration. + Common fixes: ```bash diff --git a/docs/alpha/mcp-tools.md b/docs/alpha/mcp-tools.md index 6f2685221..036aa7933 100644 --- a/docs/alpha/mcp-tools.md +++ b/docs/alpha/mcp-tools.md @@ -1083,10 +1083,10 @@ Unreleased (on main, not in v0.20.0): an until-only request also checks the uppe Unreleased (on main, not in v0.20.0): a derived memory or report keeps the most frequent restricted input label, with alphabetical ties, the highest sensitivity, and every project of every input, and none of those is lowered. An explicit request domain cannot override it. With no restricted inputs, each producer retains its prior selection. This covers briefs, weekly synthesis and its candidates, roll-ups, consolidation, connection and contradiction reports, staleness reports, open-loop reviews, project updates, promoted copies of reports, memories extracted from a source, the candidate open loops found in one, and the state a project update copies onto a project. Consolidation reports take their domain and their sensitivity over every row they name: the cluster members, the roll-up inputs, the members of the groups that a skip line names by key, and the roll-up cards they name by id. The report keeps printing the `source_refs` it copies from its cluster members, and its label also covers the sources they name, archived ones included. Open-loop reviews do the same over the sources whose ids they print. The run digest of both covers those sources, so a source that was reclassified makes a new report. A report whose inputs are all unrestricted carries the label of those inputs, so `internal` where it was `unknown`, and every profile reads the two alike. New staleness reports also inherit the highest sensitivity of the memories whose titles they include. -Unreleased (on main, not in v0.20.0): the stored-row repair resolves recorded input IDs within the same user and labels each derived row after the rows it reads, so a chain of any length settles with one read of each row, including promoted artifact copies identified by `value.kind`, `value.artifact_id` or `metadata_json.source_artifact_id`. Rows that record each other as inputs in a cycle are read again until their labels settle. Already restricted labels can change to the settled restricted label; they never become unrestricted. A cycle whose labels do not settle within a bounded number of changes aborts the migration or the restore instead of publishing intermediate labels, with an error that names up to five of the rows that kept changing and says to remove their circular input references or restore an earlier backup. The open pass does not abort the vault. A relabel and the rows that follow it commit together, at any depth, or the whole relabel is refused and nothing changes. Labels only rise. Regenerating the row is how a looser input is taken. An owner edit that would lower a derived row is held at its inputs, and the answer names `label_floor_applied`. One `labels_raised` event records the labels and carries no text. A relabel waits at most 3 s for a running write and then answers "try again". A redacted row has no inputs and is left alone. Text is never used to guess an input. A derived row with no resolvable record is not repaired but is unverified for readers. +Unreleased (on main, not in v0.20.0): the stored-row repair resolves recorded input IDs within the same user and labels each derived row after the rows it reads, so a chain of any length settles with one read of each row, including promoted artifact copies identified by `value.kind`, `value.artifact_id` or `metadata_json.source_artifact_id`. Rows that record each other as inputs in a cycle are read again until their labels settle. Already restricted labels can change to the settled restricted label; they never become unrestricted. A cycle whose labels do not settle within a bounded number of changes aborts the migration or the restore instead of publishing intermediate labels, with an error that names up to five of the rows that kept changing and says to remove their circular input references or restore an earlier backup. The open pass does not abort the vault. A relabel and the rows that follow it commit together, at any depth, or the whole relabel is refused and nothing changes. Labels only rise. Regenerating the row is how a looser input is taken. An owner edit that would lower a derived row is held at its inputs, and the answer names `label_floor_applied`. One `labels_raised` event records the labels and carries no text. A relabel waits at most 3 s for a running write and then answers "try again" with HTTP 503 and `Retry-After: 2`, with nothing changed. A whole refusal answers HTTP 409 with the cause `propagation_bound`, `row_changed`, `dependency_cycle`, `lock_order` or `database_error`, without input text or ids. A redacted row has no inputs and is left alone. Text is never used to guess an input. A derived row with no resolvable record is not repaired but is unverified for readers. -Unreleased (on main, not in v0.20.0): the repair runs when SQLite upgrades a vault, while `alice-memory import` stages a restore, in PostgreSQL migration `20261004_0095` (domain) and in PostgreSQL migration `20261005_0096` (domain, sensitivity, project scope and floor), and through `labels check` and `labels repair`. The open pass does not stop a vault from opening and leaves a row it could not repair to the read check. Each path, and what a restore can and cannot repair, is in [Backup and restore](backup-and-restore.md). +Unreleased (on main, not in v0.20.0): the repair runs when SQLite upgrades a vault, while `alice-memory import` stages a restore, in PostgreSQL migration `20261004_0095` (domain) and in PostgreSQL migration `20261005_0096` (domain, sensitivity, project scope and floor), and through `labels check` and `labels repair`. The open pass does not stop a vault from opening and leaves a row it could not repair to the read check. Explicit repair checks rows even after the open pass has stamped completion; every restore repairs its staged copy again. A failed explicit repair or restore rolls back the whole change. Each path, and what a restore can and cannot repair, is in [Backup and restore](backup-and-restore.md). -Unreleased (on main, not in v0.20.0): owner, trusted and admin callers that explicitly filter by an unrelated domain stop receiving a derived row after its label changes from `unknown` to a restricted domain; `unknown` previously matched every domain filter. New weekly candidate memories also store `input_summary`, which explain returns. These are intentional differences for unrestricted readers. Domain fields and explain policy-event labels/hashes reflect the new label; additive metadata changes context-pack token estimates. Repair audit events are additional history. The repair does not add missing historical input summaries. Correction (2026-10-05): sensitivity and project floor are repaired by migration `20261005_0096` and the SQLite v3 pass. Every reader checks a derived row, a belief and a project row against its inputs as they are now. A row whose recorded inputs cannot be found is unverified, so only the owner and an unbound admin key read it. Checking those inputs adds a read of the recorded graph. `derived_from` and `project_floor` add to pack token estimates. The artifact list, the source trace, the project list, the project row of the dashboard and the belief state route apply the caller's sensitivity ceiling, including titles, ids and counts. Counts on those screens and on the workspace and dashboards are over the rows the caller may read. Domain and project restrictions on those screens stay as they are. +Unreleased (on main, not in v0.20.0): owner, trusted and admin callers that explicitly filter by an unrelated domain stop receiving a derived row after its label changes from `unknown` to a restricted domain; `unknown` previously matched every domain filter. New weekly candidate memories also store `input_summary`, which explain returns. These are intentional differences for unrestricted readers. Domain fields and explain policy-event labels/hashes reflect the new label; additive metadata changes context-pack token estimates. Repair audit events are additional history. The repair does not add missing historical input summaries. Correction (2026-10-05): sensitivity and project floor are repaired by migration `20261005_0096` and the SQLite v3 pass. Every reader checks a derived row, a belief and a project row against its inputs as they are now. A row whose recorded inputs cannot be found is unverified, so only the owner and an unbound admin key read it. Checking those inputs adds a read of the recorded graph. `derived_from` and `project_floor` add to pack token estimates. The artifact list, the source trace, the project list, the project row of the dashboard and the belief state route apply the caller's sensitivity ceiling, including titles, ids and counts. Counts on those screens and on the workspace and dashboards are over the rows the caller may read. Each total checks the complete counted set before pagination, including rows excluded from the displayed page. Domain and project restrictions on those screens stay as they are. -Unreleased (on main, not in v0.20.0): a new derived row and a stored one take every project of every input, and a project is not dropped. A restricted key's own reports are built only from inputs that key may read. The owner's cross-project briefs stay. A report with an input that has no project is global and keeps a floor of every input's project, and it is not readable by every project. Correction (2026-10-05): the text that stood here said a reader matching one scope could potentially read a summary of other scopes. In v0.20.0 and on main before this change the reach was wider than a summary: titles and ids of other projects' artifacts, titles and ids of rows with no project, and the full text of memories with no project, in a report a bound key could read with 200 although its direct read of those rows was refused; promotion made the report recallable. +Unreleased (on main, not in v0.20.0): a new derived row and a stored one take every project of every input, and a project is not dropped. A restricted key's own reports are built only from inputs that key may read. The owner's cross-project briefs stay. A report with an input that has no project is global and keeps a floor of every input's project, and it is not readable by every project. Moving a source first returns `derived_rows_hidden_from_project_keys` and writes nothing when the count is nonzero until `confirm_label_hide` is true. On PostgreSQL, the keyless local owner or an unbound admin can call `POST /v0/vnext/sources/{source_id}/regenerate` with `user_id` to create fresh candidate memories and open loops from all stored chunks under the source's current labels, scope and provenance. HTTP 201 returns `memory_ids`, `open_loop_ids`, `memory_count` and `open_loop_count`; trusted and project-bound keys are refused. Old rows and their provenance stay intact and strict. Rerun a report's normal generation route to rebuild the report from the regenerated inputs. Correction (2026-10-05): the text that stood here said a reader matching one scope could potentially read a summary of other scopes. In v0.20.0 and on main before this change the reach was wider than a summary: titles and ids of other projects' artifacts, titles and ids of rows with no project, and the full text of memories with no project, in a report a bound key could read with 200 although its direct read of those rows was refused; promotion made the report recallable. diff --git a/tests/unit/test_derived_domain_docs.py b/tests/unit/test_derived_domain_docs.py index 2740feddb..619832b0f 100644 --- a/tests/unit/test_derived_domain_docs.py +++ b/tests/unit/test_derived_domain_docs.py @@ -68,7 +68,7 @@ def test_the_limitations_page_has_one_short_bullet_that_links_to_the_explanation Mutations, each one alone: delete ``keeps the label its inputs had when it was made``, ``follows its inputs when they are relabelled``, ``read only by the owner and an unbound admin key``, ``sensitivity - ceiling`` or the link from the bullet; point the link at ``#derived-rows``; add a ``## Derived`` heading + ceiling``, ``counting rows the caller may read`` or the link from the bullet; point the link at ``#derived-rows``; add a ``## Derived`` heading with a paragraph to the page. """ @@ -77,6 +77,7 @@ def test_the_limitations_page_has_one_short_bullet_that_links_to_the_explanation assert "follows its inputs when they are relabelled" in bullet assert "read only by the owner and an unbound admin key" in bullet assert "sensitivity ceiling" in bullet + assert "counting rows the caller may read" in bullet assert bullet.endswith(f"See [{HEADING}]({ANCHOR})") assert "\n## Derived" not in LIMITATIONS.read_text(encoding="utf-8") @@ -152,6 +153,7 @@ def test_the_mcp_tools_section_states_the_label_rule_the_repair_and_the_changes_ assert "apply no label" not in readers assert "every project of every input" in scope + assert "A restricted key's own reports are built only from inputs that key may read" in scope assert "not readable by every project" in scope assert "could potentially read a summary of other scopes" in scope diff --git a/tests/unit/test_derived_labels_docs.py b/tests/unit/test_derived_labels_docs.py index ca0c09b16..50f538065 100644 --- a/tests/unit/test_derived_labels_docs.py +++ b/tests/unit/test_derived_labels_docs.py @@ -16,8 +16,6 @@ import importlib.util from pathlib import Path -from alicebot_api.vnext_label_repair import REPAIR_STATE_KEY - ROOT = Path(__file__).resolve().parents[2] MARK = "Unreleased (on main, not in v0.20.0):" MIGRATION = ROOT / "apps/api/alembic/versions/20261005_0096_derived_label_floor.py" @@ -37,6 +35,8 @@ def _text(path: str) -> str: def test_the_pages_name_the_v3_repair_and_the_seven_table_migration() -> None: """Docs name the state key, the revision and every table the migration brackets, including projects.""" + from alicebot_api.vnext_label_repair import REPAIR_STATE_KEY + migration = _migration() tables = [item.split()[2] for item in migration._RELAX_RLS] assert migration.revision == "20261005_0096" @@ -60,6 +60,7 @@ def test_the_pages_name_the_v3_repair_and_the_seven_table_migration() -> None: assert "at most 3 s" in tools assert "apply the caller's sensitivity ceiling" in tools assert "rows the caller may read" in tools + assert "Each total checks the complete counted set before pagination" in tools assert "apply no label" not in tools assert f"`{migration.revision}`" in tools assert REPAIR_STATE_KEY not in tools @@ -95,3 +96,90 @@ def test_the_pages_name_the_v3_repair_and_the_seven_table_migration() -> None: assert "Such a row is unverified." in changelog assert "the five operator screens apply the caller's sensitivity ceiling" in changelog assert "No migration is required." in changelog.split("## Unreleased", 1)[1].split("\n- ", 2)[1] + + +def test_each_derived_label_change_has_one_complete_changelog_entry() -> None: + """One entry covers each PR, including both stores and all list and producer input doors. + + Mutations: split the repair entry by store; split producer input filtering from the list-door entry; + remove the migration role or its before-serving requirement; remove a no-migration ending. + """ + + entries = [line.removeprefix("- ") for line in _text("CHANGELOG.md").splitlines() if line.startswith("- ")] + starts = ( + "the derived-row pages", + "PostgreSQL migration `20261005_0096`", + "recall, context packs", + "an exact read", + "a locked key's consolidation", + "a memory copied from another row", + "the label of a derived row", + ) + selected: dict[str, str] = {} + for start in starts: + matches = [entry for entry in entries if entry.startswith(f"{MARK} {start}")] + assert len(matches) == 1, (start, len(matches)) + selected[start] = matches[0] + assert not any(entry.startswith(f"{MARK} opening a SQLite vault") for entry in entries) + assert not any(entry.startswith(f"{MARK} a daily brief, connection report") for entry in entries) + repair = selected[starts[1]] + assert "SQLite" in repair + assert "NOSUPERUSER NOBYPASSRLS" in repair + assert repair.endswith( + "Migration `20261005_0096` must run before serving requests as the NOSUPERUSER NOBYPASSRLS table owner." + ) + assert "producer input readers" in selected[starts[2]] + for start, entry in selected.items(): + assert entry.startswith(MARK) + assert "v0.20.0" in entry + if start != starts[1]: + assert entry.endswith("No migration is required."), start + + +def test_repair_docs_distinguish_open_restore_and_explicit_commands() -> None: + """Repair after completion and failed reads have different operator outcomes from a gated open. + + Mutations: gate explicit repair by completion; make restore one-time; turn a failed explicit repair + into partial success; remove the read-only snapshot contract; replace an unavailable doctor check + with a zero count. + """ + + backup = " ".join(_text("docs/alpha/backup-and-restore.md").split()) + assert "Explicit repair checks every time, including after the open pass has stamped completion" in backup + assert "under `BEGIN IMMEDIATE`; a failure rolls back the whole repair" in backup + assert "The open pass remains a one-time upgrade" in backup + assert "a restore always repairs its staged copy before publication" in backup + assert "Check reads a private snapshot and does not upgrade the live vault" in backup + assert "one `REPEATABLE READ READ ONLY` snapshot, set before the acting user's row-security identity" in backup + assert "an update that changes no row rolls back the whole repair" in backup + tools = _text("docs/alpha/mcp-tools.md") + assert "Explicit repair checks rows even after the open pass has stamped completion" in tools + assert "every restore repairs its staged copy again" in tools + assert "A failed explicit repair or restore rolls back the whole change" in tools + doctor = " ".join(_text("docs/alpha/doctor.md").split()) + assert f"{MARK} the doctors report" in doctor + assert "`derived labels: unavailable; run labels check`" in doctor + assert "A failed read is never reported as zero rows" in doctor + + +def test_source_move_docs_name_the_proved_recovery_and_failure_contract() -> None: + """A source move names a tested recovery route, confirmation and whole-refusal causes. + + Mutations: remove confirmation, the recovery route, current provenance, the report regeneration + step, a failure cause or the retry header. A source move must not suggest that old rows are loosened. + """ + + tools = _text("docs/alpha/mcp-tools.md") + assert "`derived_rows_hidden_from_project_keys`" in tools + assert "writes nothing when the count is nonzero until `confirm_label_hide` is true" in tools + assert "On PostgreSQL, the keyless local owner or an unbound admin" in tools + assert "`POST /v0/vnext/sources/{source_id}/regenerate` with `user_id`" in tools + assert "from all stored chunks under the source's current labels, scope and provenance" in tools + assert "Old rows and their provenance stay intact and strict" in tools + assert "Rerun a report's normal generation route" in tools + assert "HTTP 201 returns `memory_ids`, `open_loop_ids`, `memory_count` and `open_loop_count`" in tools + assert "trusted and project-bound keys are refused" in tools + assert "HTTP 503 and `Retry-After: 2`, with nothing changed" in tools + assert "HTTP 409 with the cause" in tools + for cause in ("propagation_bound", "row_changed", "dependency_cycle", "lock_order", "database_error"): + assert f"`{cause}`" in tools From 06e18f6241bbd8f0a6938468a039b06657f0888d Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:37:53 +0200 Subject: [PATCH 084/270] Make pure dependency helpers available to source recovery --- .../src/alicebot_api/vnext_derived_labels.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 97c4b3bc9..14db4f6a5 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -1219,7 +1219,54 @@ def scope_is_global(scope: object) -> bool: return is_global_scope(scope) +def input_admitted(kind: str, row: Mapping[str, object], projects: object) -> bool: + """Exact-door project test: scope and floor are both inside ``projects``.""" + + if canon_kind(kind) == "source": + scope = source_project_scope(row) + else: + scope = resolve_project_scope(row).values + shape, floor = project_floor_shape(row) + if shape == "malformed": + return False + bound = set(project_scope_identity(projects)) + scope_ids = set(project_scope_identity(scope)) + if not scope_ids or not scope_ids <= bound: + return False + return set(project_scope_identity(floor)) <= bound + + +def stamp_derived_from(payload: dict[str, object], rows_by_kind: Mapping[str, object]) -> None: + """Write the canonical dependency record onto ``payload['metadata_json']``.""" + + metadata = payload.get("metadata_json") + meta = dict(metadata) if isinstance(metadata, Mapping) else {} + record: dict[str, object] = {"v": 1} + counts: dict[str, int] = {} + for key in ("sources", "memories", "open_loops", "artifacts", "beliefs"): + raw_rows = rows_by_kind.get(key) + rows = raw_rows if isinstance(raw_rows, (list, tuple)) else [] + ids = [str(row.get("id")) for row in rows if isinstance(row, Mapping) and row.get("id") is not None] + record[key] = ids + counts[key] = len(ids) + record["counts"] = counts + meta["derived_from"] = record + payload["metadata_json"] = meta + + +def with_derived_from(metadata: Mapping[str, object], rows_by_kind: Mapping[str, object]) -> dict[str, object]: + """A copy of ``metadata`` with ``derived_from`` for the rows a producer used.""" + + payload: dict[str, object] = {"metadata_json": dict(metadata)} + stamp_derived_from(payload, rows_by_kind) + stamped = payload["metadata_json"] + return dict(stamped) if isinstance(stamped, Mapping) else {} + + __all__ = [ + "input_admitted", + "stamp_derived_from", + "with_derived_from", "DERIVED_ARTIFACT_TYPES", "DERIVED_WORKFLOWS", "HOP_BOUND", From ee6a15b6d1c554d91f2471637c68bf8c27f095a0 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:38:42 +0200 Subject: [PATCH 085/270] Follow belief aliases in propagation and refresh reviewed carrier receipts --- .../src/alicebot_api/vnext_label_writes.py | 8 +++++ apps/api/src/alicebot_api/vnext_store.py | 21 +++++++++++++- .../test_label_floor_ancestry_postgres.py | 29 +++++++++++++++++++ tests/unit/test_source_move_label_preview.py | 24 +++++++++++++++ .../unit/test_store_graph_open_loops_split.py | 10 ++++--- tests/unit/test_store_memory_access_split.py | 3 +- .../unit/test_store_memory_lifecycle_split.py | 6 ++-- 7 files changed, 93 insertions(+), 8 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index ebed03e80..5798ac0c3 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -440,6 +440,14 @@ def walk_dependants(store: Any, roots: Sequence[str]) -> list[dict[str, object]] raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") batch = pending[:200] pending = pending[200:] + belief_aliases = getattr(store, "list_belief_ids_for_memories", None) + if callable(belief_aliases): + for belief_id in belief_aliases(batch): + alias = identifier(belief_id) + if alias not in seen: + seen.add(alias) + seen.add(_compact_id(belief_id)) + pending.append(str(belief_id)) matched: list[dict[str, object]] = [] batch_ids = {identifier(item) for item in batch} | {_compact_id(item) for item in batch} for row in list_dependants(store, batch): diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 654dadaa9..6a0660847 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from datetime import UTC, datetime from typing import Any, cast -from uuid import uuid4 +from uuid import UUID, uuid4 import psycopg @@ -597,6 +597,25 @@ def iter_label_ratings(self, *, batch_size: int = 200) -> Iterator[list[VNextRow yield rows after = str(rows[-1]["id"]) + def list_belief_ids_for_memories(self, ids: Sequence[str]) -> list[str]: + """Same-user belief aliases that make a memory an indirect report input.""" + + from alicebot_api.vnext_derived_labels import identifier + + wanted = [] + for value in ids: + try: + wanted.append(str(UUID(identifier(value)))) + except ValueError: + continue + if not wanted: + return [] + rows = self._fetch_all( + "SELECT id::text AS id FROM beliefs WHERE user_id = app.current_user_id() AND memory_id = ANY(%s::uuid[]) ORDER BY id", + (wanted,), + ) + return [str(row["id"]) for row in rows] + def _fetch_one( self, operation_name: str, diff --git a/tests/integration/test_label_floor_ancestry_postgres.py b/tests/integration/test_label_floor_ancestry_postgres.py index 657f3a3b5..2a16690f1 100644 --- a/tests/integration/test_label_floor_ancestry_postgres.py +++ b/tests/integration/test_label_floor_ancestry_postgres.py @@ -66,3 +66,32 @@ def test_checked_project_review_moves_scope_and_propagates_labels(migrated_datab assert summary_after["domain"] == "health" assert summary_after["sensitivity"] == "confidential" assert beta in summary_after["metadata_json"]["project_floor"] + + +def test_a_relabel_traverses_the_belief_backing_memory(migrated_database_urls): + user_id = uuid4() + url = migrated_database_urls["app"] + with user_connection(url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"belief-relabel-{user_id}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "synthetic", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public"}) + copy = store.create_memory({"memory_key": "copy", "canonical_text": "synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])}}) + belief = store.create_belief({"memory_id": str(copy["id"]), "claim": "Synthetic belief"}) + report = store.create_artifact({"artifact_type": "contradiction_report", "title": "Synthetic", "content_markdown": "Synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"belief_ids": [str(belief["id"])]}}) + other_user = uuid4() + with user_connection(url, other_user) as conn: + ContinuityStore(conn).create_user(other_user, f"other-belief-{other_user}@example.test", "Synthetic") + other_store = PostgresVNextStore(conn) + other_memory = other_store.create_memory({"memory_key": "other", "canonical_text": "synthetic", "domain": "project", "sensitivity": "public"}) + other_store.create_belief({"memory_id": str(other_memory["id"]), "claim": "Other synthetic belief"}) + with user_connection(url, user_id) as conn: + store = PostgresVNextStore(conn) + alias = "{" + str(copy["id"]).upper() + "}" + assert store.list_belief_ids_for_memories([alias, str(other_memory["id"])]) == [str(belief["id"])] + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "regulated"}) + with user_connection(url, user_id) as conn: + store = PostgresVNextStore(conn) + assert store.get_memory(str(copy["id"]))["sensitivity"] == "regulated" + assert store.get_artifact(str(report["id"]))["sensitivity"] == "regulated" diff --git a/tests/unit/test_source_move_label_preview.py b/tests/unit/test_source_move_label_preview.py index cb6caa1ce..0ef20c1e7 100644 --- a/tests/unit/test_source_move_label_preview.py +++ b/tests/unit/test_source_move_label_preview.py @@ -51,3 +51,27 @@ def test_preview_does_not_count_a_row_already_unverified(monkeypatch): report = _report([source, _source(["alpha"])], ["alpha"]) monkeypatch.setattr(writes, "walk_dependants", lambda *_: [report]) assert writes.count_rows_hidden_by_scope_move(Store([source, report]), source, ["beta"]) == 0 + + +def test_belief_alias_is_an_intermediate_reverse_edge(monkeypatch): + source = _source(["alpha"]) + report = _report([], ["alpha"]) + belief_id = str(uuid4()) + report["metadata_json"]["belief_ids"] = [belief_id] + store = Store([source, report]) + store.list_belief_ids_for_memories = lambda ids: [belief_id] if source["id"] in ids else [] + monkeypatch.setattr(writes, "list_dependants", lambda _store, ids: [report] if belief_id in ids else []) + assert writes.walk_dependants(store, [source["id"]]) == [report] + + +def test_sqlite_reverse_walk_needs_no_unsupported_belief_table(tmp_path): + from alicebot_api.onramp import bootstrap_database + from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection + from tests.unit.test_derived_domain_fence import USER + path = tmp_path / "labels.sqlite3" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.test") + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source({"source_type": "note", "title": "synthetic", "content_hash": "synthetic", "domain": "project", "sensitivity": "public"}) + copy = store.create_memory({"memory_key": "copy", "canonical_text": "synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])}}) + assert [row["id"] for row in writes.walk_dependants(store, [str(source["id"])])] == [str(copy["id"])] diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index e126fb979..046f84aa1 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -104,6 +104,8 @@ "OPEN_LOOP_COLUMNS", ) +# Reviewed label hooks: both open-loop updaters preserve protected metadata, +# clamp derived labels and propagate a stricter label to later rows. SOURCE_RECEIPTS = { # Re-minted for the filter-before-cut fix (2026-10-03): ``list_open_loop_events`` takes ``domains`` and # ``sensitivity_allowed`` as required arguments, and the SQLite one applies them in the join before ``LIMIT`` @@ -111,7 +113,7 @@ # the shared unscoped call site can state ``None`` for both, and it refuses anything else, since the Postgres # runtime resolves no project view (reviewed change, not drift). # The file hash now matches the carrier after the label lock. Previous receipt e4724ba1... - POSTGRES_CARRIER_PATH: "87aeac394e698a0b5709fd168abfa2a8a86af674ad42370f5a40decd773554df", + POSTGRES_CARRIER_PATH: "a9fecb0462324a46610f01134146fa73a2ab63692c7a6daa44ff56760840a812", # The SQLite carrier is re-minted, with its method AST manifest below, for # ``list_open_loops`` and ``list_open_loop_events``: they bind a query through # ``literal_match_operand`` and so refuse one past the LIKE operand limit. @@ -130,15 +132,15 @@ # change (reviewed change, not drift). # Re-minted for the combined floor-aware partition read and canonical source-reference batch lookup. # Previous receipt 9a2634be... - SQLITE_CARRIER_PATH: "394cc5ae868cd967160ef0d5cd0b3340e0ee19ba256ba28d98334f22ccf6754f", + SQLITE_CARRIER_PATH: "d80d53bc64e2395f9480eb6b06338f337c44b811b0eb86974b4dbc22308a88d0", POSTGRES_COLUMNS_PATH: "5b0d972a55abf8590ce14394a37fd71b9b88ba7ab3de82d61efc1bddfc022b71", SQLITE_COLUMNS_PATH: "be81b8628d0831d3d02b280b5455fb02333db5740ebef8d85d58024384ae6556", } EXPECTED_METHOD_AST_MANIFESTS = { # Postgres manifest matches the carrier after the label lock. Previous 25580884... - POSTGRES_CARRIER_PATH: "48064a91179463a20147a8e02442f3259976752000d9aafcb51647851227c46c", + POSTGRES_CARRIER_PATH: "69d4776f91829f5dc96f751c7d513c13149f851d878476c1c2c17c245d8cf0a3", # SQLite manifest includes the floor identity on the partition read. Previous 2850ba60... - SQLITE_CARRIER_PATH: "ed3a5a9d1ac3d98239191dbb4ced020525b404e7b3f25145ec9cade331be8838", + SQLITE_CARRIER_PATH: "a4da5f218817ad3c17ad887a2b4e643307e2f6f1c5b6bb16b0e15ccf25cb017f", } EXPECTED_METADATA_MANIFESTS = { POSTGRES_CARRIER_PATH: "6edb6a10e7a37dbbbbde97e5550422718a0112257666de8e23d49c60490fa13f", diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index 2315cb2ac..469ec59e5 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -23,6 +23,7 @@ POSTGRES_FACADE_PATH = REPO_ROOT / "apps/api/src/alicebot_api/vnext_store.py" SQLITE_FACADE_PATH = REPO_ROOT / "apps/api/src/alicebot_api/sqlite_store.py" +# Reviewed lock boundary: the pending-candidate row locker takes L first. SOURCE_RECEIPTS = { "apps/api/src/alicebot_api/vnext_stores/retrieval_common.py": ( "fa1a3a90511b5c61754ba29560e91b7b3058a48d47c143b09d8505d52025b8cc" @@ -42,7 +43,7 @@ # Re-minted so the two roll-up lookups overlap scope united with floor. # Every other statement still uses the scope expression. Previous receipt 46946cc0... "apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py": ( - "057f4f0c157223fc0d94ea3afe680b66200820533ddccab7211e211f29107157" + "ad6a1a81f077fbdaf13ec414558d43a8b6ad085350aec416e8a5e856a227d7c0" ), # Re-minted for per-project memory S2 (2026-10-02): the project fence builders read the reserved global # marker and take the domains to leave out, and the single-scan partition SQL and the materialized-CTE hint diff --git a/tests/unit/test_store_memory_lifecycle_split.py b/tests/unit/test_store_memory_lifecycle_split.py index 2f8b6c208..18b15e917 100644 --- a/tests/unit/test_store_memory_lifecycle_split.py +++ b/tests/unit/test_store_memory_lifecycle_split.py @@ -82,9 +82,11 @@ # mutators take the label lock. Metadata receipts include the label_write # keyword and lock wrappers. Facades add only lock_label_writes/read_label_rows; # removing those two names reproduces each previous class-order receipt. +# Reviewed strict lock change: the graph lock reads live advisory grants, +# and the memory update checks exclusive L before changing labels. SOURCE_RECEIPTS = { COMMON_PATH: "8fc077dc71f0e631a2df81de2ebeec1fb6c768f341c2e7891309e4753eef7bb5", - POSTGRES_CARRIER_PATH: "371f92595d2f72f0cfa225a49c03aa39498caf094df4f26f4f9ac4cd926e0de1", + POSTGRES_CARRIER_PATH: "23ab87cde159a285bf8c71dbc6d2eb4e0e15035ce0f509bef38ba799f3f92de3", # SQLite carrier re-minted for the Phase 4 Stage 2 resident vector cache # (reviewed change): redaction paths that NULL a live embedding now bump # the embedding_stamp token in the same transaction (prompt eviction). @@ -98,7 +100,7 @@ SQLITE_CARRIER_PATH: "f893f1faeeb9a87135c108992aa91ff036c5f5dccb1bbdaf315ae5afe1b89b73", } EXPECTED_METHOD_AST_MANIFESTS = { - "postgres": "d4969140e86b136b29708dc3bb6b4bca635b73b4016c4e633c4d5d3da841e784", + "postgres": "827c4f391a1efe50dcb265b7bb50a5b191d2bae32c3f817dcc237d51bfb10d29", "sqlite": "d0b6024f0803ca9d3c6f6ea7f5022a28453b2b8b83833f0f05343cb3eff89a57", } EXPECTED_METADATA_MANIFESTS = { From 5aea802d5115726fd12e07278a785cb949c710fb Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:38:24 +0200 Subject: [PATCH 086/270] Require captured memories as real daily brief inputs --- .../test_derived_labels_propagation_postgres.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration/test_derived_labels_propagation_postgres.py b/tests/integration/test_derived_labels_propagation_postgres.py index 767bd4698..06ab3a8f1 100644 --- a/tests/integration/test_derived_labels_propagation_postgres.py +++ b/tests/integration/test_derived_labels_propagation_postgres.py @@ -68,6 +68,12 @@ def _build_chain(h, *, source=None, project=None, label=("project", "public")): source = store.get_source(str(capture.source_id)) copies = store.list_memories_referencing_source(source_id=str(source["id"])) assert len(copies) >= 2 + copies = [ + store.update_memory(memory_id=str(row["id"]), patch={"status": "accepted"}, actor_type="user") + if row["status"] == "candidate" + else row + for row in copies + ] brain = VNextBrainService(store) request = BrainArtifactRequest( generated_for=today(), @@ -75,6 +81,7 @@ def _build_chain(h, *, source=None, project=None, label=("project", "public")): sensitivity_allowed=("public", "internal", "private", "confidential", "regulated", "unknown"), ) daily = brain.generate_daily_brief(request) + assert {str(row["id"]) for row in copies}.issubset(set(daily["metadata_json"]["derived_from"]["memories"])) weekly = brain.generate_weekly_synthesis(request) assert str(daily["id"]) in weekly["metadata_json"]["derived_from"]["artifacts"] promoted = VNextQueueService(store, defer_embeddings=True)._promote_artifact( From bba2d900fc8f52e9a512285e08dad5a1c667fdc2 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:49:45 +0200 Subject: [PATCH 087/270] Check canonical project labels before changing legacy project pointers --- .../api/src/alicebot_api/vnext_label_writes.py | 7 +++++-- tests/unit/test_label_lock_order.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 5798ac0c3..04dba3302 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -179,7 +179,10 @@ def prepare_label_patch( proposed = dict(before or {}) proposed.update({key: value for key, value in patch.items() if value is not None}) - if before and _label_fields(before) != _label_fields(proposed): + proposed["kind"] = kind + old = _label_fields({**before, "kind": kind}) if before else None + new = _label_fields(proposed) + if old and (old[:2] != new[:2] or project_scope_identity(old[2]) != project_scope_identity(new[2]) or project_scope_identity(old[3]) != project_scope_identity(new[3])): require_exclusive_label_lock(store) return dict(patch) @@ -467,7 +470,7 @@ def walk_dependants(store: Any, roots: Sequence[str]) -> list[dict[str, object]] def _label_fields(row: Mapping[str, object]) -> tuple[str, str, tuple[str, ...], tuple[str, ...]]: metadata = row.get("metadata_json") meta = metadata if isinstance(metadata, Mapping) else {} - scope = meta.get("project_scope", ()) + scope = source_project_scope(row) if row.get("kind") == "source" or "source_type" in row else resolve_project_scope(row).values floor = meta.get("project_floor", ()) return ( str(row.get("domain") or "unknown"), diff --git a/tests/unit/test_label_lock_order.py b/tests/unit/test_label_lock_order.py index 5ac522ddc..e2e021eca 100644 --- a/tests/unit/test_label_lock_order.py +++ b/tests/unit/test_label_lock_order.py @@ -84,3 +84,21 @@ def test_compare_and_set_miss_refuses_the_whole_label_write(): store = SimpleNamespace(_fetch_optional_one=lambda *_: None) with pytest.raises(DerivedDomainRepairError, match="changed no row"): writes.write_settled_label(store, kind="memory", row_id="missing", domain="health", sensitivity="confidential", metadata={}, project_id=None, expected_domain="project", expected_sensitivity="public") + + +def test_strict_hook_checks_the_legacy_project_pointer_before_any_update(monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", True) + store = _store() + before = {"domain": "project", "sensitivity": "public", "project_id": "11111111-1111-4111-8111-111111111111"} + with pytest.raises(writes.LabelLockOrderError, match="exclusive label lock"): + writes.prepare_label_patch(store, "memory", before, {"project_id": "22222222-2222-4222-8222-222222222222"}) + + +def test_named_refusal_causes_never_disclose_error_content(): + from psycopg.errors import DivisionByZero, LockNotAvailable + from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError + for error, cause in ((writes.LabelPropagationTooLarge("synthetic-private-text"), "propagation_bound"), (DerivedDomainRepairError("changed no row: synthetic-private-text"), "row_changed"), (DerivedDomainRepairError("cycle: synthetic-private-text"), "dependency_cycle"), (writes.LabelLockOrderError("synthetic-private-text"), "lock_order"), (DivisionByZero("synthetic-private-text"), "database_error")): + status, detail, retry_after = writes.label_error_response(error) + assert status == 409 and detail.endswith("cause: " + cause) and retry_after is None + assert "synthetic-private-text" not in detail + assert writes.label_error_response(LockNotAvailable("synthetic-private-text")) == (503, writes.RETRYABLE_DETAIL, "2") From 632fd1b934afff1e2430f6f80d32e106a2bf8242 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:45:22 +0200 Subject: [PATCH 088/270] Verify producer isolation and reuse derived group cards --- apps/api/src/alicebot_api/vnext_rollups.py | 4 +- ...est_derived_labels_group_scope_postgres.py | 35 +++ .../test_derived_labels_producers_postgres.py | 244 ++++++++++++++++++ tests/unit/test_group_scope_sqlite.py | 84 ++++++ tests/unit/test_project_floor_views.py | 75 ++++++ 5 files changed, 440 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_derived_labels_group_scope_postgres.py create mode 100644 tests/integration/test_derived_labels_producers_postgres.py create mode 100644 tests/unit/test_group_scope_sqlite.py create mode 100644 tests/unit/test_project_floor_views.py diff --git a/apps/api/src/alicebot_api/vnext_rollups.py b/apps/api/src/alicebot_api/vnext_rollups.py index a4aad8eb5..c667b06fd 100644 --- a/apps/api/src/alicebot_api/vnext_rollups.py +++ b/apps/api/src/alicebot_api/vnext_rollups.py @@ -2474,11 +2474,11 @@ def _existing_rollup_state( from alicebot_api.vnext_label_guard import admit_loaded admitted_pending = {str(row.get("id")) for row in admit_loaded( self.store, kind="memory", rows=list(pending.values()), domains=domains, - sensitivity_allowed=sensitivity_allowed, projects=projects, all_of=all_of, + sensitivity_allowed=sensitivity_allowed, projects=(), all_of=all_of, )} admitted_accepted = {str(row.get("id")) for row in admit_loaded( self.store, kind="memory", rows=list(accepted.values()), domains=domains, - sensitivity_allowed=sensitivity_allowed, projects=projects, all_of=all_of, + sensitivity_allowed=sensitivity_allowed, projects=(), all_of=all_of, )} pending = {key: row for key, row in pending.items() if str(row.get("id")) in admitted_pending} accepted = {key: row for key, row in accepted.items() if str(row.get("id")) in admitted_accepted} diff --git a/tests/integration/test_derived_labels_group_scope_postgres.py b/tests/integration/test_derived_labels_group_scope_postgres.py new file mode 100644 index 000000000..39c093d19 --- /dev/null +++ b/tests/integration/test_derived_labels_group_scope_postgres.py @@ -0,0 +1,35 @@ +"""The PostgreSQL group consumers have the same controls as SQLite.""" + +from uuid import uuid4 + +import pytest + +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_derived_labels import group_scope +from alicebot_api.vnext_memory_commit import VNextMemoryCommitService +from alicebot_api.vnext_rollups import VNextRollupService +from alicebot_api.vnext_store import PostgresVNextStore +from tests.unit.test_group_scope_sqlite import ALPHA, seed_members + + +@pytest.mark.parametrize("accept", (False, True)) +def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inserts_nothing(migrated_database_urls, accept): + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") + store = PostgresVNextStore(conn) + members = seed_members(store) + service = VNextRollupService(store) + first = service.propose_rollups(projects=(ALPHA,)) + assert len(first.candidate_ids) == 1 + candidate = store.get_memory(first.candidate_ids[0]) + assert candidate["metadata_json"]["project_scope"] == [] + assert group_scope(candidate) == group_scope(members[0]) + if accept: + assert VNextMemoryCommitService(store).accept_consolidation_candidate(first.candidate_ids[0], reason="Reviewed synthetic group")["status"] == "accepted" + before = conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] + second = service.propose_rollups(projects=(ALPHA,)) + assert second.proposals == [] + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == before + assert any(group["state"] == ("already_covered_by_accepted" if accept else "existing_candidate") for group in second.groups), second diff --git a/tests/integration/test_derived_labels_producers_postgres.py b/tests/integration/test_derived_labels_producers_postgres.py new file mode 100644 index 000000000..134372736 --- /dev/null +++ b/tests/integration/test_derived_labels_producers_postgres.py @@ -0,0 +1,244 @@ +"""Authenticated producer runs exclude each unreadable input before printing it.""" + +from __future__ import annotations + +import json +from uuid import uuid4 + +import pytest + +import alicebot_api.main as main_module +from alicebot_api.config import Settings +from alicebot_api.db import user_connection +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext +from alicebot_api.routers import vnext_projects, vnext_retrieval, vnext_review +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from alicebot_api.vnext_derived_labels import with_derived_from +from alicebot_api.vnext_queue import VNextQueueService +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.test_memory_mutations_api import invoke_request + +PRODUCERS = ("daily", "weekly", "connections", "contradictions", "open_loop_review", "project_update", + "consolidation", "staleness") + + +def wire_database(monkeypatch, app_url): + for module in (main_module, vnext_projects, vnext_retrieval, vnext_review): + monkeypatch.setattr(module, "get_settings", lambda: Settings(database_url=app_url)) + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_MODEL", raising=False) + monkeypatch.setenv("ALICE_PROJECT_SCOPING", "off") + + +def seed_grid(app_url): + user_id, alpha, beta = uuid4(), str(uuid4()), str(uuid4()) + rows = [] + with user_connection(app_url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"producer-{user_id}@example.invalid", "Producer") + store = PostgresVNextStore(conn) + store.create_project({"id": alpha, "name": "Atlas", "slug": "atlas", "domain": "project", "sensitivity": "public"}) + store.create_project({"id": beta, "name": "Beta", "slug": "beta", "domain": "project", "sensitivity": "public"}) + for label, scope in (("alpha", [alpha]), ("beta", [beta]), ("shared", [alpha, beta]), ("global", [])): + marker = f"SENTINEL_{label.upper()}" + source = store.create_source({"source_type": "manual_text", "title": marker + " source Atlas", + "content_hash": str(uuid4()), "captured_at": "2026-10-05T09:00:00Z", "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": scope, "raw_text": f"Atlas does not prefer launch games. {marker} source text.\nTODO: Atlas launch review {marker}."}}) + rows.append((label, "sources", source)) + for index, game in enumerate(("Hollow Knight", "Stardew Valley", "Celeste")): + text = f"Atlas played {game} for {25 + index * 30} hours. {marker} memory {index}" + memory = store.create_memory({"memory_key": f"{label}.game.{index}", "memory_type": "episode", "title": text, + "canonical_text": text, "summary": text, "value": {"text": text}, "status": "active", "domain": "project", + "sensitivity": "public", "created_at": "2026-10-05T09:00:00Z", + "metadata_json": {"project_scope": scope, "session_date": f"2026-10-0{index + 1}"}}) + rows.append((label, "memories", memory)) + backing = store.create_memory({"memory_key": f"{label}.belief", "memory_type": "belief", "title": marker + " belief title", + "canonical_text": f"Atlas prefers launch games. {marker} belief text", "status": "active", "domain": "project", + "sensitivity": "public", "metadata_json": {"project_scope": scope}}) + belief = store.create_belief({"memory_id": str(backing["id"]), "claim": backing["canonical_text"], "confidence": .9}) + rows.extend(((label, "memories", backing), (label, "beliefs", belief))) + loop = store.create_open_loop({"title": marker + " loop Atlas", "description": marker + " loop text", + "status": "open", "domain": "project", "sensitivity": "public", "due_at": "2026-10-04T12:00:00Z", + "metadata_json": {"project_scope": scope}}) + rows.append((label, "open_loops", loop)) + artifact = store.create_artifact({"artifact_type": "research_brief", "title": marker + " artifact Atlas", + "content_markdown": marker + " artifact text Atlas launch games", "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": scope}}) + rows.append((label, "artifacts", artifact)) + prior = store.create_artifact({"artifact_type": "daily_brief", "title": marker + " prior Atlas", + "content_markdown": marker + " prior report text Atlas launch games", "domain": "project", "sensitivity": "public", + "metadata_json": with_derived_from({"workflow": "daily_brief", "project_scope": scope}, {"sources": [source]})}) + rows.append((label, "artifacts", prior)) + promoted = VNextQueueService(store).review_artifact(artifact_id=str(prior["id"]), action="promote", actor_type="user") + rows.append((label, "memories", store.get_memory(promoted["promoted_memory_id"]))) + # Fix the read windows independently of the machine clock. + conn.execute("UPDATE memories SET created_at='2026-10-05T09:00:00Z', updated_at='2026-10-05T09:00:00Z', first_seen_at='2026-10-05T09:00:00Z', last_seen_at='2026-10-05T09:00:00Z'") + conn.execute("UPDATE generated_artifacts SET created_at='2026-10-05T09:00:00Z'") + conn.execute("UPDATE open_loops SET created_at='2026-10-05T09:00:00Z'") + _, alpha_key = create_agent_key(store, user_id=user_id, agent_id="alpha", permission_profile="admin_agent", project_scope=alpha) + _, beta_key = create_agent_key(store, user_id=user_id, agent_id="beta", permission_profile="admin_agent", project_scope=beta) + _, unbound = create_agent_key(store, user_id=user_id, agent_id="unbound", permission_profile="admin_agent") + return user_id, alpha, beta, rows, alpha_key, beta_key, unbound + + +def generate(producer, user_id, alpha, key=None, *, project_scope=True): + options = {"generated_for": "2026-10-05", "source_limit": 50, "memory_limit": 50, "artifact_limit": 50, + "open_loop_limit": 50, "reference_time": "2026-10-05T12:00:00Z", "max_items": 50, + "discover_open_loops": True, "create_candidate_memories": True} + payload = {"user_id": str(user_id), "scope": {"projects": [alpha]} if project_scope else {}, "options": options} + if producer in {"daily", "weekly", "connections", "contradictions"}: + route = {"daily": "daily-brief", "weekly": "weekly-synthesis"}.get(producer, producer) + path = "/v0/vnext/artifacts/generate/" + route + elif producer == "project_update": + path = "/v0/vnext/projects/update-candidates" + payload["scope"]["project_id"] = alpha + else: + workflow = {"consolidation": "memory_consolidation", "staleness": "staleness_sweep"}.get(producer, producer) + path = f"/v0/vnext/scheduler/workflows/{workflow}/run-now" + status, body = invoke_request("POST", path, payload=payload, headers={"authorization": f"Bearer {key}"} if key else {}) + assert status == 201, (producer, status, body) + return body.get("artifact", body), body + + +def assert_canonical_printed_inputs(artifact, rows): + metadata = artifact["metadata_json"] + record = metadata["derived_from"] + assert record["v"] == 1 + for kind in ("sources", "memories", "open_loops", "artifacts", "beliefs"): + assert record["counts"][kind] == len(record[kind]) + printed = json.dumps(artifact, default=str) + any_printed = False + for _label, kind, row in rows: + if str(row["id"]) in printed: + belief_alias = kind == "memories" and any( + str(belief["memory_id"]) == str(row["id"]) and str(belief["id"]) in record["beliefs"] + for _label, belief_kind, belief in rows if belief_kind == "beliefs" + ) + assert str(row["id"]) in record[kind] or belief_alias, (artifact["artifact_type"], kind, row["id"]) + any_printed = True + assert any_printed, (artifact["artifact_type"], "fixture printed no input") + + +@pytest.mark.parametrize("producer", PRODUCERS) +def test_a_bound_key_generates_from_admitted_inputs_only(migrated_database_urls, monkeypatch, producer): + app_url = migrated_database_urls["app"] + wire_database(monkeypatch, app_url) + user_id, alpha, _beta, rows, key, _beta_key, unbound = seed_grid(app_url) + if producer == "staleness": + with user_connection(app_url, user_id) as conn: + conn.execute("UPDATE memories SET valid_to='2026-10-04T12:00:00Z'") + artifact, body = generate(producer, user_id, alpha, key) + assert_canonical_printed_inputs(artifact, rows) + artifact_id = str(artifact["id"]) + surfaces = [body] + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + surfaces.extend((store.get_artifact(artifact_id), store.list_events(target_type="artifact", target_id=artifact_id))) + if producer == "staleness": + for label, kind, row in rows: + if label == "shared" and kind == "memories": + assert store.get_memory(str(row["id"]))["status"] == "active" + for path in (f"/v0/vnext/artifacts/{artifact_id}", f"/v0/vnext/traces/artifacts/{artifact_id}"): + status, surface = invoke_request("GET", path, query_params={"user_id": str(user_id)}, headers={"authorization": f"Bearer {key}"}) + assert status == 200, (producer, status, surface) + surfaces.append(surface) + # Project updates have a coupled candidate lifecycle, exercised by their review adapter. + review_path = (f"/v0/vnext/projects/update-candidates/{artifact_id}/review" if producer == "project_update" + else f"/v0/vnext/artifacts/{artifact_id}/review") + status, promoted = invoke_request("POST", review_path, payload={"user_id": str(user_id), "action": "accept" if producer == "project_update" else "promote"}, headers={"authorization": f"Bearer {unbound}"}) + assert status == 200, (producer, status, promoted) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + memory_id = (artifact["metadata_json"]["candidate_memory_id"] if producer == "project_update" + else promoted["promoted_memory_id"]) + surfaces.append(store.get_memory(memory_id)) + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + context = MCPRuntimeContext(database_url=app_url, user_id=user_id) + recalled = call_mcp_tool(context, name="alice_recall", arguments={"query": "Atlas", "limit": 50}) + def copies(value): + if isinstance(value, dict): + if str(value.get("id")) == str(memory_id): + return [value] + return [row for child in value.values() for row in copies(child)] + return [row for child in value for row in copies(child)] if isinstance(value, list) else [] + recalled_copies = copies(recalled) + assert recalled_copies, (producer, "promoted copy was not recalled", recalled) + surfaces.extend(recalled_copies) + surfaces.append(call_mcp_tool(context, name="alice_explain", arguments={"memory_id": memory_id})) + text = json.dumps(surfaces, default=str) + assert any(str(row["id"]) in text for label, _kind, row in rows if label == "alpha") + for label, _kind, row in rows: + if label != "alpha": + assert str(row["id"]) not in text, (producer, label, row["id"]) + for field in ("title", "canonical_text", "claim", "description"): + if row.get(field): + assert str(row[field]) not in text, (producer, label, field) + assert f"SENTINEL_{label.upper()}" not in text + + +def test_input_selection_uses_effective_labels_including_the_owner_default_ceiling(migrated_database_urls, monkeypatch): + app_url = migrated_database_urls["app"] + wire_database(monkeypatch, app_url) + user_id, alpha, _beta, rows, _key, _beta_key, _unbound = seed_grid(app_url) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = next(row for label, kind, row in rows if label == "alpha" and kind == "sources") + copy = store.create_memory({"memory_key": "stale.source.copy", "canonical_text": "STALE_SOURCE_COPY Atlas secret", + "title": "STALE_SOURCE_COPY", "status": "active", "domain": "project", "sensitivity": "public", + "metadata_json": {"source_id": str(source["id"]), "project_scope": [alpha]}}) + promoted = next(row for label, kind, row in rows if label == "alpha" and kind == "memories" + and row["metadata_json"].get("source_artifact_id")) + conn.execute("UPDATE sources SET sensitivity='confidential' WHERE id=%s", (source["id"],)) + conn.execute("UPDATE memories SET created_at='2026-10-05T09:00:00Z' WHERE id=%s", (copy["id"],)) + assert store.get_memory(str(copy["id"]))["sensitivity"] == "public" + assert store.get_memory(str(promoted["id"]))["sensitivity"] == "public" + _, trusted = create_agent_key(store, user_id=user_id, agent_id="trusted-alpha", permission_profile="trusted_local_agent", project_scope=alpha) + owner = VNextBrainService(store).generate_daily_brief(BrainArtifactRequest(generated_for="2026-10-05", + source_limit=50, memory_limit=50, artifact_limit=50, open_loop_limit=50, discover_open_loops=False, + create_candidate_memories=False)) + bound, _ = generate("daily", user_id, alpha, trusted) + for report in (owner, bound): + text = json.dumps(report, default=str) + assert str(copy["id"]) not in text + assert copy["canonical_text"] not in text + assert str(promoted["id"]) not in text + assert str(source["id"]) not in text + assert "SENTINEL_ALPHA prior report text" not in text + assert any(str(row["id"]) in text for label, kind, row in rows if label == "alpha" and kind == "memories" + and row["memory_type"] == "episode") + + +def test_the_owner_keeps_the_cross_project_brief_and_bound_keys_cannot_read_it(migrated_database_urls, monkeypatch): + app_url = migrated_database_urls["app"] + wire_database(monkeypatch, app_url) + user_id, alpha, beta, rows, alpha_key, beta_key, unbound = seed_grid(app_url) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + owner = VNextBrainService(store).generate_daily_brief(BrainArtifactRequest(generated_for="2026-10-05", + source_limit=50, memory_limit=50, artifact_limit=50, open_loop_limit=50, discover_open_loops=False, + create_candidate_memories=False)) + text = json.dumps(owner, default=str) + assert all(f"SENTINEL_{label.upper()}" in text for label in ("alpha", "beta", "shared", "global")) + assert owner["metadata_json"]["project_scope"] == [] + assert set(owner["metadata_json"]["project_floor"]) == {alpha, beta} + artifact_id = str(owner["id"]) + for key in (alpha_key, beta_key, unbound): + for path in (f"/v0/vnext/artifacts/{artifact_id}", f"/v0/vnext/traces/artifacts/{artifact_id}"): + status, body = invoke_request("GET", path, query_params={"user_id": str(user_id)}, headers={"authorization": f"Bearer {key}"}) + assert status == (200 if key == unbound else 403), (status, body) + status, promoted = invoke_request("POST", f"/v0/vnext/artifacts/{artifact_id}/review", + payload={"user_id": str(user_id), "action": "promote"}, headers={"authorization": f"Bearer {unbound}"}) + assert status == 200, promoted + memory_id = promoted["promoted_memory_id"] + with user_connection(app_url, user_id) as conn: + copy = PostgresVNextStore(conn).get_memory(memory_id) + assert copy["metadata_json"]["project_scope"] == [] + assert set(copy["metadata_json"]["project_floor"]) == {alpha, beta} + for key in (alpha_key, beta_key, unbound): + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + recalled = call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name="alice_recall", + arguments={"query": "Atlas", "limit": 50}) + assert (memory_id in json.dumps(recalled, default=str)) == (key == unbound) diff --git a/tests/unit/test_group_scope_sqlite.py b/tests/unit/test_group_scope_sqlite.py new file mode 100644 index 000000000..a982ae6ed --- /dev/null +++ b/tests/unit/test_group_scope_sqlite.py @@ -0,0 +1,84 @@ +"""Group consumers keep global aggregate cards usable without widening reads.""" + +from __future__ import annotations + +import sqlite3 +import json +from uuid import uuid4 + +import pytest + +from alicebot_api.sqlite_schema import bootstrap_sqlite_schema +from alicebot_api.sqlite_store import SQLiteVNextStore, ensure_sqlite_user +from alicebot_api.vnext_derived_labels import group_scope +from alicebot_api.vnext_memory_commit import VNextMemoryCommitService, VNextMemoryCommitValidationError +from alicebot_api.vnext_rollups import VNextRollupService + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +def seed_members(store): + members = [] + for index, (text, day) in enumerate((("I played Hollow Knight for 25 hours", "2023-06-02"), + ("I played Stardew Valley for 85 hours", "2023-06-20"), + ("I played Celeste for 10 hours", "2023-07-01"))): + members.append(store.create_memory({"memory_key": f"group-{index}", "memory_type": "episode", + "title": text, "canonical_text": text, "summary": text, "status": "active", "value": {"text": text}, + "domain": "personal", "sensitivity": "internal", + "metadata_json": {"session_date": day, "project_scope": [ALPHA, BETA]}})) + return members + + +@pytest.fixture +def sqlite_group_store(): + conn = sqlite3.connect(":memory:") + bootstrap_sqlite_schema(conn) + user_id = str(uuid4()) + ensure_sqlite_user(conn, user_id, "group@example.invalid", "Group") + yield SQLiteVNextStore(conn, user_id) + conn.close() + + +def test_a_consolidation_candidate_over_a_two_project_group_is_accepted(sqlite_group_store): + store = sqlite_group_store + members = seed_members(store) + first = VNextRollupService(store).propose_rollups(projects=(ALPHA,)) + assert len(first.candidate_ids) == 1 + candidate = store.get_memory(first.candidate_ids[0]) + assert candidate["metadata_json"]["project_scope"] == [] + assert group_scope(candidate) == group_scope(members[0]) + accepted = VNextMemoryCommitService(store).accept_consolidation_candidate(str(candidate["id"]), reason="Reviewed synthetic group") + assert accepted["status"] == "accepted" + + +@pytest.mark.parametrize("accept", (False, True)) +def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inserts_nothing(sqlite_group_store, accept): + store = sqlite_group_store + seed_members(store) + service = VNextRollupService(store) + first = service.propose_rollups(projects=(ALPHA,)) + assert len(first.candidate_ids) == 1 + if accept: + VNextMemoryCommitService(store).accept_consolidation_candidate(first.candidate_ids[0], reason="Reviewed synthetic group") + before = store.conn.execute("SELECT count(*) FROM memories").fetchone()[0] + second = service.propose_rollups(projects=(ALPHA,)) + assert second.proposals == [] + assert store.conn.execute("SELECT count(*) FROM memories").fetchone()[0] == before + assert any(group["state"] == ("already_covered_by_accepted" if accept else "existing_candidate") for group in second.groups), second + + +def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossing_project_scopes(sqlite_group_store): + store = sqlite_group_store + members = seed_members(store) + first = VNextRollupService(store).propose_rollups(projects=(ALPHA,)) + candidate_id = first.candidate_ids[0] + # Preserve the snapshot apart from the group labels so the scope check is reached. + member = members[0] + metadata = dict(member["metadata_json"]) + metadata["project_floor"] = [ALPHA] + metadata["project_scope"] = [] + store.conn.execute("UPDATE memories SET metadata_json=? WHERE id=?", (json.dumps(metadata), member["id"])) + candidate = store.get_memory(candidate_id) + with pytest.raises(VNextMemoryCommitValidationError, match="crosses project scopes"): + VNextMemoryCommitService(store).accept_consolidation_candidate(candidate_id, reason="Reviewed synthetic group") diff --git a/tests/unit/test_project_floor_views.py b/tests/unit/test_project_floor_views.py new file mode 100644 index 000000000..0914388c4 --- /dev/null +++ b/tests/unit/test_project_floor_views.py @@ -0,0 +1,75 @@ +"""Project views agree across both SQL builders and every Python mirror.""" + +from __future__ import annotations + +import inspect +import itertools +import json +import sqlite3 + +import pytest + +from alicebot_api.mcp.retrieval_shared import _resource_matches_project_scope +from alicebot_api.session_briefing import _memory_honours_fence +from alicebot_api.vnext_project_scope import GLOBAL_PROJECT_MARKER, project_scopes_overlap +from alicebot_api.vnext_retrieval import _ResolvedRetrievalScope, _project_scope_meets, _row_matches_scope +from alicebot_api.vnext_stores.sqlite.query_predicates import ( + _ensure_project_scope_identity_sqlite, + _project_view_sql, + _view_membership_sql, +) + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 +SCOPES = ((), (ALPHA,), (BETA,), (ALPHA, BETA), ("Alice",), (ALPHA.upper(),)) +VIEWS = ((), (ALPHA,), (BETA,), (ALPHA, GLOBAL_PROJECT_MARKER), + (BETA, GLOBAL_PROJECT_MARKER), (GLOBAL_PROJECT_MARKER,), (ALPHA, BETA, GLOBAL_PROJECT_MARKER)) + + +def expected(scope, floor, view): + if not view: + return True + ids = {value.lower() for value in view if value != GLOBAL_PROJECT_MARKER} + stored = {value.lower() for value in scope} + alice = lambda values: {value for value in values if value in {ALPHA, BETA}} + return bool(stored & ids) or ( + GLOBAL_PROJECT_MARKER in view and not alice(stored) and alice(value.lower() for value in floor) <= ids + ) + + +@pytest.mark.parametrize("scope,floor,view", itertools.product(SCOPES, SCOPES, VIEWS)) +def test_sql_and_python_project_view_membership_grid(scope, floor, view): + row = {"domain": "project", "sensitivity": "public", "metadata_json": { + "project_scope": list(scope), "project_floor": list(floor), + }} + want = expected(scope, floor, view) + assert project_scopes_overlap(scope, view, floor=floor) == (want if view else False) + assert _project_scope_meets(set(scope), view, floor=floor) == (want if view else False) + assert _resource_matches_project_scope(row, view) == want + resolved = _ResolvedRetrievalScope(frozenset(view), frozenset(), None, None, frozenset()) + assert _row_matches_scope(row, resolved) == want + assert _memory_honours_fence(row, effective_domains=(), effective_sensitivity_allowed=("public",), + effective_project_scope=view, exclude_global_domains=frozenset()) == want + with sqlite3.connect(":memory:") as conn: + _ensure_project_scope_identity_sqlite(conn) + conn.execute("CREATE TABLE rows(metadata_json TEXT, project_id TEXT, domain TEXT)") + conn.execute("INSERT INTO rows VALUES (?, NULL, 'project')", (json.dumps(row["metadata_json"]),)) + placeholders = lambda values: ",".join("?" for _ in values) + kwargs = dict(placeholders=placeholders, scope_expression="alice_project_scope_identity(metadata_json,project_id)", + text_expressions=("metadata_json", "project_id"), domain_expression="domain", + floor_expression="alice_project_floor_identity(metadata_json)") + clause, params = _project_view_sql(**kwargs, projects=view, global_excluded_domains=()) + assert bool(conn.execute("SELECT count(*) FROM rows WHERE 1=1" + clause, params).fetchone()[0]) == want + if view: + ids = tuple(value.lower() for value in view if value != GLOBAL_PROJECT_MARKER) + for partition in (False, True): + sql, params = _view_membership_sql(**kwargs, ids=ids, wants_global=GLOBAL_PROJECT_MARKER in view, + global_excluded_domains=(), partition=partition) + got = conn.execute("SELECT " + sql + " FROM rows", params).fetchone()[0] + assert (got is not None if partition else bool(got)) == want + + +def test_every_marker_aware_python_mirror_passes_the_floor(): + for function in (_project_scope_meets, _row_matches_scope, _resource_matches_project_scope, _memory_honours_fence, + _view_membership_sql, _project_view_sql): + assert "floor" in inspect.getsource(function), function.__name__ From 8ba9cf29f5654b53febdc07a182ee474898adb55 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:00:40 +0200 Subject: [PATCH 089/270] Pin reviewed label reader and recovery facade additions --- tests/unit/test_store_events_revisions_split.py | 4 ++-- tests/unit/test_store_graph_open_loops_split.py | 4 ++-- tests/unit/test_store_memory_access_split.py | 4 ++-- tests/unit/test_store_memory_lifecycle_split.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_store_events_revisions_split.py b/tests/unit/test_store_events_revisions_split.py index dde8c9474..7c8963a19 100644 --- a/tests/unit/test_store_events_revisions_split.py +++ b/tests/unit/test_store_events_revisions_split.py @@ -148,7 +148,7 @@ # Re-minted for the per-file importer savepoint (2026-10-02): ``savepoint`` is appended last on both façades. # Previous receipt: 650e2e0ff088d67c... Proof: the class key list equals the list at origin/main 040a2a10 # plus ``savepoint`` before ``__dict__``, with every other key in the same order. - "postgres": "dda7be1a316b3c525195f4682a608d0bab24b57f635db2a7aebc0c7ea187fc68", + "postgres": "3751e3fd145562485233765a14ae4e7bc7f396e6e72356a079e266fd98025839", # Re-minted for the merge of #500 and #502: ``check_literal_match_query``. # Re-minted again for per-project memory S2 (2026-10-02): the two single-scan partition reads # ``list_memories_view_partitions`` and ``list_open_loops_view_partitions``, SQLite only on purpose @@ -156,7 +156,7 @@ # Re-minted again for the per-file importer savepoint (2026-10-02), the same appended method. # Previous receipt: 365b7a01acf7a8b5... Proof: as for postgres, one added key and no other change. # Re-minted for the derived-label lock: ``lock_label_writes`` and ``read_label_rows`` on both facades. - "sqlite": "3212a93f2011cecddb6b0002a3f9b3abe37819d3582b2a1173846871981a5ff9", + "sqlite": "265db9bfe184e712eb3bbb64356c7a9601b130c93ca847962697c35f68b84ecd", } EXPECTED_SUPPORT_AST_SHA256 = { "postgres_columns": { diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index 046f84aa1..a9f07ceaa 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -159,7 +159,7 @@ # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # lock_label_writes and read_label_rows follow __init__. Previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (174, "095250b8a77d6c0a32d2783343916b947bcae8ae86e3a1693e47c2fb11802211"), + "PostgresVNextStore": (179, "71998141cd00c0639209285accbc29faa775b2e0250422c96101ee2f144df832"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -175,7 +175,7 @@ # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. # lock_label_writes and read_label_rows follow __init__. Previous receipt (134, 13012720...). - "SQLiteVNextStore": (136, "1d99dbaf69e4ea888ca7beb2389ac176650a2e573c067bf0686adc9e609132f5"), + "SQLiteVNextStore": (138, "783c07140c4015bcf8e43a3c68f3c2837ad14ce3525f8bd56a6f9c319d38eada"), } EXPECTED_COLUMN_AST = { POSTGRES_COLUMNS_PATH: { diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index 469ec59e5..fb1f7f04c 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -218,7 +218,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (174, "095250b8a77d6c0a32d2783343916b947bcae8ae86e3a1693e47c2fb11802211"), + "PostgresVNextStore": (179, "71998141cd00c0639209285accbc29faa775b2e0250422c96101ee2f144df832"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -235,7 +235,7 @@ # prunable_sources here; every pre-existing class member keeps its order. # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (134, 13012720...). - "SQLiteVNextStore": (136, "1d99dbaf69e4ea888ca7beb2389ac176650a2e573c067bf0686adc9e609132f5"), + "SQLiteVNextStore": (138, "783c07140c4015bcf8e43a3c68f3c2837ad14ce3525f8bd56a6f9c319d38eada"), } diff --git a/tests/unit/test_store_memory_lifecycle_split.py b/tests/unit/test_store_memory_lifecycle_split.py index 18b15e917..c8e45d634 100644 --- a/tests/unit/test_store_memory_lifecycle_split.py +++ b/tests/unit/test_store_memory_lifecycle_split.py @@ -113,7 +113,7 @@ # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (174, "095250b8a77d6c0a32d2783343916b947bcae8ae86e3a1693e47c2fb11802211"), + "PostgresVNextStore": (179, "71998141cd00c0639209285accbc29faa775b2e0250422c96101ee2f144df832"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -128,7 +128,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. - "SQLiteVNextStore": (136, "1d99dbaf69e4ea888ca7beb2389ac176650a2e573c067bf0686adc9e609132f5"), + "SQLiteVNextStore": (138, "783c07140c4015bcf8e43a3c68f3c2837ad14ce3525f8bd56a6f9c319d38eada"), } EXPECTED_FACADE_COMMENT_DIGESTS = { POSTGRES_FACADE_PATH: "d8599a46ee26dc35a3ae52c1a98a416509add9ae4a42ece780c5c5ed7e132b93", From 468abc7c390b2323b60d1fea276d7cf63f44842f Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:58:36 +0200 Subject: [PATCH 090/270] Document source regeneration as a creation response --- apps/api/src/alicebot_api/routers/vnext_memories.py | 2 +- .../integration/test_source_move_label_preview_postgres.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index 037c16fa6..b8e09b65e 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -779,7 +779,7 @@ def get_vnext_source(source_id: UUID, user_id: UUID) -> JSONResponse: ) -@source_review_router.post("/v0/vnext/sources/{source_id}/regenerate", summary="Regenerate fresh candidates from a stored source", description="The local owner or an unbound admin can regenerate candidate memories and open loops from all stored chunks using the source's current labels. Existing sources and outputs remain unchanged. Rerun the report's generation route to rebuild a report.") +@source_review_router.post("/v0/vnext/sources/{source_id}/regenerate", status_code=201, summary="Regenerate fresh candidates from a stored source", description="The local owner or an unbound admin can regenerate candidate memories and open loops from all stored chunks using the source's current labels. Existing sources and outputs remain unchanged. Rerun the report's generation route to rebuild a report.") def regenerate_vnext_source(source_id: UUID, request: VNextSourceRegenerateRequest, authorization: str | None = Header(default=None)) -> JSONResponse: from alicebot_api.vnext_label_writes import label_error_response from alicebot_api.vnext_source_regeneration import regenerate_source_inputs diff --git a/tests/integration/test_source_move_label_preview_postgres.py b/tests/integration/test_source_move_label_preview_postgres.py index 14056987b..a280c7f4c 100644 --- a/tests/integration/test_source_move_label_preview_postgres.py +++ b/tests/integration/test_source_move_label_preview_postgres.py @@ -19,6 +19,12 @@ BETA = "prj_" + "b" * 16 +def test_regeneration_openapi_documents_creation_status(): + responses = main.app.openapi()["paths"]["/v0/vnext/sources/{source_id}/regenerate"]["post"]["responses"] + assert "201" in responses + assert responses["201"]["content"]["application/json"]["schema"]["$ref"].endswith("RegenerateVnextSourceSuccessResponse") + + def _request(path, payload, raw_key): messages = [] body = json.dumps(payload).encode() From 6a49ea6353f49342cb3cd887d3b489608b6c07f4 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:01:37 +0200 Subject: [PATCH 091/270] Refresh the reviewed SQLite owner clamp carrier receipt --- tests/unit/test_store_events_revisions_split.py | 2 ++ tests/unit/test_store_graph_open_loops_split.py | 2 ++ tests/unit/test_store_memory_access_split.py | 2 ++ tests/unit/test_store_memory_lifecycle_split.py | 6 ++++-- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_store_events_revisions_split.py b/tests/unit/test_store_events_revisions_split.py index 7c8963a19..c718a97b6 100644 --- a/tests/unit/test_store_events_revisions_split.py +++ b/tests/unit/test_store_events_revisions_split.py @@ -139,6 +139,8 @@ }, } EXPECTED_CLASS_KEY_SHA256 = { + # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. + # Existing facade members retain their relative order. # Source owner methods are additive; the existing member order is unchanged. # Re-minted for the paired browser-clip capability façade methods. # The sqlite hash is re-minted again for ``check_source_search_query``. It is diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index a9f07ceaa..0e0249075 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -153,6 +153,8 @@ SQLITE_CARRIER_PATH: (6, "970028b5c929f0e749d8b40bdee571c872600de7a713e58d60e0da86f022af8a"), } EXPECTED_CLASS_ORDERS = { + # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. + # Existing facade members retain their relative order. # Two paired browser-clip capability methods extend both façades, and one # more paired method, ``list_memories_referencing_sources``. # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index fb1f7f04c..1dc1d3837 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -209,6 +209,8 @@ ) EXPECTED_CLASS_ORDERS = { + # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. + # Existing facade members retain their relative order. # Two paired browser-clip capability methods extend both façades. One more # paired method, ``list_memories_referencing_sources``, is the batched form # of ``list_memories_referencing_source``; both carrier receipts above were diff --git a/tests/unit/test_store_memory_lifecycle_split.py b/tests/unit/test_store_memory_lifecycle_split.py index c8e45d634..05a69e847 100644 --- a/tests/unit/test_store_memory_lifecycle_split.py +++ b/tests/unit/test_store_memory_lifecycle_split.py @@ -97,17 +97,19 @@ # back between two reads cannot fail memories_seen_range_check. Previous # sqlite receipt 67adaa61..., method AST 3f134ac9...; the metadata # manifests are unchanged. - SQLITE_CARRIER_PATH: "f893f1faeeb9a87135c108992aa91ff036c5f5dccb1bbdaf315ae5afe1b89b73", + SQLITE_CARRIER_PATH: "759cf44762c388e599b4e8c377fa3415ca2fdf9ba7884698259171ef3d2b8138", } EXPECTED_METHOD_AST_MANIFESTS = { "postgres": "827c4f391a1efe50dcb265b7bb50a5b191d2bae32c3f817dcc237d51bfb10d29", - "sqlite": "d0b6024f0803ca9d3c6f6ea7f5022a28453b2b8b83833f0f05343cb3eff89a57", + "sqlite": "3577659fcf9e583bdb957bb1a959ac1d8cae07ce8b50321bc36697dec47acec4", } EXPECTED_METADATA_MANIFESTS = { "postgres": "07a567e26d0f7c4f51ae2a1910d059397b513a575d85f06c2f8c0396ac1bb2ef", "sqlite": "1270ef0115988552418349aa9e94a7442ba04be41443f278f68a1fa81857903a", } EXPECTED_CLASS_ORDERS = { + # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. + # Existing facade members retain their relative order. # Two paired browser-clip capability methods extend both façades, and one # more paired method, ``list_memories_referencing_sources``. # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. From 265d0a4e8bdebcb1cb3b4c4973c15ae9ebd61d23 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:00:27 +0200 Subject: [PATCH 092/270] Check ordered repair statements and diagnose stale dependent reports --- .../test_derived_labels_migration_postgres.py | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_derived_labels_migration_postgres.py b/tests/integration/test_derived_labels_migration_postgres.py index 0c9152a5e..3f24a8a19 100644 --- a/tests/integration/test_derived_labels_migration_postgres.py +++ b/tests/integration/test_derived_labels_migration_postgres.py @@ -451,6 +451,47 @@ def raise_source(): repaired.result(timeout=10) relabelled.result(timeout=10) rows, _events = stored(urls, user, targets) - assert all(row["sensitivity"] == "regulated" for values in rows.values() for row in values) + assert all(row["sensitivity"] == "regulated" for values in rows.values() for row in values), [ + (table, row["metadata_json"], row["sensitivity"]) + for table, values in rows.items() + for row in values + if row["sensitivity"] != "regulated" + ] + restricted_reads(urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(urls, user, targets, monkeypatch) + + +def test_repair_takes_ordered_row_locks_and_preserves_nonlabel_fields(migrated_database_urls, monkeypatch): + urls = migrated_database_urls + user, targets = seed_stale(urls) + with user_connection(urls["app"], user) as conn: + before = conn.execute("SELECT * FROM memories ORDER BY id").fetchall() + original = psycopg.Cursor.execute + locks = [] + + def record(cursor, query, *args, **kwargs): + text = query.as_string(cursor.connection) if hasattr(query, "as_string") else str(query) + normalized = " ".join(text.split()) + if normalized.startswith("SELECT id FROM ") and "ORDER BY id FOR UPDATE" in normalized: + locks.append(normalized.split()[3]) + return original(cursor, query, *args, **kwargs) + + monkeypatch.setattr(psycopg.Cursor, "execute", record) + labels._run_vnext_labels_repair(cli_context(urls, user), None) + assert locks == ["generated_artifacts", "projects", "open_loops", "memories"] + with user_connection(urls["app"], user) as conn: + after = conn.execute("SELECT * FROM memories ORDER BY id").fetchall() + for previous, new in zip(before, after, strict=True): + assert { + key: value + for key, value in previous.items() + if key not in {"domain", "sensitivity", "metadata_json", "project_id"} + } == { + key: value + for key, value in new.items() + if key not in {"domain", "sensitivity", "metadata_json", "project_id"} + } + assert new["metadata_json"]["source_id"] == previous["metadata_json"]["source_id"] + assert_repaired(urls, user, targets) restricted_reads(urls, user, targets, monkeypatch, guard_off=True) restricted_reads(urls, user, targets, monkeypatch) From 719c32bd82727504205fc369a61cebe67c382ff3 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:02:01 +0200 Subject: [PATCH 093/270] Guard workspace activity cards with current target labels --- .../src/alicebot_api/routers/workspaces.py | 51 ++++++++++++------- apps/api/src/alicebot_api/vnext_dogfooding.py | 24 ++++----- ...derived_labels_read_acceptance_postgres.py | 35 +++++++++++++ tests/unit/test_complete_readable_counts.py | 21 ++++++++ tests/unit/test_label_door_registry.py | 1 + 5 files changed, 100 insertions(+), 32 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index c378393e5..bc85a15e5 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -3,7 +3,7 @@ from collections.abc import Mapping, Sequence from uuid import UUID -from fastapi import APIRouter, Request +from fastapi import APIRouter, Header, Request from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse import psycopg @@ -18,13 +18,14 @@ ) from alicebot_api.public_errors import public_exception_response from alicebot_api.routers._api_shared import _resolve_authenticated_v1_user_id -from alicebot_api.routers._vnext_shared import _vnext_int, _vnext_source_trace +from alicebot_api.routers._vnext_shared import _vnext_agent_auth_error_response, _vnext_int, _vnext_source_trace from alicebot_api.routers.providers import ( _discover_provider_capability, _persist_discovered_provider_capability, _seed_workspace_provider_configs, ) -from alicebot_api.vnext_agent_control import summarize_agent_policy_telemetry +from alicebot_api.vnext_agent_control import AgentIdentity, summarize_agent_policy_telemetry +from alicebot_api.vnext_agent_keys import AgentKeyAuthenticationError, agent_key_from_authorization, resolve_protected_agent_identity from alicebot_api.vnext_connectors import VNextConnectorService from alicebot_api.vnext_dogfooding import VNextDogfoodingService from alicebot_api.vnext_doctor import VNextDoctorService @@ -84,14 +85,19 @@ def _workspace_rows(store: PostgresVNextStore, kind: str, rows: Sequence[Mapping ) -def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: - from alicebot_api.vnext_label_guard import LabelGuard +def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdentity | None = None) -> dict[str, object]: + from alicebot_api.vnext_label_guard import LabelGuard, sensitivity_ceiling sensitivity_allowed = ["public", "internal", "private", "unknown"] - guard = LabelGuard.for_filters(store, (), sensitivity_allowed, ()) + ceiling = sensitivity_ceiling(identity) + if ceiling is not None: + sensitivity_allowed = [value for value in sensitivity_allowed if value in ceiling] + projects = identity.project_scope if identity is not None else () + all_of = projects if identity is not None and identity.project_scope_locked else None + guard = LabelGuard.for_filters(store, (), sensitivity_allowed, projects, all_of=all_of) review_statuses = ["candidate", "needs_review", "private_only", "accepted", "rejected"] fetched_sources = store.list_sources(sensitivity_allowed=sensitivity_allowed, limit=20) - sources = _workspace_rows(store, "source", fetched_sources, sensitivity_allowed) + sources = guard.admit_rows("source", fetched_sources) source_count = sum(guard.readable_status_counts("source").values()) list_memories_by_statuses = getattr(store, "list_memories_by_statuses", None) if callable(list_memories_by_statuses): @@ -104,11 +110,11 @@ def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: fetched_memories = [ memory for memory in store.list_memories(status=None) if str(memory.get("status")) in set(review_statuses) ][:30] - review_memories = _workspace_rows(store, "memory", fetched_memories, sensitivity_allowed) + review_memories = guard.admit_rows("memory", fetched_memories) memory_status_counts = guard.readable_status_counts("memory") review_memory_total = sum(memory_status_counts.get(status, 0) for status in review_statuses) fetched_artifacts = store.list_artifacts(sensitivity_allowed=sensitivity_allowed, limit=30) - artifacts = _workspace_rows(store, "artifact", fetched_artifacts, sensitivity_allowed) + artifacts = guard.admit_rows("artifact", fetched_artifacts) artifact_status_counts = guard.readable_status_counts("artifact") artifact_count = sum(artifact_status_counts.values()) quality_evals = guard.admit_related_rows(store.list_artifact_quality_ratings(limit=50), kind="artifact", field="artifact_id") @@ -117,22 +123,22 @@ def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: for batch in store.iter_label_ratings() ) fetched_projects = store.list_projects(status=None, sensitivity_allowed=sensitivity_allowed, limit=20) - projects = _workspace_rows(store, "project", fetched_projects, sensitivity_allowed) + projects = guard.admit_rows("project", fetched_projects) project_count = sum(guard.readable_status_counts("project").values()) fetched_loops = store.list_open_loops(status=None, sensitivity_allowed=sensitivity_allowed, limit=30) - open_loops = _workspace_rows(store, "open_loop", fetched_loops, sensitivity_allowed) + open_loops = guard.admit_rows("open_loop", fetched_loops) open_loop_status_counts = guard.readable_status_counts("open_loop") open_loop_count = open_loop_status_counts.get("open", 0) people = store.list_people(sensitivity_allowed=sensitivity_allowed, limit=12) fetched_beliefs = store.list_beliefs(status=None, sensitivity_allowed=sensitivity_allowed, limit=12) - beliefs = LabelGuard.for_filters(store, (), sensitivity_allowed, ()).admit_beliefs(fetched_beliefs) + beliefs = guard.admit_beliefs(fetched_beliefs) tasks = store.list_tasks(status=None, limit=12) fetched_events = store.list_events(limit=20) recent_events = guard.admit_events(fetched_events) event_count = guard.readable_event_count() agent_identities = store.list_agent_identities(limit=20) agent_count = store.count_agent_identities() - agent_events = store.list_agent_events(limit=50) + agent_events = guard.admit_events(store.list_agent_events(limit=50)) list_recent_agentic_commits = getattr(store, "list_recent_agentic_commits", None) list_pending_inline_confirmations = getattr(store, "list_pending_inline_confirmations", None) memory_commit_service = VNextMemoryCommitService(store) @@ -146,10 +152,12 @@ def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: if callable(list_pending_inline_confirmations) else memory_commit_service.inline_confirmations(limit=20) ) + recent_memory_commits = guard.admit_rows("memory", recent_memory_commits) + inline_confirmations = guard.admit_rows("memory", inline_confirmations) scheduler_status = VNextSchedulerService(store).status() scheduler_status = {**scheduler_status, "daemon": daemon_status()} connector_health = VNextConnectorService(store).connector_health_all() - dogfooding = VNextDogfoodingService(store).dashboard() + dogfooding = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(sensitivity_allowed), label_guard=guard) doctor = VNextDoctorService(store).run(ci=True) policy_telemetry = summarize_agent_policy_telemetry( agent_events=agent_events, @@ -160,7 +168,7 @@ def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: project_dashboards: list[dict[str, object]] = [] for project in projects[:5]: try: - project_dashboards.append(project_service.project_dashboard(project_id=str(project["id"]))) + project_dashboards.append(project_service.project_dashboard(project_id=str(project["id"]), identity=identity)) except VNextProjectValidationError: continue trace_items = [ @@ -291,11 +299,18 @@ def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: @core_router.get("/v0/vnext/workspace") -def get_vnext_workspace(user_id: UUID) -> JSONResponse: +def get_vnext_workspace(user_id: UUID, authorization: str | None = Header(default=None)) -> JSONResponse: settings = get_settings() - with user_connection(settings.database_url, user_id) as conn: - payload = _vnext_workspace_payload(PostgresVNextStore(conn)) + try: + with user_connection(settings.database_url, user_id) as conn: + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, user_id=user_id, raw_key=agent_key_from_authorization(authorization), payload={}, + ) + payload = _vnext_workspace_payload(store, identity=identity) + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) return JSONResponse(status_code=200, content=jsonable_encoder(payload)) diff --git a/apps/api/src/alicebot_api/vnext_dogfooding.py b/apps/api/src/alicebot_api/vnext_dogfooding.py index 5a14906ed..cf7e5f118 100644 --- a/apps/api/src/alicebot_api/vnext_dogfooding.py +++ b/apps/api/src/alicebot_api/vnext_dogfooding.py @@ -3,13 +3,15 @@ from collections import Counter from datetime import UTC, datetime, timedelta from statistics import mean -from typing import Protocol +from typing import TYPE_CHECKING, Protocol from alicebot_api.vnext_connectors import VNextConnectorService, VNextConnectorStore from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_repositories import JsonObject from alicebot_api.vnext_store import is_redacted_project_update_artifact +if TYPE_CHECKING: + from alicebot_api.vnext_label_guard import LabelGuard class VNextDogfoodingStore(VNextConnectorStore, Protocol): def append_event(self, event: JsonObject) -> JsonObject: ... @@ -170,9 +172,9 @@ class VNextDogfoodingService: def __init__(self, store: VNextDogfoodingStore) -> None: self.store = store - def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None) -> JsonObject: + def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None, label_guard: LabelGuard | None = None) -> JsonObject: from alicebot_api.vnext_agent_control import ALL_SENSITIVITY - from alicebot_api.vnext_label_guard import LabelGuard, admit_loaded + from alicebot_api.vnext_label_guard import LabelGuard sources = self.store.list_sources(limit=500) try: @@ -185,17 +187,11 @@ def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None) -> Js # None is the owner and an admin key: every sensitivity, so the guard # reads nothing and the lists stay as the store returned them. ceiling = sensitivity_allowed if sensitivity_allowed is not None else ALL_SENSITIVITY - guard = LabelGuard.for_filters(self.store, (), ceiling, ()) - sources = admit_loaded(self.store, kind="source", rows=sources, domains=(), sensitivity_allowed=ceiling, projects=()) - memories = admit_loaded( - self.store, kind="memory", rows=memories, domains=(), sensitivity_allowed=ceiling, projects=() - ) - artifacts = admit_loaded( - self.store, kind="artifact", rows=artifacts, domains=(), sensitivity_allowed=ceiling, projects=() - ) - open_loops = admit_loaded( - self.store, kind="open_loop", rows=open_loops, domains=(), sensitivity_allowed=ceiling, projects=() - ) + guard = label_guard if label_guard is not None else LabelGuard.for_filters(self.store, (), ceiling, ()) + sources = guard.admit_rows("source", sources) + memories = guard.admit_rows("memory", memories) + artifacts = guard.admit_rows("artifact", artifacts) + open_loops = guard.admit_rows("open_loop", open_loops) memory_status_counts = guard.readable_status_counts("memory") try: events = self.store.list_events(limit=5_000) diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py index b52c7074d..696655293 100644 --- a/tests/integration/test_derived_labels_read_acceptance_postgres.py +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -14,6 +14,7 @@ from alicebot_api.routers import vnext_review, vnext_retrieval, vnext_projects, vnext_memories, workspaces from alicebot_api.store import ContinuityStore from alicebot_api.vnext_agent_control import ALL_SENSITIVITY +from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_label_writes import without_insert_floor from alicebot_api.vnext_store import PostgresVNextStore from tests.unit.test_derived_labels_real_keys import READERS, expected_read, real_reader_key, seed_read_rows @@ -130,3 +131,37 @@ def test_workspace_counts_full_population_with_sql_hidden_and_stale_rows(migrate batches = list(store.iter_label_rows("source", batch_size=100)) assert [len(batch) for batch in batches] == [100, 100, 6] assert all("content_markdown" not in row and "title" not in row for batch in batches for row in batch) + + +@pytest.mark.parametrize("reader", ("owner", "admin", "trusted")) +def test_workspace_activity_uses_actual_key_and_current_targets(migrated_database_urls, monkeypatch, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + monkeypatch.setattr(workspaces, "get_settings", lambda: Settings(database_url=app_url)) + monkeypatch.setattr(workspaces.VNextDoctorService, "run", lambda self, **kwargs: {}) + hidden_ids = [] + visible_ids = [] + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Cedar hidden parent", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential"}) + for hidden in (False, True): + metadata = {"agentic_memory": {"kind": "agentic_memory_commit", "confirmation": {"status": "pending"}}} + if hidden: + metadata["source_id"] = str(source["id"]) + with without_insert_floor(): + memory = store.create_memory({"memory_key": f"activity-{hidden}", "canonical_text": "Cedar hidden activity" if hidden else "Public activity", "status": "needs_review", "confirmation_status": "unconfirmed", "domain": "project", "sensitivity": "public", "metadata_json": metadata}) + (hidden_ids if hidden else visible_ids).append(str(memory["id"])) + event = append_event(store, event_type="agent.policy_blocked", actor_type="agent", actor_id="synthetic-reader", target_type="memory", target_id=str(memory["id"]), payload={"decision": {"decision": "blocked", "target_id": str(memory["id"])}}) + (hidden_ids if hidden else visible_ids).append(str(event["id"])) + key = real_reader_key(store, user_id, reader) + response = workspaces.get_vnext_workspace(user_id, authorization=f"Bearer {key}" if key else None) + assert response.status_code == 200 + rendered = response.body.decode() + assert all(identifier not in rendered for identifier in hidden_ids) + assert all(identifier in rendered for identifier in visible_ids) + assert "Cedar hidden" not in rendered + body = json.loads(rendered) + assert len(body["agent_activity"]["policy_blocks"]) == 1 + assert len(body["agent_activity"]["recent_commits"]) == 1 + assert len(body["agent_activity"]["inline_confirmations"]) == 1 + assert body["dogfooding"]["sample_scope"]["memories"]["total_count"] == 1 diff --git a/tests/unit/test_complete_readable_counts.py b/tests/unit/test_complete_readable_counts.py index 97eb1bbea..ad31e0cf4 100644 --- a/tests/unit/test_complete_readable_counts.py +++ b/tests/unit/test_complete_readable_counts.py @@ -107,6 +107,27 @@ def test_all_sql_prefiltered_rows_leave_zero_totals(monkeypatch): assert all(not sample["has_more"] for sample in body["samples"].values()) +def test_workspace_activity_and_nested_dashboard_share_the_guard(monkeypatch): + _quiet_services(monkeypatch) + store = PopulationStore() + visible = _row("visible-memory") + hidden = _row("hidden-memory", "confidential") + store.rows["memory"] = [visible, hidden] + store.events = [{"id": "visible-event", "target_type": "memory", "target_id": visible["id"], "event_type": "agent.policy_blocked"}, + {"id": "hidden-event", "target_type": "memory", "target_id": hidden["id"], "event_type": "agent.policy_blocked"}] + store.list_agent_events = lambda **kwargs: store.events + store.list_recent_agentic_commits = lambda **kwargs: [visible, hidden] + store.list_pending_inline_confirmations = lambda **kwargs: [visible, hidden] + body = workspaces._vnext_workspace_payload(store) + assert "hidden-memory" not in str(body) + assert "hidden-event" not in str(body) + activity = body["agent_activity"] + assert [row["id"] for row in activity["recent_commits"]] == [visible["id"]] + assert [row["id"] for row in activity["inline_confirmations"]] == [visible["id"]] + assert [row["id"] for row in activity["policy_blocks"]] == ["visible-event"] + assert body["dogfooding"]["sample_scope"]["memories"]["total_count"] == 1 + + def test_dogfooding_counts_hidden_rows_beyond_500(monkeypatch): _quiet_services(monkeypatch) store = PopulationStore() diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py index e82773f3e..f48dae1c7 100644 --- a/tests/unit/test_label_door_registry.py +++ b/tests/unit/test_label_door_registry.py @@ -113,6 +113,7 @@ } NOT_A_DOOR = { + "routers/vnext_memories.py:regenerate_vnext_source": "operator-only regeneration rejects every profile except owner and unbound admin before the source lookup; real-profile rejection tests pin this gate", "mcp/evidence_artifacts.py:_handle_alice_vnext_review_items": "legacy review list has no policy check", "vnext_projects.py:VNextProjectService.review_project_update": "write path; the route authorizes before this mutation", "vnext_projects.py:VNextProjectService.review_open_loop": "write path; the route and the open-loop tool settle the loop first", From 02bcc4506dc34cb2083902607a8f2478b673deff Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:38:42 +0200 Subject: [PATCH 094/270] Follow belief aliases in propagation and refresh reviewed carrier receipts --- .../src/alicebot_api/vnext_label_writes.py | 8 +++++ apps/api/src/alicebot_api/vnext_store.py | 21 +++++++++++++- .../test_label_floor_ancestry_postgres.py | 29 +++++++++++++++++++ tests/unit/test_source_move_label_preview.py | 24 +++++++++++++++ .../unit/test_store_graph_open_loops_split.py | 10 ++++--- tests/unit/test_store_memory_access_split.py | 3 +- .../unit/test_store_memory_lifecycle_split.py | 6 ++-- 7 files changed, 93 insertions(+), 8 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index ebed03e80..5798ac0c3 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -440,6 +440,14 @@ def walk_dependants(store: Any, roots: Sequence[str]) -> list[dict[str, object]] raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") batch = pending[:200] pending = pending[200:] + belief_aliases = getattr(store, "list_belief_ids_for_memories", None) + if callable(belief_aliases): + for belief_id in belief_aliases(batch): + alias = identifier(belief_id) + if alias not in seen: + seen.add(alias) + seen.add(_compact_id(belief_id)) + pending.append(str(belief_id)) matched: list[dict[str, object]] = [] batch_ids = {identifier(item) for item in batch} | {_compact_id(item) for item in batch} for row in list_dependants(store, batch): diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 59aeee2f9..f172b2e6a 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from datetime import UTC, datetime from typing import Any, cast -from uuid import uuid4 +from uuid import UUID, uuid4 import psycopg @@ -527,6 +527,25 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: (wanted,), ) + def list_belief_ids_for_memories(self, ids: Sequence[str]) -> list[str]: + """Same-user belief aliases that make a memory an indirect report input.""" + + from alicebot_api.vnext_derived_labels import identifier + + wanted = [] + for value in ids: + try: + wanted.append(str(UUID(identifier(value)))) + except ValueError: + continue + if not wanted: + return [] + rows = self._fetch_all( + "SELECT id::text AS id FROM beliefs WHERE user_id = app.current_user_id() AND memory_id = ANY(%s::uuid[]) ORDER BY id", + (wanted,), + ) + return [str(row["id"]) for row in rows] + def _fetch_one( self, operation_name: str, diff --git a/tests/integration/test_label_floor_ancestry_postgres.py b/tests/integration/test_label_floor_ancestry_postgres.py index 657f3a3b5..2a16690f1 100644 --- a/tests/integration/test_label_floor_ancestry_postgres.py +++ b/tests/integration/test_label_floor_ancestry_postgres.py @@ -66,3 +66,32 @@ def test_checked_project_review_moves_scope_and_propagates_labels(migrated_datab assert summary_after["domain"] == "health" assert summary_after["sensitivity"] == "confidential" assert beta in summary_after["metadata_json"]["project_floor"] + + +def test_a_relabel_traverses_the_belief_backing_memory(migrated_database_urls): + user_id = uuid4() + url = migrated_database_urls["app"] + with user_connection(url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"belief-relabel-{user_id}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "synthetic", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public"}) + copy = store.create_memory({"memory_key": "copy", "canonical_text": "synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])}}) + belief = store.create_belief({"memory_id": str(copy["id"]), "claim": "Synthetic belief"}) + report = store.create_artifact({"artifact_type": "contradiction_report", "title": "Synthetic", "content_markdown": "Synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"belief_ids": [str(belief["id"])]}}) + other_user = uuid4() + with user_connection(url, other_user) as conn: + ContinuityStore(conn).create_user(other_user, f"other-belief-{other_user}@example.test", "Synthetic") + other_store = PostgresVNextStore(conn) + other_memory = other_store.create_memory({"memory_key": "other", "canonical_text": "synthetic", "domain": "project", "sensitivity": "public"}) + other_store.create_belief({"memory_id": str(other_memory["id"]), "claim": "Other synthetic belief"}) + with user_connection(url, user_id) as conn: + store = PostgresVNextStore(conn) + alias = "{" + str(copy["id"]).upper() + "}" + assert store.list_belief_ids_for_memories([alias, str(other_memory["id"])]) == [str(belief["id"])] + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "regulated"}) + with user_connection(url, user_id) as conn: + store = PostgresVNextStore(conn) + assert store.get_memory(str(copy["id"]))["sensitivity"] == "regulated" + assert store.get_artifact(str(report["id"]))["sensitivity"] == "regulated" diff --git a/tests/unit/test_source_move_label_preview.py b/tests/unit/test_source_move_label_preview.py index cb6caa1ce..0ef20c1e7 100644 --- a/tests/unit/test_source_move_label_preview.py +++ b/tests/unit/test_source_move_label_preview.py @@ -51,3 +51,27 @@ def test_preview_does_not_count_a_row_already_unverified(monkeypatch): report = _report([source, _source(["alpha"])], ["alpha"]) monkeypatch.setattr(writes, "walk_dependants", lambda *_: [report]) assert writes.count_rows_hidden_by_scope_move(Store([source, report]), source, ["beta"]) == 0 + + +def test_belief_alias_is_an_intermediate_reverse_edge(monkeypatch): + source = _source(["alpha"]) + report = _report([], ["alpha"]) + belief_id = str(uuid4()) + report["metadata_json"]["belief_ids"] = [belief_id] + store = Store([source, report]) + store.list_belief_ids_for_memories = lambda ids: [belief_id] if source["id"] in ids else [] + monkeypatch.setattr(writes, "list_dependants", lambda _store, ids: [report] if belief_id in ids else []) + assert writes.walk_dependants(store, [source["id"]]) == [report] + + +def test_sqlite_reverse_walk_needs_no_unsupported_belief_table(tmp_path): + from alicebot_api.onramp import bootstrap_database + from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection + from tests.unit.test_derived_domain_fence import USER + path = tmp_path / "labels.sqlite3" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.test") + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source({"source_type": "note", "title": "synthetic", "content_hash": "synthetic", "domain": "project", "sensitivity": "public"}) + copy = store.create_memory({"memory_key": "copy", "canonical_text": "synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])}}) + assert [row["id"] for row in writes.walk_dependants(store, [str(source["id"])])] == [str(copy["id"])] diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index eae9d1a4d..7013c4cff 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -104,13 +104,15 @@ "OPEN_LOOP_COLUMNS", ) +# Reviewed label hooks: both open-loop updaters preserve protected metadata, +# clamp derived labels and propagate a stricter label to later rows. SOURCE_RECEIPTS = { # Re-minted for the filter-before-cut fix (2026-10-03): ``list_open_loop_events`` takes ``domains`` and # ``sensitivity_allowed`` as required arguments, and the SQLite one applies them in the join before ``LIMIT`` # (an empty ceiling returns no rows without a query). The Postgres reader takes the same two arguments so that # the shared unscoped call site can state ``None`` for both, and it refuses anything else, since the Postgres # runtime resolves no project view (reviewed change, not drift). - POSTGRES_CARRIER_PATH: "e4724ba1ec3b8917c5be74b619259ddf1c282825b8e938ce4fa9947491a90a6f", + POSTGRES_CARRIER_PATH: "a9fecb0462324a46610f01134146fa73a2ab63692c7a6daa44ff56760840a812", # The SQLite carrier is re-minted, with its method AST manifest below, for # ``list_open_loops`` and ``list_open_loop_events``: they bind a query through # ``literal_match_operand`` and so refuse one past the LIKE operand limit. @@ -128,13 +130,13 @@ # delete preview and the scrub count and blank the same loops. The rule text moved unchanged to # ``vnext_stores/sqlite/open_loop_source_reference.py``; only the reader function and the receipt of the file # change (reviewed change, not drift). - SQLITE_CARRIER_PATH: "9a2634bef621d32262b845c046820d8b19c64801ec9f9b462e978f364f16f643", + SQLITE_CARRIER_PATH: "ea8a177e5037cfe40682def81b4cc1d6116b754b01e942d08a4367e5e54c8d2b", POSTGRES_COLUMNS_PATH: "5b0d972a55abf8590ce14394a37fd71b9b88ba7ab3de82d61efc1bddfc022b71", SQLITE_COLUMNS_PATH: "be81b8628d0831d3d02b280b5455fb02333db5740ebef8d85d58024384ae6556", } EXPECTED_METHOD_AST_MANIFESTS = { - POSTGRES_CARRIER_PATH: "2558088459f1b9a565e1b366ffe0b7c4025c623a9e2ea78007d06a46793ce1b8", - SQLITE_CARRIER_PATH: "2850ba6057b1510759613aaa3798a226808a42470ee11cfb9c6e3afbf3e98e66", + POSTGRES_CARRIER_PATH: "69d4776f91829f5dc96f751c7d513c13149f851d878476c1c2c17c245d8cf0a3", + SQLITE_CARRIER_PATH: "2d29f3668f52984f860b25fb6db5b37b04a2e4395631e584f70c93fa016540a1", } EXPECTED_METADATA_MANIFESTS = { POSTGRES_CARRIER_PATH: "6edb6a10e7a37dbbbbde97e5550422718a0112257666de8e23d49c60490fa13f", diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index 7b2647bbf..0e669eb6c 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -23,6 +23,7 @@ POSTGRES_FACADE_PATH = REPO_ROOT / "apps/api/src/alicebot_api/vnext_store.py" SQLITE_FACADE_PATH = REPO_ROOT / "apps/api/src/alicebot_api/sqlite_store.py" +# Reviewed lock boundary: the pending-candidate row locker takes L first. SOURCE_RECEIPTS = { "apps/api/src/alicebot_api/vnext_stores/retrieval_common.py": ( "fa1a3a90511b5c61754ba29560e91b7b3058a48d47c143b09d8505d52025b8cc" @@ -37,7 +38,7 @@ # which takes a keyword-only ``include_deleted`` (false by default, so every caller reads what it read before) # and drops the ``deleted_at IS NULL`` clause only when it is true. Previous Postgres receipt f642880f... "apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py": ( - "46946cc087de35f54474adf47cadcd67b862685bfa67a38be277d8e58a00c47e" + "9d42ac3a44b33f067306e7903f711cddf7ba538529cf03c16baf00473d861839" ), # Re-minted for per-project memory S2 (2026-10-02): the project fence builders read the reserved global # marker and take the domains to leave out, and the single-scan partition SQL and the materialized-CTE hint diff --git a/tests/unit/test_store_memory_lifecycle_split.py b/tests/unit/test_store_memory_lifecycle_split.py index 2f8b6c208..18b15e917 100644 --- a/tests/unit/test_store_memory_lifecycle_split.py +++ b/tests/unit/test_store_memory_lifecycle_split.py @@ -82,9 +82,11 @@ # mutators take the label lock. Metadata receipts include the label_write # keyword and lock wrappers. Facades add only lock_label_writes/read_label_rows; # removing those two names reproduces each previous class-order receipt. +# Reviewed strict lock change: the graph lock reads live advisory grants, +# and the memory update checks exclusive L before changing labels. SOURCE_RECEIPTS = { COMMON_PATH: "8fc077dc71f0e631a2df81de2ebeec1fb6c768f341c2e7891309e4753eef7bb5", - POSTGRES_CARRIER_PATH: "371f92595d2f72f0cfa225a49c03aa39498caf094df4f26f4f9ac4cd926e0de1", + POSTGRES_CARRIER_PATH: "23ab87cde159a285bf8c71dbc6d2eb4e0e15035ce0f509bef38ba799f3f92de3", # SQLite carrier re-minted for the Phase 4 Stage 2 resident vector cache # (reviewed change): redaction paths that NULL a live embedding now bump # the embedding_stamp token in the same transaction (prompt eviction). @@ -98,7 +100,7 @@ SQLITE_CARRIER_PATH: "f893f1faeeb9a87135c108992aa91ff036c5f5dccb1bbdaf315ae5afe1b89b73", } EXPECTED_METHOD_AST_MANIFESTS = { - "postgres": "d4969140e86b136b29708dc3bb6b4bca635b73b4016c4e633c4d5d3da841e784", + "postgres": "827c4f391a1efe50dcb265b7bb50a5b191d2bae32c3f817dcc237d51bfb10d29", "sqlite": "d0b6024f0803ca9d3c6f6ea7f5022a28453b2b8b83833f0f05343cb3eff89a57", } EXPECTED_METADATA_MANIFESTS = { From 7da4765673c2bd5d278106de085fd306cf903344 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:49:45 +0200 Subject: [PATCH 095/270] Check canonical project labels before changing legacy project pointers --- .../api/src/alicebot_api/vnext_label_writes.py | 7 +++++-- tests/unit/test_label_lock_order.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 5798ac0c3..04dba3302 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -179,7 +179,10 @@ def prepare_label_patch( proposed = dict(before or {}) proposed.update({key: value for key, value in patch.items() if value is not None}) - if before and _label_fields(before) != _label_fields(proposed): + proposed["kind"] = kind + old = _label_fields({**before, "kind": kind}) if before else None + new = _label_fields(proposed) + if old and (old[:2] != new[:2] or project_scope_identity(old[2]) != project_scope_identity(new[2]) or project_scope_identity(old[3]) != project_scope_identity(new[3])): require_exclusive_label_lock(store) return dict(patch) @@ -467,7 +470,7 @@ def walk_dependants(store: Any, roots: Sequence[str]) -> list[dict[str, object]] def _label_fields(row: Mapping[str, object]) -> tuple[str, str, tuple[str, ...], tuple[str, ...]]: metadata = row.get("metadata_json") meta = metadata if isinstance(metadata, Mapping) else {} - scope = meta.get("project_scope", ()) + scope = source_project_scope(row) if row.get("kind") == "source" or "source_type" in row else resolve_project_scope(row).values floor = meta.get("project_floor", ()) return ( str(row.get("domain") or "unknown"), diff --git a/tests/unit/test_label_lock_order.py b/tests/unit/test_label_lock_order.py index 5ac522ddc..e2e021eca 100644 --- a/tests/unit/test_label_lock_order.py +++ b/tests/unit/test_label_lock_order.py @@ -84,3 +84,21 @@ def test_compare_and_set_miss_refuses_the_whole_label_write(): store = SimpleNamespace(_fetch_optional_one=lambda *_: None) with pytest.raises(DerivedDomainRepairError, match="changed no row"): writes.write_settled_label(store, kind="memory", row_id="missing", domain="health", sensitivity="confidential", metadata={}, project_id=None, expected_domain="project", expected_sensitivity="public") + + +def test_strict_hook_checks_the_legacy_project_pointer_before_any_update(monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", True) + store = _store() + before = {"domain": "project", "sensitivity": "public", "project_id": "11111111-1111-4111-8111-111111111111"} + with pytest.raises(writes.LabelLockOrderError, match="exclusive label lock"): + writes.prepare_label_patch(store, "memory", before, {"project_id": "22222222-2222-4222-8222-222222222222"}) + + +def test_named_refusal_causes_never_disclose_error_content(): + from psycopg.errors import DivisionByZero, LockNotAvailable + from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError + for error, cause in ((writes.LabelPropagationTooLarge("synthetic-private-text"), "propagation_bound"), (DerivedDomainRepairError("changed no row: synthetic-private-text"), "row_changed"), (DerivedDomainRepairError("cycle: synthetic-private-text"), "dependency_cycle"), (writes.LabelLockOrderError("synthetic-private-text"), "lock_order"), (DivisionByZero("synthetic-private-text"), "database_error")): + status, detail, retry_after = writes.label_error_response(error) + assert status == 409 and detail.endswith("cause: " + cause) and retry_after is None + assert "synthetic-private-text" not in detail + assert writes.label_error_response(LockNotAvailable("synthetic-private-text")) == (503, writes.RETRYABLE_DETAIL, "2") From faf905a756ab1bbd0dc02b7a461b845bc0abc930 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:03:15 +0200 Subject: [PATCH 096/270] Exercise real workspace diagnostics with repaired label readers --- .../test_derived_labels_read_acceptance_postgres.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py index 696655293..3eaac2692 100644 --- a/tests/integration/test_derived_labels_read_acceptance_postgres.py +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -106,9 +106,6 @@ def test_all_five_operator_screens_with_real_keys(migrated_database_urls, monkey def test_workspace_counts_full_population_with_sql_hidden_and_stale_rows(migrated_database_urls, monkeypatch): app_url = migrated_database_urls["app"] user_id = _user(app_url) - # The doctor has its own store-specific repair acceptance test. Keep this - # count probe independent of that ancillary diagnostic implementation. - monkeypatch.setattr(workspaces.VNextDoctorService, "run", lambda self, **kwargs: {}) with user_connection(app_url, user_id) as conn: store = PostgresVNextStore(conn) secret = store.create_source({"source_type": "note", "title": "Cedar hidden", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential"}) @@ -138,7 +135,6 @@ def test_workspace_activity_uses_actual_key_and_current_targets(migrated_databas app_url = migrated_database_urls["app"] user_id = _user(app_url) monkeypatch.setattr(workspaces, "get_settings", lambda: Settings(database_url=app_url)) - monkeypatch.setattr(workspaces.VNextDoctorService, "run", lambda self, **kwargs: {}) hidden_ids = [] visible_ids = [] with user_connection(app_url, user_id) as conn: From a3711a6c8be2d5cdc6473eecead8bc20ce160b16 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:58:36 +0200 Subject: [PATCH 097/270] Document source regeneration as a creation response --- apps/api/src/alicebot_api/routers/vnext_memories.py | 2 +- .../integration/test_source_move_label_preview_postgres.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index c58712977..f8d44a3ca 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -760,7 +760,7 @@ def get_vnext_source(source_id: UUID, user_id: UUID) -> JSONResponse: ) -@source_review_router.post("/v0/vnext/sources/{source_id}/regenerate", summary="Regenerate fresh candidates from a stored source", description="The local owner or an unbound admin can regenerate candidate memories and open loops from all stored chunks using the source's current labels. Existing sources and outputs remain unchanged. Rerun the report's generation route to rebuild a report.") +@source_review_router.post("/v0/vnext/sources/{source_id}/regenerate", status_code=201, summary="Regenerate fresh candidates from a stored source", description="The local owner or an unbound admin can regenerate candidate memories and open loops from all stored chunks using the source's current labels. Existing sources and outputs remain unchanged. Rerun the report's generation route to rebuild a report.") def regenerate_vnext_source(source_id: UUID, request: VNextSourceRegenerateRequest, authorization: str | None = Header(default=None)) -> JSONResponse: from alicebot_api.vnext_label_writes import label_error_response from alicebot_api.vnext_source_regeneration import regenerate_source_inputs diff --git a/tests/integration/test_source_move_label_preview_postgres.py b/tests/integration/test_source_move_label_preview_postgres.py index 14056987b..a280c7f4c 100644 --- a/tests/integration/test_source_move_label_preview_postgres.py +++ b/tests/integration/test_source_move_label_preview_postgres.py @@ -19,6 +19,12 @@ BETA = "prj_" + "b" * 16 +def test_regeneration_openapi_documents_creation_status(): + responses = main.app.openapi()["paths"]["/v0/vnext/sources/{source_id}/regenerate"]["post"]["responses"] + assert "201" in responses + assert responses["201"]["content"]["application/json"]["schema"]["$ref"].endswith("RegenerateVnextSourceSuccessResponse") + + def _request(path, payload, raw_key): messages = [] body = json.dumps(payload).encode() From 9b0bebb6a44f0b3f0f6e35a0778f1c8bc1b81152 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:06:05 +0200 Subject: [PATCH 098/270] Refresh reviewed write backport carrier receipts --- tests/unit/test_store_events_revisions_split.py | 4 +++- tests/unit/test_store_graph_open_loops_split.py | 6 ++++-- tests/unit/test_store_memory_access_split.py | 6 ++++-- tests/unit/test_store_memory_lifecycle_split.py | 10 +++++++--- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_store_events_revisions_split.py b/tests/unit/test_store_events_revisions_split.py index dde8c9474..ee09f4e66 100644 --- a/tests/unit/test_store_events_revisions_split.py +++ b/tests/unit/test_store_events_revisions_split.py @@ -139,6 +139,8 @@ }, } EXPECTED_CLASS_KEY_SHA256 = { + # Reviewed recovery and reverse propagation add only the two Postgres + # methods; all prior runtime class keys retain their order. # Source owner methods are additive; the existing member order is unchanged. # Re-minted for the paired browser-clip capability façade methods. # The sqlite hash is re-minted again for ``check_source_search_query``. It is @@ -148,7 +150,7 @@ # Re-minted for the per-file importer savepoint (2026-10-02): ``savepoint`` is appended last on both façades. # Previous receipt: 650e2e0ff088d67c... Proof: the class key list equals the list at origin/main 040a2a10 # plus ``savepoint`` before ``__dict__``, with every other key in the same order. - "postgres": "dda7be1a316b3c525195f4682a608d0bab24b57f635db2a7aebc0c7ea187fc68", + "postgres": "e4a921e06bc707a362fb9c2b3050d292b119cd0b92b346ce5997a679b3bd61f0", # Re-minted for the merge of #500 and #502: ``check_literal_match_query``. # Re-minted again for per-project memory S2 (2026-10-02): the two single-scan partition reads # ``list_memories_view_partitions`` and ``list_open_loops_view_partitions``, SQLite only on purpose diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index 7013c4cff..ec64e2cba 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -149,12 +149,14 @@ SQLITE_CARRIER_PATH: (6, "970028b5c929f0e749d8b40bdee571c872600de7a713e58d60e0da86f022af8a"), } EXPECTED_CLASS_ORDERS = { + # Reviewed source recovery and belief propagation add only two Postgres + # methods. The SQLite facade and every existing member retain their order. # Two paired browser-clip capability methods extend both façades, and one # more paired method, ``list_memories_referencing_sources``. # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (172, "6f1a459fcf4319cf4281f6cc0d4e81679c3e05d851fd0a874a2d90298d7c2569"), + "PostgresVNextStore": (176, "3549d6ea179f4ebc2341f1c251f139dfb747d56700a4554aef114150efed5c3f"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -169,7 +171,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. - "SQLiteVNextStore": (134, "1301272026897057cf071009cc21787543ddc326f1e06f1a75a763f3e344767e"), + "SQLiteVNextStore": (136, "1d99dbaf69e4ea888ca7beb2389ac176650a2e573c067bf0686adc9e609132f5"), } EXPECTED_COLUMN_AST = { POSTGRES_COLUMNS_PATH: { diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index 0e669eb6c..90a0dc8bb 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -194,6 +194,8 @@ ) EXPECTED_CLASS_ORDERS = { + # Reviewed source recovery and belief propagation add only two Postgres + # methods. The SQLite facade and every existing member retain their order. # Two paired browser-clip capability methods extend both façades. One more # paired method, ``list_memories_referencing_sources``, is the batched form # of ``list_memories_referencing_source``; both carrier receipts above were @@ -201,7 +203,7 @@ # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (172, "6f1a459fcf4319cf4281f6cc0d4e81679c3e05d851fd0a874a2d90298d7c2569"), + "PostgresVNextStore": (176, "3549d6ea179f4ebc2341f1c251f139dfb747d56700a4554aef114150efed5c3f"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -216,7 +218,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. - "SQLiteVNextStore": (134, "1301272026897057cf071009cc21787543ddc326f1e06f1a75a763f3e344767e"), + "SQLiteVNextStore": (136, "1d99dbaf69e4ea888ca7beb2389ac176650a2e573c067bf0686adc9e609132f5"), } diff --git a/tests/unit/test_store_memory_lifecycle_split.py b/tests/unit/test_store_memory_lifecycle_split.py index 18b15e917..4f9f9489d 100644 --- a/tests/unit/test_store_memory_lifecycle_split.py +++ b/tests/unit/test_store_memory_lifecycle_split.py @@ -82,6 +82,8 @@ # mutators take the label lock. Metadata receipts include the label_write # keyword and lock wrappers. Facades add only lock_label_writes/read_label_rows; # removing those two names reproduces each previous class-order receipt. +# Reviewed SQLite owner clamp: update_memory settles an edit at its inputs +# and records whether the clamp applied; its signature and metadata stay fixed. # Reviewed strict lock change: the graph lock reads live advisory grants, # and the memory update checks exclusive L before changing labels. SOURCE_RECEIPTS = { @@ -97,23 +99,25 @@ # back between two reads cannot fail memories_seen_range_check. Previous # sqlite receipt 67adaa61..., method AST 3f134ac9...; the metadata # manifests are unchanged. - SQLITE_CARRIER_PATH: "f893f1faeeb9a87135c108992aa91ff036c5f5dccb1bbdaf315ae5afe1b89b73", + SQLITE_CARRIER_PATH: "759cf44762c388e599b4e8c377fa3415ca2fdf9ba7884698259171ef3d2b8138", } EXPECTED_METHOD_AST_MANIFESTS = { "postgres": "827c4f391a1efe50dcb265b7bb50a5b191d2bae32c3f817dcc237d51bfb10d29", - "sqlite": "d0b6024f0803ca9d3c6f6ea7f5022a28453b2b8b83833f0f05343cb3eff89a57", + "sqlite": "3577659fcf9e583bdb957bb1a959ac1d8cae07ce8b50321bc36697dec47acec4", } EXPECTED_METADATA_MANIFESTS = { "postgres": "07a567e26d0f7c4f51ae2a1910d059397b513a575d85f06c2f8c0396ac1bb2ef", "sqlite": "1270ef0115988552418349aa9e94a7442ba04be41443f278f68a1fa81857903a", } EXPECTED_CLASS_ORDERS = { + # Reviewed source recovery and belief propagation add only two Postgres + # methods. The SQLite facade and every existing member retain their order. # Two paired browser-clip capability methods extend both façades, and one # more paired method, ``list_memories_referencing_sources``. # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (174, "095250b8a77d6c0a32d2783343916b947bcae8ae86e3a1693e47c2fb11802211"), + "PostgresVNextStore": (176, "3549d6ea179f4ebc2341f1c251f139dfb747d56700a4554aef114150efed5c3f"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, From a860d07963f2e394d69ffd14b294e58ce935faf8 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:27:20 +0200 Subject: [PATCH 099/270] Enforce canonical dependency record completeness --- apps/api/src/alicebot_api/vnext_derived_labels.py | 7 +------ tests/unit/test_derived_labels_record_completeness.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 3391fb4b9..97c4b3bc9 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -534,16 +534,13 @@ def _derived_from_deps(record: object) -> tuple[str, set[tuple[str, str]]]: return "malformed", set() found: set[tuple[str, str]] = set() lists: dict[str, list[str]] = {} - repeated = False for key, kind in _DERIVED_FROM_KIND.items(): if key not in record: lists[key] = [] continue raw = record.get(key) - if not isinstance(raw, list) or any(not isinstance(item, str) for item in raw): + if not isinstance(raw, list) or any(not isinstance(item, str) or not item.strip() for item in raw): return "malformed", set() - if len(raw) != len(set(raw)): - repeated = True ids = _strings(raw) lists[key] = ids _add_ids(found, kind, ids) @@ -554,8 +551,6 @@ def _derived_from_deps(record: object) -> tuple[str, set[tuple[str, str]]]: return "", found if not isinstance(counts, Mapping): return "malformed", found - if repeated: - return "", found for key, ids in lists.items(): if key not in counts: if ids: diff --git a/tests/unit/test_derived_labels_record_completeness.py b/tests/unit/test_derived_labels_record_completeness.py index efac96c30..ff9d6b679 100644 --- a/tests/unit/test_derived_labels_record_completeness.py +++ b/tests/unit/test_derived_labels_record_completeness.py @@ -34,3 +34,13 @@ def test_a_missing_or_malformed_legacy_completeness_record_is_unverified(workflo def test_a_legitimate_empty_legacy_record_is_verified(workflow, lists, counts): row = legacy_report(workflow, lists, counts) assert settle_labels([row]).by_stored("artifact", "report").unverified is False + + +@pytest.mark.parametrize("ids,count", ((["source", "source"], 1), ([""], 1), ([" "], 1))) +def test_canonical_counts_and_identifiers_do_not_use_the_legacy_duplicate_exception(ids, count): + source = {"kind": "source", "id": "source", "domain": "project", "sensitivity": "public", "metadata_json": {}} + row = {"kind": "artifact", "id": "report", "metadata_json": {"derived_from": { + "v": 1, "sources": ids, "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], + "counts": {"sources": count, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, + }}} + assert settle_labels([source, row]).by_stored("artifact", "report").unverified is True From 68ffd47816a76b456785065567e7972050476dc7 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:09:56 +0200 Subject: [PATCH 100/270] Guard policy telemetry and quality ratings by current labels --- .../alicebot_api/routers/vnext_projects.py | 29 ++++++++++---- .../src/alicebot_api/routers/vnext_review.py | 27 ++++++++++--- ...derived_labels_read_acceptance_postgres.py | 39 +++++++++++++++++++ tests/unit/test_label_door_registry.py | 14 ++++++- 4 files changed, 95 insertions(+), 14 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/vnext_projects.py b/apps/api/src/alicebot_api/routers/vnext_projects.py index 0e6485081..cf87ac8ed 100644 --- a/apps/api/src/alicebot_api/routers/vnext_projects.py +++ b/apps/api/src/alicebot_api/routers/vnext_projects.py @@ -382,17 +382,32 @@ def get_vnext_agent_policy_telemetry( user_id: UUID, agent_id: str | None = None, limit: Annotated[int, Query(ge=1, le=200)] = 200, + authorization: str | None = Header(default=None), ) -> JSONResponse: + from alicebot_api.vnext_agent_control import ALL_SENSITIVITY + from alicebot_api.vnext_label_guard import LabelGuard, sensitivity_ceiling + settings = get_settings() bounded_limit = min(max(limit, 1), 200) - with user_connection(settings.database_url, user_id) as conn: - store = PostgresVNextStore(conn) - payload = summarize_agent_policy_telemetry( - agent_events=store.list_agent_events(agent_id=agent_id, limit=bounded_limit), - artifacts=store.list_agent_policy_artifacts(agent_id=agent_id, limit=bounded_limit), - memories=store.list_agent_policy_memories(agent_id=agent_id, limit=bounded_limit), - ) + try: + with user_connection(settings.database_url, user_id) as conn: + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, user_id=user_id, raw_key=agent_key_from_authorization(authorization), payload={}, + ) + projects = identity.project_scope if identity is not None else () + guard = LabelGuard.for_filters( + store, (), sensitivity_ceiling(identity) or ALL_SENSITIVITY, projects, + all_of=projects if identity is not None and identity.project_scope_locked else None, + ) + payload = summarize_agent_policy_telemetry( + agent_events=guard.admit_events(store.list_agent_events(agent_id=agent_id, limit=bounded_limit)), + artifacts=guard.admit_rows("artifact", store.list_agent_policy_artifacts(agent_id=agent_id, limit=bounded_limit)), + memories=guard.admit_rows("memory", store.list_agent_policy_memories(agent_id=agent_id, limit=bounded_limit)), + ) + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) return JSONResponse(status_code=200, content=jsonable_encoder({"summary": payload})) diff --git a/apps/api/src/alicebot_api/routers/vnext_review.py b/apps/api/src/alicebot_api/routers/vnext_review.py index 6805c0d9f..9a94db0bc 100644 --- a/apps/api/src/alicebot_api/routers/vnext_review.py +++ b/apps/api/src/alicebot_api/routers/vnext_review.py @@ -792,14 +792,29 @@ def rate_vnext_artifact_quality( return JSONResponse(status_code=201, content=jsonable_encoder(payload)) @review_router.get("/v0/vnext/quality-evals") -def list_vnext_quality_evals(user_id: UUID, artifact_id: UUID | None = None, limit: int = 100) -> JSONResponse: +def list_vnext_quality_evals(user_id: UUID, artifact_id: UUID | None = None, limit: int = 100, authorization: str | None = Header(default=None)) -> JSONResponse: + from alicebot_api.vnext_agent_control import ALL_SENSITIVITY + from alicebot_api.vnext_label_guard import LabelGuard, sensitivity_ceiling + settings = get_settings() bounded_limit = max(1, min(limit, 200)) - with user_connection(settings.database_url, user_id) as conn: - rows = PostgresVNextStore(conn).list_artifact_quality_ratings( - artifact_id=str(artifact_id) if artifact_id is not None else None, - limit=bounded_limit, - ) + try: + with user_connection(settings.database_url, user_id) as conn: + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, user_id=user_id, raw_key=agent_key_from_authorization(authorization), payload={}, + ) + projects = identity.project_scope if identity is not None else () + guard = LabelGuard.for_filters( + store, (), sensitivity_ceiling(identity) or ALL_SENSITIVITY, projects, + all_of=projects if identity is not None and identity.project_scope_locked else None, + ) + rows = guard.admit_related_rows(store.list_artifact_quality_ratings( + artifact_id=str(artifact_id) if artifact_id is not None else None, + limit=bounded_limit, + ), kind="artifact", field="artifact_id") + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) return JSONResponse( status_code=200, content=jsonable_encoder( diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py index 3eaac2692..a0c899d53 100644 --- a/tests/integration/test_derived_labels_read_acceptance_postgres.py +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -161,3 +161,42 @@ def test_workspace_activity_uses_actual_key_and_current_targets(migrated_databas assert len(body["agent_activity"]["recent_commits"]) == 1 assert len(body["agent_activity"]["inline_confirmations"]) == 1 assert body["dogfooding"]["sample_scope"]["memories"]["total_count"] == 1 + + +@pytest.mark.parametrize("reader", ("owner", "admin", "trusted")) +def test_telemetry_and_quality_ratings_use_current_target_labels(migrated_database_urls, monkeypatch, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + for module in (vnext_projects, vnext_review): + monkeypatch.setattr(module, "get_settings", lambda: Settings(database_url=app_url)) + hidden_ids = [] + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Cedar hidden parent", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential"}) + for hidden in (False, True): + metadata = {"agent_id": "synthetic-reader"} + if hidden: + metadata["source_id"] = str(source["id"]) + derived = {"v": 1, "sources": [str(source["id"])] if hidden else [], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": int(hidden), "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}} + with without_insert_floor(): + memory = store.create_memory({"memory_key": f"telemetry-{hidden}", "canonical_text": "Cedar hidden telemetry" if hidden else "Public telemetry", "domain": "project", "sensitivity": "public", "metadata_json": metadata}) + artifact = store.create_artifact({"artifact_type": "daily_brief", "title": "Cedar hidden artifact" if hidden else "Public artifact", "content_markdown": "Public stored copy", "domain": "project", "sensitivity": "public", "metadata_json": {"agent_id": "synthetic-reader", "generated_by": "agent", "derived_from": derived}}) + rating = store.create_artifact_quality_rating({"artifact_id": str(artifact["id"]), "reviewer_id": "synthetic-reviewer", "verbosity": "right_sized", "comments": "Cedar hidden feedback" if hidden else "Public feedback"}) + append_event(store, event_type="agent.policy_blocked", actor_type="agent", actor_id="synthetic-reader", target_type="memory", target_id=str(memory["id"]), payload={}) + if hidden: + hidden_ids.extend((str(artifact["id"]), str(rating["id"]))) + key = real_reader_key(store, user_id, reader) + auth = f"Bearer {key}" if key else None + telemetry = vnext_projects.get_vnext_agent_policy_telemetry(user_id, authorization=auth) + summary = json.loads(telemetry.body)["summary"] + admitted_count = 1 if reader == "trusted" else 2 + assert summary["total_agent_events"] == admitted_count + assert summary["memory_proposals_by_agent"] == [{"agent_id": "synthetic-reader", "count": admitted_count}] + assert summary["artifact_generation_by_agent"] == [{"agent_id": "synthetic-reader", "count": admitted_count}] + ratings = vnext_review.list_vnext_quality_evals(user_id, authorization=auth) + assert json.loads(ratings.body)["count"] == admitted_count + if reader == "trusted": + assert all(identifier not in ratings.body.decode() for identifier in hidden_ids) + assert "Cedar hidden feedback" not in ratings.body.decode() + else: + assert all(identifier in ratings.body.decode() for identifier in hidden_ids) diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py index f48dae1c7..3e0350f58 100644 --- a/tests/unit/test_label_door_registry.py +++ b/tests/unit/test_label_door_registry.py @@ -36,6 +36,8 @@ "list_events_for_source_trace", "list_recent_agentic_commits", "list_pending_inline_confirmations", "count_sources", "count_artifacts", "count_artifacts_by_status", "count_projects", "count_memories_by_status", "count_open_loops", "count_open_loops_by_status", "count_events", "iter_label_rows", "iter_label_events", "iter_label_ratings", + "list_agent_events", "list_agent_policy_artifacts", "list_agent_policy_memories", + "list_artifact_quality_ratings", "count_artifact_quality_ratings", } GUARD_CALLS = { @@ -72,6 +74,8 @@ "routers/_vnext_shared.py:_vnext_load_source_trace": None, "routers/vnext_projects.py:list_vnext_projects": None, "routers/vnext_review.py:list_vnext_artifacts": None, + "routers/vnext_review.py:list_vnext_quality_evals": None, + "routers/vnext_projects.py:get_vnext_agent_policy_telemetry": None, "routers/vnext_review.py:get_vnext_belief_state": None, "routers/vnext_memories.py:get_vnext_dogfooding_dashboard": None, "vnext_projects.py:VNextProjectService.project_dashboard": None, @@ -85,6 +89,7 @@ "session_briefing.py:_event_target_honours_fence": None, "session_briefing.py:_memory_honours_fence": None, "routers/workspaces.py:_vnext_workspace_payload": None, + "routers/workspaces.py:_workspace_event_visible": "_workspace_rows", "vnext_context_tree.py:VNextContextTreeService.build_tree": None, "vnext_dogfooding.py:VNextDogfoodingService.dashboard": None, "vnext_contradictions.py:VNextContradictionService.belief_state": None, @@ -164,6 +169,7 @@ "vnext_connectors.py:VNextConnectorService.get_cursor": "connector cursor events only; no labelled targets", "vnext_connectors.py:VNextConnectorService.get_config": "connector configuration events only; no labelled targets", "vnext_connectors.py:VNextConnectorService.connector_health": "connector state telemetry only; no labels_raised events", + "vnext_dogfooding.py:VNextDogfoodingStore.list_artifact_quality_ratings": "store protocol declaration; no execution or response", "vnext_artifact_review.py:dispatch_vnext_artifact_review": "writer entry; calling route or MCP authorizes the artifact before dispatch", "vnext_memory_commit.py:VNextMemoryCommitService._guard_supersession_acyclic": "write validation traverses pointers without exposing their content", } @@ -224,6 +230,11 @@ def _reader_names(node: ast.AST) -> set[str]: elif isinstance(child, ast.Call) and isinstance(child.func, ast.Name) and child.func.id == "getattr": if len(child.args) > 1 and isinstance(child.args[1], ast.Constant) and child.args[1].value in READS: names.add(child.args[1].value) + elif len(child.args) > 1 and not isinstance(child.args[1], ast.Constant): + # Target-kind dispatch maps choose a method name dynamically. + # Resolve their possible reader names from this function's AST. + names.update(value.value for value in ast.walk(node) if isinstance(value, ast.Constant) + and isinstance(value.value, str) and value.value in READS) return names @@ -266,7 +277,8 @@ def test_every_scanned_reader_is_classified() -> None: def test_discovery_catches_new_direct_and_dynamic_readers() -> None: - for source in ("def added(store): return store.list_events()", "def added(store): return getattr(store, 'list_memories')()", "def added(store): return invoke(store.list_beliefs)"): + for source in ("def added(store): return store.list_events()", "def added(store): return getattr(store, 'list_memories')()", "def added(store): return invoke(store.list_beliefs)", + "def added(store, kind):\n methods = {'memory': 'get_memory', 'artifact': 'get_artifact'}\n return getattr(store, methods[kind])()"): node = ast.parse(source).body[0] assert _reader_names(node) assert "added" not in DOORS and "added" not in NOT_A_DOOR From 1431ad7b4350f9d0215de2cdf7d29a3493dc6d9b Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:13:34 +0200 Subject: [PATCH 101/270] Verify replacement memories preserve dependency edges --- .../test_sqlite_derived_labels_write_path.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/unit/test_sqlite_derived_labels_write_path.py b/tests/unit/test_sqlite_derived_labels_write_path.py index ad231c68a..ee6a00024 100644 --- a/tests/unit/test_sqlite_derived_labels_write_path.py +++ b/tests/unit/test_sqlite_derived_labels_write_path.py @@ -232,6 +232,45 @@ def test_merge_protected_metadata_keeps_label_keys_unless_the_write_is_a_relabel assert relabel["consolidation"] == {"cluster_member_ids": ["m"]} +def test_a_replacement_memory_keeps_dependencies_and_floor(tmp_path: Path, monkeypatch) -> None: + from uuid import UUID + + from alicebot_api.mcp.registry import call_mcp_tool + from alicebot_api.mcp.types import MCPRuntimeContext + from alicebot_api.onramp import sqlite_url_for_path + from alicebot_api.vnext_derived_labels import with_derived_from + + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + path = tmp_path / "replacement.sqlite3" + with _vault(path) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source({"source_type": "note", "title": "Synthetic observation", "content_hash": "replacement", "domain": "health", "sensitivity": "confidential", "metadata_json": {"project_scope": [ALPHA]}}) + metadata = with_derived_from({"source_id": str(source["id"]), "project_scope": [ALPHA]}, {"sources": [source]}) + original = store.create_memory({"memory_key": "replacement-original", "canonical_text": "Synthetic original observation", "status": "active", "domain": "health", "sensitivity": "confidential", "metadata_json": metadata}) + link = store.create_provenance_link({"target_type": "memory", "target_id": str(original["id"]), "source_id": str(source["id"]), "evidence_role": "supports", "confidence": 1.0}) + result = call_mcp_tool( + MCPRuntimeContext(database_url=sqlite_url_for_path(path), user_id=UUID(USER)), + name="alice_memory_correct", + arguments={"review_item_id": str(original["id"]), "action": "supersede-existing", "replacement_title": "Synthetic corrected observation", "replacement_provenance": {"source_id": str(source["id"]), "evidence_role": "supports", "confidence": 1.0}}, + ) + replacement_id = str(result["replacement_object"]["id"]) + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + replacement = store.get_memory(replacement_id) + assert replacement["metadata_json"]["derived_from"] == metadata["derived_from"] + assert replacement["metadata_json"]["source_id"] == source["id"] + assert replacement["metadata_json"]["project_scope"] == [ALPHA] + assert replacement["metadata_json"]["project_floor"] == [ALPHA] + assert replacement["sensitivity"] == "confidential" + assert store.get_memory(str(original["id"]))["canonical_text"] == original["canonical_text"] + assert str(store.list_provenance_links(target_type="memory", target_id=str(original["id"]))[0]["id"]) == str(link["id"]) + assert str(store.list_provenance_links(target_type="memory", target_id=replacement_id)[0]["source_id"]) == str(source["id"]) + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "regulated"}) + assert store.get_memory(replacement_id)["sensitivity"] == "regulated" + + def test_insert_floor_survives_two_hops(tmp_path: Path): db = tmp_path / 'labels.sqlite3' bootstrap_database(db, user_id=USER, user_email='synthetic@example.test') From 3d3dc36abf3137d3155e6d97e59e0a77dfcf8d42 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:12:47 +0200 Subject: [PATCH 102/270] Pin the scope-free project state and preserve moved source floors --- .../test_derived_labels_propagation_postgres.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_derived_labels_propagation_postgres.py b/tests/integration/test_derived_labels_propagation_postgres.py index 06ab3a8f1..237a50fe4 100644 --- a/tests/integration/test_derived_labels_propagation_postgres.py +++ b/tests/integration/test_derived_labels_propagation_postgres.py @@ -200,6 +200,11 @@ def test_a_source_relabel_reaches_the_extracted_memories_the_loop_every_report_a for kind, row_id in rows: row = _read_row(store, kind, row_id) assert_raised(row) - assert beta in row["metadata_json"]["project_floor"] + if kind == "project": + # A project state has domain and sensitivity but carries no project scope. + assert not row["metadata_json"].get("project_scope") + assert not row["metadata_json"].get("project_floor") + else: + assert {str(project["id"]), beta}.issubset(row["metadata_json"]["project_floor"]), (kind, row) if kind == "artifact": assert h.request("GET", f"/v0/vnext/artifacts/{row_id}", key=bound)[0] == 403 From 81e778ea74a2aa4a35064973f87fb3952d0b5dd3 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:14:57 +0200 Subject: [PATCH 103/270] Keep legacy repair proof isolated under separated database roles --- tests/integration/test_derived_domain_postgres.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_derived_domain_postgres.py b/tests/integration/test_derived_domain_postgres.py index 3c68bc2dc..b248e0aca 100644 --- a/tests/integration/test_derived_domain_postgres.py +++ b/tests/integration/test_derived_domain_postgres.py @@ -1,5 +1,6 @@ """Restricted-domain generation and migration on the role-separated database.""" +import os from uuid import uuid4 from urllib.parse import urlsplit, urlunsplit @@ -183,8 +184,11 @@ def test_postgres_repair_as_documented_nobypassrls_owner(database_urls, monkeypa role_url = urlunsplit( parsed._replace(netloc=f"{role}:fixture-role-password@{parsed.hostname}:{parsed.port or 5432}") ) - with psycopg.connect(database_urls["admin"], autocommit=True) as admin: - owner = admin.execute("SELECT current_user").fetchone()[0] + lifecycle = urlsplit(os.getenv("DATABASE_LIFECYCLE_URL", database_urls["admin"])) + lifecycle_url = urlunsplit(lifecycle._replace(path=parsed.path)) + with psycopg.connect(lifecycle_url, autocommit=True) as admin: + owner = parsed.username + assert owner is not None admin.execute( sql.SQL("CREATE ROLE {} LOGIN NOSUPERUSER NOBYPASSRLS PASSWORD {}").format( sql.Identifier(role), sql.Literal("fixture-role-password") @@ -210,10 +214,11 @@ def injected_failure(*args): with monkeypatch.context() as patch: patch.setattr(repair, "relabel_event", injected_failure) with pytest.raises(RuntimeError, match="injected audit failure"): - command.upgrade(make_alembic_config(role_url), "head") + command.upgrade(make_alembic_config(role_url), "20261004_0095") assert admin.execute("SELECT version_num FROM alembic_version").fetchone()[0] == "20260721_0094" else: - command.upgrade(make_alembic_config(role_url), "head") + # Keep this v2 guard proof independent of the later v3 repair. + command.upgrade(make_alembic_config(role_url), "20261004_0095") assert all( row[0] for row in admin.execute( From da309661a059bb70f9bdb8b3b903d0f101411da7 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:16:10 +0200 Subject: [PATCH 104/270] Collect encoded memory and trace references before lookup --- CHANGELOG.md | 2 +- .../vnext_open_loop_references.py | 4 +++ .../test_open_loop_references_read_fence.py | 35 +++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60fca387c..de7b75f40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings, including JSON escapes and repeated keys. Admitted references and surrounding values stay, and the owner receives live references unchanged. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. +- Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings, including JSON escapes and repeated keys. Lookup and removal both decode JSON text under source and memory references and inside trace fields. Admitted references and surrounding values stay, and the owner receives live references unchanged. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. - Unreleased (on main, not in v0.20.0): the derived-row pages state the three-part label, who reads an unverified row, and that the five operator screens apply the caller's sensitivity ceiling. The v0.20.0 notes omitted both limits. No migration is required. - Unreleased (on main, not in v0.20.0): PostgreSQL migration `20261005_0096` raises a stored derived row to the labels of its inputs, and SQLite does the same on the next open and before a restore is published. The open pass can leave a vault open with its completion key unstamped after a failed repair; explicit repair and restore refuse the whole change on failure. `alicebot vnext labels` and `alice-memory labels` check and repair the same rows, including rows added after an earlier repair. The doctors print how many derived rows are below their inputs or unverified, as a warning, and report failed checks as unavailable. v0.20.0 left the stored label where it was written. Migration `20261005_0096` must run before serving requests as the NOSUPERUSER NOBYPASSRLS table owner. - Unreleased (on main, not in v0.20.0): recall, context packs, resume, recent decisions, the session brief, the review queue, open-loop lists, and the workspace, dashboard, context tree, and dogfooding lists drop a loaded row whose effective label is outside the request. The artifact list, the source trace, the project list, the dashboard project row, and the belief state route apply the caller's sensitivity ceiling, including titles, ids, and counts. Resume, recent decisions, the session brief, audit, and explain apply the target row's current effective label, so an excluded key does not learn of a label change. The producer input readers also drop an input whose effective label is outside the request before it appears in report text. v0.20.0 returned rows by their stored labels and could copy a public row with confidential inputs into a new report. No migration is required. diff --git a/apps/api/src/alicebot_api/vnext_open_loop_references.py b/apps/api/src/alicebot_api/vnext_open_loop_references.py index 80100c097..0dfe79d33 100644 --- a/apps/api/src/alicebot_api/vnext_open_loop_references.py +++ b/apps/api/src/alicebot_api/vnext_open_loop_references.py @@ -280,6 +280,10 @@ def _collect_ids(value: object, found: set[str], referenced: set[str], *, at_ref if depth > _METADATA_MAX_DEPTH: return if isinstance(value, str): + decoded = _json_container(value) + if decoded is not None: + _collect_ids(decoded, found, referenced, at_reference=at_reference, depth=depth + 1) + return ids = _ids_in_text(value) if at_reference: ids |= set(cited_source_ids(value).named) diff --git a/tests/unit/test_open_loop_references_read_fence.py b/tests/unit/test_open_loop_references_read_fence.py index e77423303..1de252c54 100644 --- a/tests/unit/test_open_loop_references_read_fence.py +++ b/tests/unit/test_open_loop_references_read_fence.py @@ -352,6 +352,41 @@ def test_encoded_json_references_are_withheld_on_the_real_key_list(world: _World assert "amber" in str(checked[reference_key]) +@pytest.mark.parametrize("case", ("memory_reference", "nested_source_reference", "duplicate_memory_key")) +def test_encoded_memory_and_trace_references_are_collected_before_the_real_key_list(world: _World, case: str) -> None: + """The collection and removal walk the same decoded JSON, with admitted and owner controls. + + Mutation: skip JSON decoding in ``_collect_ids``. Memory references and source references inside a trace then stay. + """ + + encode = lambda value: "".join("\\u%04x" % ord(char) for char in value) + if case == "nested_source_reference": + key, hidden, admitted = "trace", world.sources["beta"], world.sources["own"] + text = '{"source_ids": ["' + encode(hidden) + '", "' + encode(admitted) + '"], "kept": "cobalt"}' + else: + key, hidden, admitted = "memory_refs", world.memories["beta"], world.memories["own"] + if case == "duplicate_memory_key": + text = '{"memory_id": "memory:' + encode(hidden) + '", "memory_id": "memory:' + encode(admitted) + '", "kept": "cobalt"}' + else: + text = '["memory:' + encode(hidden) + '", "memory:' + encode(admitted) + '", "cobalt"]' + metadata = {"project_scope": ["alpha"], key: text, "kept": "control"} + world._plant("encoded_collection", metadata=metadata) + loop_id = world.loops["encoded_collection"] + world.vault.sql("DELETE FROM open_loops WHERE id != ?", (loop_id,)) + owner = world.vault.wire("alice_open_loops", {"status": "all", "limit": 100}, key=None) + assert owner["is_error"] is False + assert len(owner["payload"]["items"]) == 1 + assert owner["payload"]["items"][0]["metadata_json"] == metadata + items = world.list_items("alpha_project") + assert len(items) == 1 and str(items[0]["id"]) == loop_id + checked = items[0]["metadata_json"] + decoded = json.dumps(json.loads(checked[key])) + assert hidden not in decoded + assert admitted in decoded + assert "cobalt" in decoded + assert checked["kept"] == "control" + + @pytest.mark.parametrize("reader", _UPDATERS) @pytest.mark.parametrize("name", sorted(("own", "health", "confidential", "old_beta", "old_global", "old_deleted", "old_ghost", "old_upper", "old_metadata"))) def test_the_open_loop_update_actions_withhold_what_the_reader_may_not_read(world: _World, reader: str, name: str) -> None: From e7478e2485178a05adc03f9eba3cbd8ff77ba4c6 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:19:08 +0200 Subject: [PATCH 105/270] Cover missing canonical dependency count records --- tests/unit/test_derived_labels_record_completeness.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/test_derived_labels_record_completeness.py b/tests/unit/test_derived_labels_record_completeness.py index ff9d6b679..6bc06813d 100644 --- a/tests/unit/test_derived_labels_record_completeness.py +++ b/tests/unit/test_derived_labels_record_completeness.py @@ -44,3 +44,13 @@ def test_canonical_counts_and_identifiers_do_not_use_the_legacy_duplicate_except "counts": {"sources": count, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, }}} assert settle_labels([source, row]).by_stored("artifact", "report").unverified is True + + +@pytest.mark.parametrize("counts", (None, {}, {"sources": True}, {"sources": "1"}, [])) +def test_a_nonempty_canonical_record_requires_well_formed_complete_counts(counts): + source = {"kind": "source", "id": "source", "domain": "project", "sensitivity": "public", "metadata_json": {}} + row = {"kind": "artifact", "id": "report", "metadata_json": {"derived_from": { + "v": 1, "sources": ["source"], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], + "counts": counts, + }}} + assert settle_labels([source, row]).by_stored("artifact", "report").unverified is True From 4bb982668f04d80a15ce908ba22ac17fae88ee9f Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:19:08 +0200 Subject: [PATCH 106/270] Expand effective producer inputs and promoted group controls --- ...est_derived_labels_group_scope_postgres.py | 20 +++++++++- .../test_derived_labels_producers_postgres.py | 19 +++++++--- tests/unit/test_group_scope_sqlite.py | 37 ++++++++++++++++++- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_derived_labels_group_scope_postgres.py b/tests/integration/test_derived_labels_group_scope_postgres.py index 39c093d19..3b6f07192 100644 --- a/tests/integration/test_derived_labels_group_scope_postgres.py +++ b/tests/integration/test_derived_labels_group_scope_postgres.py @@ -1,16 +1,17 @@ """The PostgreSQL group consumers have the same controls as SQLite.""" from uuid import uuid4 +import json import pytest from alicebot_api.db import user_connection from alicebot_api.store import ContinuityStore from alicebot_api.vnext_derived_labels import group_scope -from alicebot_api.vnext_memory_commit import VNextMemoryCommitService +from alicebot_api.vnext_memory_commit import VNextMemoryCommitService, VNextMemoryCommitValidationError from alicebot_api.vnext_rollups import VNextRollupService from alicebot_api.vnext_store import PostgresVNextStore -from tests.unit.test_group_scope_sqlite import ALPHA, seed_members +from tests.unit.test_group_scope_sqlite import ALPHA, seed_members, seed_promoted_members @pytest.mark.parametrize("accept", (False, True)) @@ -33,3 +34,18 @@ def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inser assert second.proposals == [] assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == before assert any(group["state"] == ("already_covered_by_accepted" if accept else "existing_candidate") for group in second.groups), second + + +def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossing_project_scopes(migrated_database_urls): + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") + store = PostgresVNextStore(conn) + members = seed_promoted_members(store) + first = VNextRollupService(store).propose_rollups() + candidate_id = first.candidate_ids[0] + member = members[0] + metadata = dict(member["metadata_json"], project_scope=[], project_floor=[ALPHA]) + conn.execute("UPDATE memories SET metadata_json=%s::jsonb WHERE id=%s", (json.dumps(metadata), member["id"])) + with pytest.raises(VNextMemoryCommitValidationError, match="crosses project scopes"): + VNextMemoryCommitService(store).accept_consolidation_candidate(candidate_id, reason="Reviewed synthetic group") diff --git a/tests/integration/test_derived_labels_producers_postgres.py b/tests/integration/test_derived_labels_producers_postgres.py index 134372736..dcc57c021 100644 --- a/tests/integration/test_derived_labels_producers_postgres.py +++ b/tests/integration/test_derived_labels_producers_postgres.py @@ -179,36 +179,45 @@ def copies(value): assert f"SENTINEL_{label.upper()}" not in text -def test_input_selection_uses_effective_labels_including_the_owner_default_ceiling(migrated_database_urls, monkeypatch): +@pytest.mark.parametrize("producer", PRODUCERS) +def test_input_selection_uses_effective_labels_including_the_owner_default_ceiling(migrated_database_urls, monkeypatch, producer): app_url = migrated_database_urls["app"] wire_database(monkeypatch, app_url) user_id, alpha, _beta, rows, _key, _beta_key, _unbound = seed_grid(app_url) with user_connection(app_url, user_id) as conn: store = PostgresVNextStore(conn) source = next(row for label, kind, row in rows if label == "alpha" and kind == "sources") - copy = store.create_memory({"memory_key": "stale.source.copy", "canonical_text": "STALE_SOURCE_COPY Atlas secret", + copy = store.create_memory({"memory_key": "stale.source.copy", "canonical_text": "Atlas played Hollow Knight for 25 hours. STALE_SOURCE_COPY Atlas secret", "title": "STALE_SOURCE_COPY", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"]), "project_scope": [alpha]}}) promoted = next(row for label, kind, row in rows if label == "alpha" and kind == "memories" and row["metadata_json"].get("source_artifact_id")) + derived_loop = store.create_open_loop({"title": "STALE_SOURCE_LOOP Atlas", "description": "STALE_SOURCE_LOOP secret", + "source_id": str(source["id"]), "status": "open", "domain": "project", "sensitivity": "public", + "due_at": "2026-10-04T12:00:00Z", "metadata_json": {"project_scope": [alpha], + "discovered_by": "vnext_daily_brief", "source_id": str(source["id"])}}) conn.execute("UPDATE sources SET sensitivity='confidential' WHERE id=%s", (source["id"],)) conn.execute("UPDATE memories SET created_at='2026-10-05T09:00:00Z' WHERE id=%s", (copy["id"],)) + if producer == "staleness": + conn.execute("UPDATE memories SET valid_to='2026-10-04T12:00:00Z'") assert store.get_memory(str(copy["id"]))["sensitivity"] == "public" assert store.get_memory(str(promoted["id"]))["sensitivity"] == "public" _, trusted = create_agent_key(store, user_id=user_id, agent_id="trusted-alpha", permission_profile="trusted_local_agent", project_scope=alpha) owner = VNextBrainService(store).generate_daily_brief(BrainArtifactRequest(generated_for="2026-10-05", source_limit=50, memory_limit=50, artifact_limit=50, open_loop_limit=50, discover_open_loops=False, create_candidate_memories=False)) - bound, _ = generate("daily", user_id, alpha, trusted) + bound, _ = generate(producer, user_id, alpha, trusted) for report in (owner, bound): text = json.dumps(report, default=str) assert str(copy["id"]) not in text assert copy["canonical_text"] not in text assert str(promoted["id"]) not in text assert str(source["id"]) not in text + assert str(derived_loop["id"]) not in text + assert "STALE_SOURCE_LOOP" not in text assert "SENTINEL_ALPHA prior report text" not in text - assert any(str(row["id"]) in text for label, kind, row in rows if label == "alpha" and kind == "memories" - and row["memory_type"] == "episode") + assert any(str(row["id"]) in text for label, kind, row in rows if label == "alpha" and + (kind == "open_loops" or (kind == "memories" and row["memory_type"] == "episode"))), (producer, report) def test_the_owner_keeps_the_cross_project_brief_and_bound_keys_cannot_read_it(migrated_database_urls, monkeypatch): diff --git a/tests/unit/test_group_scope_sqlite.py b/tests/unit/test_group_scope_sqlite.py index a982ae6ed..795bd2a6d 100644 --- a/tests/unit/test_group_scope_sqlite.py +++ b/tests/unit/test_group_scope_sqlite.py @@ -11,8 +11,10 @@ from alicebot_api.sqlite_schema import bootstrap_sqlite_schema from alicebot_api.sqlite_store import SQLiteVNextStore, ensure_sqlite_user from alicebot_api.vnext_derived_labels import group_scope +from alicebot_api.vnext_derived_labels import with_derived_from from alicebot_api.vnext_memory_commit import VNextMemoryCommitService, VNextMemoryCommitValidationError from alicebot_api.vnext_rollups import VNextRollupService +from alicebot_api.vnext_queue import VNextQueueService ALPHA = "prj_" + "a" * 16 BETA = "prj_" + "b" * 16 @@ -30,6 +32,37 @@ def seed_members(store): return members +def seed_promoted_members(store): + members = [] + for index, (text, day) in enumerate((("I played Hollow Knight for 25 hours", "2023-06-02"), + ("I played Stardew Valley for 85 hours", "2023-06-20"), + ("I played Celeste for 10 hours", "2023-07-01"))): + if not hasattr(store, "create_artifact"): + # SQLite has no artifact table. A reviewed weekly copy uses the same aggregate rule. + parent = store.create_memory({"memory_key": f"promoted.parent.{index}", "canonical_text": text, + "status": "superseded", "domain": "personal", "sensitivity": "internal", + "metadata_json": {"project_scope": [ALPHA, BETA]}}) + memory = store.create_memory({"memory_key": f"promoted.copy.{index}", "title": text, + "canonical_text": text, "summary": text, "value": {"text": text}, "status": "active", + "memory_type": "semantic", "domain": "personal", "sensitivity": "internal", + "metadata_json": with_derived_from({"discovered_by": "vnext_weekly_synthesis", + "promotion_reviewed": True, "project_scope": [], "session_date": day}, {"memories": [parent]})}) + assert set(memory["metadata_json"]["project_floor"]) == {ALPHA, BETA} + members.append(memory) + continue + artifact = store.create_artifact({"artifact_type": "research_brief", "title": text, + "content_markdown": text, "domain": "personal", "sensitivity": "internal", + "metadata_json": {"project_scope": [ALPHA, BETA]}}) + promotion = VNextQueueService(store).review_artifact(artifact_id=str(artifact["id"]), action="promote") + memory = store.get_memory(promotion["promoted_memory_id"]) + metadata = dict(memory["metadata_json"], session_date=day) + memory = store.update_memory(memory_id=str(memory["id"]), patch={"metadata_json": metadata}) + assert memory["metadata_json"]["project_scope"] == [] + assert set(memory["metadata_json"]["project_floor"]) == {ALPHA, BETA} + members.append(memory) + return members + + @pytest.fixture def sqlite_group_store(): conn = sqlite3.connect(":memory:") @@ -70,8 +103,8 @@ def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inser def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossing_project_scopes(sqlite_group_store): store = sqlite_group_store - members = seed_members(store) - first = VNextRollupService(store).propose_rollups(projects=(ALPHA,)) + members = seed_promoted_members(store) + first = VNextRollupService(store).propose_rollups() candidate_id = first.candidate_ids[0] # Preserve the snapshot apart from the group labels so the scope check is reached. member = members[0] From 18cd836c43e5bd8555a861d58b6ab024a7f0fb1a Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:19:50 +0200 Subject: [PATCH 107/270] Test project floor views and derived group card lookup --- ...est_derived_labels_group_scope_postgres.py | 35 ++++++++ tests/unit/test_group_scope_sqlite.py | 84 +++++++++++++++++++ tests/unit/test_project_floor_views.py | 75 +++++++++++++++++ 3 files changed, 194 insertions(+) create mode 100644 tests/integration/test_derived_labels_group_scope_postgres.py create mode 100644 tests/unit/test_group_scope_sqlite.py create mode 100644 tests/unit/test_project_floor_views.py diff --git a/tests/integration/test_derived_labels_group_scope_postgres.py b/tests/integration/test_derived_labels_group_scope_postgres.py new file mode 100644 index 000000000..39c093d19 --- /dev/null +++ b/tests/integration/test_derived_labels_group_scope_postgres.py @@ -0,0 +1,35 @@ +"""The PostgreSQL group consumers have the same controls as SQLite.""" + +from uuid import uuid4 + +import pytest + +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_derived_labels import group_scope +from alicebot_api.vnext_memory_commit import VNextMemoryCommitService +from alicebot_api.vnext_rollups import VNextRollupService +from alicebot_api.vnext_store import PostgresVNextStore +from tests.unit.test_group_scope_sqlite import ALPHA, seed_members + + +@pytest.mark.parametrize("accept", (False, True)) +def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inserts_nothing(migrated_database_urls, accept): + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") + store = PostgresVNextStore(conn) + members = seed_members(store) + service = VNextRollupService(store) + first = service.propose_rollups(projects=(ALPHA,)) + assert len(first.candidate_ids) == 1 + candidate = store.get_memory(first.candidate_ids[0]) + assert candidate["metadata_json"]["project_scope"] == [] + assert group_scope(candidate) == group_scope(members[0]) + if accept: + assert VNextMemoryCommitService(store).accept_consolidation_candidate(first.candidate_ids[0], reason="Reviewed synthetic group")["status"] == "accepted" + before = conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] + second = service.propose_rollups(projects=(ALPHA,)) + assert second.proposals == [] + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == before + assert any(group["state"] == ("already_covered_by_accepted" if accept else "existing_candidate") for group in second.groups), second diff --git a/tests/unit/test_group_scope_sqlite.py b/tests/unit/test_group_scope_sqlite.py new file mode 100644 index 000000000..a982ae6ed --- /dev/null +++ b/tests/unit/test_group_scope_sqlite.py @@ -0,0 +1,84 @@ +"""Group consumers keep global aggregate cards usable without widening reads.""" + +from __future__ import annotations + +import sqlite3 +import json +from uuid import uuid4 + +import pytest + +from alicebot_api.sqlite_schema import bootstrap_sqlite_schema +from alicebot_api.sqlite_store import SQLiteVNextStore, ensure_sqlite_user +from alicebot_api.vnext_derived_labels import group_scope +from alicebot_api.vnext_memory_commit import VNextMemoryCommitService, VNextMemoryCommitValidationError +from alicebot_api.vnext_rollups import VNextRollupService + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +def seed_members(store): + members = [] + for index, (text, day) in enumerate((("I played Hollow Knight for 25 hours", "2023-06-02"), + ("I played Stardew Valley for 85 hours", "2023-06-20"), + ("I played Celeste for 10 hours", "2023-07-01"))): + members.append(store.create_memory({"memory_key": f"group-{index}", "memory_type": "episode", + "title": text, "canonical_text": text, "summary": text, "status": "active", "value": {"text": text}, + "domain": "personal", "sensitivity": "internal", + "metadata_json": {"session_date": day, "project_scope": [ALPHA, BETA]}})) + return members + + +@pytest.fixture +def sqlite_group_store(): + conn = sqlite3.connect(":memory:") + bootstrap_sqlite_schema(conn) + user_id = str(uuid4()) + ensure_sqlite_user(conn, user_id, "group@example.invalid", "Group") + yield SQLiteVNextStore(conn, user_id) + conn.close() + + +def test_a_consolidation_candidate_over_a_two_project_group_is_accepted(sqlite_group_store): + store = sqlite_group_store + members = seed_members(store) + first = VNextRollupService(store).propose_rollups(projects=(ALPHA,)) + assert len(first.candidate_ids) == 1 + candidate = store.get_memory(first.candidate_ids[0]) + assert candidate["metadata_json"]["project_scope"] == [] + assert group_scope(candidate) == group_scope(members[0]) + accepted = VNextMemoryCommitService(store).accept_consolidation_candidate(str(candidate["id"]), reason="Reviewed synthetic group") + assert accepted["status"] == "accepted" + + +@pytest.mark.parametrize("accept", (False, True)) +def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inserts_nothing(sqlite_group_store, accept): + store = sqlite_group_store + seed_members(store) + service = VNextRollupService(store) + first = service.propose_rollups(projects=(ALPHA,)) + assert len(first.candidate_ids) == 1 + if accept: + VNextMemoryCommitService(store).accept_consolidation_candidate(first.candidate_ids[0], reason="Reviewed synthetic group") + before = store.conn.execute("SELECT count(*) FROM memories").fetchone()[0] + second = service.propose_rollups(projects=(ALPHA,)) + assert second.proposals == [] + assert store.conn.execute("SELECT count(*) FROM memories").fetchone()[0] == before + assert any(group["state"] == ("already_covered_by_accepted" if accept else "existing_candidate") for group in second.groups), second + + +def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossing_project_scopes(sqlite_group_store): + store = sqlite_group_store + members = seed_members(store) + first = VNextRollupService(store).propose_rollups(projects=(ALPHA,)) + candidate_id = first.candidate_ids[0] + # Preserve the snapshot apart from the group labels so the scope check is reached. + member = members[0] + metadata = dict(member["metadata_json"]) + metadata["project_floor"] = [ALPHA] + metadata["project_scope"] = [] + store.conn.execute("UPDATE memories SET metadata_json=? WHERE id=?", (json.dumps(metadata), member["id"])) + candidate = store.get_memory(candidate_id) + with pytest.raises(VNextMemoryCommitValidationError, match="crosses project scopes"): + VNextMemoryCommitService(store).accept_consolidation_candidate(candidate_id, reason="Reviewed synthetic group") diff --git a/tests/unit/test_project_floor_views.py b/tests/unit/test_project_floor_views.py new file mode 100644 index 000000000..0914388c4 --- /dev/null +++ b/tests/unit/test_project_floor_views.py @@ -0,0 +1,75 @@ +"""Project views agree across both SQL builders and every Python mirror.""" + +from __future__ import annotations + +import inspect +import itertools +import json +import sqlite3 + +import pytest + +from alicebot_api.mcp.retrieval_shared import _resource_matches_project_scope +from alicebot_api.session_briefing import _memory_honours_fence +from alicebot_api.vnext_project_scope import GLOBAL_PROJECT_MARKER, project_scopes_overlap +from alicebot_api.vnext_retrieval import _ResolvedRetrievalScope, _project_scope_meets, _row_matches_scope +from alicebot_api.vnext_stores.sqlite.query_predicates import ( + _ensure_project_scope_identity_sqlite, + _project_view_sql, + _view_membership_sql, +) + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 +SCOPES = ((), (ALPHA,), (BETA,), (ALPHA, BETA), ("Alice",), (ALPHA.upper(),)) +VIEWS = ((), (ALPHA,), (BETA,), (ALPHA, GLOBAL_PROJECT_MARKER), + (BETA, GLOBAL_PROJECT_MARKER), (GLOBAL_PROJECT_MARKER,), (ALPHA, BETA, GLOBAL_PROJECT_MARKER)) + + +def expected(scope, floor, view): + if not view: + return True + ids = {value.lower() for value in view if value != GLOBAL_PROJECT_MARKER} + stored = {value.lower() for value in scope} + alice = lambda values: {value for value in values if value in {ALPHA, BETA}} + return bool(stored & ids) or ( + GLOBAL_PROJECT_MARKER in view and not alice(stored) and alice(value.lower() for value in floor) <= ids + ) + + +@pytest.mark.parametrize("scope,floor,view", itertools.product(SCOPES, SCOPES, VIEWS)) +def test_sql_and_python_project_view_membership_grid(scope, floor, view): + row = {"domain": "project", "sensitivity": "public", "metadata_json": { + "project_scope": list(scope), "project_floor": list(floor), + }} + want = expected(scope, floor, view) + assert project_scopes_overlap(scope, view, floor=floor) == (want if view else False) + assert _project_scope_meets(set(scope), view, floor=floor) == (want if view else False) + assert _resource_matches_project_scope(row, view) == want + resolved = _ResolvedRetrievalScope(frozenset(view), frozenset(), None, None, frozenset()) + assert _row_matches_scope(row, resolved) == want + assert _memory_honours_fence(row, effective_domains=(), effective_sensitivity_allowed=("public",), + effective_project_scope=view, exclude_global_domains=frozenset()) == want + with sqlite3.connect(":memory:") as conn: + _ensure_project_scope_identity_sqlite(conn) + conn.execute("CREATE TABLE rows(metadata_json TEXT, project_id TEXT, domain TEXT)") + conn.execute("INSERT INTO rows VALUES (?, NULL, 'project')", (json.dumps(row["metadata_json"]),)) + placeholders = lambda values: ",".join("?" for _ in values) + kwargs = dict(placeholders=placeholders, scope_expression="alice_project_scope_identity(metadata_json,project_id)", + text_expressions=("metadata_json", "project_id"), domain_expression="domain", + floor_expression="alice_project_floor_identity(metadata_json)") + clause, params = _project_view_sql(**kwargs, projects=view, global_excluded_domains=()) + assert bool(conn.execute("SELECT count(*) FROM rows WHERE 1=1" + clause, params).fetchone()[0]) == want + if view: + ids = tuple(value.lower() for value in view if value != GLOBAL_PROJECT_MARKER) + for partition in (False, True): + sql, params = _view_membership_sql(**kwargs, ids=ids, wants_global=GLOBAL_PROJECT_MARKER in view, + global_excluded_domains=(), partition=partition) + got = conn.execute("SELECT " + sql + " FROM rows", params).fetchone()[0] + assert (got is not None if partition else bool(got)) == want + + +def test_every_marker_aware_python_mirror_passes_the_floor(): + for function in (_project_scope_meets, _row_matches_scope, _resource_matches_project_scope, _memory_honours_fence, + _view_membership_sql, _project_view_sql): + assert "floor" in inspect.getsource(function), function.__name__ From ab382aaaf0a979f93f04d80323583e194b5ae1a4 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:19:42 +0200 Subject: [PATCH 108/270] Report the changed row count from explicit SQLite label repair --- apps/api/src/alicebot_api/label_commands.py | 4 ++-- apps/api/src/alicebot_api/vnext_label_repair.py | 5 +++-- tests/unit/test_sqlite_derived_labels_v3.py | 8 ++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/api/src/alicebot_api/label_commands.py b/apps/api/src/alicebot_api/label_commands.py index f6d523716..c780585d4 100644 --- a/apps/api/src/alicebot_api/label_commands.py +++ b/apps/api/src/alicebot_api/label_commands.py @@ -68,9 +68,9 @@ def run_labels(args) -> int: return 1 if below or unverified or raised_on_next_open else 0 try: with sqlite_user_connection(db, args.user_id) as conn: - relabel_labels_sqlite(conn, explicit=True) + changed = relabel_labels_sqlite(conn, explicit=True) except DerivedDomainRepairError as exc: print(f"labels repair failed: {exc}") return 2 - print("labels repair finished") + print(f"labels repair updated {changed}") return 0 diff --git a/apps/api/src/alicebot_api/vnext_label_repair.py b/apps/api/src/alicebot_api/vnext_label_repair.py index 3dad92c6d..e3163aaa4 100644 --- a/apps/api/src/alicebot_api/vnext_label_repair.py +++ b/apps/api/src/alicebot_api/vnext_label_repair.py @@ -154,7 +154,7 @@ def _stamped(conn) -> bool: ) -def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = False) -> None: +def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = False) -> int: """Raise stored derived labels once, or always for a restore or owner repair. When no transaction is open this begins one and reads the state key inside @@ -169,7 +169,7 @@ def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = Fal if not restoring and not explicit and _stamped(conn): if owns: conn.commit() - return + return 0 changes = plan_label_repairs(_load_tables(conn)) for table, user, stored, previous, new, node in changes: metadata = {"project_scope": list(new["project_scope"]), "project_floor": list(new["project_floor"])} @@ -217,6 +217,7 @@ def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = Fal conn.execute("INSERT OR REPLACE INTO alice_schema_state (key, value) VALUES (?, ?)", (REPAIR_STATE_KEY, "1")) if owns: conn.commit() + return len(changes) except Exception: if owns: conn.rollback() diff --git a/tests/unit/test_sqlite_derived_labels_v3.py b/tests/unit/test_sqlite_derived_labels_v3.py index b07162b93..f7856e023 100644 --- a/tests/unit/test_sqlite_derived_labels_v3.py +++ b/tests/unit/test_sqlite_derived_labels_v3.py @@ -181,15 +181,19 @@ def test_old_backup_with_cross_project_aggregates(tmp_path, monkeypatch): restricted_reads(target, ids, monkeypatch, project="alpha") -def test_explicit_repair_ignores_completion_stamp(tmp_path, monkeypatch): +def test_explicit_repair_ignores_completion_stamp(tmp_path, monkeypatch, capsys): path = tmp_path / "stamped.db" ids = old_vault(path, monkeypatch, domain="project") with sqlite3.connect(path) as conn: conn.execute("INSERT OR REPLACE INTO alice_schema_state VALUES (?, '1')", (repair.REPAIR_STATE_KEY,)) assert onramp_main(["labels", "repair", "--db", str(path), "--user-id", USER]) == 0 - rows, events, _ = read_stored_columns(path, ids) + assert "labels repair updated 1" in capsys.readouterr().out + rows, events, state = read_stored_columns(path, ids) assert rows[0][1] == "confidential", "explicit repair must revisit a stamped vault" assert len(events) == 1 + assert onramp_main(["labels", "repair", "--db", str(path), "--user-id", USER]) == 0 + assert "labels repair updated 0" in capsys.readouterr().out + assert read_stored_columns(path, ids) == (rows, events, state) restricted_reads(path, ids, monkeypatch, guard_off=True) restricted_reads(path, ids, monkeypatch) From 9ddcb6ab3d668e05c291d8cc712b788fc27649c8 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:19:08 +0200 Subject: [PATCH 109/270] Cover missing canonical dependency count records --- tests/unit/test_derived_labels_record_completeness.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/test_derived_labels_record_completeness.py b/tests/unit/test_derived_labels_record_completeness.py index ff9d6b679..6bc06813d 100644 --- a/tests/unit/test_derived_labels_record_completeness.py +++ b/tests/unit/test_derived_labels_record_completeness.py @@ -44,3 +44,13 @@ def test_canonical_counts_and_identifiers_do_not_use_the_legacy_duplicate_except "counts": {"sources": count, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}, }}} assert settle_labels([source, row]).by_stored("artifact", "report").unverified is True + + +@pytest.mark.parametrize("counts", (None, {}, {"sources": True}, {"sources": "1"}, [])) +def test_a_nonempty_canonical_record_requires_well_formed_complete_counts(counts): + source = {"kind": "source", "id": "source", "domain": "project", "sensitivity": "public", "metadata_json": {}} + row = {"kind": "artifact", "id": "report", "metadata_json": {"derived_from": { + "v": 1, "sources": ["source"], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], + "counts": counts, + }}} + assert settle_labels([source, row]).by_stored("artifact", "report").unverified is True From 1ba95930b46ab71279bd75f0b5faf712825c94bb Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:23:12 +0200 Subject: [PATCH 110/270] Retain reviewed facade additions in producer carrier receipts --- tests/unit/test_store_graph_open_loops_split.py | 2 +- tests/unit/test_store_memory_access_split.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index fade2ff5b..91cea70ad 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -162,7 +162,7 @@ # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # lock_label_writes and read_label_rows follow __init__. Previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (174, "095250b8a77d6c0a32d2783343916b947bcae8ae86e3a1693e47c2fb11802211"), + "PostgresVNextStore": (176, "3549d6ea179f4ebc2341f1c251f139dfb747d56700a4554aef114150efed5c3f"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index 8fde85850..205f6510b 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -220,7 +220,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (174, "095250b8a77d6c0a32d2783343916b947bcae8ae86e3a1693e47c2fb11802211"), + "PostgresVNextStore": (176, "3549d6ea179f4ebc2341f1c251f139dfb747d56700a4554aef114150efed5c3f"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, From bf42c2a5fae6f63885e2e6f383addbb4c9cd2831 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:33:00 +0200 Subject: [PATCH 111/270] Fix effective project scope and exact audit authorization --- .../alicebot_api/routers/vnext_memories.py | 29 +++++++++++++++++-- apps/api/src/alicebot_api/sqlite_store.py | 2 +- .../api/src/alicebot_api/vnext_label_guard.py | 6 ++++ apps/api/src/alicebot_api/vnext_store.py | 2 +- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index 27ff748e4..ce23ff584 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Mapping from datetime import UTC, datetime from typing import Literal from uuid import UUID @@ -42,6 +43,7 @@ _vnext_agent_identity, _vnext_agent_record, _vnext_authenticated_agent_identity, + _vnext_exact_resource_policy, _vnext_load_source_trace, _vnext_metadata, _vnext_permission_response, @@ -1855,12 +1857,35 @@ def list_vnext_recent_memory_commits(user_id: UUID, limit: int = Query(default=2 @memory_router.get("/v0/vnext/memories/{memory_id}/audit") -def get_vnext_memory_audit(memory_id: UUID, user_id: UUID) -> JSONResponse: +def get_vnext_memory_audit( + memory_id: UUID, + user_id: UUID, + authorization: str | None = Header(default=None), +) -> JSONResponse: + from alicebot_api.vnext_label_guard import apply_unverified_rule, effective_row_for_fence + settings = get_settings() try: with user_connection(settings.database_url, user_id) as conn: store = PostgresVNextStore(conn) - payload = VNextMemoryCommitService(store).audit(memory_id=str(memory_id)) + identity = resolve_protected_agent_identity( + store, user_id=user_id, raw_key=agent_key_from_authorization(authorization), payload={}, + ) + + def authorize_memory(memory: Mapping[str, object]) -> None: + effective = effective_row_for_fence(store, identity, "memory", memory) + decision = _vnext_exact_resource_policy(identity=identity, action="memory.audit", resource=dict(effective)) + decision = apply_unverified_rule(decision, effective, identity) + append_policy_events(store, identity=identity, decision=decision, target_type="memory", target_id=str(memory["id"])) + if decision.decision == "blocked": + raise AgentPolicyBlockedError(decision) + + try: + payload = VNextMemoryCommitService(store).audit(memory_id=str(memory_id), authorize_memory=authorize_memory) + except AgentPolicyBlockedError as exc: + return _vnext_permission_response(exc.decision) + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) except VNextMemoryCommitValidationError as exc: return public_exception_response(exc, status_code=404) return JSONResponse(status_code=200, content=jsonable_encoder(payload)) diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index 4808aceb8..05d3be249 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -419,7 +419,7 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: return [] extra = "" if table == "memories": - extra = ", value, project_id" + extra = ", value, project_id, source_event_ids, deleted_at, status" elif table == "open_loops": extra = ", project_id, source_id, memory_id" from uuid import UUID diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index e6c3afb4f..2723ce2be 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -110,6 +110,12 @@ def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[ metadata["project_floor"] = list(label.project_floor) copy["unverified"] = False copy["metadata_json"] = metadata + # Store records expose scope and floor at the top level as well. The + # resolver reads those first, so both representations must agree. + copy["project_scope"] = list(metadata["project_scope"]) + copy["project_floor"] = list(metadata["project_floor"]) + if not copy["project_scope"]: + copy["project_id"] = None return copy def admit_rows(self, kind: str, rows: Sequence[Mapping[str, object]]) -> list[Mapping[str, object]]: diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 0baadf2a7..5ab808b1a 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -514,7 +514,7 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: ) extra = "" if table == "memories": - extra = ", value, project_id" + extra = ", value, project_id, source_event_ids, deleted_at, status" elif table == "open_loops": extra = ", project_id, source_id, memory_id" elif table == "beliefs": From 66edbd9b6fc7f1770670324ed3c02ffc5a056ff7 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:22:29 +0200 Subject: [PATCH 112/270] Test missing encoded source aliases at the read boundary --- .../test_open_loop_references_read_fence.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/unit/test_open_loop_references_read_fence.py b/tests/unit/test_open_loop_references_read_fence.py index 1de252c54..dcba0624a 100644 --- a/tests/unit/test_open_loop_references_read_fence.py +++ b/tests/unit/test_open_loop_references_read_fence.py @@ -387,6 +387,28 @@ def test_encoded_memory_and_trace_references_are_collected_before_the_real_key_l assert checked["kept"] == "control" +def test_a_missing_encoded_sources_alias_is_withheld_for_the_key_and_owner(world: _World) -> None: + """The canonical alias keeps a missing source in a reference position after JSON decoding. + + Mutation: stop adding canonical metadata names to the lookup. The generic scan then leaves the missing id. + """ + + missing, admitted = str(uuid4()), world.sources["own"] + encode = lambda value: "".join("\\u%04x" % ord(char) for char in value) + metadata = {"project_scope": ["alpha"], "sources": '["' + encode(missing) + '", "' + encode(admitted) + '"]', "kept": "control"} + world._plant("missing_encoded_alias", metadata=metadata) + loop_id = world.loops["missing_encoded_alias"] + world.vault.sql("DELETE FROM open_loops WHERE id != ?", (loop_id,)) + for key in (world.vault.keys["alpha_project"], None): + response = world.vault.wire("alice_open_loops", {"status": "all", "limit": 100}, key=key) + assert response["is_error"] is False + assert len(response["payload"]["items"]) == 1 + item = response["payload"]["items"][0] + assert str(item["id"]) == loop_id + assert item["metadata_json"]["kept"] == "control" + assert json.loads(item["metadata_json"]["sources"]) == [admitted] + + @pytest.mark.parametrize("reader", _UPDATERS) @pytest.mark.parametrize("name", sorted(("own", "health", "confidential", "old_beta", "old_global", "old_deleted", "old_ghost", "old_upper", "old_metadata"))) def test_the_open_loop_update_actions_withhold_what_the_reader_may_not_read(world: _World, reader: str, name: str) -> None: From 066991a2251423ea28ec442a0363713af57e3efc Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:22:51 +0200 Subject: [PATCH 113/270] Fence context events and skip filtered workspace content diagnostics --- .../src/alicebot_api/routers/workspaces.py | 2 +- .../src/alicebot_api/vnext_context_tree.py | 26 ++-------- apps/api/src/alicebot_api/vnext_doctor.py | 49 ++++++++++------- ...derived_labels_read_acceptance_postgres.py | 49 +++++++++++++++++ tests/unit/test_complete_readable_counts.py | 52 +++++++++++++++++++ tests/unit/test_derived_labels_real_keys.py | 29 +++++++++++ 6 files changed, 163 insertions(+), 44 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index bc85a15e5..70fa58851 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -158,7 +158,7 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti scheduler_status = {**scheduler_status, "daemon": daemon_status()} connector_health = VNextConnectorService(store).connector_health_all() dogfooding = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(sensitivity_allowed), label_guard=guard) - doctor = VNextDoctorService(store).run(ci=True) + doctor = VNextDoctorService(store).run(ci=True, include_content_diagnostics=False) policy_telemetry = summarize_agent_policy_telemetry( agent_events=agent_events, artifacts=artifacts, diff --git a/apps/api/src/alicebot_api/vnext_context_tree.py b/apps/api/src/alicebot_api/vnext_context_tree.py index 3706404a9..583839cfc 100644 --- a/apps/api/src/alicebot_api/vnext_context_tree.py +++ b/apps/api/src/alicebot_api/vnext_context_tree.py @@ -163,29 +163,9 @@ def _label(row: JsonObject, *keys: str, fallback: str) -> str: def _tree_event_visible(store: object, event: JsonObject, domains: list[str] | None, sensitivity: list[str], projects: tuple[str, ...]) -> bool: - from alicebot_api.vnext_label_guard import admit_loaded - - kind = str(event.get("target_type") or "") - target_id = event.get("target_id") - getters = { - "memory": "get_memory", - "open_loop": "get_open_loop", - "artifact": "get_artifact", - "project": "get_project", - "source": "get_source", - } - getter_name = getters.get(kind) - if getter_name is None or not isinstance(target_id, str) or target_id == "": - return True - getter = getattr(store, getter_name, None) - if not callable(getter): - return True - row = getter(target_id) - if not isinstance(row, dict) and not hasattr(row, "get"): - return True - return bool( - admit_loaded(store, kind=kind, rows=[row], domains=domains, sensitivity_allowed=sensitivity, projects=projects) - ) + from alicebot_api.vnext_label_guard import LabelGuard + + return bool(LabelGuard.for_filters(store, domains, sensitivity, projects).admit_events([event])) def _row_node(prefix: str, row: JsonObject, *, label_keys: tuple[str, ...], fallback: str) -> JsonObject: diff --git a/apps/api/src/alicebot_api/vnext_doctor.py b/apps/api/src/alicebot_api/vnext_doctor.py index 7081eda06..5eda25f00 100644 --- a/apps/api/src/alicebot_api/vnext_doctor.py +++ b/apps/api/src/alicebot_api/vnext_doctor.py @@ -126,7 +126,7 @@ def migration_status(self) -> JsonObject: def local_live_cors_status(self, settings: Settings | None = None) -> JsonObject: return local_live_cors_status(settings=settings or get_settings(), env=self.env, cwd=self.cwd) - def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: + def run(self, *, fix_safe: bool = False, ci: bool = False, include_content_diagnostics: bool = True) -> JsonObject: if fix_safe: VNextConnectorService(cast(Any, self.store), secret_provider=self.secret_provider).ensure_default_settings() @@ -262,6 +262,34 @@ def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: details=cast(JsonObject, local_cors), ) + if include_content_diagnostics: + self._content_checks(checks) + else: + for name in ("flagged_sources", "derived_labels"): + checks.append(DoctorCheck( + name=name, status="skipped", severity="info", + message="Content diagnostics are omitted from this filtered workspace view. Run doctor for a full report.", + details={"scope": "filtered_workspace", "evaluated": False}, + )) + + blocking = [check for check in checks if check.status == "fail" and check.severity == "blocking"] + warnings = [check for check in checks if check.status == "fail" and check.severity == "warning"] + payload = { + "status": "fail" if blocking else "warn" if warnings else "pass", + "fix_safe_applied": fix_safe, + "ci_mode": ci, + "blocking_failure_count": len(blocking), + "warning_count": len(warnings), + "checks": [check.to_record() for check in checks], + "recommended_fixes": [ + check.recommended_fix for check in checks if check.status == "fail" and check.recommended_fix is not None + ], + "migration_status": migration_status, + "connector_health": health, + } + return cast(JsonObject, payload) + + def _content_checks(self, checks: list[DoctorCheck]) -> None: flagged_ids, stopped_early = _flagged_source_scan(self.store) remedy = _flagged_source_remedy(self.store) if flagged_ids: @@ -305,25 +333,6 @@ def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: recommended_fix="alicebot vnext labels repair", ) - blocking = [check for check in checks if check.status == "fail" and check.severity == "blocking"] - warnings = [check for check in checks if check.status == "fail" and check.severity == "warning"] - payload = { - "status": "fail" if blocking else "warn" if warnings else "pass", - "fix_safe_applied": fix_safe, - "ci_mode": ci, - "blocking_failure_count": len(blocking), - "warning_count": len(warnings), - "checks": [check.to_record() for check in checks], - "recommended_fixes": [ - check.recommended_fix - for check in checks - if check.status == "fail" and check.recommended_fix is not None - ], - "migration_status": migration_status, - "connector_health": health, - } - return cast(JsonObject, payload) - def _flagged_source_remedy(store: object) -> str: """Keep the backend-specific owner remedy.""" diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py index a0c899d53..699272e96 100644 --- a/tests/integration/test_derived_labels_read_acceptance_postgres.py +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import asyncio from uuid import UUID, uuid4 import pytest @@ -200,3 +201,51 @@ def test_telemetry_and_quality_ratings_use_current_target_labels(migrated_databa assert "Cedar hidden feedback" not in ratings.body.decode() else: assert all(identifier in ratings.body.decode() for identifier in hidden_ids) + + +def test_real_trusted_http_workspace_omits_full_content_doctor_counts(migrated_database_urls, monkeypatch): + from alicebot_api import main + + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + settings = Settings(database_url=app_url) + monkeypatch.setattr(workspaces, "get_settings", lambda: settings) + monkeypatch.setattr(main, "get_settings", lambda: settings) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + key = real_reader_key(store, user_id, "trusted") + headers = {"Authorization": f"Bearer {key}"} + async def request_workspace(): + messages = [] + request_sent = False + async def receive(): + nonlocal request_sent + if not request_sent: + request_sent = True + return {"type": "http.request", "body": b"", "more_body": False} + await asyncio.Event().wait() + async def send(message): + messages.append(message) + scope = {"type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", "method": "GET", + "scheme": "http", "path": "/v0/vnext/workspace", "raw_path": b"/v0/vnext/workspace", "root_path": "", + "query_string": f"user_id={user_id}".encode(), "headers": [(name.lower().encode(), value.encode()) for name, value in headers.items()], + "client": ("127.0.0.1", 1), "server": ("127.0.0.1", 80)} + await main.app(scope, receive, send) + status = next(message["status"] for message in messages if message["type"] == "http.response.start") + body = b"".join(message.get("body", b"") for message in messages if message["type"] == "http.response.body") + return status, json.loads(body), body.decode() + before = asyncio.run(request_workspace()) + assert before[0] == 200, before[2] + with user_connection(app_url, user_id) as conn: + rows = seed_read_rows(PostgresVNextStore(conn)) + after = asyncio.run(request_workspace()) + assert after[0] == 200, after[2] + assert all(str(rows[state]["id"]) not in after[2] for state in ("verified_confidential", "unverified")) + diagnostic = after[1]["doctor"] + for check in diagnostic["checks"]: + if check["name"] in {"derived_labels", "flagged_sources"}: + assert check["status"] == "skipped" + assert check["details"] == {"scope": "filtered_workspace", "evaluated": False} + assert "below their inputs" not in check["message"] + for field in ("status", "warning_count", "blocking_failure_count", "recommended_fixes"): + assert diagnostic[field] == before[1]["doctor"][field] diff --git a/tests/unit/test_complete_readable_counts.py b/tests/unit/test_complete_readable_counts.py index ad31e0cf4..869c6071a 100644 --- a/tests/unit/test_complete_readable_counts.py +++ b/tests/unit/test_complete_readable_counts.py @@ -7,9 +7,12 @@ import pytest from alicebot_api.routers import _vnext_shared, workspaces +from alicebot_api.mcp.retrieval import _resume_event_honours_policy_fence +from alicebot_api.session_briefing import _event_target_honours_fence from alicebot_api.vnext_agent_control import ALL_SENSITIVITY, AgentIdentity from alicebot_api.vnext_dogfooding import VNextDogfoodingService from alicebot_api.vnext_label_guard import LabelGuard +from alicebot_api.vnext_context_tree import _tree_event_visible class PopulationStore: @@ -46,6 +49,9 @@ def list_events(self, **kwargs): def __getattr__(self, name): kinds = {"sources": "source", "artifacts": "artifact", "projects": "project", "open_loops": "open_loop"} suffix = name.removeprefix("list_") + if name.startswith("get_") and name.removeprefix("get_") in self.rows: + kind = name.removeprefix("get_") + return lambda identifier: next((row for row in self.rows[kind] if row["id"] == identifier), None) if suffix in kinds: def listed(**kwargs): rows = self.rows[kinds[suffix]] @@ -181,3 +187,49 @@ def test_trace_event_completeness_uses_admitted_targets(monkeypatch): ) assert [row["id"] for row in admitted] == ["0", "1"] assert complete is True + + +@pytest.mark.parametrize("kind", ("source", "memory", "open_loop", "artifact", "project")) +def test_context_events_use_current_effective_target_of_every_kind(kind): + store = PopulationStore() + source_id = "11111111-1111-4111-8111-111111111111" + row_id = source_id if kind == "source" else "22222222-2222-4222-8222-222222222222" + store.rows["source"] = [_row(source_id, "confidential")] + if kind != "source": + refs = {"sources": [source_id], "memories": [], "open_loops": [], "artifacts": [], "beliefs": []} + store.rows[kind] = [_row(row_id, metadata_json={"derived_from": {"v": 1, **refs, "counts": {name: len(ids) for name, ids in refs.items()}}})] + if kind == "open_loop": + store.rows[kind][0]["metadata_json"]["discovered_by"] = "vnext_daily_brief" + store.rows[kind][0]["metadata_json"]["source_id"] = source_id + event = {"target_type": kind, "target_id": row_id, "event_type": f"{kind}.labels_raised", "payload_json": {"cause": "repair_v3"}} + trusted = ["public", "internal", "private", "unknown"] + assert _tree_event_visible(store, event, None, trusted, ()) is False + assert _tree_event_visible(store, event, None, list(ALL_SENSITIVITY), ()) is True + store.rows[kind] = [_row(row_id)] + assert _tree_event_visible(store, event, None, trusted, ()) is True + + +def test_context_event_missing_and_unknown_label_targets_fail_closed(): + store = PopulationStore() + sensitivities = ["public", "internal", "private", "unknown"] + for kind in ("source", "memory", "open_loop", "artifact", "project", "not-a-label-kind"): + event = {"target_type": kind, "target_id": "missing", "event_type": f"{kind}.labels_raised"} + assert _tree_event_visible(store, event, None, sensitivities, ()) is False + assert _tree_event_visible(store, {"target_type": "connector", "event_type": "connector.heartbeat"}, None, sensitivities, ()) is True + + +@pytest.mark.parametrize("kind", ("memory", "open_loop")) +@pytest.mark.parametrize("reader", (_resume_event_honours_policy_fence, _event_target_honours_fence)) +def test_resume_and_session_events_use_current_target_labels(kind, reader): + store = PopulationStore() + source_id = "11111111-1111-4111-8111-111111111111" + row_id = "22222222-2222-4222-8222-222222222222" + store.rows["source"] = [_row(source_id, "confidential")] + store.rows[kind] = [_row(row_id, metadata_json={"source_id": source_id})] + if kind == "open_loop": + store.rows[kind][0]["metadata_json"]["discovered_by"] = "vnext_daily_brief" + event = {"target_type": kind, "target_id": row_id, "event_type": f"{kind}.labels_raised", "payload_json": {"cause": "repair_v3"}} + arguments = {"effective_domains": ("project", "health"), "effective_sensitivity_allowed": ("public", "internal", "private", "unknown"), "effective_project_scope": (), "exclude_global_domains": frozenset()} + assert reader(store, event, **arguments) is False + arguments["effective_sensitivity_allowed"] = ALL_SENSITIVITY + assert reader(store, event, **arguments) is True diff --git a/tests/unit/test_derived_labels_real_keys.py b/tests/unit/test_derived_labels_real_keys.py index 1e10d640c..88da4a5e1 100644 --- a/tests/unit/test_derived_labels_real_keys.py +++ b/tests/unit/test_derived_labels_real_keys.py @@ -14,7 +14,10 @@ from alicebot_api.vnext_agent_control import ALL_SENSITIVITY from alicebot_api.vnext_agent_keys import create_agent_key from alicebot_api.vnext_event_log import append_event +from alicebot_api.vnext_doctor import VNextDoctorService from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_secrets import InMemorySecretProvider +from tests.unit.test_vnext_doctor import DoctorStore READERS = ("owner", "admin", "trusted", "read_only", "bound_admin", "bound_trusted", "bound_read_only") @@ -80,3 +83,29 @@ def test_sqlite_real_key_core_doors(tmp_path, monkeypatch, reader): assert (str(row["id"]) in rendered) is admitted, (reader, state, tool, rendered) if not admitted: assert str(row["title"]) not in rendered + + +def test_full_owner_doctor_keeps_true_counts_and_filtered_view_skips_content(tmp_path): + user_id = uuid4() + path = tmp_path / "doctor.sqlite3" + bootstrap_database(path, user_id=str(user_id), user_email="synthetic@example.invalid") + with sqlite_user_connection(path, user_id) as conn: + store = SQLiteVNextStore(conn, user_id) + # SQLite has no provider/connector doctor protocol. Keep those synthetic + # operations fixed while the real connection supplies content diagnostics. + diagnostic_store = DoctorStore() + diagnostic_store.conn = store.conn + diagnostic_store.list_sources = lambda **kwargs: [row for batch in store.iter_label_rows("source") for row in batch] + service = VNextDoctorService(diagnostic_store, secret_provider=InMemorySecretProvider(), env={}, cwd=tmp_path) + before = service.run(include_content_diagnostics=False) + rows = seed_read_rows(store) + full = service.run() + scoped = service.run(include_content_diagnostics=False) + derived = next(check for check in full["checks"] if check["name"] == "derived_labels") + assert derived["message"] == "derived labels: 1 below their inputs, 1 unverified" + skipped = [check for check in scoped["checks"] if check["name"] in {"derived_labels", "flagged_sources"}] + assert len(skipped) == 2 + assert all(check["status"] == "skipped" and check["details"] == {"scope": "filtered_workspace", "evaluated": False} for check in skipped) + assert all(str(row["id"]) not in json.dumps(scoped) for row in rows.values()) + for field in ("status", "blocking_failure_count", "warning_count", "recommended_fixes"): + assert scoped[field] == before[field] From a9c8373d1c29aee32da782ac70601be699b3b3af Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:23:30 +0200 Subject: [PATCH 114/270] docs: explain filtered workspace diagnostics --- docs/alpha/mcp-tools.md | 2 +- tests/unit/test_derived_domain_docs.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/alpha/mcp-tools.md b/docs/alpha/mcp-tools.md index 036aa7933..69a92bc92 100644 --- a/docs/alpha/mcp-tools.md +++ b/docs/alpha/mcp-tools.md @@ -1087,6 +1087,6 @@ Unreleased (on main, not in v0.20.0): the stored-row repair resolves recorded in Unreleased (on main, not in v0.20.0): the repair runs when SQLite upgrades a vault, while `alice-memory import` stages a restore, in PostgreSQL migration `20261004_0095` (domain) and in PostgreSQL migration `20261005_0096` (domain, sensitivity, project scope and floor), and through `labels check` and `labels repair`. The open pass does not stop a vault from opening and leaves a row it could not repair to the read check. Explicit repair checks rows even after the open pass has stamped completion; every restore repairs its staged copy again. A failed explicit repair or restore rolls back the whole change. Each path, and what a restore can and cannot repair, is in [Backup and restore](backup-and-restore.md). -Unreleased (on main, not in v0.20.0): owner, trusted and admin callers that explicitly filter by an unrelated domain stop receiving a derived row after its label changes from `unknown` to a restricted domain; `unknown` previously matched every domain filter. New weekly candidate memories also store `input_summary`, which explain returns. These are intentional differences for unrestricted readers. Domain fields and explain policy-event labels/hashes reflect the new label; additive metadata changes context-pack token estimates. Repair audit events are additional history. The repair does not add missing historical input summaries. Correction (2026-10-05): sensitivity and project floor are repaired by migration `20261005_0096` and the SQLite v3 pass. Every reader checks a derived row, a belief and a project row against its inputs as they are now. A row whose recorded inputs cannot be found is unverified, so only the owner and an unbound admin key read it. Checking those inputs adds a read of the recorded graph. `derived_from` and `project_floor` add to pack token estimates. The artifact list, the source trace, the project list, the project row of the dashboard and the belief state route apply the caller's sensitivity ceiling, including titles, ids and counts. Counts on those screens and on the workspace and dashboards are over the rows the caller may read. Each total checks the complete counted set before pagination, including rows excluded from the displayed page. Domain and project restrictions on those screens stay as they are. +Unreleased (on main, not in v0.20.0): owner, trusted and admin callers that explicitly filter by an unrelated domain stop receiving a derived row after its label changes from `unknown` to a restricted domain; `unknown` previously matched every domain filter. New weekly candidate memories also store `input_summary`, which explain returns. These are intentional differences for unrestricted readers. Domain fields and explain policy-event labels/hashes reflect the new label; additive metadata changes context-pack token estimates. Repair audit events are additional history. The repair does not add missing historical input summaries. Correction (2026-10-05): sensitivity and project floor are repaired by migration `20261005_0096` and the SQLite v3 pass. Every reader checks a derived row, a belief and a project row against its inputs as they are now. A row whose recorded inputs cannot be found is unverified, so only the owner and an unbound admin key read it. Checking those inputs adds a read of the recorded graph. `derived_from` and `project_floor` add to pack token estimates. The artifact list, the source trace, the project list, the project row of the dashboard and the belief state route apply the caller's sensitivity ceiling, including titles, ids and counts. Counts on those screens and on the workspace and dashboards are over the rows the caller may read. Each total checks the complete counted set before pagination, including rows excluded from the displayed page. The filtered workspace skips content diagnostics; run doctor for the full derived-label and flagged-source report. Domain and project restrictions on those screens stay as they are. Unreleased (on main, not in v0.20.0): a new derived row and a stored one take every project of every input, and a project is not dropped. A restricted key's own reports are built only from inputs that key may read. The owner's cross-project briefs stay. A report with an input that has no project is global and keeps a floor of every input's project, and it is not readable by every project. Moving a source first returns `derived_rows_hidden_from_project_keys` and writes nothing when the count is nonzero until `confirm_label_hide` is true. On PostgreSQL, the keyless local owner or an unbound admin can call `POST /v0/vnext/sources/{source_id}/regenerate` with `user_id` to create fresh candidate memories and open loops from all stored chunks under the source's current labels, scope and provenance. HTTP 201 returns `memory_ids`, `open_loop_ids`, `memory_count` and `open_loop_count`; trusted and project-bound keys are refused. Old rows and their provenance stay intact and strict. Rerun a report's normal generation route to rebuild the report from the regenerated inputs. Correction (2026-10-05): the text that stood here said a reader matching one scope could potentially read a summary of other scopes. In v0.20.0 and on main before this change the reach was wider than a summary: titles and ids of other projects' artifacts, titles and ids of rows with no project, and the full text of memories with no project, in a report a bound key could read with 200 although its direct read of those rows was refused; promotion made the report recallable. diff --git a/tests/unit/test_derived_domain_docs.py b/tests/unit/test_derived_domain_docs.py index 619832b0f..2924d9e5a 100644 --- a/tests/unit/test_derived_domain_docs.py +++ b/tests/unit/test_derived_domain_docs.py @@ -150,6 +150,8 @@ def test_the_mcp_tools_section_states_the_label_rule_the_repair_and_the_changes_ assert "or repair historical sensitivity values" not in readers assert "apply the caller's sensitivity ceiling" in readers assert "rows the caller may read" in readers + assert "The filtered workspace skips content diagnostics" in readers + assert "run doctor for the full derived-label and flagged-source report" in readers assert "apply no label" not in readers assert "every project of every input" in scope From b0583e35216fd531d9c5103f39194f6cf9b70409 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:27:11 +0200 Subject: [PATCH 115/270] Follow canonical encoded references during label propagation --- .../src/alicebot_api/vnext_label_writes.py | 18 +++- apps/api/src/alicebot_api/vnext_store.py | 2 +- ...est_encoded_label_dependencies_postgres.py | 95 +++++++++++++++++++ tests/unit/test_encoded_label_dependencies.py | 76 +++++++++++++++ 4 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 tests/integration/test_encoded_label_dependencies_postgres.py create mode 100644 tests/unit/test_encoded_label_dependencies.py diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 04dba3302..9265da97e 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -366,11 +366,14 @@ def _sqlite_dependants(store: Any, table: str, kind: str, compacts: Sequence[str if table == "memories": extra = ", value, project_id, NULL AS source_id, NULL AS memory_id" text_clause, _ = _like_clause("metadata_json", len(compacts), qmark=True) + # A nested JSON string may encode every character of an id. Keep all + # escaped candidates for the canonical dependency parser below. + text_clause += " OR instr(coalesce(metadata_json, ''), char(92)) > 0" params: list[object] = [store.user_id, *[f"%{item}%" for item in compacts]] value_sql = "" if with_value: value_clause, _ = _like_clause("value", len(compacts), qmark=True) - value_sql = f" OR {value_clause}" + value_sql = f" OR {value_clause} OR instr(coalesce(value, ''), char(92)) > 0" params.extend(f"%{item}%" for item in compacts) column_sql = "" if table == "open_loops": @@ -396,23 +399,30 @@ def _postgres_dependants(store: Any, table: str, kind: str, compacts: Sequence[s if table == "memories": extra = ", value, project_id" elif table == "open_loops": - extra = ", NULL::jsonb AS value, project_id, source_id, memory_id" + extra = ", NULL::jsonb AS value, project_id, source_id::text AS source_id, memory_id::text AS memory_id" elif table == "generated_artifacts": extra = ", NULL::jsonb AS value, artifact_type" else: extra = ", NULL::jsonb AS value" text_clause, _ = _like_clause("metadata_json::text", len(compacts), qmark=False) + text_clause += " OR strpos(coalesce(metadata_json::text, ''), chr(92)) > 0" params: list[object] = [f"%{item}%" for item in compacts] value_sql = "" if with_value: value_clause, _ = _like_clause("value::text", len(compacts), qmark=False) - value_sql = f" OR {value_clause}" + value_sql = f" OR {value_clause} OR strpos(coalesce(value::text, ''), chr(92)) > 0" params.extend(f"%{item}%" for item in compacts) + column_sql = "" + if table == "open_loops": + for column in ("source_id", "memory_id"): + clause, _ = _like_clause(f"{column}::text", len(compacts), qmark=False) + column_sql += f" OR {clause}" + params.extend(f"%{item}%" for item in compacts) rows = store._fetch_all( f""" SELECT id::text AS id, user_id::text AS user_id, domain, sensitivity, metadata_json{extra} FROM {table} - WHERE ({text_clause}{value_sql}) + WHERE ({text_clause}{value_sql}{column_sql}) """, # nosec B608 # internal literal table/columns; every external value is bound tuple(params), ) diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 6a0660847..7ee1a2f93 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -516,7 +516,7 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: if table == "memories": extra = ", value, project_id, source_event_ids, deleted_at, status" elif table == "open_loops": - extra = ", project_id, source_id, memory_id" + extra = ", project_id, source_id::text AS source_id, memory_id::text AS memory_id" elif table == "beliefs": extra = ", memory_id" elif table == "generated_artifacts": diff --git a/tests/integration/test_encoded_label_dependencies_postgres.py b/tests/integration/test_encoded_label_dependencies_postgres.py new file mode 100644 index 000000000..cd7c5f701 --- /dev/null +++ b/tests/integration/test_encoded_label_dependencies_postgres.py @@ -0,0 +1,95 @@ +"""Actual PostgreSQL encoded dependency and confirmation controls.""" +import json +from uuid import UUID, uuid4 + +import pytest +from psycopg.types.json import Jsonb + +from alicebot_api.config import Settings +from alicebot_api.db import user_connection +from alicebot_api.routers import vnext_memories as router +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_derived_labels import dependencies_of +from alicebot_api.vnext_label_writes import walk_dependants +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.test_source_move_label_preview_postgres import ALPHA, BETA, _snapshot +from tests.unit.test_encoded_label_dependencies import encoded_object + + +def test_encoded_source_move_previews_without_writes_then_confirms(migrated_database_urls, monkeypatch): + url = migrated_database_urls["app"] + user = uuid4() + with user_connection(url, user) as conn: + ContinuityStore(conn).create_user(user, f"encoded-{user}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Synthetic", "content_hash": str(user), "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA]}}) + reference = encoded_object("source_id", str(source["id"])) + copy = store.create_memory({"memory_key": "encoded", "canonical_text": "Synthetic", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": reference, "project_scope": [ALPHA]}}) + unrelated = store.create_memory({"memory_key": "unrelated", "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"note": encoded_object("source_id", str(uuid4()))}}) + assert ("source", str(source["id"])) in dependencies_of("memory", copy) + assert {str(row["id"]) for row in walk_dependants(store, [str(source["id"])])} == {str(copy["id"])} + monkeypatch.setattr(router, "get_settings", lambda: Settings(database_url=url)) + before = _snapshot(url, user, source) + request = router.VNextSourceReviewRequest(user_id=user, action="assign_project", project_id=BETA, sensitivity="confidential") + preview = router.review_vnext_source(UUID(str(source["id"])), request) + payload = json.loads(preview.body) + assert preview.status_code == 200 and payload["preview"] is True + assert payload["derived_rows_hidden_from_project_keys"] == 1 + assert payload["confirm_required"] is True + assert _snapshot(url, user, source) == before + confirmed = router.review_vnext_source(UUID(str(source["id"])), request.model_copy(update={"confirm_label_hide": True})) + assert confirmed.status_code == 200 + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + stored = store.get_memory(str(copy["id"])) + assert stored["sensitivity"] == "confidential" + assert stored["metadata_json"]["source_id"] == reference + assert BETA in stored["metadata_json"]["project_floor"] + assert store.get_memory(str(unrelated["id"]))["sensitivity"] == "public" + assert _snapshot(url, user, source)["provenance_links"] == before["provenance_links"] + + +@pytest.mark.parametrize("key,kind", [("source_id", "source"), ("memory_id", "memory"), ("artifact_id", "artifact"), ("belief_ids", "belief")]) +def test_encoded_metadata_references_keep_the_canonical_reverse_edge(migrated_database_urls, key, kind): + user = uuid4() + root_id = str(uuid4()) + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"encoded-keys-{user}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + row = store.create_memory({"memory_key": "encoded-key", "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public"}) + metadata = {key: [root_id] if key.endswith("ids") else root_id, "consolidation": {}} + raw = json.dumps(metadata) + encoded = "".join("\\u%04x" % ord(char) if char.isalnum() or char == "_" else char for char in raw) + conn.execute("UPDATE memories SET metadata_json = %s::jsonb WHERE id = %s", (encoded, row["id"])) + stored = store.get_memory(str(row["id"])) + assert (kind, root_id) in dependencies_of("memory", stored) + assert {str(item["id"]) for item in walk_dependants(store, [root_id])} == {str(row["id"])} + + +@pytest.mark.parametrize("key,kind", [("source_id", "source"), ("artifact_id", "artifact")]) +def test_encoded_value_object_keeps_its_canonical_reverse_edge(migrated_database_urls, key, kind): + user = uuid4() + root_id = str(uuid4()) + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"encoded-value-{user}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + row = store.create_memory({"memory_key": "encoded-value", "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public"}) + conn.execute("UPDATE memories SET value = %s, metadata_json = %s WHERE id = %s", (Jsonb(encoded_object(key, root_id)), Jsonb({"consolidation": {}}), row["id"])) + stored = store.get_memory(str(row["id"])) + assert (kind, root_id) in dependencies_of("memory", stored) + assert {str(item["id"]) for item in walk_dependants(store, [root_id])} == {str(row["id"])} + + +def test_a_candidate_open_loop_direct_source_column_is_a_reverse_edge(migrated_database_urls): + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"loop-edge-{user}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + source = store.create_source({"source_type": "note", "title": "Synthetic", "content_hash": str(user), "domain": "project", "sensitivity": "public"}) + loop = store.create_open_loop({"title": "Synthetic", "loop_type": "task", "source_id": str(source["id"]), "domain": "project", "sensitivity": "public", "metadata_json": {"discovered_by": "synthetic"}}) + assert {str(row["id"]) for row in walk_dependants(store, [str(source["id"])])} == {str(loop["id"])} + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) + assert store.get_open_loop(str(loop["id"]))["sensitivity"] == "confidential" diff --git a/tests/unit/test_encoded_label_dependencies.py b/tests/unit/test_encoded_label_dependencies.py new file mode 100644 index 000000000..fb94635ff --- /dev/null +++ b/tests/unit/test_encoded_label_dependencies.py @@ -0,0 +1,76 @@ +"""The reverse lookup remains a superset of canonical encoded references.""" +import json +from uuid import uuid4 + +import pytest + +from alicebot_api.onramp import bootstrap_database +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_derived_labels import dependencies_of +from alicebot_api.vnext_label_writes import count_rows_hidden_by_scope_move, walk_dependants +from tests.unit.test_derived_domain_fence import USER + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +def encoded_object(key, row_id): + escaped = "".join("\\u%04x" % ord(character) for character in row_id) + return '{"' + key + '":"' + escaped + '"}' + + +def test_an_encoded_source_is_previewed_and_raised_without_losing_its_reference(tmp_path): + path = tmp_path / "encoded.sqlite3" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.test") + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source({"source_type": "note", "title": "Synthetic", "content_hash": "encoded", "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA]}}) + reference = encoded_object("source_id", str(source["id"])) + copy = store.create_memory({"memory_key": "encoded", "canonical_text": "Synthetic", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": reference, "project_scope": [ALPHA]}}) + unrelated = store.create_memory({"memory_key": "unrelated", "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"note": encoded_object("source_id", str(uuid4()))}}) + before = {table: list(conn.execute(f"SELECT * FROM {table} ORDER BY id")) for table in ("sources", "memories", "event_log", "provenance_links")} + assert ("source", str(source["id"])) in dependencies_of("memory", copy) + assert {str(row["id"]) for row in walk_dependants(store, [str(source["id"])])} == {str(copy["id"])} + assert count_rows_hidden_by_scope_move(store, source, [BETA]) == 1 + assert {table: list(conn.execute(f"SELECT * FROM {table} ORDER BY id")) for table in before} == before + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential", "metadata_json": {"project_scope": [BETA]}}) + stored = store.get_memory(str(copy["id"])) + assert stored["sensitivity"] == "confidential" + assert BETA in stored["metadata_json"]["project_floor"] + assert stored["metadata_json"]["source_id"] == reference + assert store.get_memory(str(unrelated["id"]))["sensitivity"] == "public" + + +@pytest.mark.parametrize("key,kind", [("source_id", "source"), ("memory_id", "memory"), ("artifact_id", "artifact"), ("belief_ids", "belief")]) +def test_unicode_encoded_metadata_keys_and_ids_are_not_cut_before_the_parser(tmp_path, key, kind): + path = tmp_path / "encoded-keys.sqlite3" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.test") + row_id = str(uuid4()) + value = [row_id] if key.endswith("ids") else row_id + metadata = json.dumps({key: value, "consolidation": {}}) + encoded = "".join("\\u%04x" % ord(character) if character.isalnum() or character == "_" else character for character in metadata) + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + row = store.create_memory({"memory_key": "raw-encoded", "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public"}) + conn.execute("UPDATE memories SET metadata_json = ? WHERE id = ?", (encoded, row["id"])) + stored = store.get_memory(str(row["id"])) + assert (kind, row_id) in dependencies_of("memory", stored) + assert {str(item["id"]) for item in walk_dependants(store, [row_id])} == {str(row["id"])} + + +@pytest.mark.parametrize("key,kind", [("source_id", "source"), ("artifact_id", "artifact")]) +def test_encoded_value_object_keeps_the_same_reverse_edge(tmp_path, key, kind): + path = tmp_path / "encoded-value.sqlite3" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.test") + row_id = str(uuid4()) + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + row = store.create_memory({"memory_key": "encoded-value", "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public"}) + # The dependency lives in the encoded value alone. A marker without + # the root id states the row class without making SQL's id term pass. + metadata = {"consolidation": {}} + conn.execute("UPDATE memories SET value = ?, metadata_json = ? WHERE id = ?", (json.dumps(encoded_object(key, row_id)), json.dumps(metadata), row["id"])) + stored = store.get_memory(str(row["id"])) + assert (kind, row_id) in dependencies_of("memory", stored) + assert {str(item["id"]) for item in walk_dependants(store, [row_id])} == {str(row["id"])} + From 79d7ba8c1c51ac0257f13c48dcb91fca073c64d5 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:27:25 +0200 Subject: [PATCH 116/270] Keep workspace binding and returned rows separately typed --- apps/api/src/alicebot_api/routers/workspaces.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index 70fa58851..4948bc285 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -92,9 +92,9 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti ceiling = sensitivity_ceiling(identity) if ceiling is not None: sensitivity_allowed = [value for value in sensitivity_allowed if value in ceiling] - projects = identity.project_scope if identity is not None else () - all_of = projects if identity is not None and identity.project_scope_locked else None - guard = LabelGuard.for_filters(store, (), sensitivity_allowed, projects, all_of=all_of) + requested_projects = identity.project_scope if identity is not None else () + all_of = requested_projects if identity is not None and identity.project_scope_locked else None + guard = LabelGuard.for_filters(store, (), sensitivity_allowed, requested_projects, all_of=all_of) review_statuses = ["candidate", "needs_review", "private_only", "accepted", "rejected"] fetched_sources = store.list_sources(sensitivity_allowed=sensitivity_allowed, limit=20) sources = guard.admit_rows("source", fetched_sources) @@ -152,7 +152,7 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti if callable(list_pending_inline_confirmations) else memory_commit_service.inline_confirmations(limit=20) ) - recent_memory_commits = guard.admit_rows("memory", recent_memory_commits) + recent_memory_commits = guard.admit_rows("memory", recent_memory_commits if isinstance(recent_memory_commits, list) else []) inline_confirmations = guard.admit_rows("memory", inline_confirmations) scheduler_status = VNextSchedulerService(store).status() scheduler_status = {**scheduler_status, "daemon": daemon_status()} From b572439db2737d3c203bf9acdff61bcbbfd2a5bc Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:24:50 +0200 Subject: [PATCH 117/270] Prove consolidation admission before provider input --- .../test_derived_labels_producers_postgres.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/integration/test_derived_labels_producers_postgres.py b/tests/integration/test_derived_labels_producers_postgres.py index dcc57c021..9a1dbc79b 100644 --- a/tests/integration/test_derived_labels_producers_postgres.py +++ b/tests/integration/test_derived_labels_producers_postgres.py @@ -16,6 +16,7 @@ from alicebot_api.store import ContinuityStore from alicebot_api.vnext_agent_keys import create_agent_key from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from alicebot_api.vnext_consolidation import MemoryConsolidationRequest, VNextConsolidationService, _clustering_options from alicebot_api.vnext_derived_labels import with_derived_from from alicebot_api.vnext_queue import VNextQueueService from alicebot_api.vnext_store import PostgresVNextStore @@ -192,6 +193,7 @@ def test_input_selection_uses_effective_labels_including_the_owner_default_ceili "metadata_json": {"source_id": str(source["id"]), "project_scope": [alpha]}}) promoted = next(row for label, kind, row in rows if label == "alpha" and kind == "memories" and row["metadata_json"].get("source_artifact_id")) + prior_id = promoted["metadata_json"]["source_artifact_id"] derived_loop = store.create_open_loop({"title": "STALE_SOURCE_LOOP Atlas", "description": "STALE_SOURCE_LOOP secret", "source_id": str(source["id"]), "status": "open", "domain": "project", "sensitivity": "public", "due_at": "2026-10-04T12:00:00Z", "metadata_json": {"project_scope": [alpha], @@ -212,6 +214,7 @@ def test_input_selection_uses_effective_labels_including_the_owner_default_ceili assert str(copy["id"]) not in text assert copy["canonical_text"] not in text assert str(promoted["id"]) not in text + assert str(prior_id) not in text assert str(source["id"]) not in text assert str(derived_loop["id"]) not in text assert "STALE_SOURCE_LOOP" not in text @@ -251,3 +254,32 @@ def test_the_owner_keeps_the_cross_project_brief_and_bound_keys_cannot_read_it(m recalled = call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name="alice_recall", arguments={"query": "Atlas", "limit": 50}) assert (memory_id in json.dumps(recalled, default=str)) == (key == unbound) + + +def test_consolidation_admits_effective_memory_labels_before_the_embedding_provider(migrated_database_urls, monkeypatch): + app_url = migrated_database_urls["app"] + user_id, alpha, _beta, rows, _alpha_key, _beta_key, _unbound = seed_grid(app_url) + printed = [] + + class RecordingProvider: + def embed_batch(self, texts): + printed.extend(texts) + return [[1.0, 0.0, 0.0] for _ in texts] + + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = next(row for label, kind, row in rows if label == "alpha" and kind == "sources") + store.create_memory({"memory_key": "provider.stale.copy", "canonical_text": "PROVIDER_SECRET Atlas played Hollow Knight for 25 hours", + "title": "PROVIDER_SECRET", "status": "active", "domain": "project", "sensitivity": "public", + "metadata_json": {"source_id": str(source["id"]), "project_scope": [alpha]}}) + conn.execute("UPDATE sources SET sensitivity='confidential' WHERE id=%s", (source["id"],)) + # Synthetic vectors are local. Presence is forced for all selected rows so selection reaches the provider. + monkeypatch.setattr(store, "list_memory_ids_with_embeddings", lambda ids: set(ids)) + service = VNextConsolidationService(store, embedding_provider=RecordingProvider()) + service._cluster_memories(domains=None, sensitivity=["public", "internal", "private", "unknown"], + projects=(alpha,), all_of=(alpha,), options=_clustering_options(MemoryConsolidationRequest())) + text = json.dumps(printed) + assert printed + assert "SENTINEL_ALPHA memory" in text + assert "PROVIDER_SECRET" not in text + assert "SENTINEL_ALPHA prior report text" not in text From 7ccd11a53e6871679f8d22d5edbaf181e4bedfb8 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:31:46 +0200 Subject: [PATCH 118/270] Fence session events with current target labels --- apps/api/src/alicebot_api/mcp/retrieval.py | 4 +- tests/unit/test_derived_labels_real_keys.py | 82 +++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_derived_labels_real_keys.py diff --git a/apps/api/src/alicebot_api/mcp/retrieval.py b/apps/api/src/alicebot_api/mcp/retrieval.py index 7290d2a09..ee9db7cb8 100644 --- a/apps/api/src/alicebot_api/mcp/retrieval.py +++ b/apps/api/src/alicebot_api/mcp/retrieval.py @@ -809,6 +809,7 @@ def _resume_event_honours_policy_fence( # row it points at is the thing to test (the event queries leave them out in SQL # too, which keeps a held-back event from using up a place). exclude_global_domains: frozenset[str], + effective_project_scope: tuple[str, ...] = (), ) -> bool: target_type = event.get("target_type") target_id = event.get("target_id") @@ -836,7 +837,7 @@ def _resume_event_honours_policy_fence( rows=[row], domains=effective_domains, sensitivity_allowed=effective_sensitivity_allowed, - projects=(), + projects=effective_project_scope, ): return False return _resource_matches_domains(row, effective_domains) and _resource_matches_sensitivity( @@ -1226,6 +1227,7 @@ def read_events(scope: tuple[str, ...], excluded: frozenset[str], count: int) -> effective_domains=effective_domains, effective_sensitivity_allowed=effective_sensitivity_allowed, exclude_global_domains=held_back, + effective_project_scope=effective_project_scope, ) ] event_rows.sort(key=_event_recency, reverse=True) diff --git a/tests/unit/test_derived_labels_real_keys.py b/tests/unit/test_derived_labels_real_keys.py new file mode 100644 index 000000000..1e10d640c --- /dev/null +++ b/tests/unit/test_derived_labels_real_keys.py @@ -0,0 +1,82 @@ +"""Core read doors authenticate actual SQLite keys before effective admission.""" + +from __future__ import annotations + +import json +from uuid import UUID, uuid4 + +import pytest + +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError +from alicebot_api.onramp import bootstrap_database, sqlite_url_for_path +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_agent_control import ALL_SENSITIVITY +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_event_log import append_event +from alicebot_api.vnext_label_writes import without_insert_floor + + +READERS = ("owner", "admin", "trusted", "read_only", "bound_admin", "bound_trusted", "bound_read_only") +ALPHA = "prj_" + "a" * 16 + + +def seed_read_rows(store): + source = store.create_source({"source_type": "note", "title": "Restricted parent", "content_hash": str(uuid4()), + "domain": "health", "sensitivity": "confidential", "metadata_json": {"project_scope": [ALPHA]}}) + rows = {} + for state in ("verified_public", "verified_confidential", "unverified"): + metadata = {"project_scope": [ALPHA]} + if state != "verified_public": + metadata["source_id"] = str(source["id"]) if state == "verified_confidential" else str(uuid4()) + with without_insert_floor(): + rows[state] = store.create_memory({"memory_key": state, "canonical_text": f"Cedar sentinel {state}", + "title": f"Hidden title {state}", "memory_type": "decision", "status": "active", + "domain": "project", "sensitivity": "public", "metadata_json": metadata}) + append_event(store, event_type="memory.labels_raised", actor_type="system", + target_type="memory", target_id=str(rows[state]["id"]), payload={"cause": "repair_v3"}) + return rows + + +def real_reader_key(store, user_id, reader): + if reader == "owner": + return None + profile = {"admin": "admin_agent", "trusted": "trusted_local_agent", "read_only": "read_only_agent"}[reader.removeprefix("bound_")] + _record, raw = create_agent_key(store, user_id=user_id, agent_id=reader, permission_profile=profile, + project_scope=ALPHA if reader.startswith("bound_") else None) + return raw + + +def expected_read(reader, state): + return state == "verified_public" or reader in {"owner", "admin"} or (reader == "bound_admin" and state == "verified_confidential") + + +@pytest.mark.parametrize("reader", READERS) +def test_sqlite_real_key_core_doors(tmp_path, monkeypatch, reader): + user_id = uuid4() + path = tmp_path / "reads.sqlite3" + bootstrap_database(path, user_id=str(user_id), user_email="synthetic@example.invalid") + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.setenv("ALICE_PROJECT_SCOPING", "off") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + with sqlite_user_connection(path, user_id) as conn: + store = SQLiteVNextStore(conn, user_id) + rows = seed_read_rows(store) + key = real_reader_key(store, user_id, reader) + if key: + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + context = MCPRuntimeContext(database_url=sqlite_url_for_path(path), user_id=UUID(str(user_id))) + for state, row in rows.items(): + admitted = expected_read(reader, state) + for tool in ("alice_recall", "alice_context_pack", "alice_recent_decisions", "alice_explain", "alice_resume"): + arguments = {"memory_id": str(row["id"])} if tool == "alice_explain" else {"query": str(row["canonical_text"]), "sensitivity_allowed": list(ALL_SENSITIVITY)} + try: + result = call_mcp_tool(context, name=tool, arguments=arguments) + except MCPToolError: + assert not admitted, (reader, state, tool) + continue + rendered = json.dumps(result, default=str) + assert (str(row["id"]) in rendered) is admitted, (reader, state, tool, rendered) + if not admitted: + assert str(row["title"]) not in rendered From 2c12abb60e1adec60bab4fe6884b4a4168a80257 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:33:32 +0200 Subject: [PATCH 119/270] Count complete readable populations and register read boundaries --- .../src/alicebot_api/routers/_vnext_shared.py | 49 +++--- .../src/alicebot_api/routers/workspaces.py | 51 ++---- apps/api/src/alicebot_api/sqlite_store.py | 44 +++++ apps/api/src/alicebot_api/vnext_dogfooding.py | 12 +- .../api/src/alicebot_api/vnext_label_guard.py | 97 ++++++----- apps/api/src/alicebot_api/vnext_store.py | 67 ++++++++ ...derived_labels_read_acceptance_postgres.py | 132 ++++++++++++++ tests/unit/test_complete_readable_counts.py | 162 ++++++++++++++++++ tests/unit/test_label_door_registry.py | 92 +++++++++- 9 files changed, 592 insertions(+), 114 deletions(-) create mode 100644 tests/integration/test_derived_labels_read_acceptance_postgres.py create mode 100644 tests/unit/test_complete_readable_counts.py diff --git a/apps/api/src/alicebot_api/routers/_vnext_shared.py b/apps/api/src/alicebot_api/routers/_vnext_shared.py index f0f0a8bfa..99fddf907 100644 --- a/apps/api/src/alicebot_api/routers/_vnext_shared.py +++ b/apps/api/src/alicebot_api/routers/_vnext_shared.py @@ -280,6 +280,21 @@ def _vnext_source_trace( } +def _vnext_readable_trace_rows(store, kind, fetch, identity, *, admit=None): + """Deepen the prefix until readable truncation can be answered.""" + + from alicebot_api.vnext_label_guard import apply_sensitivity_ceiling + + limit = _VNEXT_SOURCE_TRACE_COLLECTION_LIMIT + prefix = limit + 1 + while True: + fetched = list(fetch(prefix)) + admitted = list(admit(fetched)) if admit is not None else apply_sensitivity_ceiling(store, kind=kind, rows=fetched, identity=identity) + if len(admitted) > limit or len(fetched) < prefix: + return _vnext_bounded_trace_rows(admitted) + prefix *= 2 + + def _vnext_load_source_trace( *, store: PostgresVNextStore, @@ -299,39 +314,27 @@ def _vnext_load_source_trace( if not apply_sensitivity_ceiling(store, kind="source", rows=[source], identity=caller): return None source_id = str(source["id"]) - memories, memories_complete = _vnext_bounded_trace_rows( - store.list_memories_referencing_source( - source_id=source_id, - limit=_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT + 1, - ) + memories, memories_complete = _vnext_readable_trace_rows( + store, "memory", lambda limit: store.list_memories_referencing_source(source_id=source_id, limit=limit), caller ) - artifacts, artifacts_complete = _vnext_bounded_trace_rows( - store.list_artifacts_referencing_source( - source_id=source_id, - limit=_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT + 1, - ) + artifacts, artifacts_complete = _vnext_readable_trace_rows( + store, "artifact", lambda limit: store.list_artifacts_referencing_source(source_id=source_id, limit=limit), caller ) - open_loops, open_loops_complete = _vnext_bounded_trace_rows( - store.list_open_loops_referencing_source( - source_id=source_id, - limit=_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT + 1, - ) + open_loops, open_loops_complete = _vnext_readable_trace_rows( + store, "open_loop", lambda limit: store.list_open_loops_referencing_source(source_id=source_id, limit=limit), caller ) - memories = apply_sensitivity_ceiling(store, kind="memory", rows=memories, identity=caller) - artifacts = apply_sensitivity_ceiling(store, kind="artifact", rows=artifacts, identity=caller) - open_loops = apply_sensitivity_ceiling(store, kind="open_loop", rows=open_loops, identity=caller) kept_ids = {str(row.get("id")) for row in (*memories, *artifacts, *open_loops)} kept_ids.add(source_id) - events, direct_events_complete = _vnext_bounded_trace_rows( - store.list_events_for_source_trace( + events, direct_events_complete = _vnext_readable_trace_rows( + store, "event", lambda limit: store.list_events_for_source_trace( source_id=source_id, memory_ids=[str(memory["id"]) for memory in memories], artifact_ids=[str(artifact["id"]) for artifact in artifacts], open_loop_ids=[str(open_loop["id"]) for open_loop in open_loops], - limit=_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT + 1, - ) + limit=limit, + ), caller, + admit=lambda rows: [event for event in rows if str(event.get("target_id") or "") in kept_ids], ) - events = [event for event in events if str(event.get("target_id") or "") in kept_ids] events_complete = direct_events_complete and memories_complete and artifacts_complete and open_loops_complete return _vnext_source_trace( store=store, diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index 1ae7c314a..c378393e5 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -85,13 +85,14 @@ def _workspace_rows(store: PostgresVNextStore, kind: str, rows: Sequence[Mapping def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: - from alicebot_api.vnext_label_guard import LabelGuard, readable_count, readable_status_counts + from alicebot_api.vnext_label_guard import LabelGuard sensitivity_allowed = ["public", "internal", "private", "unknown"] + guard = LabelGuard.for_filters(store, (), sensitivity_allowed, ()) review_statuses = ["candidate", "needs_review", "private_only", "accepted", "rejected"] fetched_sources = store.list_sources(sensitivity_allowed=sensitivity_allowed, limit=20) sources = _workspace_rows(store, "source", fetched_sources, sensitivity_allowed) - source_count = readable_count(store.count_sources(), len(fetched_sources), len(sources)) + source_count = sum(guard.readable_status_counts("source").values()) list_memories_by_statuses = getattr(store, "list_memories_by_statuses", None) if callable(list_memories_by_statuses): fetched_memories = list_memories_by_statuses( @@ -104,53 +105,31 @@ def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: memory for memory in store.list_memories(status=None) if str(memory.get("status")) in set(review_statuses) ][:30] review_memories = _workspace_rows(store, "memory", fetched_memories, sensitivity_allowed) - count_memories_by_status = getattr(store, "count_memories_by_status", None) - memory_status_counts = ( - count_memories_by_status(sensitivity_allowed=sensitivity_allowed) - if callable(count_memories_by_status) - else _vnext_status_counts(fetched_memories) - ) - memory_status_counts = readable_status_counts(memory_status_counts, fetched_memories, review_memories) + memory_status_counts = guard.readable_status_counts("memory") review_memory_total = sum(memory_status_counts.get(status, 0) for status in review_statuses) fetched_artifacts = store.list_artifacts(sensitivity_allowed=sensitivity_allowed, limit=30) artifacts = _workspace_rows(store, "artifact", fetched_artifacts, sensitivity_allowed) - artifact_count = readable_count(store.count_artifacts(), len(fetched_artifacts), len(artifacts)) - artifact_status_counts = readable_status_counts( - store.count_artifacts_by_status(), - fetched_artifacts, - artifacts, + artifact_status_counts = guard.readable_status_counts("artifact") + artifact_count = sum(artifact_status_counts.values()) + quality_evals = guard.admit_related_rows(store.list_artifact_quality_ratings(limit=50), kind="artifact", field="artifact_id") + quality_eval_count = sum( + len(guard.admit_related_rows(batch, kind="artifact", field="artifact_id")) + for batch in store.iter_label_ratings() ) - quality_evals = store.list_artifact_quality_ratings(limit=50) - quality_eval_count = store.count_artifact_quality_ratings() fetched_projects = store.list_projects(status=None, sensitivity_allowed=sensitivity_allowed, limit=20) projects = _workspace_rows(store, "project", fetched_projects, sensitivity_allowed) - project_count = readable_count(store.count_projects(), len(fetched_projects), len(projects)) + project_count = sum(guard.readable_status_counts("project").values()) fetched_loops = store.list_open_loops(status=None, sensitivity_allowed=sensitivity_allowed, limit=30) open_loops = _workspace_rows(store, "open_loop", fetched_loops, sensitivity_allowed) - open_loop_status_counts = readable_status_counts( - store.count_open_loops_by_status(), - fetched_loops, - open_loops, - ) - stored_open_loop_count = store.count_open_loops(status="open") - open_loop_count = ( - stored_open_loop_count - if len(fetched_loops) == len(open_loops) - else int(open_loop_status_counts.get("open", stored_open_loop_count)) - ) + open_loop_status_counts = guard.readable_status_counts("open_loop") + open_loop_count = open_loop_status_counts.get("open", 0) people = store.list_people(sensitivity_allowed=sensitivity_allowed, limit=12) fetched_beliefs = store.list_beliefs(status=None, sensitivity_allowed=sensitivity_allowed, limit=12) beliefs = LabelGuard.for_filters(store, (), sensitivity_allowed, ()).admit_beliefs(fetched_beliefs) tasks = store.list_tasks(status=None, limit=12) fetched_events = store.list_events(limit=20) - recent_events = [ - event - for event in fetched_events - if _workspace_event_visible(store, event, sensitivity_allowed) - ] - count_events = getattr(store, "count_events", None) - stored_event_count = count_events() if callable(count_events) else len(fetched_events) - event_count = readable_count(stored_event_count, len(fetched_events), len(recent_events)) + recent_events = guard.admit_events(fetched_events) + event_count = guard.readable_event_count() agent_identities = store.list_agent_identities(limit=20) agent_count = store.count_agent_identities() agent_events = store.list_agent_events(limit=50) diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index 05d3be249..b43d456aa 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -443,6 +443,50 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: (self.user_id, *wanted, *canonical), ) + def iter_label_rows(self, kind: str, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + """Complete counted population, in narrow tenant-bound keyset batches.""" + + if batch_size < 1: + raise ValueError("batch_size must be positive") + table = {"source": "sources", "memory": "memories", "open_loop": "open_loops"}.get(kind) + if table is None: + raise ValueError("unsupported label kind") + extra = "" + if kind == "memory": + extra = ", status, value, project_id, source_event_ids, deleted_at" + elif kind == "open_loop": + extra = ", status, project_id, source_id, memory_id" + live = " AND deleted_at IS NULL" if kind in {"source", "memory"} else "" + after = "" + while True: + rows = self._fetch_all( + f"""SELECT id, user_id, domain, sensitivity, metadata_json{extra} + FROM {table} WHERE user_id = ? AND id > ?{live} + ORDER BY id LIMIT ?""", + (self.user_id, after, batch_size), + ) + if not rows: + return + yield rows + after = str(rows[-1]["id"]) + + def iter_label_events(self, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + """Complete event targets for readable counts, without event payloads.""" + + if batch_size < 1: + raise ValueError("batch_size must be positive") + after = "" + while True: + rows = self._fetch_all( + """SELECT id, target_type, target_id, event_type FROM event_log + WHERE user_id = ? AND id > ? ORDER BY id LIMIT ?""", + (self.user_id, after, batch_size), + ) + if not rows: + return + yield rows + after = str(rows[-1]["id"]) + # -- fetch helpers (mirror PostgresVNextStore conventions) ------------ def _execute(self, query: str, params: tuple[object, ...] = ()) -> sqlite3.Cursor: diff --git a/apps/api/src/alicebot_api/vnext_dogfooding.py b/apps/api/src/alicebot_api/vnext_dogfooding.py index 8620af6e9..5a14906ed 100644 --- a/apps/api/src/alicebot_api/vnext_dogfooding.py +++ b/apps/api/src/alicebot_api/vnext_dogfooding.py @@ -172,24 +172,20 @@ def __init__(self, store: VNextDogfoodingStore) -> None: def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None) -> JsonObject: from alicebot_api.vnext_agent_control import ALL_SENSITIVITY - from alicebot_api.vnext_label_guard import admit_loaded, readable_status_counts + from alicebot_api.vnext_label_guard import LabelGuard, admit_loaded sources = self.store.list_sources(limit=500) try: memories = self.store.list_memories(status=None, limit=500) except TypeError: # Compatibility for external/test stores on the old protocol. memories = self.store.list_memories(status=None)[:500] - count_memories_by_status = getattr(self.store, "count_memories_by_status", None) - memory_status_counts = ( - count_memories_by_status() if callable(count_memories_by_status) else _status_counts(memories) - ) artifacts = self.store.list_artifacts(limit=500) ratings = self.store.list_artifact_quality_ratings(limit=500) open_loops = self.store.list_open_loops(status=None, limit=500) # None is the owner and an admin key: every sensitivity, so the guard # reads nothing and the lists stay as the store returned them. ceiling = sensitivity_allowed if sensitivity_allowed is not None else ALL_SENSITIVITY - fetched_memories = memories + guard = LabelGuard.for_filters(self.store, (), ceiling, ()) sources = admit_loaded(self.store, kind="source", rows=sources, domains=(), sensitivity_allowed=ceiling, projects=()) memories = admit_loaded( self.store, kind="memory", rows=memories, domains=(), sensitivity_allowed=ceiling, projects=() @@ -200,11 +196,13 @@ def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None) -> Js open_loops = admit_loaded( self.store, kind="open_loop", rows=open_loops, domains=(), sensitivity_allowed=ceiling, projects=() ) - memory_status_counts = readable_status_counts(memory_status_counts, fetched_memories, memories) + memory_status_counts = guard.readable_status_counts("memory") try: events = self.store.list_events(limit=5_000) except TypeError: # Compatibility for external/test stores on the old protocol. events = self.store.list_events()[:5_000] + events = guard.admit_events(events) + ratings = guard.admit_related_rows(ratings, kind="artifact", field="artifact_id") scheduler_runs = self.store.list_scheduler_runs(limit=20) now = datetime.now(UTC) today_cutoff = now.replace(hour=0, minute=0, second=0, microsecond=0) diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index ec57e7ebe..a9e4de7db 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -129,10 +129,6 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: if not self.active: return [row for row in rows if isinstance(row, Mapping)] - # Compatibility stores without an ancestry reader may retain SQL-filtered - # rows only when no locked all-of binding needs verification. - if not callable(getattr(self.store, "read_label_rows", None)): - return [] if self.all_of is not None else [row for row in rows if isinstance(row, Mapping)] kept: list[_Row] = [] for row in rows: if not isinstance(row, Mapping): @@ -142,6 +138,61 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: kept.append(row) return kept + def readable_status_counts(self, kind: str) -> dict[str, int]: + """Count the complete population through the same effective admission. + + Display pages and stored-label SQL counts cannot establish this total. + Stores must expose the complete narrow population rather than guessing + a restricted total from a sample. + """ + + iterator = getattr(self.store, "iter_label_rows", None) + if not callable(iterator): + raise TypeError("readable counts require complete label enumeration") + counts: dict[str, int] = {} + for batch in iterator(kind): + for row in self.admit_rows(kind, batch): + status = str(row.get("status", "unknown")) + counts[status] = counts.get(status, 0) + 1 + return counts + + def admit_related_rows(self, rows: Sequence[_Row], *, kind: str, field: str) -> list[_Row]: + """Admit an event or rating by its target's current effective label.""" + + if not self.active: + return [row for row in rows if isinstance(row, Mapping)] + reader = getattr(self.store, "read_label_rows", None) + if not callable(reader): + return [] + ids = list(dict.fromkeys(str(row.get(field)) for row in rows if row.get(field))) + found = list(reader(kind, ids)) if ids else [] + admitted = {str(row.get("id")) for row in ( + self.admit_beliefs(found) if kind == "belief" else self.admit_rows(kind, found) + )} + return [row for row in rows if str(row.get(field) or "") in admitted] + + def admit_events(self, rows: Sequence[_Row]) -> list[_Row]: + """Known label targets are admitted before an event exposes their IDs.""" + + if not self.active: + return [row for row in rows if isinstance(row, Mapping)] + admitted: set[int] = set() + kinds = {"source", "memory", "open_loop", "artifact", "project", "belief"} + for kind in kinds: + targets = [row for row in rows if str(row.get("target_type")) == kind] + admitted.update(id(row) for row in self.admit_related_rows(targets, kind=kind, field="target_id")) + return [row for row in rows if id(row) in admitted or ( + str(row.get("target_type")) not in kinds and not str(row.get("event_type", "")).endswith(".labels_raised") + )] + + def readable_event_count(self) -> int: + """Complete event count after current target admission.""" + + iterator = getattr(self.store, "iter_label_events", None) + if not callable(iterator): + raise TypeError("readable counts require complete event enumeration") + return sum(len(self.admit_events(batch)) for batch in iterator()) + def admit_beliefs(self, beliefs: Sequence[_Row]) -> list[_Row]: """Beliefs whose backing memory the filters admit. One batched read.""" @@ -279,44 +330,6 @@ def apply_sensitivity_ceiling( ) -def readable_count(sql_count: int, fetched: int, admitted: int) -> int: - """A stored count, reduced only by rows this page's guard dropped. - - When the guard drops nothing the stored count is returned unchanged. - When the fetched page is the whole set, the count is the admitted length. - """ - - dropped = fetched - admitted - if dropped <= 0: - return sql_count - if sql_count <= fetched: - return admitted - return max(0, sql_count - dropped) - - -def readable_status_counts( - counts: Mapping[str, int], - fetched: Sequence[Mapping[str, object]], - admitted: Sequence[Mapping[str, object]], - *, - field: str = "status", -) -> dict[str, int]: - """Status counts with one taken off for each row the guard dropped.""" - - if len(fetched) == len(admitted): - return dict(counts) - kept = {id(row) for row in admitted} - updated = dict(counts) - for row in fetched: - if id(row) in kept: - continue - status = str(row.get(field, "unknown")) - current = int(updated.get(status, 0)) - if current > 0: - updated[status] = current - 1 - return updated - - def apply_unverified_rule( decision: PolicyDecision, row: Mapping[str, object] | None, diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 5ab808b1a..6a0660847 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -530,6 +530,73 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: (wanted,), ) + def iter_label_rows(self, kind: str, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + """Complete counted population, in narrow keyset batches under tenant RLS.""" + + if batch_size < 1: + raise ValueError("batch_size must be positive") + table = {"source": "sources", "memory": "memories", "open_loop": "open_loops", + "artifact": "generated_artifacts", "project": "projects"}.get(kind) + if table is None: + raise ValueError("unsupported label kind") + extra = "" + if kind == "memory": + extra = ", status, value, project_id, source_event_ids, deleted_at" + elif kind == "open_loop": + extra = ", status, project_id, source_id, memory_id" + elif kind == "artifact": + extra = ", status, artifact_type" + elif kind == "project": + extra = ", status" + live = " AND deleted_at IS NULL" if kind in {"source", "memory"} else "" + after: str | None = None + while True: + rows = self._fetch_all( + f"""SELECT id, user_id, domain, sensitivity, metadata_json{extra} + FROM {table} + WHERE (%s::uuid IS NULL OR id > %s::uuid){live} + ORDER BY id LIMIT %s""", + (after, after, batch_size), + ) + if not rows: + return + yield rows + after = str(rows[-1]["id"]) + + def iter_label_events(self, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + """Complete event targets for readable counts, without event payloads.""" + + if batch_size < 1: + raise ValueError("batch_size must be positive") + after: str | None = None + while True: + rows = self._fetch_all( + """SELECT id, target_type, target_id, event_type FROM event_log + WHERE (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s""", + (after, after, batch_size), + ) + if not rows: + return + yield rows + after = str(rows[-1]["id"]) + + def iter_label_ratings(self, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + """Complete rating targets for counts, without feedback text.""" + + if batch_size < 1: + raise ValueError("batch_size must be positive") + after: str | None = None + while True: + rows = self._fetch_all( + """SELECT id, artifact_id FROM artifact_quality_ratings + WHERE (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s""", + (after, after, batch_size), + ) + if not rows: + return + yield rows + after = str(rows[-1]["id"]) + def list_belief_ids_for_memories(self, ids: Sequence[str]) -> list[str]: """Same-user belief aliases that make a memory an indirect report input.""" diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py new file mode 100644 index 000000000..b52c7074d --- /dev/null +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -0,0 +1,132 @@ +"""Real keys, operator screens and complete readable counts on PostgreSQL.""" + +from __future__ import annotations + +import json +from uuid import UUID, uuid4 + +import pytest + +from alicebot_api.config import Settings +from alicebot_api.db import user_connection +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError +from alicebot_api.routers import vnext_review, vnext_retrieval, vnext_projects, vnext_memories, workspaces +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_agent_control import ALL_SENSITIVITY +from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_store import PostgresVNextStore +from tests.unit.test_derived_labels_real_keys import READERS, expected_read, real_reader_key, seed_read_rows + + +def _user(app_url): + user_id = uuid4() + with user_connection(app_url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, "synthetic@example.invalid", "Synthetic") + return user_id + + +@pytest.mark.parametrize("reader", READERS) +def test_postgres_real_key_core_doors(migrated_database_urls, monkeypatch, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + monkeypatch.setattr(vnext_memories, "get_settings", lambda: Settings(database_url=app_url)) + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + rows = seed_read_rows(store) + key = real_reader_key(store, user_id, reader) + if key: + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + context = MCPRuntimeContext(database_url=app_url, user_id=user_id) + for state, row in rows.items(): + admitted = expected_read(reader, state) + audit = vnext_memories.get_vnext_memory_audit(UUID(str(row["id"])), user_id, authorization=f"Bearer {key}" if key else None) + assert audit.status_code == (200 if admitted else 403), (reader, state, audit.body) + if not admitted: + assert str(row["id"]) not in audit.body.decode() + for tool in ("alice_recall", "alice_context_pack", "alice_recent_decisions", "alice_explain", "alice_resume"): + arguments = {"memory_id": str(row["id"])} if tool == "alice_explain" else {"query": str(row["canonical_text"]), "sensitivity_allowed": list(ALL_SENSITIVITY)} + try: + result = call_mcp_tool(context, name=tool, arguments=arguments) + except MCPToolError: + assert not admitted, (reader, state, tool) + continue + rendered = json.dumps(result, default=str) + assert (str(row["id"]) in rendered) is admitted, (reader, state, tool, rendered) + if not admitted: + assert str(row["title"]) not in rendered + + +@pytest.mark.parametrize("reader", ("owner", "admin", "trusted")) +def test_all_five_operator_screens_with_real_keys(migrated_database_urls, monkeypatch, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + for module in (vnext_review, vnext_retrieval, vnext_projects): + monkeypatch.setattr(module, "get_settings", lambda: Settings(database_url=app_url)) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Cedar hidden source", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential"}) + public_source = store.create_source({"source_type": "note", "title": "Public source", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public"}) + memory = store.create_memory({"memory_key": "belief", "canonical_text": "Cedar hidden belief", "status": "active", "domain": "project", "sensitivity": "confidential"}) + artifact = store.create_artifact({"artifact_type": "daily_brief", "title": "Cedar hidden artifact", "content_markdown": "Cedar hidden artifact", "domain": "project", "sensitivity": "confidential", "metadata_json": {"derived_from": {"v": 1, "sources": [str(public_source["id"])], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}}, "source_refs": [str(public_source["id"])]}}) + project = store.create_project({"name": "Cedar hidden project", "slug": "cedar-hidden", "current_state": "Cedar hidden state", "domain": "project", "sensitivity": "confidential"}) + belief_id = uuid4() + conn.execute("INSERT INTO beliefs(id,user_id,memory_id,claim) VALUES (%s,%s,%s,%s)", (belief_id, user_id, memory["id"], "Cedar hidden belief")) + key = real_reader_key(store, user_id, reader) + auth = f"Bearer {key}" if key else None + responses = ( + (vnext_review.list_vnext_artifacts(user_id, authorization=auth), str(artifact["id"])), + (vnext_retrieval.get_vnext_source_trace(UUID(str(source["id"])), user_id, authorization=auth), str(source["id"])), + (vnext_projects.list_vnext_projects(user_id, authorization=auth), str(project["id"])), + (vnext_projects.get_vnext_project_dashboard(str(project["id"]), user_id, authorization=auth), str(project["id"])), + (vnext_review.get_vnext_belief_state(str(belief_id), user_id, authorization=auth), str(belief_id)), + ) + for response, identifier in responses: + body = response.body.decode() + if reader == "trusted": + assert identifier not in body + assert "Cedar hidden" not in body + assert response.status_code in {200, 404} + if response.status_code == 200: + assert json.loads(body)["count"] == 0 + else: + assert response.status_code == 200, body + assert identifier in body + # A visible source trace must not carry a hidden artifact or its count. + trace = vnext_retrieval.get_vnext_source_trace(UUID(str(public_source["id"])), user_id, authorization=auth) + body = json.loads(trace.body) + assert body["summary"]["artifact_count"] == (0 if reader == "trusted" else 1) + assert (str(artifact["id"]) in trace.body.decode()) is (reader != "trusted") + + +def test_workspace_counts_full_population_with_sql_hidden_and_stale_rows(migrated_database_urls, monkeypatch): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + # The doctor has its own store-specific repair acceptance test. Keep this + # count probe independent of that ancillary diagnostic implementation. + monkeypatch.setattr(workspaces.VNextDoctorService, "run", lambda self, **kwargs: {}) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + secret = store.create_source({"source_type": "note", "title": "Cedar hidden", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential"}) + for index in range(205): + store.create_source({"source_type": "note", "title": "Public source", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public"}) + for index in range(35): + store.create_memory({"memory_key": f"visible-{index}", "canonical_text": "Public fact", "status": "candidate", "domain": "project", "sensitivity": "public"}) + with without_insert_floor(): + for index in range(40): + store.create_memory({"memory_key": f"hidden-{index}", "canonical_text": "Cedar hidden", "status": "candidate", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(secret["id"])}}) + store.create_artifact({"artifact_type": "daily_brief", "title": "Cedar hidden", "content_markdown": "Cedar hidden", "domain": "project", "sensitivity": "confidential", "metadata_json": {"derived_from": {"v": 1, "sources": [], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": 0, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}}}}) + store.create_project({"name": "Cedar hidden", "slug": "hidden", "domain": "project", "sensitivity": "confidential"}) + body = workspaces._vnext_workspace_payload(store) + assert body["summary"]["source_count"] == 205 + assert body["summary"]["candidate_memory_count"] == 35 + assert body["summary"]["artifact_count"] == 0 + assert body["summary"]["project_count"] == 0 + assert body["samples"]["sources"]["has_more"] is True + assert "Cedar hidden" not in json.dumps(body, default=str) + batches = list(store.iter_label_rows("source", batch_size=100)) + assert [len(batch) for batch in batches] == [100, 100, 6] + assert all("content_markdown" not in row and "title" not in row for batch in batches for row in batch) diff --git a/tests/unit/test_complete_readable_counts.py b/tests/unit/test_complete_readable_counts.py new file mode 100644 index 000000000..97eb1bbea --- /dev/null +++ b/tests/unit/test_complete_readable_counts.py @@ -0,0 +1,162 @@ +"""Totals and trace completeness use the full effectively readable population.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from alicebot_api.routers import _vnext_shared, workspaces +from alicebot_api.vnext_agent_control import ALL_SENSITIVITY, AgentIdentity +from alicebot_api.vnext_dogfooding import VNextDogfoodingService +from alicebot_api.vnext_label_guard import LabelGuard + + +class PopulationStore: + def __init__(self): + self.rows = {kind: [] for kind in ("source", "memory", "artifact", "project", "open_loop")} + self.events = [] + + def read_label_rows(self, kind, ids): + return [row for row in self.rows[kind] if row["id"] in ids] + + def iter_label_rows(self, kind, *, batch_size=200): + for start in range(0, len(self.rows[kind]), batch_size): + yield self.rows[kind][start:start + batch_size] + + def iter_label_events(self, *, batch_size=200): + for start in range(0, len(self.events), batch_size): + yield self.events[start:start + batch_size] + + def iter_label_ratings(self): + return iter(()) + + def list_memories(self, *, status=None, limit=None, **kwargs): + return self.rows["memory"][:limit] + + def list_memories_by_statuses(self, *, statuses, sensitivity_allowed, limit): + return [row for row in self.rows["memory"] if row["status"] in statuses and row["sensitivity"] in sensitivity_allowed][:limit] + + def count_memories_by_status(self, **kwargs): + return {"candidate": len(self.rows["memory"])} + + def list_events(self, **kwargs): + return self.events[:kwargs.get("limit")] + + def __getattr__(self, name): + kinds = {"sources": "source", "artifacts": "artifact", "projects": "project", "open_loops": "open_loop"} + suffix = name.removeprefix("list_") + if suffix in kinds: + def listed(**kwargs): + rows = self.rows[kinds[suffix]] + sensitivities = kwargs.get("sensitivity_allowed") + if sensitivities: + rows = [row for row in rows if row["sensitivity"] in sensitivities] + return rows[:kwargs.get("limit")] + return listed + if name.startswith("list_"): + return lambda *args, **kwargs: [] + if name.startswith("count_"): + return lambda **kwargs: len(self.rows[kinds[name.removeprefix("count_")]]) if name.removeprefix("count_") in kinds else 0 + if name == "get_brain_charter": + return lambda: {} + raise AttributeError(name) + + +def _row(identifier, sensitivity="public", **kwargs): + return {"id": identifier, "domain": "project", "sensitivity": sensitivity, "status": "candidate", "metadata_json": {}, **kwargs} + + +def _quiet_services(monkeypatch): + for name in ("VNextSchedulerService", "VNextConnectorService", "VNextDoctorService", "VNextProjectService", "VNextMemoryCommitService"): + monkeypatch.setattr(workspaces, name, lambda store: SimpleNamespace( + status=lambda: {}, connector_health_all=lambda: [], run=lambda **kwargs: {}, + project_dashboard=lambda **kwargs: {}, recent_commits=lambda **kwargs: {"recent_commits": []}, + inline_confirmations=lambda **kwargs: [])) + monkeypatch.setattr(workspaces, "daemon_status", lambda: {}) + monkeypatch.setattr("alicebot_api.vnext_dogfooding.VNextConnectorService.connector_health_all", lambda self: []) + + +def test_workspace_counts_sql_hidden_and_beyond_display_page(monkeypatch): + _quiet_services(monkeypatch) + store = PopulationStore() + for kind in store.rows: + store.rows[kind] = [_row(f"{kind}-{index}") for index in range(35)] + store.rows[kind] += [_row(f"{kind}-hidden-{index}", "confidential") for index in range(205)] + store.rows["open_loop"] = [{**row, "status": "open"} for row in store.rows["open_loop"]] + store.events = [{"id": str(index), "target_type": "memory", "target_id": row["id"], "event_type": "memory.labels_raised"} for index, row in enumerate(store.rows["memory"])] + body = workspaces._vnext_workspace_payload(store) + summary = body["summary"] + for field in ("source_count", "artifact_count", "project_count", "open_loop_count", "event_count", "candidate_memory_count"): + assert summary[field] == 35, (field, summary[field]) + assert summary["memory_status_counts"] == {"candidate": 35} + assert summary["artifact_status_counts"] == {"candidate": 35} + assert summary["open_loop_status_counts"] == {"open": 35} + assert body["samples"]["sources"]["has_more"] is True + assert "hidden" not in str(body) + + +def test_all_sql_prefiltered_rows_leave_zero_totals(monkeypatch): + _quiet_services(monkeypatch) + store = PopulationStore() + for kind in store.rows: + store.rows[kind] = [_row(f"{kind}-hidden", "confidential")] + body = workspaces._vnext_workspace_payload(store) + for field in ("source_count", "artifact_count", "project_count", "open_loop_count", "candidate_memory_count"): + assert body["summary"][field] == 0 + assert all(not sample["has_more"] for sample in body["samples"].values()) + + +def test_dogfooding_counts_hidden_rows_beyond_500(monkeypatch): + _quiet_services(monkeypatch) + store = PopulationStore() + store.rows["memory"] = [_row(str(index)) for index in range(500)] + [_row("hidden", "confidential")] + trusted = VNextDogfoodingService(store).dashboard(sensitivity_allowed=("public", "internal", "private", "unknown")) + owner = VNextDogfoodingService(store).dashboard() + assert trusted["memory_status_counts"] == {"candidate": 500} + assert trusted["sample_scope"]["memories"]["total_count"] == 500 + assert owner["memory_status_counts"] == {"candidate": 501} + + +def test_count_rejects_a_store_without_complete_enumeration(): + with pytest.raises(TypeError, match="complete label enumeration"): + LabelGuard.for_filters(object(), (), ("public",)).readable_status_counts("memory") + + +def test_derived_totals_use_effective_labels_and_keep_owner_control(): + store = PopulationStore() + store.rows["source"] = [_row("11111111-1111-4111-8111-111111111111", "confidential")] + store.rows["memory"] = [_row("22222222-2222-4222-8222-222222222222", metadata_json={"source_id": store.rows["source"][0]["id"]})] + trusted = LabelGuard.for_filters(store, (), ("public", "internal", "private", "unknown")) + owner = LabelGuard.for_filters(store, (), ALL_SENSITIVITY) + assert trusted.readable_status_counts("memory") == {} + assert owner.readable_status_counts("memory") == {"candidate": 1} + + +def test_trace_completeness_does_not_reveal_hidden_501st_row(monkeypatch): + monkeypatch.setattr(_vnext_shared, "_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT", 2) + store = PopulationStore() + rows = [_row(str(index)) for index in range(2)] + [_row("hidden", "confidential")] + fetches = [] + def fetch(limit): + fetches.append(limit) + return rows[:limit] + trusted = AgentIdentity(agent_id="reader", permission_profile="trusted_local_agent", agent_type="unknown") + admitted, complete = _vnext_shared._vnext_readable_trace_rows(store, "memory", fetch, trusted) + assert [row["id"] for row in admitted] == ["0", "1"] + assert complete is True + assert fetches == [3, 6] + owner, complete = _vnext_shared._vnext_readable_trace_rows(store, "memory", fetch, None) + assert len(owner) == 2 + assert complete is False + + +def test_trace_event_completeness_uses_admitted_targets(monkeypatch): + monkeypatch.setattr(_vnext_shared, "_VNEXT_SOURCE_TRACE_COLLECTION_LIMIT", 2) + events = [{"id": str(index), "target_id": "visible"} for index in range(2)] + [{"id": "hidden-event", "target_id": "hidden"}] + admitted, complete = _vnext_shared._vnext_readable_trace_rows( + PopulationStore(), "event", lambda limit: events[:limit], None, + admit=lambda rows: [row for row in rows if row["target_id"] == "visible"], + ) + assert [row["id"] for row in admitted] == ["0", "1"] + assert complete is True diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py index f04a8be44..e82773f3e 100644 --- a/tests/unit/test_label_door_registry.py +++ b/tests/unit/test_label_door_registry.py @@ -9,6 +9,7 @@ SRC = ROOT / "apps/api/src" READS = { + "get_source", "list_sources", "get_sources_by_ids", "get_memory", "get_memory_for_update", "get_memories_by_ids", @@ -31,6 +32,10 @@ "list_projects", "get_project", "get_project_for_update", + "list_open_loops_referencing_source", "list_events", "list_memory_events", "list_open_loop_events", + "list_events_for_source_trace", "list_recent_agentic_commits", "list_pending_inline_confirmations", + "count_sources", "count_artifacts", "count_artifacts_by_status", "count_projects", "count_memories_by_status", + "count_open_loops", "count_open_loops_by_status", "count_events", "iter_label_rows", "iter_label_events", "iter_label_ratings", } GUARD_CALLS = { @@ -43,6 +48,7 @@ "admit_loaded", "apply_sensitivity_ceiling", "sensitivity_ceiling", + "admit_events", "admit_related_rows", "readable_status_counts", "readable_event_count", } # function -> helper that holds the guard call, or None when the function calls it @@ -57,6 +63,7 @@ "mcp/review.py:_vnext_memory_review": None, "mcp/review.py:_vnext_memory_correct": None, "routers/vnext_memories.py:review_vnext_memory": None, + "routers/vnext_memories.py:get_vnext_memory_audit": None, "mcp/memories.py:redact_memory_flow": None, "routers/vnext_projects.py:review_vnext_open_loop": None, "mcp/retrieval.py:_handle_alice_open_loops": None, @@ -91,6 +98,18 @@ "vnext_retrieval.py:VNextRetrievalService._contradicting_evidence": None, "vnext_retrieval.py:VNextRetrievalService._recent_changes": None, "vnext_retrieval.py:VNextRetrievalService.memory_visibility": None, + "vnext_brain.py:VNextBrainService._load_inputs": None, + "vnext_connections.py:VNextConnectionService.generate_connection_report": None, + "vnext_contradictions.py:VNextContradictionService.generate_contradiction_report": None, + "vnext_consolidation.py:VNextConsolidationService._cluster_memories": None, + "vnext_consolidation.py:VNextConsolidationService.generate_memory_consolidation": None, + "vnext_rollups.py:VNextRollupService._collect_rows": None, + "vnext_rollups.py:VNextRollupService._existing_rollup_state": None, + "vnext_scheduler.py:VNextSchedulerService._run_staleness_sweep": None, + "vnext_scheduler.py:VNextSchedulerService._generate_open_loop_review_artifact": None, + "vnext_context_tree.py:_tree_event_visible": None, + "routers/vnext_retrieval.py:get_vnext_source_trace": "routers/_vnext_shared.py:_vnext_load_source_trace", + "routers/vnext_retrieval.py:get_vnext_artifact_trace": "routers/_vnext_shared.py:_vnext_authorized_artifact", } NOT_A_DOOR = { @@ -115,9 +134,40 @@ "vnext_queue.py:VNextQueueService.review_artifact": "the HTTP review route authorizes through _vnext_authorized_artifact first", "vnext_queue.py:VNextQueueService._promote_artifact": "the HTTP review route authorizes through _vnext_authorized_artifact first", "vnext_queue.py:VNextQueueService.export_artifact_markdown": "the HTTP export route authorizes through _vnext_authorized_artifact first", + "mcp/evidence_artifacts.py:_authorize_memory_audit_provenance": "original source pointers use SourceReadFence.admits before disclosure", + "routers/vnext_memories.py:get_vnext_source": "original source operator route; existing domain and project exemptions are preserved", + "routers/vnext_memories.py:get_vnext_connector_status": "operator connector telemetry; original-source labels retain existing behavior", + "routers/vnext_memories.py:review_vnext_source": "write path over an original source; existing exact policy applies", + "routers/vnext_memories.py:delete_vnext_source": "owner mutation of an original source", + "vnext_memory_commit.py:VNextMemoryCommitService.auto_promoted_by_agent": "write sweep; every target is authorized by expire before mutation", + "vnext_memory_commit.py:VNextMemoryCommitService._transition_memory": "writer checks source validity; no new read response", + "vnext_source_fence.py:_rows_by_id": "narrow loader; SavedProvenanceReader applies effective labels before presenting", + "vnext_source_fence.py:source_rows_including_archived": "original-source loader for provenance and producer label computation", + "vnext_source_fence.py:SavedProvenanceReader._row_for": "private loader; SavedProvenanceReader._admits settles each row", + "vnext_retrieval.py:_current_memory_id": "pointer loader; caller memory_visibility settles before exposing the pointer", + "vnext_retrieval.py:_memories_referencing_sources": "internal loader; expand_provenance_once applies effective admission", + "vnext_retrieval.py:VNextRetrievalService._sources_by_ids": "original-source lookup; SourceReadFence admits before formatting", + "vnext_retrieval.py:VNextRetrievalService._query_embedding": "store capability discovery only; does not execute a row reader", + "vnext_retrieval.py:VNextRetrievalService._source_stage_lists": "original-source stage; existing source admission remains", + "vnext_retrieval.py:VNextRetrievalService._supersession_context": "pointer metadata is fenced through memory_visibility at output", + "session_briefing.py:_merge_recent_change_targets": "private loader checks _event_target_honours_fence before adding a target", + "session_briefing.py:_resolve_excerpt_query": "original source lookup; source fence and withheld event targets constrain excerpts", + "vnext_context_tree.py:VNextContextTreeService._scoped_events": "build_tree filters every resulting event through _tree_event_visible", + "vnext_dogfooding.py:VNextDogfoodingService.record_insight_feedback": "write route authorizes artifact through _vnext_authorized_artifact", + "vnext_contradictions.py:_project_scoped_beliefs": "internal loader; generate_contradiction_report admits through backing memories", + "vnext_consolidation.py:_list_memories_bounded": "private loader; _cluster_memories applies effective admission", + "vnext_consolidation.py:_existing_cluster_candidates": "owner acceptance/idempotency lookup; group-scope consumers retain existing behavior", + "vnext_scheduler.py:_StagedSchedulerStore.update_memory": "staged write replay; returned row is not a disclosure door", + "vnext_scheduler.py:VNextSchedulerService.status": "scheduler telemetry; events are constrained to scheduler targets", + "vnext_scheduler.py:VNextSchedulerService._generate_project_update_scan_artifact": "internal project scan; generate_project_update_candidate applies effective admission", + "vnext_connectors.py:VNextConnectorService.get_cursor": "connector cursor events only; no labelled targets", + "vnext_connectors.py:VNextConnectorService.get_config": "connector configuration events only; no labelled targets", + "vnext_connectors.py:VNextConnectorService.connector_health": "connector state telemetry only; no labels_raised events", + "vnext_artifact_review.py:dispatch_vnext_artifact_review": "writer entry; calling route or MCP authorizes the artifact before dispatch", + "vnext_memory_commit.py:VNextMemoryCommitService._guard_supersession_acyclic": "write validation traverses pointers without exposing their content", } -SCAN_MODULES = ( +SCAN_MODULES = tuple(sorted({ "alicebot_api/routers/_vnext_shared.py", "alicebot_api/mcp/evidence_artifacts.py", "alicebot_api/mcp/review.py", @@ -130,7 +180,13 @@ "alicebot_api/vnext_open_loop_references.py", "alicebot_api/vnext_queue.py", "alicebot_api/mcp/retrieval.py", -) + "alicebot_api/vnext_retrieval.py", "alicebot_api/session_briefing.py", + "alicebot_api/routers/workspaces.py", "alicebot_api/vnext_context_tree.py", "alicebot_api/vnext_dogfooding.py", + "alicebot_api/vnext_brain.py", "alicebot_api/vnext_connections.py", "alicebot_api/vnext_contradictions.py", + "alicebot_api/vnext_consolidation.py", "alicebot_api/vnext_rollups.py", "alicebot_api/vnext_scheduler.py", + "alicebot_api/vnext_connectors.py", "alicebot_api/vnext_artifact_review.py", + *(str(path.relative_to(SRC)) for directory in ("routers", "mcp") for path in (SRC / "alicebot_api" / directory).glob("*.py")), +})) def _functions(tree: ast.AST) -> list[tuple[str, ast.FunctionDef]]: @@ -157,6 +213,19 @@ def _called_names(node: ast.AST) -> set[str]: return names +def _reader_names(node: ast.AST) -> set[str]: + """Actual AST calls, bound-method callbacks, and dynamic reader lookup.""" + + names = _called_names(node) & READS + for child in ast.walk(node): + if isinstance(child, ast.Attribute) and child.attr in READS: + names.add(child.attr) + elif isinstance(child, ast.Call) and isinstance(child.func, ast.Name) and child.func.id == "getattr": + if len(child.args) > 1 and isinstance(child.args[1], ast.Constant) and child.args[1].value in READS: + names.add(child.args[1].value) + return names + + def _has_guard(node: ast.AST) -> bool: return bool(_called_names(node) & GUARD_CALLS) @@ -172,9 +241,13 @@ def test_every_exact_door_calls_the_guard() -> None: if helper is None: assert _has_guard(functions[name]), key else: - assert helper in _called_names(functions[name]), key - assert helper in functions, key - assert _has_guard(functions[helper]), helper + helper_path, helper_name = helper.split(":", 1) if ":" in helper else (path, helper) + assert helper_name in _called_names(functions[name]), key + if helper_path not in modules: + modules[helper_path] = ast.parse((SRC / "alicebot_api" / helper_path).read_text(encoding="utf-8")) + helper_functions = dict(_functions(modules[helper_path])) + assert helper_name in helper_functions, key + assert _has_guard(helper_functions[helper_name]), helper def test_every_scanned_reader_is_classified() -> None: @@ -184,8 +257,15 @@ def test_every_scanned_reader_is_classified() -> None: tree = ast.parse((SRC / relative).read_text(encoding="utf-8")) short = relative.removeprefix("alicebot_api/") for name, node in _functions(tree): - if _called_names(node) & READS: + if _reader_names(node): key = f"{short}:{name}" if key not in classified: missing.append(key) assert missing == [] + + +def test_discovery_catches_new_direct_and_dynamic_readers() -> None: + for source in ("def added(store): return store.list_events()", "def added(store): return getattr(store, 'list_memories')()", "def added(store): return invoke(store.list_beliefs)"): + node = ast.parse(source).body[0] + assert _reader_names(node) + assert "added" not in DOORS and "added" not in NOT_A_DOOR From a8496c16291272136111a60220ffcd8b5ac46446 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:02:01 +0200 Subject: [PATCH 120/270] Guard workspace activity cards with current target labels --- .../src/alicebot_api/routers/workspaces.py | 51 ++++++++++++------- apps/api/src/alicebot_api/vnext_dogfooding.py | 24 ++++----- ...derived_labels_read_acceptance_postgres.py | 35 +++++++++++++ tests/unit/test_complete_readable_counts.py | 21 ++++++++ tests/unit/test_label_door_registry.py | 1 + 5 files changed, 100 insertions(+), 32 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index c378393e5..bc85a15e5 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -3,7 +3,7 @@ from collections.abc import Mapping, Sequence from uuid import UUID -from fastapi import APIRouter, Request +from fastapi import APIRouter, Header, Request from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse import psycopg @@ -18,13 +18,14 @@ ) from alicebot_api.public_errors import public_exception_response from alicebot_api.routers._api_shared import _resolve_authenticated_v1_user_id -from alicebot_api.routers._vnext_shared import _vnext_int, _vnext_source_trace +from alicebot_api.routers._vnext_shared import _vnext_agent_auth_error_response, _vnext_int, _vnext_source_trace from alicebot_api.routers.providers import ( _discover_provider_capability, _persist_discovered_provider_capability, _seed_workspace_provider_configs, ) -from alicebot_api.vnext_agent_control import summarize_agent_policy_telemetry +from alicebot_api.vnext_agent_control import AgentIdentity, summarize_agent_policy_telemetry +from alicebot_api.vnext_agent_keys import AgentKeyAuthenticationError, agent_key_from_authorization, resolve_protected_agent_identity from alicebot_api.vnext_connectors import VNextConnectorService from alicebot_api.vnext_dogfooding import VNextDogfoodingService from alicebot_api.vnext_doctor import VNextDoctorService @@ -84,14 +85,19 @@ def _workspace_rows(store: PostgresVNextStore, kind: str, rows: Sequence[Mapping ) -def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: - from alicebot_api.vnext_label_guard import LabelGuard +def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdentity | None = None) -> dict[str, object]: + from alicebot_api.vnext_label_guard import LabelGuard, sensitivity_ceiling sensitivity_allowed = ["public", "internal", "private", "unknown"] - guard = LabelGuard.for_filters(store, (), sensitivity_allowed, ()) + ceiling = sensitivity_ceiling(identity) + if ceiling is not None: + sensitivity_allowed = [value for value in sensitivity_allowed if value in ceiling] + projects = identity.project_scope if identity is not None else () + all_of = projects if identity is not None and identity.project_scope_locked else None + guard = LabelGuard.for_filters(store, (), sensitivity_allowed, projects, all_of=all_of) review_statuses = ["candidate", "needs_review", "private_only", "accepted", "rejected"] fetched_sources = store.list_sources(sensitivity_allowed=sensitivity_allowed, limit=20) - sources = _workspace_rows(store, "source", fetched_sources, sensitivity_allowed) + sources = guard.admit_rows("source", fetched_sources) source_count = sum(guard.readable_status_counts("source").values()) list_memories_by_statuses = getattr(store, "list_memories_by_statuses", None) if callable(list_memories_by_statuses): @@ -104,11 +110,11 @@ def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: fetched_memories = [ memory for memory in store.list_memories(status=None) if str(memory.get("status")) in set(review_statuses) ][:30] - review_memories = _workspace_rows(store, "memory", fetched_memories, sensitivity_allowed) + review_memories = guard.admit_rows("memory", fetched_memories) memory_status_counts = guard.readable_status_counts("memory") review_memory_total = sum(memory_status_counts.get(status, 0) for status in review_statuses) fetched_artifacts = store.list_artifacts(sensitivity_allowed=sensitivity_allowed, limit=30) - artifacts = _workspace_rows(store, "artifact", fetched_artifacts, sensitivity_allowed) + artifacts = guard.admit_rows("artifact", fetched_artifacts) artifact_status_counts = guard.readable_status_counts("artifact") artifact_count = sum(artifact_status_counts.values()) quality_evals = guard.admit_related_rows(store.list_artifact_quality_ratings(limit=50), kind="artifact", field="artifact_id") @@ -117,22 +123,22 @@ def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: for batch in store.iter_label_ratings() ) fetched_projects = store.list_projects(status=None, sensitivity_allowed=sensitivity_allowed, limit=20) - projects = _workspace_rows(store, "project", fetched_projects, sensitivity_allowed) + projects = guard.admit_rows("project", fetched_projects) project_count = sum(guard.readable_status_counts("project").values()) fetched_loops = store.list_open_loops(status=None, sensitivity_allowed=sensitivity_allowed, limit=30) - open_loops = _workspace_rows(store, "open_loop", fetched_loops, sensitivity_allowed) + open_loops = guard.admit_rows("open_loop", fetched_loops) open_loop_status_counts = guard.readable_status_counts("open_loop") open_loop_count = open_loop_status_counts.get("open", 0) people = store.list_people(sensitivity_allowed=sensitivity_allowed, limit=12) fetched_beliefs = store.list_beliefs(status=None, sensitivity_allowed=sensitivity_allowed, limit=12) - beliefs = LabelGuard.for_filters(store, (), sensitivity_allowed, ()).admit_beliefs(fetched_beliefs) + beliefs = guard.admit_beliefs(fetched_beliefs) tasks = store.list_tasks(status=None, limit=12) fetched_events = store.list_events(limit=20) recent_events = guard.admit_events(fetched_events) event_count = guard.readable_event_count() agent_identities = store.list_agent_identities(limit=20) agent_count = store.count_agent_identities() - agent_events = store.list_agent_events(limit=50) + agent_events = guard.admit_events(store.list_agent_events(limit=50)) list_recent_agentic_commits = getattr(store, "list_recent_agentic_commits", None) list_pending_inline_confirmations = getattr(store, "list_pending_inline_confirmations", None) memory_commit_service = VNextMemoryCommitService(store) @@ -146,10 +152,12 @@ def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: if callable(list_pending_inline_confirmations) else memory_commit_service.inline_confirmations(limit=20) ) + recent_memory_commits = guard.admit_rows("memory", recent_memory_commits) + inline_confirmations = guard.admit_rows("memory", inline_confirmations) scheduler_status = VNextSchedulerService(store).status() scheduler_status = {**scheduler_status, "daemon": daemon_status()} connector_health = VNextConnectorService(store).connector_health_all() - dogfooding = VNextDogfoodingService(store).dashboard() + dogfooding = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(sensitivity_allowed), label_guard=guard) doctor = VNextDoctorService(store).run(ci=True) policy_telemetry = summarize_agent_policy_telemetry( agent_events=agent_events, @@ -160,7 +168,7 @@ def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: project_dashboards: list[dict[str, object]] = [] for project in projects[:5]: try: - project_dashboards.append(project_service.project_dashboard(project_id=str(project["id"]))) + project_dashboards.append(project_service.project_dashboard(project_id=str(project["id"]), identity=identity)) except VNextProjectValidationError: continue trace_items = [ @@ -291,11 +299,18 @@ def _vnext_workspace_payload(store: PostgresVNextStore) -> dict[str, object]: @core_router.get("/v0/vnext/workspace") -def get_vnext_workspace(user_id: UUID) -> JSONResponse: +def get_vnext_workspace(user_id: UUID, authorization: str | None = Header(default=None)) -> JSONResponse: settings = get_settings() - with user_connection(settings.database_url, user_id) as conn: - payload = _vnext_workspace_payload(PostgresVNextStore(conn)) + try: + with user_connection(settings.database_url, user_id) as conn: + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, user_id=user_id, raw_key=agent_key_from_authorization(authorization), payload={}, + ) + payload = _vnext_workspace_payload(store, identity=identity) + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) return JSONResponse(status_code=200, content=jsonable_encoder(payload)) diff --git a/apps/api/src/alicebot_api/vnext_dogfooding.py b/apps/api/src/alicebot_api/vnext_dogfooding.py index 5a14906ed..cf7e5f118 100644 --- a/apps/api/src/alicebot_api/vnext_dogfooding.py +++ b/apps/api/src/alicebot_api/vnext_dogfooding.py @@ -3,13 +3,15 @@ from collections import Counter from datetime import UTC, datetime, timedelta from statistics import mean -from typing import Protocol +from typing import TYPE_CHECKING, Protocol from alicebot_api.vnext_connectors import VNextConnectorService, VNextConnectorStore from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_repositories import JsonObject from alicebot_api.vnext_store import is_redacted_project_update_artifact +if TYPE_CHECKING: + from alicebot_api.vnext_label_guard import LabelGuard class VNextDogfoodingStore(VNextConnectorStore, Protocol): def append_event(self, event: JsonObject) -> JsonObject: ... @@ -170,9 +172,9 @@ class VNextDogfoodingService: def __init__(self, store: VNextDogfoodingStore) -> None: self.store = store - def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None) -> JsonObject: + def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None, label_guard: LabelGuard | None = None) -> JsonObject: from alicebot_api.vnext_agent_control import ALL_SENSITIVITY - from alicebot_api.vnext_label_guard import LabelGuard, admit_loaded + from alicebot_api.vnext_label_guard import LabelGuard sources = self.store.list_sources(limit=500) try: @@ -185,17 +187,11 @@ def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None) -> Js # None is the owner and an admin key: every sensitivity, so the guard # reads nothing and the lists stay as the store returned them. ceiling = sensitivity_allowed if sensitivity_allowed is not None else ALL_SENSITIVITY - guard = LabelGuard.for_filters(self.store, (), ceiling, ()) - sources = admit_loaded(self.store, kind="source", rows=sources, domains=(), sensitivity_allowed=ceiling, projects=()) - memories = admit_loaded( - self.store, kind="memory", rows=memories, domains=(), sensitivity_allowed=ceiling, projects=() - ) - artifacts = admit_loaded( - self.store, kind="artifact", rows=artifacts, domains=(), sensitivity_allowed=ceiling, projects=() - ) - open_loops = admit_loaded( - self.store, kind="open_loop", rows=open_loops, domains=(), sensitivity_allowed=ceiling, projects=() - ) + guard = label_guard if label_guard is not None else LabelGuard.for_filters(self.store, (), ceiling, ()) + sources = guard.admit_rows("source", sources) + memories = guard.admit_rows("memory", memories) + artifacts = guard.admit_rows("artifact", artifacts) + open_loops = guard.admit_rows("open_loop", open_loops) memory_status_counts = guard.readable_status_counts("memory") try: events = self.store.list_events(limit=5_000) diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py index b52c7074d..696655293 100644 --- a/tests/integration/test_derived_labels_read_acceptance_postgres.py +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -14,6 +14,7 @@ from alicebot_api.routers import vnext_review, vnext_retrieval, vnext_projects, vnext_memories, workspaces from alicebot_api.store import ContinuityStore from alicebot_api.vnext_agent_control import ALL_SENSITIVITY +from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_label_writes import without_insert_floor from alicebot_api.vnext_store import PostgresVNextStore from tests.unit.test_derived_labels_real_keys import READERS, expected_read, real_reader_key, seed_read_rows @@ -130,3 +131,37 @@ def test_workspace_counts_full_population_with_sql_hidden_and_stale_rows(migrate batches = list(store.iter_label_rows("source", batch_size=100)) assert [len(batch) for batch in batches] == [100, 100, 6] assert all("content_markdown" not in row and "title" not in row for batch in batches for row in batch) + + +@pytest.mark.parametrize("reader", ("owner", "admin", "trusted")) +def test_workspace_activity_uses_actual_key_and_current_targets(migrated_database_urls, monkeypatch, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + monkeypatch.setattr(workspaces, "get_settings", lambda: Settings(database_url=app_url)) + monkeypatch.setattr(workspaces.VNextDoctorService, "run", lambda self, **kwargs: {}) + hidden_ids = [] + visible_ids = [] + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Cedar hidden parent", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential"}) + for hidden in (False, True): + metadata = {"agentic_memory": {"kind": "agentic_memory_commit", "confirmation": {"status": "pending"}}} + if hidden: + metadata["source_id"] = str(source["id"]) + with without_insert_floor(): + memory = store.create_memory({"memory_key": f"activity-{hidden}", "canonical_text": "Cedar hidden activity" if hidden else "Public activity", "status": "needs_review", "confirmation_status": "unconfirmed", "domain": "project", "sensitivity": "public", "metadata_json": metadata}) + (hidden_ids if hidden else visible_ids).append(str(memory["id"])) + event = append_event(store, event_type="agent.policy_blocked", actor_type="agent", actor_id="synthetic-reader", target_type="memory", target_id=str(memory["id"]), payload={"decision": {"decision": "blocked", "target_id": str(memory["id"])}}) + (hidden_ids if hidden else visible_ids).append(str(event["id"])) + key = real_reader_key(store, user_id, reader) + response = workspaces.get_vnext_workspace(user_id, authorization=f"Bearer {key}" if key else None) + assert response.status_code == 200 + rendered = response.body.decode() + assert all(identifier not in rendered for identifier in hidden_ids) + assert all(identifier in rendered for identifier in visible_ids) + assert "Cedar hidden" not in rendered + body = json.loads(rendered) + assert len(body["agent_activity"]["policy_blocks"]) == 1 + assert len(body["agent_activity"]["recent_commits"]) == 1 + assert len(body["agent_activity"]["inline_confirmations"]) == 1 + assert body["dogfooding"]["sample_scope"]["memories"]["total_count"] == 1 diff --git a/tests/unit/test_complete_readable_counts.py b/tests/unit/test_complete_readable_counts.py index 97eb1bbea..ad31e0cf4 100644 --- a/tests/unit/test_complete_readable_counts.py +++ b/tests/unit/test_complete_readable_counts.py @@ -107,6 +107,27 @@ def test_all_sql_prefiltered_rows_leave_zero_totals(monkeypatch): assert all(not sample["has_more"] for sample in body["samples"].values()) +def test_workspace_activity_and_nested_dashboard_share_the_guard(monkeypatch): + _quiet_services(monkeypatch) + store = PopulationStore() + visible = _row("visible-memory") + hidden = _row("hidden-memory", "confidential") + store.rows["memory"] = [visible, hidden] + store.events = [{"id": "visible-event", "target_type": "memory", "target_id": visible["id"], "event_type": "agent.policy_blocked"}, + {"id": "hidden-event", "target_type": "memory", "target_id": hidden["id"], "event_type": "agent.policy_blocked"}] + store.list_agent_events = lambda **kwargs: store.events + store.list_recent_agentic_commits = lambda **kwargs: [visible, hidden] + store.list_pending_inline_confirmations = lambda **kwargs: [visible, hidden] + body = workspaces._vnext_workspace_payload(store) + assert "hidden-memory" not in str(body) + assert "hidden-event" not in str(body) + activity = body["agent_activity"] + assert [row["id"] for row in activity["recent_commits"]] == [visible["id"]] + assert [row["id"] for row in activity["inline_confirmations"]] == [visible["id"]] + assert [row["id"] for row in activity["policy_blocks"]] == ["visible-event"] + assert body["dogfooding"]["sample_scope"]["memories"]["total_count"] == 1 + + def test_dogfooding_counts_hidden_rows_beyond_500(monkeypatch): _quiet_services(monkeypatch) store = PopulationStore() diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py index e82773f3e..f48dae1c7 100644 --- a/tests/unit/test_label_door_registry.py +++ b/tests/unit/test_label_door_registry.py @@ -113,6 +113,7 @@ } NOT_A_DOOR = { + "routers/vnext_memories.py:regenerate_vnext_source": "operator-only regeneration rejects every profile except owner and unbound admin before the source lookup; real-profile rejection tests pin this gate", "mcp/evidence_artifacts.py:_handle_alice_vnext_review_items": "legacy review list has no policy check", "vnext_projects.py:VNextProjectService.review_project_update": "write path; the route authorizes before this mutation", "vnext_projects.py:VNextProjectService.review_open_loop": "write path; the route and the open-loop tool settle the loop first", From a786db8645ef7ca0ee56ef8c8e34bc439c9c0d94 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:09:56 +0200 Subject: [PATCH 121/270] Guard policy telemetry and quality ratings by current labels --- .../alicebot_api/routers/vnext_projects.py | 29 ++++++++++---- .../src/alicebot_api/routers/vnext_review.py | 27 ++++++++++--- ...derived_labels_read_acceptance_postgres.py | 39 +++++++++++++++++++ tests/unit/test_label_door_registry.py | 14 ++++++- 4 files changed, 95 insertions(+), 14 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/vnext_projects.py b/apps/api/src/alicebot_api/routers/vnext_projects.py index 0e6485081..cf87ac8ed 100644 --- a/apps/api/src/alicebot_api/routers/vnext_projects.py +++ b/apps/api/src/alicebot_api/routers/vnext_projects.py @@ -382,17 +382,32 @@ def get_vnext_agent_policy_telemetry( user_id: UUID, agent_id: str | None = None, limit: Annotated[int, Query(ge=1, le=200)] = 200, + authorization: str | None = Header(default=None), ) -> JSONResponse: + from alicebot_api.vnext_agent_control import ALL_SENSITIVITY + from alicebot_api.vnext_label_guard import LabelGuard, sensitivity_ceiling + settings = get_settings() bounded_limit = min(max(limit, 1), 200) - with user_connection(settings.database_url, user_id) as conn: - store = PostgresVNextStore(conn) - payload = summarize_agent_policy_telemetry( - agent_events=store.list_agent_events(agent_id=agent_id, limit=bounded_limit), - artifacts=store.list_agent_policy_artifacts(agent_id=agent_id, limit=bounded_limit), - memories=store.list_agent_policy_memories(agent_id=agent_id, limit=bounded_limit), - ) + try: + with user_connection(settings.database_url, user_id) as conn: + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, user_id=user_id, raw_key=agent_key_from_authorization(authorization), payload={}, + ) + projects = identity.project_scope if identity is not None else () + guard = LabelGuard.for_filters( + store, (), sensitivity_ceiling(identity) or ALL_SENSITIVITY, projects, + all_of=projects if identity is not None and identity.project_scope_locked else None, + ) + payload = summarize_agent_policy_telemetry( + agent_events=guard.admit_events(store.list_agent_events(agent_id=agent_id, limit=bounded_limit)), + artifacts=guard.admit_rows("artifact", store.list_agent_policy_artifacts(agent_id=agent_id, limit=bounded_limit)), + memories=guard.admit_rows("memory", store.list_agent_policy_memories(agent_id=agent_id, limit=bounded_limit)), + ) + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) return JSONResponse(status_code=200, content=jsonable_encoder({"summary": payload})) diff --git a/apps/api/src/alicebot_api/routers/vnext_review.py b/apps/api/src/alicebot_api/routers/vnext_review.py index 6805c0d9f..9a94db0bc 100644 --- a/apps/api/src/alicebot_api/routers/vnext_review.py +++ b/apps/api/src/alicebot_api/routers/vnext_review.py @@ -792,14 +792,29 @@ def rate_vnext_artifact_quality( return JSONResponse(status_code=201, content=jsonable_encoder(payload)) @review_router.get("/v0/vnext/quality-evals") -def list_vnext_quality_evals(user_id: UUID, artifact_id: UUID | None = None, limit: int = 100) -> JSONResponse: +def list_vnext_quality_evals(user_id: UUID, artifact_id: UUID | None = None, limit: int = 100, authorization: str | None = Header(default=None)) -> JSONResponse: + from alicebot_api.vnext_agent_control import ALL_SENSITIVITY + from alicebot_api.vnext_label_guard import LabelGuard, sensitivity_ceiling + settings = get_settings() bounded_limit = max(1, min(limit, 200)) - with user_connection(settings.database_url, user_id) as conn: - rows = PostgresVNextStore(conn).list_artifact_quality_ratings( - artifact_id=str(artifact_id) if artifact_id is not None else None, - limit=bounded_limit, - ) + try: + with user_connection(settings.database_url, user_id) as conn: + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, user_id=user_id, raw_key=agent_key_from_authorization(authorization), payload={}, + ) + projects = identity.project_scope if identity is not None else () + guard = LabelGuard.for_filters( + store, (), sensitivity_ceiling(identity) or ALL_SENSITIVITY, projects, + all_of=projects if identity is not None and identity.project_scope_locked else None, + ) + rows = guard.admit_related_rows(store.list_artifact_quality_ratings( + artifact_id=str(artifact_id) if artifact_id is not None else None, + limit=bounded_limit, + ), kind="artifact", field="artifact_id") + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) return JSONResponse( status_code=200, content=jsonable_encoder( diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py index 696655293..30a1b9702 100644 --- a/tests/integration/test_derived_labels_read_acceptance_postgres.py +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -165,3 +165,42 @@ def test_workspace_activity_uses_actual_key_and_current_targets(migrated_databas assert len(body["agent_activity"]["recent_commits"]) == 1 assert len(body["agent_activity"]["inline_confirmations"]) == 1 assert body["dogfooding"]["sample_scope"]["memories"]["total_count"] == 1 + + +@pytest.mark.parametrize("reader", ("owner", "admin", "trusted")) +def test_telemetry_and_quality_ratings_use_current_target_labels(migrated_database_urls, monkeypatch, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + for module in (vnext_projects, vnext_review): + monkeypatch.setattr(module, "get_settings", lambda: Settings(database_url=app_url)) + hidden_ids = [] + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Cedar hidden parent", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential"}) + for hidden in (False, True): + metadata = {"agent_id": "synthetic-reader"} + if hidden: + metadata["source_id"] = str(source["id"]) + derived = {"v": 1, "sources": [str(source["id"])] if hidden else [], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": int(hidden), "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}} + with without_insert_floor(): + memory = store.create_memory({"memory_key": f"telemetry-{hidden}", "canonical_text": "Cedar hidden telemetry" if hidden else "Public telemetry", "domain": "project", "sensitivity": "public", "metadata_json": metadata}) + artifact = store.create_artifact({"artifact_type": "daily_brief", "title": "Cedar hidden artifact" if hidden else "Public artifact", "content_markdown": "Public stored copy", "domain": "project", "sensitivity": "public", "metadata_json": {"agent_id": "synthetic-reader", "generated_by": "agent", "derived_from": derived}}) + rating = store.create_artifact_quality_rating({"artifact_id": str(artifact["id"]), "reviewer_id": "synthetic-reviewer", "verbosity": "right_sized", "comments": "Cedar hidden feedback" if hidden else "Public feedback"}) + append_event(store, event_type="agent.policy_blocked", actor_type="agent", actor_id="synthetic-reader", target_type="memory", target_id=str(memory["id"]), payload={}) + if hidden: + hidden_ids.extend((str(artifact["id"]), str(rating["id"]))) + key = real_reader_key(store, user_id, reader) + auth = f"Bearer {key}" if key else None + telemetry = vnext_projects.get_vnext_agent_policy_telemetry(user_id, authorization=auth) + summary = json.loads(telemetry.body)["summary"] + admitted_count = 1 if reader == "trusted" else 2 + assert summary["total_agent_events"] == admitted_count + assert summary["memory_proposals_by_agent"] == [{"agent_id": "synthetic-reader", "count": admitted_count}] + assert summary["artifact_generation_by_agent"] == [{"agent_id": "synthetic-reader", "count": admitted_count}] + ratings = vnext_review.list_vnext_quality_evals(user_id, authorization=auth) + assert json.loads(ratings.body)["count"] == admitted_count + if reader == "trusted": + assert all(identifier not in ratings.body.decode() for identifier in hidden_ids) + assert "Cedar hidden feedback" not in ratings.body.decode() + else: + assert all(identifier in ratings.body.decode() for identifier in hidden_ids) diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py index f48dae1c7..3e0350f58 100644 --- a/tests/unit/test_label_door_registry.py +++ b/tests/unit/test_label_door_registry.py @@ -36,6 +36,8 @@ "list_events_for_source_trace", "list_recent_agentic_commits", "list_pending_inline_confirmations", "count_sources", "count_artifacts", "count_artifacts_by_status", "count_projects", "count_memories_by_status", "count_open_loops", "count_open_loops_by_status", "count_events", "iter_label_rows", "iter_label_events", "iter_label_ratings", + "list_agent_events", "list_agent_policy_artifacts", "list_agent_policy_memories", + "list_artifact_quality_ratings", "count_artifact_quality_ratings", } GUARD_CALLS = { @@ -72,6 +74,8 @@ "routers/_vnext_shared.py:_vnext_load_source_trace": None, "routers/vnext_projects.py:list_vnext_projects": None, "routers/vnext_review.py:list_vnext_artifacts": None, + "routers/vnext_review.py:list_vnext_quality_evals": None, + "routers/vnext_projects.py:get_vnext_agent_policy_telemetry": None, "routers/vnext_review.py:get_vnext_belief_state": None, "routers/vnext_memories.py:get_vnext_dogfooding_dashboard": None, "vnext_projects.py:VNextProjectService.project_dashboard": None, @@ -85,6 +89,7 @@ "session_briefing.py:_event_target_honours_fence": None, "session_briefing.py:_memory_honours_fence": None, "routers/workspaces.py:_vnext_workspace_payload": None, + "routers/workspaces.py:_workspace_event_visible": "_workspace_rows", "vnext_context_tree.py:VNextContextTreeService.build_tree": None, "vnext_dogfooding.py:VNextDogfoodingService.dashboard": None, "vnext_contradictions.py:VNextContradictionService.belief_state": None, @@ -164,6 +169,7 @@ "vnext_connectors.py:VNextConnectorService.get_cursor": "connector cursor events only; no labelled targets", "vnext_connectors.py:VNextConnectorService.get_config": "connector configuration events only; no labelled targets", "vnext_connectors.py:VNextConnectorService.connector_health": "connector state telemetry only; no labels_raised events", + "vnext_dogfooding.py:VNextDogfoodingStore.list_artifact_quality_ratings": "store protocol declaration; no execution or response", "vnext_artifact_review.py:dispatch_vnext_artifact_review": "writer entry; calling route or MCP authorizes the artifact before dispatch", "vnext_memory_commit.py:VNextMemoryCommitService._guard_supersession_acyclic": "write validation traverses pointers without exposing their content", } @@ -224,6 +230,11 @@ def _reader_names(node: ast.AST) -> set[str]: elif isinstance(child, ast.Call) and isinstance(child.func, ast.Name) and child.func.id == "getattr": if len(child.args) > 1 and isinstance(child.args[1], ast.Constant) and child.args[1].value in READS: names.add(child.args[1].value) + elif len(child.args) > 1 and not isinstance(child.args[1], ast.Constant): + # Target-kind dispatch maps choose a method name dynamically. + # Resolve their possible reader names from this function's AST. + names.update(value.value for value in ast.walk(node) if isinstance(value, ast.Constant) + and isinstance(value.value, str) and value.value in READS) return names @@ -266,7 +277,8 @@ def test_every_scanned_reader_is_classified() -> None: def test_discovery_catches_new_direct_and_dynamic_readers() -> None: - for source in ("def added(store): return store.list_events()", "def added(store): return getattr(store, 'list_memories')()", "def added(store): return invoke(store.list_beliefs)"): + for source in ("def added(store): return store.list_events()", "def added(store): return getattr(store, 'list_memories')()", "def added(store): return invoke(store.list_beliefs)", + "def added(store, kind):\n methods = {'memory': 'get_memory', 'artifact': 'get_artifact'}\n return getattr(store, methods[kind])()"): node = ast.parse(source).body[0] assert _reader_names(node) assert "added" not in DOORS and "added" not in NOT_A_DOOR From 0cf105db4884847ef7650909ccc09eb384245a7c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:22:51 +0200 Subject: [PATCH 122/270] Fence context events and skip filtered workspace content diagnostics --- .../src/alicebot_api/routers/workspaces.py | 2 +- .../src/alicebot_api/vnext_context_tree.py | 26 ++------- apps/api/src/alicebot_api/vnext_doctor.py | 47 +++++++++------- ...derived_labels_read_acceptance_postgres.py | 53 +++++++++++++++++-- tests/unit/test_complete_readable_counts.py | 52 ++++++++++++++++++ tests/unit/test_derived_labels_real_keys.py | 29 ++++++++++ 6 files changed, 163 insertions(+), 46 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index bc85a15e5..70fa58851 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -158,7 +158,7 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti scheduler_status = {**scheduler_status, "daemon": daemon_status()} connector_health = VNextConnectorService(store).connector_health_all() dogfooding = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(sensitivity_allowed), label_guard=guard) - doctor = VNextDoctorService(store).run(ci=True) + doctor = VNextDoctorService(store).run(ci=True, include_content_diagnostics=False) policy_telemetry = summarize_agent_policy_telemetry( agent_events=agent_events, artifacts=artifacts, diff --git a/apps/api/src/alicebot_api/vnext_context_tree.py b/apps/api/src/alicebot_api/vnext_context_tree.py index 3706404a9..583839cfc 100644 --- a/apps/api/src/alicebot_api/vnext_context_tree.py +++ b/apps/api/src/alicebot_api/vnext_context_tree.py @@ -163,29 +163,9 @@ def _label(row: JsonObject, *keys: str, fallback: str) -> str: def _tree_event_visible(store: object, event: JsonObject, domains: list[str] | None, sensitivity: list[str], projects: tuple[str, ...]) -> bool: - from alicebot_api.vnext_label_guard import admit_loaded - - kind = str(event.get("target_type") or "") - target_id = event.get("target_id") - getters = { - "memory": "get_memory", - "open_loop": "get_open_loop", - "artifact": "get_artifact", - "project": "get_project", - "source": "get_source", - } - getter_name = getters.get(kind) - if getter_name is None or not isinstance(target_id, str) or target_id == "": - return True - getter = getattr(store, getter_name, None) - if not callable(getter): - return True - row = getter(target_id) - if not isinstance(row, dict) and not hasattr(row, "get"): - return True - return bool( - admit_loaded(store, kind=kind, rows=[row], domains=domains, sensitivity_allowed=sensitivity, projects=projects) - ) + from alicebot_api.vnext_label_guard import LabelGuard + + return bool(LabelGuard.for_filters(store, domains, sensitivity, projects).admit_events([event])) def _row_node(prefix: str, row: JsonObject, *, label_keys: tuple[str, ...], fallback: str) -> JsonObject: diff --git a/apps/api/src/alicebot_api/vnext_doctor.py b/apps/api/src/alicebot_api/vnext_doctor.py index 6df43e805..85e0b2a6a 100644 --- a/apps/api/src/alicebot_api/vnext_doctor.py +++ b/apps/api/src/alicebot_api/vnext_doctor.py @@ -128,7 +128,7 @@ def migration_status(self) -> JsonObject: def local_live_cors_status(self, settings: Settings | None = None) -> JsonObject: return local_live_cors_status(settings=settings or get_settings(), env=self.env, cwd=self.cwd) - def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: + def run(self, *, fix_safe: bool = False, ci: bool = False, include_content_diagnostics: bool = True) -> JsonObject: if fix_safe: VNextConnectorService(cast(Any, self.store), secret_provider=self.secret_provider).ensure_default_settings() @@ -263,6 +263,34 @@ def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: details=cast(JsonObject, local_cors), ) + if include_content_diagnostics: + self._content_checks(checks) + else: + for name in ("flagged_sources", "derived_labels"): + checks.append(DoctorCheck( + name=name, status="skipped", severity="info", + message="Content diagnostics are omitted from this filtered workspace view. Run doctor for a full report.", + details={"scope": "filtered_workspace", "evaluated": False}, + )) + + blocking = [check for check in checks if check.status == "fail" and check.severity == "blocking"] + warnings = [check for check in checks if check.status == "fail" and check.severity == "warning"] + payload = { + "status": "fail" if blocking else "warn" if warnings else "pass", + "fix_safe_applied": fix_safe, + "ci_mode": ci, + "blocking_failure_count": len(blocking), + "warning_count": len(warnings), + "checks": [check.to_record() for check in checks], + "recommended_fixes": [ + check.recommended_fix for check in checks if check.status == "fail" and check.recommended_fix is not None + ], + "migration_status": migration_status, + "connector_health": health, + } + return cast(JsonObject, payload) + + def _content_checks(self, checks: list[DoctorCheck]) -> None: flagged_ids, stopped_early = _flagged_source_scan(self.store) remedy = _flagged_source_remedy(self.store) if flagged_ids: @@ -288,23 +316,6 @@ def run(self, *, fix_safe: bool = False, ci: bool = False) -> JsonObject: }, ) - blocking = [check for check in checks if check.status == "fail" and check.severity == "blocking"] - warnings = [check for check in checks if check.status == "fail" and check.severity == "warning"] - payload = { - "status": "fail" if blocking else "warn" if warnings else "pass", - "fix_safe_applied": fix_safe, - "ci_mode": ci, - "blocking_failure_count": len(blocking), - "warning_count": len(warnings), - "checks": [check.to_record() for check in checks], - "recommended_fixes": [ - check.recommended_fix for check in checks if check.status == "fail" and check.recommended_fix is not None - ], - "migration_status": migration_status, - "connector_health": health, - } - return cast(JsonObject, payload) - def _flagged_source_remedy(store: object) -> str: """Keep the backend-specific owner remedy.""" diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py index 30a1b9702..699272e96 100644 --- a/tests/integration/test_derived_labels_read_acceptance_postgres.py +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import asyncio from uuid import UUID, uuid4 import pytest @@ -106,9 +107,6 @@ def test_all_five_operator_screens_with_real_keys(migrated_database_urls, monkey def test_workspace_counts_full_population_with_sql_hidden_and_stale_rows(migrated_database_urls, monkeypatch): app_url = migrated_database_urls["app"] user_id = _user(app_url) - # The doctor has its own store-specific repair acceptance test. Keep this - # count probe independent of that ancillary diagnostic implementation. - monkeypatch.setattr(workspaces.VNextDoctorService, "run", lambda self, **kwargs: {}) with user_connection(app_url, user_id) as conn: store = PostgresVNextStore(conn) secret = store.create_source({"source_type": "note", "title": "Cedar hidden", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential"}) @@ -138,7 +136,6 @@ def test_workspace_activity_uses_actual_key_and_current_targets(migrated_databas app_url = migrated_database_urls["app"] user_id = _user(app_url) monkeypatch.setattr(workspaces, "get_settings", lambda: Settings(database_url=app_url)) - monkeypatch.setattr(workspaces.VNextDoctorService, "run", lambda self, **kwargs: {}) hidden_ids = [] visible_ids = [] with user_connection(app_url, user_id) as conn: @@ -204,3 +201,51 @@ def test_telemetry_and_quality_ratings_use_current_target_labels(migrated_databa assert "Cedar hidden feedback" not in ratings.body.decode() else: assert all(identifier in ratings.body.decode() for identifier in hidden_ids) + + +def test_real_trusted_http_workspace_omits_full_content_doctor_counts(migrated_database_urls, monkeypatch): + from alicebot_api import main + + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + settings = Settings(database_url=app_url) + monkeypatch.setattr(workspaces, "get_settings", lambda: settings) + monkeypatch.setattr(main, "get_settings", lambda: settings) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + key = real_reader_key(store, user_id, "trusted") + headers = {"Authorization": f"Bearer {key}"} + async def request_workspace(): + messages = [] + request_sent = False + async def receive(): + nonlocal request_sent + if not request_sent: + request_sent = True + return {"type": "http.request", "body": b"", "more_body": False} + await asyncio.Event().wait() + async def send(message): + messages.append(message) + scope = {"type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", "method": "GET", + "scheme": "http", "path": "/v0/vnext/workspace", "raw_path": b"/v0/vnext/workspace", "root_path": "", + "query_string": f"user_id={user_id}".encode(), "headers": [(name.lower().encode(), value.encode()) for name, value in headers.items()], + "client": ("127.0.0.1", 1), "server": ("127.0.0.1", 80)} + await main.app(scope, receive, send) + status = next(message["status"] for message in messages if message["type"] == "http.response.start") + body = b"".join(message.get("body", b"") for message in messages if message["type"] == "http.response.body") + return status, json.loads(body), body.decode() + before = asyncio.run(request_workspace()) + assert before[0] == 200, before[2] + with user_connection(app_url, user_id) as conn: + rows = seed_read_rows(PostgresVNextStore(conn)) + after = asyncio.run(request_workspace()) + assert after[0] == 200, after[2] + assert all(str(rows[state]["id"]) not in after[2] for state in ("verified_confidential", "unverified")) + diagnostic = after[1]["doctor"] + for check in diagnostic["checks"]: + if check["name"] in {"derived_labels", "flagged_sources"}: + assert check["status"] == "skipped" + assert check["details"] == {"scope": "filtered_workspace", "evaluated": False} + assert "below their inputs" not in check["message"] + for field in ("status", "warning_count", "blocking_failure_count", "recommended_fixes"): + assert diagnostic[field] == before[1]["doctor"][field] diff --git a/tests/unit/test_complete_readable_counts.py b/tests/unit/test_complete_readable_counts.py index ad31e0cf4..869c6071a 100644 --- a/tests/unit/test_complete_readable_counts.py +++ b/tests/unit/test_complete_readable_counts.py @@ -7,9 +7,12 @@ import pytest from alicebot_api.routers import _vnext_shared, workspaces +from alicebot_api.mcp.retrieval import _resume_event_honours_policy_fence +from alicebot_api.session_briefing import _event_target_honours_fence from alicebot_api.vnext_agent_control import ALL_SENSITIVITY, AgentIdentity from alicebot_api.vnext_dogfooding import VNextDogfoodingService from alicebot_api.vnext_label_guard import LabelGuard +from alicebot_api.vnext_context_tree import _tree_event_visible class PopulationStore: @@ -46,6 +49,9 @@ def list_events(self, **kwargs): def __getattr__(self, name): kinds = {"sources": "source", "artifacts": "artifact", "projects": "project", "open_loops": "open_loop"} suffix = name.removeprefix("list_") + if name.startswith("get_") and name.removeprefix("get_") in self.rows: + kind = name.removeprefix("get_") + return lambda identifier: next((row for row in self.rows[kind] if row["id"] == identifier), None) if suffix in kinds: def listed(**kwargs): rows = self.rows[kinds[suffix]] @@ -181,3 +187,49 @@ def test_trace_event_completeness_uses_admitted_targets(monkeypatch): ) assert [row["id"] for row in admitted] == ["0", "1"] assert complete is True + + +@pytest.mark.parametrize("kind", ("source", "memory", "open_loop", "artifact", "project")) +def test_context_events_use_current_effective_target_of_every_kind(kind): + store = PopulationStore() + source_id = "11111111-1111-4111-8111-111111111111" + row_id = source_id if kind == "source" else "22222222-2222-4222-8222-222222222222" + store.rows["source"] = [_row(source_id, "confidential")] + if kind != "source": + refs = {"sources": [source_id], "memories": [], "open_loops": [], "artifacts": [], "beliefs": []} + store.rows[kind] = [_row(row_id, metadata_json={"derived_from": {"v": 1, **refs, "counts": {name: len(ids) for name, ids in refs.items()}}})] + if kind == "open_loop": + store.rows[kind][0]["metadata_json"]["discovered_by"] = "vnext_daily_brief" + store.rows[kind][0]["metadata_json"]["source_id"] = source_id + event = {"target_type": kind, "target_id": row_id, "event_type": f"{kind}.labels_raised", "payload_json": {"cause": "repair_v3"}} + trusted = ["public", "internal", "private", "unknown"] + assert _tree_event_visible(store, event, None, trusted, ()) is False + assert _tree_event_visible(store, event, None, list(ALL_SENSITIVITY), ()) is True + store.rows[kind] = [_row(row_id)] + assert _tree_event_visible(store, event, None, trusted, ()) is True + + +def test_context_event_missing_and_unknown_label_targets_fail_closed(): + store = PopulationStore() + sensitivities = ["public", "internal", "private", "unknown"] + for kind in ("source", "memory", "open_loop", "artifact", "project", "not-a-label-kind"): + event = {"target_type": kind, "target_id": "missing", "event_type": f"{kind}.labels_raised"} + assert _tree_event_visible(store, event, None, sensitivities, ()) is False + assert _tree_event_visible(store, {"target_type": "connector", "event_type": "connector.heartbeat"}, None, sensitivities, ()) is True + + +@pytest.mark.parametrize("kind", ("memory", "open_loop")) +@pytest.mark.parametrize("reader", (_resume_event_honours_policy_fence, _event_target_honours_fence)) +def test_resume_and_session_events_use_current_target_labels(kind, reader): + store = PopulationStore() + source_id = "11111111-1111-4111-8111-111111111111" + row_id = "22222222-2222-4222-8222-222222222222" + store.rows["source"] = [_row(source_id, "confidential")] + store.rows[kind] = [_row(row_id, metadata_json={"source_id": source_id})] + if kind == "open_loop": + store.rows[kind][0]["metadata_json"]["discovered_by"] = "vnext_daily_brief" + event = {"target_type": kind, "target_id": row_id, "event_type": f"{kind}.labels_raised", "payload_json": {"cause": "repair_v3"}} + arguments = {"effective_domains": ("project", "health"), "effective_sensitivity_allowed": ("public", "internal", "private", "unknown"), "effective_project_scope": (), "exclude_global_domains": frozenset()} + assert reader(store, event, **arguments) is False + arguments["effective_sensitivity_allowed"] = ALL_SENSITIVITY + assert reader(store, event, **arguments) is True diff --git a/tests/unit/test_derived_labels_real_keys.py b/tests/unit/test_derived_labels_real_keys.py index 1e10d640c..88da4a5e1 100644 --- a/tests/unit/test_derived_labels_real_keys.py +++ b/tests/unit/test_derived_labels_real_keys.py @@ -14,7 +14,10 @@ from alicebot_api.vnext_agent_control import ALL_SENSITIVITY from alicebot_api.vnext_agent_keys import create_agent_key from alicebot_api.vnext_event_log import append_event +from alicebot_api.vnext_doctor import VNextDoctorService from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_secrets import InMemorySecretProvider +from tests.unit.test_vnext_doctor import DoctorStore READERS = ("owner", "admin", "trusted", "read_only", "bound_admin", "bound_trusted", "bound_read_only") @@ -80,3 +83,29 @@ def test_sqlite_real_key_core_doors(tmp_path, monkeypatch, reader): assert (str(row["id"]) in rendered) is admitted, (reader, state, tool, rendered) if not admitted: assert str(row["title"]) not in rendered + + +def test_full_owner_doctor_keeps_true_counts_and_filtered_view_skips_content(tmp_path): + user_id = uuid4() + path = tmp_path / "doctor.sqlite3" + bootstrap_database(path, user_id=str(user_id), user_email="synthetic@example.invalid") + with sqlite_user_connection(path, user_id) as conn: + store = SQLiteVNextStore(conn, user_id) + # SQLite has no provider/connector doctor protocol. Keep those synthetic + # operations fixed while the real connection supplies content diagnostics. + diagnostic_store = DoctorStore() + diagnostic_store.conn = store.conn + diagnostic_store.list_sources = lambda **kwargs: [row for batch in store.iter_label_rows("source") for row in batch] + service = VNextDoctorService(diagnostic_store, secret_provider=InMemorySecretProvider(), env={}, cwd=tmp_path) + before = service.run(include_content_diagnostics=False) + rows = seed_read_rows(store) + full = service.run() + scoped = service.run(include_content_diagnostics=False) + derived = next(check for check in full["checks"] if check["name"] == "derived_labels") + assert derived["message"] == "derived labels: 1 below their inputs, 1 unverified" + skipped = [check for check in scoped["checks"] if check["name"] in {"derived_labels", "flagged_sources"}] + assert len(skipped) == 2 + assert all(check["status"] == "skipped" and check["details"] == {"scope": "filtered_workspace", "evaluated": False} for check in skipped) + assert all(str(row["id"]) not in json.dumps(scoped) for row in rows.values()) + for field in ("status", "blocking_failure_count", "warning_count", "recommended_fixes"): + assert scoped[field] == before[field] From 06331cf200e9d36e6a874f8f7f2c3ab94d846bf6 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:27:25 +0200 Subject: [PATCH 123/270] Keep workspace binding and returned rows separately typed --- apps/api/src/alicebot_api/routers/workspaces.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index 70fa58851..4948bc285 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -92,9 +92,9 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti ceiling = sensitivity_ceiling(identity) if ceiling is not None: sensitivity_allowed = [value for value in sensitivity_allowed if value in ceiling] - projects = identity.project_scope if identity is not None else () - all_of = projects if identity is not None and identity.project_scope_locked else None - guard = LabelGuard.for_filters(store, (), sensitivity_allowed, projects, all_of=all_of) + requested_projects = identity.project_scope if identity is not None else () + all_of = requested_projects if identity is not None and identity.project_scope_locked else None + guard = LabelGuard.for_filters(store, (), sensitivity_allowed, requested_projects, all_of=all_of) review_statuses = ["candidate", "needs_review", "private_only", "accepted", "rejected"] fetched_sources = store.list_sources(sensitivity_allowed=sensitivity_allowed, limit=20) sources = guard.admit_rows("source", fetched_sources) @@ -152,7 +152,7 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti if callable(list_pending_inline_confirmations) else memory_commit_service.inline_confirmations(limit=20) ) - recent_memory_commits = guard.admit_rows("memory", recent_memory_commits) + recent_memory_commits = guard.admit_rows("memory", recent_memory_commits if isinstance(recent_memory_commits, list) else []) inline_confirmations = guard.admit_rows("memory", inline_confirmations) scheduler_status = VNextSchedulerService(store).status() scheduler_status = {**scheduler_status, "daemon": daemon_status()} From d51bb57d6b5750c700c889f751442f7526b609bd Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:32:52 +0200 Subject: [PATCH 124/270] Verify producer input isolation and reuse derived group cards --- apps/api/src/alicebot_api/vnext_rollups.py | 4 +- .../test_derived_labels_producers_postgres.py | 244 ++++++++++++++++++ 2 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_derived_labels_producers_postgres.py diff --git a/apps/api/src/alicebot_api/vnext_rollups.py b/apps/api/src/alicebot_api/vnext_rollups.py index a4aad8eb5..c667b06fd 100644 --- a/apps/api/src/alicebot_api/vnext_rollups.py +++ b/apps/api/src/alicebot_api/vnext_rollups.py @@ -2474,11 +2474,11 @@ def _existing_rollup_state( from alicebot_api.vnext_label_guard import admit_loaded admitted_pending = {str(row.get("id")) for row in admit_loaded( self.store, kind="memory", rows=list(pending.values()), domains=domains, - sensitivity_allowed=sensitivity_allowed, projects=projects, all_of=all_of, + sensitivity_allowed=sensitivity_allowed, projects=(), all_of=all_of, )} admitted_accepted = {str(row.get("id")) for row in admit_loaded( self.store, kind="memory", rows=list(accepted.values()), domains=domains, - sensitivity_allowed=sensitivity_allowed, projects=projects, all_of=all_of, + sensitivity_allowed=sensitivity_allowed, projects=(), all_of=all_of, )} pending = {key: row for key, row in pending.items() if str(row.get("id")) in admitted_pending} accepted = {key: row for key, row in accepted.items() if str(row.get("id")) in admitted_accepted} diff --git a/tests/integration/test_derived_labels_producers_postgres.py b/tests/integration/test_derived_labels_producers_postgres.py new file mode 100644 index 000000000..134372736 --- /dev/null +++ b/tests/integration/test_derived_labels_producers_postgres.py @@ -0,0 +1,244 @@ +"""Authenticated producer runs exclude each unreadable input before printing it.""" + +from __future__ import annotations + +import json +from uuid import uuid4 + +import pytest + +import alicebot_api.main as main_module +from alicebot_api.config import Settings +from alicebot_api.db import user_connection +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext +from alicebot_api.routers import vnext_projects, vnext_retrieval, vnext_review +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from alicebot_api.vnext_derived_labels import with_derived_from +from alicebot_api.vnext_queue import VNextQueueService +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.test_memory_mutations_api import invoke_request + +PRODUCERS = ("daily", "weekly", "connections", "contradictions", "open_loop_review", "project_update", + "consolidation", "staleness") + + +def wire_database(monkeypatch, app_url): + for module in (main_module, vnext_projects, vnext_retrieval, vnext_review): + monkeypatch.setattr(module, "get_settings", lambda: Settings(database_url=app_url)) + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_MODEL", raising=False) + monkeypatch.setenv("ALICE_PROJECT_SCOPING", "off") + + +def seed_grid(app_url): + user_id, alpha, beta = uuid4(), str(uuid4()), str(uuid4()) + rows = [] + with user_connection(app_url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"producer-{user_id}@example.invalid", "Producer") + store = PostgresVNextStore(conn) + store.create_project({"id": alpha, "name": "Atlas", "slug": "atlas", "domain": "project", "sensitivity": "public"}) + store.create_project({"id": beta, "name": "Beta", "slug": "beta", "domain": "project", "sensitivity": "public"}) + for label, scope in (("alpha", [alpha]), ("beta", [beta]), ("shared", [alpha, beta]), ("global", [])): + marker = f"SENTINEL_{label.upper()}" + source = store.create_source({"source_type": "manual_text", "title": marker + " source Atlas", + "content_hash": str(uuid4()), "captured_at": "2026-10-05T09:00:00Z", "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": scope, "raw_text": f"Atlas does not prefer launch games. {marker} source text.\nTODO: Atlas launch review {marker}."}}) + rows.append((label, "sources", source)) + for index, game in enumerate(("Hollow Knight", "Stardew Valley", "Celeste")): + text = f"Atlas played {game} for {25 + index * 30} hours. {marker} memory {index}" + memory = store.create_memory({"memory_key": f"{label}.game.{index}", "memory_type": "episode", "title": text, + "canonical_text": text, "summary": text, "value": {"text": text}, "status": "active", "domain": "project", + "sensitivity": "public", "created_at": "2026-10-05T09:00:00Z", + "metadata_json": {"project_scope": scope, "session_date": f"2026-10-0{index + 1}"}}) + rows.append((label, "memories", memory)) + backing = store.create_memory({"memory_key": f"{label}.belief", "memory_type": "belief", "title": marker + " belief title", + "canonical_text": f"Atlas prefers launch games. {marker} belief text", "status": "active", "domain": "project", + "sensitivity": "public", "metadata_json": {"project_scope": scope}}) + belief = store.create_belief({"memory_id": str(backing["id"]), "claim": backing["canonical_text"], "confidence": .9}) + rows.extend(((label, "memories", backing), (label, "beliefs", belief))) + loop = store.create_open_loop({"title": marker + " loop Atlas", "description": marker + " loop text", + "status": "open", "domain": "project", "sensitivity": "public", "due_at": "2026-10-04T12:00:00Z", + "metadata_json": {"project_scope": scope}}) + rows.append((label, "open_loops", loop)) + artifact = store.create_artifact({"artifact_type": "research_brief", "title": marker + " artifact Atlas", + "content_markdown": marker + " artifact text Atlas launch games", "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": scope}}) + rows.append((label, "artifacts", artifact)) + prior = store.create_artifact({"artifact_type": "daily_brief", "title": marker + " prior Atlas", + "content_markdown": marker + " prior report text Atlas launch games", "domain": "project", "sensitivity": "public", + "metadata_json": with_derived_from({"workflow": "daily_brief", "project_scope": scope}, {"sources": [source]})}) + rows.append((label, "artifacts", prior)) + promoted = VNextQueueService(store).review_artifact(artifact_id=str(prior["id"]), action="promote", actor_type="user") + rows.append((label, "memories", store.get_memory(promoted["promoted_memory_id"]))) + # Fix the read windows independently of the machine clock. + conn.execute("UPDATE memories SET created_at='2026-10-05T09:00:00Z', updated_at='2026-10-05T09:00:00Z', first_seen_at='2026-10-05T09:00:00Z', last_seen_at='2026-10-05T09:00:00Z'") + conn.execute("UPDATE generated_artifacts SET created_at='2026-10-05T09:00:00Z'") + conn.execute("UPDATE open_loops SET created_at='2026-10-05T09:00:00Z'") + _, alpha_key = create_agent_key(store, user_id=user_id, agent_id="alpha", permission_profile="admin_agent", project_scope=alpha) + _, beta_key = create_agent_key(store, user_id=user_id, agent_id="beta", permission_profile="admin_agent", project_scope=beta) + _, unbound = create_agent_key(store, user_id=user_id, agent_id="unbound", permission_profile="admin_agent") + return user_id, alpha, beta, rows, alpha_key, beta_key, unbound + + +def generate(producer, user_id, alpha, key=None, *, project_scope=True): + options = {"generated_for": "2026-10-05", "source_limit": 50, "memory_limit": 50, "artifact_limit": 50, + "open_loop_limit": 50, "reference_time": "2026-10-05T12:00:00Z", "max_items": 50, + "discover_open_loops": True, "create_candidate_memories": True} + payload = {"user_id": str(user_id), "scope": {"projects": [alpha]} if project_scope else {}, "options": options} + if producer in {"daily", "weekly", "connections", "contradictions"}: + route = {"daily": "daily-brief", "weekly": "weekly-synthesis"}.get(producer, producer) + path = "/v0/vnext/artifacts/generate/" + route + elif producer == "project_update": + path = "/v0/vnext/projects/update-candidates" + payload["scope"]["project_id"] = alpha + else: + workflow = {"consolidation": "memory_consolidation", "staleness": "staleness_sweep"}.get(producer, producer) + path = f"/v0/vnext/scheduler/workflows/{workflow}/run-now" + status, body = invoke_request("POST", path, payload=payload, headers={"authorization": f"Bearer {key}"} if key else {}) + assert status == 201, (producer, status, body) + return body.get("artifact", body), body + + +def assert_canonical_printed_inputs(artifact, rows): + metadata = artifact["metadata_json"] + record = metadata["derived_from"] + assert record["v"] == 1 + for kind in ("sources", "memories", "open_loops", "artifacts", "beliefs"): + assert record["counts"][kind] == len(record[kind]) + printed = json.dumps(artifact, default=str) + any_printed = False + for _label, kind, row in rows: + if str(row["id"]) in printed: + belief_alias = kind == "memories" and any( + str(belief["memory_id"]) == str(row["id"]) and str(belief["id"]) in record["beliefs"] + for _label, belief_kind, belief in rows if belief_kind == "beliefs" + ) + assert str(row["id"]) in record[kind] or belief_alias, (artifact["artifact_type"], kind, row["id"]) + any_printed = True + assert any_printed, (artifact["artifact_type"], "fixture printed no input") + + +@pytest.mark.parametrize("producer", PRODUCERS) +def test_a_bound_key_generates_from_admitted_inputs_only(migrated_database_urls, monkeypatch, producer): + app_url = migrated_database_urls["app"] + wire_database(monkeypatch, app_url) + user_id, alpha, _beta, rows, key, _beta_key, unbound = seed_grid(app_url) + if producer == "staleness": + with user_connection(app_url, user_id) as conn: + conn.execute("UPDATE memories SET valid_to='2026-10-04T12:00:00Z'") + artifact, body = generate(producer, user_id, alpha, key) + assert_canonical_printed_inputs(artifact, rows) + artifact_id = str(artifact["id"]) + surfaces = [body] + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + surfaces.extend((store.get_artifact(artifact_id), store.list_events(target_type="artifact", target_id=artifact_id))) + if producer == "staleness": + for label, kind, row in rows: + if label == "shared" and kind == "memories": + assert store.get_memory(str(row["id"]))["status"] == "active" + for path in (f"/v0/vnext/artifacts/{artifact_id}", f"/v0/vnext/traces/artifacts/{artifact_id}"): + status, surface = invoke_request("GET", path, query_params={"user_id": str(user_id)}, headers={"authorization": f"Bearer {key}"}) + assert status == 200, (producer, status, surface) + surfaces.append(surface) + # Project updates have a coupled candidate lifecycle, exercised by their review adapter. + review_path = (f"/v0/vnext/projects/update-candidates/{artifact_id}/review" if producer == "project_update" + else f"/v0/vnext/artifacts/{artifact_id}/review") + status, promoted = invoke_request("POST", review_path, payload={"user_id": str(user_id), "action": "accept" if producer == "project_update" else "promote"}, headers={"authorization": f"Bearer {unbound}"}) + assert status == 200, (producer, status, promoted) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + memory_id = (artifact["metadata_json"]["candidate_memory_id"] if producer == "project_update" + else promoted["promoted_memory_id"]) + surfaces.append(store.get_memory(memory_id)) + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + context = MCPRuntimeContext(database_url=app_url, user_id=user_id) + recalled = call_mcp_tool(context, name="alice_recall", arguments={"query": "Atlas", "limit": 50}) + def copies(value): + if isinstance(value, dict): + if str(value.get("id")) == str(memory_id): + return [value] + return [row for child in value.values() for row in copies(child)] + return [row for child in value for row in copies(child)] if isinstance(value, list) else [] + recalled_copies = copies(recalled) + assert recalled_copies, (producer, "promoted copy was not recalled", recalled) + surfaces.extend(recalled_copies) + surfaces.append(call_mcp_tool(context, name="alice_explain", arguments={"memory_id": memory_id})) + text = json.dumps(surfaces, default=str) + assert any(str(row["id"]) in text for label, _kind, row in rows if label == "alpha") + for label, _kind, row in rows: + if label != "alpha": + assert str(row["id"]) not in text, (producer, label, row["id"]) + for field in ("title", "canonical_text", "claim", "description"): + if row.get(field): + assert str(row[field]) not in text, (producer, label, field) + assert f"SENTINEL_{label.upper()}" not in text + + +def test_input_selection_uses_effective_labels_including_the_owner_default_ceiling(migrated_database_urls, monkeypatch): + app_url = migrated_database_urls["app"] + wire_database(monkeypatch, app_url) + user_id, alpha, _beta, rows, _key, _beta_key, _unbound = seed_grid(app_url) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = next(row for label, kind, row in rows if label == "alpha" and kind == "sources") + copy = store.create_memory({"memory_key": "stale.source.copy", "canonical_text": "STALE_SOURCE_COPY Atlas secret", + "title": "STALE_SOURCE_COPY", "status": "active", "domain": "project", "sensitivity": "public", + "metadata_json": {"source_id": str(source["id"]), "project_scope": [alpha]}}) + promoted = next(row for label, kind, row in rows if label == "alpha" and kind == "memories" + and row["metadata_json"].get("source_artifact_id")) + conn.execute("UPDATE sources SET sensitivity='confidential' WHERE id=%s", (source["id"],)) + conn.execute("UPDATE memories SET created_at='2026-10-05T09:00:00Z' WHERE id=%s", (copy["id"],)) + assert store.get_memory(str(copy["id"]))["sensitivity"] == "public" + assert store.get_memory(str(promoted["id"]))["sensitivity"] == "public" + _, trusted = create_agent_key(store, user_id=user_id, agent_id="trusted-alpha", permission_profile="trusted_local_agent", project_scope=alpha) + owner = VNextBrainService(store).generate_daily_brief(BrainArtifactRequest(generated_for="2026-10-05", + source_limit=50, memory_limit=50, artifact_limit=50, open_loop_limit=50, discover_open_loops=False, + create_candidate_memories=False)) + bound, _ = generate("daily", user_id, alpha, trusted) + for report in (owner, bound): + text = json.dumps(report, default=str) + assert str(copy["id"]) not in text + assert copy["canonical_text"] not in text + assert str(promoted["id"]) not in text + assert str(source["id"]) not in text + assert "SENTINEL_ALPHA prior report text" not in text + assert any(str(row["id"]) in text for label, kind, row in rows if label == "alpha" and kind == "memories" + and row["memory_type"] == "episode") + + +def test_the_owner_keeps_the_cross_project_brief_and_bound_keys_cannot_read_it(migrated_database_urls, monkeypatch): + app_url = migrated_database_urls["app"] + wire_database(monkeypatch, app_url) + user_id, alpha, beta, rows, alpha_key, beta_key, unbound = seed_grid(app_url) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + owner = VNextBrainService(store).generate_daily_brief(BrainArtifactRequest(generated_for="2026-10-05", + source_limit=50, memory_limit=50, artifact_limit=50, open_loop_limit=50, discover_open_loops=False, + create_candidate_memories=False)) + text = json.dumps(owner, default=str) + assert all(f"SENTINEL_{label.upper()}" in text for label in ("alpha", "beta", "shared", "global")) + assert owner["metadata_json"]["project_scope"] == [] + assert set(owner["metadata_json"]["project_floor"]) == {alpha, beta} + artifact_id = str(owner["id"]) + for key in (alpha_key, beta_key, unbound): + for path in (f"/v0/vnext/artifacts/{artifact_id}", f"/v0/vnext/traces/artifacts/{artifact_id}"): + status, body = invoke_request("GET", path, query_params={"user_id": str(user_id)}, headers={"authorization": f"Bearer {key}"}) + assert status == (200 if key == unbound else 403), (status, body) + status, promoted = invoke_request("POST", f"/v0/vnext/artifacts/{artifact_id}/review", + payload={"user_id": str(user_id), "action": "promote"}, headers={"authorization": f"Bearer {unbound}"}) + assert status == 200, promoted + memory_id = promoted["promoted_memory_id"] + with user_connection(app_url, user_id) as conn: + copy = PostgresVNextStore(conn).get_memory(memory_id) + assert copy["metadata_json"]["project_scope"] == [] + assert set(copy["metadata_json"]["project_floor"]) == {alpha, beta} + for key in (alpha_key, beta_key, unbound): + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + recalled = call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name="alice_recall", + arguments={"query": "Atlas", "limit": 50}) + assert (memory_id in json.dumps(recalled, default=str)) == (key == unbound) From de748c4a990d336f67513cbb55082675735fbaff Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:19:08 +0200 Subject: [PATCH 125/270] Expand effective producer inputs and promoted group controls --- ...est_derived_labels_group_scope_postgres.py | 20 +++++++++- .../test_derived_labels_producers_postgres.py | 19 +++++++--- tests/unit/test_group_scope_sqlite.py | 37 ++++++++++++++++++- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_derived_labels_group_scope_postgres.py b/tests/integration/test_derived_labels_group_scope_postgres.py index 39c093d19..3b6f07192 100644 --- a/tests/integration/test_derived_labels_group_scope_postgres.py +++ b/tests/integration/test_derived_labels_group_scope_postgres.py @@ -1,16 +1,17 @@ """The PostgreSQL group consumers have the same controls as SQLite.""" from uuid import uuid4 +import json import pytest from alicebot_api.db import user_connection from alicebot_api.store import ContinuityStore from alicebot_api.vnext_derived_labels import group_scope -from alicebot_api.vnext_memory_commit import VNextMemoryCommitService +from alicebot_api.vnext_memory_commit import VNextMemoryCommitService, VNextMemoryCommitValidationError from alicebot_api.vnext_rollups import VNextRollupService from alicebot_api.vnext_store import PostgresVNextStore -from tests.unit.test_group_scope_sqlite import ALPHA, seed_members +from tests.unit.test_group_scope_sqlite import ALPHA, seed_members, seed_promoted_members @pytest.mark.parametrize("accept", (False, True)) @@ -33,3 +34,18 @@ def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inser assert second.proposals == [] assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == before assert any(group["state"] == ("already_covered_by_accepted" if accept else "existing_candidate") for group in second.groups), second + + +def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossing_project_scopes(migrated_database_urls): + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") + store = PostgresVNextStore(conn) + members = seed_promoted_members(store) + first = VNextRollupService(store).propose_rollups() + candidate_id = first.candidate_ids[0] + member = members[0] + metadata = dict(member["metadata_json"], project_scope=[], project_floor=[ALPHA]) + conn.execute("UPDATE memories SET metadata_json=%s::jsonb WHERE id=%s", (json.dumps(metadata), member["id"])) + with pytest.raises(VNextMemoryCommitValidationError, match="crosses project scopes"): + VNextMemoryCommitService(store).accept_consolidation_candidate(candidate_id, reason="Reviewed synthetic group") diff --git a/tests/integration/test_derived_labels_producers_postgres.py b/tests/integration/test_derived_labels_producers_postgres.py index 134372736..dcc57c021 100644 --- a/tests/integration/test_derived_labels_producers_postgres.py +++ b/tests/integration/test_derived_labels_producers_postgres.py @@ -179,36 +179,45 @@ def copies(value): assert f"SENTINEL_{label.upper()}" not in text -def test_input_selection_uses_effective_labels_including_the_owner_default_ceiling(migrated_database_urls, monkeypatch): +@pytest.mark.parametrize("producer", PRODUCERS) +def test_input_selection_uses_effective_labels_including_the_owner_default_ceiling(migrated_database_urls, monkeypatch, producer): app_url = migrated_database_urls["app"] wire_database(monkeypatch, app_url) user_id, alpha, _beta, rows, _key, _beta_key, _unbound = seed_grid(app_url) with user_connection(app_url, user_id) as conn: store = PostgresVNextStore(conn) source = next(row for label, kind, row in rows if label == "alpha" and kind == "sources") - copy = store.create_memory({"memory_key": "stale.source.copy", "canonical_text": "STALE_SOURCE_COPY Atlas secret", + copy = store.create_memory({"memory_key": "stale.source.copy", "canonical_text": "Atlas played Hollow Knight for 25 hours. STALE_SOURCE_COPY Atlas secret", "title": "STALE_SOURCE_COPY", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"]), "project_scope": [alpha]}}) promoted = next(row for label, kind, row in rows if label == "alpha" and kind == "memories" and row["metadata_json"].get("source_artifact_id")) + derived_loop = store.create_open_loop({"title": "STALE_SOURCE_LOOP Atlas", "description": "STALE_SOURCE_LOOP secret", + "source_id": str(source["id"]), "status": "open", "domain": "project", "sensitivity": "public", + "due_at": "2026-10-04T12:00:00Z", "metadata_json": {"project_scope": [alpha], + "discovered_by": "vnext_daily_brief", "source_id": str(source["id"])}}) conn.execute("UPDATE sources SET sensitivity='confidential' WHERE id=%s", (source["id"],)) conn.execute("UPDATE memories SET created_at='2026-10-05T09:00:00Z' WHERE id=%s", (copy["id"],)) + if producer == "staleness": + conn.execute("UPDATE memories SET valid_to='2026-10-04T12:00:00Z'") assert store.get_memory(str(copy["id"]))["sensitivity"] == "public" assert store.get_memory(str(promoted["id"]))["sensitivity"] == "public" _, trusted = create_agent_key(store, user_id=user_id, agent_id="trusted-alpha", permission_profile="trusted_local_agent", project_scope=alpha) owner = VNextBrainService(store).generate_daily_brief(BrainArtifactRequest(generated_for="2026-10-05", source_limit=50, memory_limit=50, artifact_limit=50, open_loop_limit=50, discover_open_loops=False, create_candidate_memories=False)) - bound, _ = generate("daily", user_id, alpha, trusted) + bound, _ = generate(producer, user_id, alpha, trusted) for report in (owner, bound): text = json.dumps(report, default=str) assert str(copy["id"]) not in text assert copy["canonical_text"] not in text assert str(promoted["id"]) not in text assert str(source["id"]) not in text + assert str(derived_loop["id"]) not in text + assert "STALE_SOURCE_LOOP" not in text assert "SENTINEL_ALPHA prior report text" not in text - assert any(str(row["id"]) in text for label, kind, row in rows if label == "alpha" and kind == "memories" - and row["memory_type"] == "episode") + assert any(str(row["id"]) in text for label, kind, row in rows if label == "alpha" and + (kind == "open_loops" or (kind == "memories" and row["memory_type"] == "episode"))), (producer, report) def test_the_owner_keeps_the_cross_project_brief_and_bound_keys_cannot_read_it(migrated_database_urls, monkeypatch): diff --git a/tests/unit/test_group_scope_sqlite.py b/tests/unit/test_group_scope_sqlite.py index a982ae6ed..795bd2a6d 100644 --- a/tests/unit/test_group_scope_sqlite.py +++ b/tests/unit/test_group_scope_sqlite.py @@ -11,8 +11,10 @@ from alicebot_api.sqlite_schema import bootstrap_sqlite_schema from alicebot_api.sqlite_store import SQLiteVNextStore, ensure_sqlite_user from alicebot_api.vnext_derived_labels import group_scope +from alicebot_api.vnext_derived_labels import with_derived_from from alicebot_api.vnext_memory_commit import VNextMemoryCommitService, VNextMemoryCommitValidationError from alicebot_api.vnext_rollups import VNextRollupService +from alicebot_api.vnext_queue import VNextQueueService ALPHA = "prj_" + "a" * 16 BETA = "prj_" + "b" * 16 @@ -30,6 +32,37 @@ def seed_members(store): return members +def seed_promoted_members(store): + members = [] + for index, (text, day) in enumerate((("I played Hollow Knight for 25 hours", "2023-06-02"), + ("I played Stardew Valley for 85 hours", "2023-06-20"), + ("I played Celeste for 10 hours", "2023-07-01"))): + if not hasattr(store, "create_artifact"): + # SQLite has no artifact table. A reviewed weekly copy uses the same aggregate rule. + parent = store.create_memory({"memory_key": f"promoted.parent.{index}", "canonical_text": text, + "status": "superseded", "domain": "personal", "sensitivity": "internal", + "metadata_json": {"project_scope": [ALPHA, BETA]}}) + memory = store.create_memory({"memory_key": f"promoted.copy.{index}", "title": text, + "canonical_text": text, "summary": text, "value": {"text": text}, "status": "active", + "memory_type": "semantic", "domain": "personal", "sensitivity": "internal", + "metadata_json": with_derived_from({"discovered_by": "vnext_weekly_synthesis", + "promotion_reviewed": True, "project_scope": [], "session_date": day}, {"memories": [parent]})}) + assert set(memory["metadata_json"]["project_floor"]) == {ALPHA, BETA} + members.append(memory) + continue + artifact = store.create_artifact({"artifact_type": "research_brief", "title": text, + "content_markdown": text, "domain": "personal", "sensitivity": "internal", + "metadata_json": {"project_scope": [ALPHA, BETA]}}) + promotion = VNextQueueService(store).review_artifact(artifact_id=str(artifact["id"]), action="promote") + memory = store.get_memory(promotion["promoted_memory_id"]) + metadata = dict(memory["metadata_json"], session_date=day) + memory = store.update_memory(memory_id=str(memory["id"]), patch={"metadata_json": metadata}) + assert memory["metadata_json"]["project_scope"] == [] + assert set(memory["metadata_json"]["project_floor"]) == {ALPHA, BETA} + members.append(memory) + return members + + @pytest.fixture def sqlite_group_store(): conn = sqlite3.connect(":memory:") @@ -70,8 +103,8 @@ def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inser def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossing_project_scopes(sqlite_group_store): store = sqlite_group_store - members = seed_members(store) - first = VNextRollupService(store).propose_rollups(projects=(ALPHA,)) + members = seed_promoted_members(store) + first = VNextRollupService(store).propose_rollups() candidate_id = first.candidate_ids[0] # Preserve the snapshot apart from the group labels so the scope check is reached. member = members[0] From 22f4d2a7f40ab6a21441d1324ffcf289b1537aae Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:24:50 +0200 Subject: [PATCH 126/270] Prove consolidation admission before provider input --- .../test_derived_labels_producers_postgres.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/integration/test_derived_labels_producers_postgres.py b/tests/integration/test_derived_labels_producers_postgres.py index dcc57c021..9a1dbc79b 100644 --- a/tests/integration/test_derived_labels_producers_postgres.py +++ b/tests/integration/test_derived_labels_producers_postgres.py @@ -16,6 +16,7 @@ from alicebot_api.store import ContinuityStore from alicebot_api.vnext_agent_keys import create_agent_key from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from alicebot_api.vnext_consolidation import MemoryConsolidationRequest, VNextConsolidationService, _clustering_options from alicebot_api.vnext_derived_labels import with_derived_from from alicebot_api.vnext_queue import VNextQueueService from alicebot_api.vnext_store import PostgresVNextStore @@ -192,6 +193,7 @@ def test_input_selection_uses_effective_labels_including_the_owner_default_ceili "metadata_json": {"source_id": str(source["id"]), "project_scope": [alpha]}}) promoted = next(row for label, kind, row in rows if label == "alpha" and kind == "memories" and row["metadata_json"].get("source_artifact_id")) + prior_id = promoted["metadata_json"]["source_artifact_id"] derived_loop = store.create_open_loop({"title": "STALE_SOURCE_LOOP Atlas", "description": "STALE_SOURCE_LOOP secret", "source_id": str(source["id"]), "status": "open", "domain": "project", "sensitivity": "public", "due_at": "2026-10-04T12:00:00Z", "metadata_json": {"project_scope": [alpha], @@ -212,6 +214,7 @@ def test_input_selection_uses_effective_labels_including_the_owner_default_ceili assert str(copy["id"]) not in text assert copy["canonical_text"] not in text assert str(promoted["id"]) not in text + assert str(prior_id) not in text assert str(source["id"]) not in text assert str(derived_loop["id"]) not in text assert "STALE_SOURCE_LOOP" not in text @@ -251,3 +254,32 @@ def test_the_owner_keeps_the_cross_project_brief_and_bound_keys_cannot_read_it(m recalled = call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name="alice_recall", arguments={"query": "Atlas", "limit": 50}) assert (memory_id in json.dumps(recalled, default=str)) == (key == unbound) + + +def test_consolidation_admits_effective_memory_labels_before_the_embedding_provider(migrated_database_urls, monkeypatch): + app_url = migrated_database_urls["app"] + user_id, alpha, _beta, rows, _alpha_key, _beta_key, _unbound = seed_grid(app_url) + printed = [] + + class RecordingProvider: + def embed_batch(self, texts): + printed.extend(texts) + return [[1.0, 0.0, 0.0] for _ in texts] + + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = next(row for label, kind, row in rows if label == "alpha" and kind == "sources") + store.create_memory({"memory_key": "provider.stale.copy", "canonical_text": "PROVIDER_SECRET Atlas played Hollow Knight for 25 hours", + "title": "PROVIDER_SECRET", "status": "active", "domain": "project", "sensitivity": "public", + "metadata_json": {"source_id": str(source["id"]), "project_scope": [alpha]}}) + conn.execute("UPDATE sources SET sensitivity='confidential' WHERE id=%s", (source["id"],)) + # Synthetic vectors are local. Presence is forced for all selected rows so selection reaches the provider. + monkeypatch.setattr(store, "list_memory_ids_with_embeddings", lambda ids: set(ids)) + service = VNextConsolidationService(store, embedding_provider=RecordingProvider()) + service._cluster_memories(domains=None, sensitivity=["public", "internal", "private", "unknown"], + projects=(alpha,), all_of=(alpha,), options=_clustering_options(MemoryConsolidationRequest())) + text = json.dumps(printed) + assert printed + assert "SENTINEL_ALPHA memory" in text + assert "PROVIDER_SECRET" not in text + assert "SENTINEL_ALPHA prior report text" not in text From 7dad5486db28b15fbbe77baadc0d9afd880fbd74 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:34:15 +0200 Subject: [PATCH 127/270] Pin reviewed read facade shape and stage doctor census test with repair --- tests/unit/test_derived_labels_real_keys.py | 25 ------------------- .../unit/test_store_events_revisions_split.py | 8 +++--- .../unit/test_store_graph_open_loops_split.py | 8 +++--- tests/unit/test_store_memory_access_split.py | 8 +++--- .../unit/test_store_memory_lifecycle_split.py | 8 +++--- 5 files changed, 16 insertions(+), 41 deletions(-) diff --git a/tests/unit/test_derived_labels_real_keys.py b/tests/unit/test_derived_labels_real_keys.py index 88da4a5e1..991454b09 100644 --- a/tests/unit/test_derived_labels_real_keys.py +++ b/tests/unit/test_derived_labels_real_keys.py @@ -84,28 +84,3 @@ def test_sqlite_real_key_core_doors(tmp_path, monkeypatch, reader): if not admitted: assert str(row["title"]) not in rendered - -def test_full_owner_doctor_keeps_true_counts_and_filtered_view_skips_content(tmp_path): - user_id = uuid4() - path = tmp_path / "doctor.sqlite3" - bootstrap_database(path, user_id=str(user_id), user_email="synthetic@example.invalid") - with sqlite_user_connection(path, user_id) as conn: - store = SQLiteVNextStore(conn, user_id) - # SQLite has no provider/connector doctor protocol. Keep those synthetic - # operations fixed while the real connection supplies content diagnostics. - diagnostic_store = DoctorStore() - diagnostic_store.conn = store.conn - diagnostic_store.list_sources = lambda **kwargs: [row for batch in store.iter_label_rows("source") for row in batch] - service = VNextDoctorService(diagnostic_store, secret_provider=InMemorySecretProvider(), env={}, cwd=tmp_path) - before = service.run(include_content_diagnostics=False) - rows = seed_read_rows(store) - full = service.run() - scoped = service.run(include_content_diagnostics=False) - derived = next(check for check in full["checks"] if check["name"] == "derived_labels") - assert derived["message"] == "derived labels: 1 below their inputs, 1 unverified" - skipped = [check for check in scoped["checks"] if check["name"] in {"derived_labels", "flagged_sources"}] - assert len(skipped) == 2 - assert all(check["status"] == "skipped" and check["details"] == {"scope": "filtered_workspace", "evaluated": False} for check in skipped) - assert all(str(row["id"]) not in json.dumps(scoped) for row in rows.values()) - for field in ("status", "blocking_failure_count", "warning_count", "recommended_fixes"): - assert scoped[field] == before[field] diff --git a/tests/unit/test_store_events_revisions_split.py b/tests/unit/test_store_events_revisions_split.py index ee09f4e66..c718a97b6 100644 --- a/tests/unit/test_store_events_revisions_split.py +++ b/tests/unit/test_store_events_revisions_split.py @@ -139,8 +139,8 @@ }, } EXPECTED_CLASS_KEY_SHA256 = { - # Reviewed recovery and reverse propagation add only the two Postgres - # methods; all prior runtime class keys retain their order. + # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. + # Existing facade members retain their relative order. # Source owner methods are additive; the existing member order is unchanged. # Re-minted for the paired browser-clip capability façade methods. # The sqlite hash is re-minted again for ``check_source_search_query``. It is @@ -150,7 +150,7 @@ # Re-minted for the per-file importer savepoint (2026-10-02): ``savepoint`` is appended last on both façades. # Previous receipt: 650e2e0ff088d67c... Proof: the class key list equals the list at origin/main 040a2a10 # plus ``savepoint`` before ``__dict__``, with every other key in the same order. - "postgres": "e4a921e06bc707a362fb9c2b3050d292b119cd0b92b346ce5997a679b3bd61f0", + "postgres": "3751e3fd145562485233765a14ae4e7bc7f396e6e72356a079e266fd98025839", # Re-minted for the merge of #500 and #502: ``check_literal_match_query``. # Re-minted again for per-project memory S2 (2026-10-02): the two single-scan partition reads # ``list_memories_view_partitions`` and ``list_open_loops_view_partitions``, SQLite only on purpose @@ -158,7 +158,7 @@ # Re-minted again for the per-file importer savepoint (2026-10-02), the same appended method. # Previous receipt: 365b7a01acf7a8b5... Proof: as for postgres, one added key and no other change. # Re-minted for the derived-label lock: ``lock_label_writes`` and ``read_label_rows`` on both facades. - "sqlite": "3212a93f2011cecddb6b0002a3f9b3abe37819d3582b2a1173846871981a5ff9", + "sqlite": "265db9bfe184e712eb3bbb64356c7a9601b130c93ca847962697c35f68b84ecd", } EXPECTED_SUPPORT_AST_SHA256 = { "postgres_columns": { diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index 91cea70ad..d37f2ddb2 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -154,15 +154,15 @@ SQLITE_CARRIER_PATH: (6, "970028b5c929f0e749d8b40bdee571c872600de7a713e58d60e0da86f022af8a"), } EXPECTED_CLASS_ORDERS = { - # Reviewed source recovery and belief propagation add only two Postgres - # methods. The SQLite facade and every existing member retain their order. + # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. + # Existing facade members retain their relative order. # Two paired browser-clip capability methods extend both façades, and one # more paired method, ``list_memories_referencing_sources``. # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # lock_label_writes and read_label_rows follow __init__. Previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (176, "3549d6ea179f4ebc2341f1c251f139dfb747d56700a4554aef114150efed5c3f"), + "PostgresVNextStore": (179, "71998141cd00c0639209285accbc29faa775b2e0250422c96101ee2f144df832"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -178,7 +178,7 @@ # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. # lock_label_writes and read_label_rows follow __init__. Previous receipt (134, 13012720...). - "SQLiteVNextStore": (136, "1d99dbaf69e4ea888ca7beb2389ac176650a2e573c067bf0686adc9e609132f5"), + "SQLiteVNextStore": (138, "783c07140c4015bcf8e43a3c68f3c2837ad14ce3525f8bd56a6f9c319d38eada"), } EXPECTED_COLUMN_AST = { POSTGRES_COLUMNS_PATH: { diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index 205f6510b..1dc1d3837 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -209,8 +209,8 @@ ) EXPECTED_CLASS_ORDERS = { - # Reviewed source recovery and belief propagation add only two Postgres - # methods. The SQLite facade and every existing member retain their order. + # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. + # Existing facade members retain their relative order. # Two paired browser-clip capability methods extend both façades. One more # paired method, ``list_memories_referencing_sources``, is the batched form # of ``list_memories_referencing_source``; both carrier receipts above were @@ -220,7 +220,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (176, "3549d6ea179f4ebc2341f1c251f139dfb747d56700a4554aef114150efed5c3f"), + "PostgresVNextStore": (179, "71998141cd00c0639209285accbc29faa775b2e0250422c96101ee2f144df832"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -237,7 +237,7 @@ # prunable_sources here; every pre-existing class member keeps its order. # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (134, 13012720...). - "SQLiteVNextStore": (136, "1d99dbaf69e4ea888ca7beb2389ac176650a2e573c067bf0686adc9e609132f5"), + "SQLiteVNextStore": (138, "783c07140c4015bcf8e43a3c68f3c2837ad14ce3525f8bd56a6f9c319d38eada"), } diff --git a/tests/unit/test_store_memory_lifecycle_split.py b/tests/unit/test_store_memory_lifecycle_split.py index 4f9f9489d..0c18c352d 100644 --- a/tests/unit/test_store_memory_lifecycle_split.py +++ b/tests/unit/test_store_memory_lifecycle_split.py @@ -110,14 +110,14 @@ "sqlite": "1270ef0115988552418349aa9e94a7442ba04be41443f278f68a1fa81857903a", } EXPECTED_CLASS_ORDERS = { - # Reviewed source recovery and belief propagation add only two Postgres - # methods. The SQLite facade and every existing member retain their order. + # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. + # Existing facade members retain their relative order. # Two paired browser-clip capability methods extend both façades, and one # more paired method, ``list_memories_referencing_sources``. # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (176, "3549d6ea179f4ebc2341f1c251f139dfb747d56700a4554aef114150efed5c3f"), + "PostgresVNextStore": (179, "71998141cd00c0639209285accbc29faa775b2e0250422c96101ee2f144df832"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -132,7 +132,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. - "SQLiteVNextStore": (136, "1d99dbaf69e4ea888ca7beb2389ac176650a2e573c067bf0686adc9e609132f5"), + "SQLiteVNextStore": (138, "783c07140c4015bcf8e43a3c68f3c2837ad14ce3525f8bd56a6f9c319d38eada"), } EXPECTED_FACADE_COMMENT_DIGESTS = { POSTGRES_FACADE_PATH: "d8599a46ee26dc35a3ae52c1a98a416509add9ae4a42ece780c5c5ed7e132b93", From 4b3a79ee6b5093ec1e8e8c8ba999b72440413be0 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:12:01 +0200 Subject: [PATCH 128/270] Repair stored labels atomically and report owner checks accurately --- .../20261005_0096_derived_label_floor.py | 19 +++-- apps/api/src/alicebot_api/cli/labels.py | 61 ++++++++------- apps/api/src/alicebot_api/db.py | 15 ++++ apps/api/src/alicebot_api/label_commands.py | 2 +- apps/api/src/alicebot_api/vault_doctor.py | 12 ++- apps/api/src/alicebot_api/vnext_doctor.py | 14 +++- .../src/alicebot_api/vnext_label_repair.py | 76 ++++++++++++++----- 7 files changed, 135 insertions(+), 64 deletions(-) diff --git a/apps/api/alembic/versions/20261005_0096_derived_label_floor.py b/apps/api/alembic/versions/20261005_0096_derived_label_floor.py index 94e52e144..1f8c128a8 100644 --- a/apps/api/alembic/versions/20261005_0096_derived_label_floor.py +++ b/apps/api/alembic/versions/20261005_0096_derived_label_floor.py @@ -14,7 +14,7 @@ from alicebot_api.vnext_derived_domain_backfill import require_changed from alicebot_api.vnext_derived_labels import labels_raised_payload from alicebot_api.vnext_event_log import build_event_log_record -from alicebot_api.vnext_label_repair import INPUT_SELECTS_V3, plan_label_repairs +from alicebot_api.vnext_label_repair import INPUT_SELECTS_V3, plan_label_repairs, label_project_id revision = "20261005_0096" down_revision = "20261004_0095" @@ -47,15 +47,17 @@ def upgrade() -> None: for table, statement in INPUT_SELECTS_V3.items(): tables[table] = list(connection.execute(text(statement)).mappings()) for table, user, row_id, previous, new, node in plan_label_repairs(tables): - metadata = dict(node.get("metadata_json") or {}) - metadata["project_scope"] = list(new["project_scope"]) - metadata["project_floor"] = list(new["project_floor"]) + metadata = {"project_scope": list(new["project_scope"]), "project_floor": list(new["project_floor"])} + project_sql = ", project_id = :project_id" if table in {"memories", "open_loops"} else "" + project_guard = " AND project_id IS NOT DISTINCT FROM :previous_project_id" if project_sql else "" require_changed( connection.execute( text( f"UPDATE {table} SET domain = :domain, sensitivity = :sensitivity, " - "metadata_json = CAST(:metadata AS jsonb) " - "WHERE user_id = CAST(:user AS uuid) AND id = CAST(:id AS uuid)" + f"metadata_json = metadata_json || CAST(:metadata AS jsonb){project_sql} " + "WHERE user_id = CAST(:user AS uuid) AND id = CAST(:id AS uuid) " + "AND domain = :previous_domain AND sensitivity = :previous_sensitivity " + f"AND metadata_json = CAST(:previous_metadata AS jsonb){project_guard}" ), { "domain": new["domain"], @@ -63,6 +65,11 @@ def upgrade() -> None: "metadata": json.dumps(metadata), "user": user, "id": row_id, + "previous_domain": previous["domain"], + "previous_sensitivity": previous["sensitivity"], + "previous_metadata": json.dumps(node.get("metadata_json") or {}), + "project_id": label_project_id(new["project_scope"]), + "previous_project_id": node.get("project_id"), }, ).rowcount, table, diff --git a/apps/api/src/alicebot_api/cli/labels.py b/apps/api/src/alicebot_api/cli/labels.py index 7f98472b5..fc3b92b37 100644 --- a/apps/api/src/alicebot_api/cli/labels.py +++ b/apps/api/src/alicebot_api/cli/labels.py @@ -3,29 +3,30 @@ from __future__ import annotations from alicebot_api.cli.shared import CLIContext, _vnext_store_context -from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError -from alicebot_api.vnext_label_repair import classify_stored_labels, format_label_check, plan_label_repairs +from alicebot_api.db import user_read_snapshot_connection +from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError, require_changed +from alicebot_api.vnext_label_repair import classify_stored_labels, format_label_check, plan_label_repairs, load_postgres_label_tables, label_project_id from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock -def _postgres_tables(conn) -> dict[str, list[dict[str, object]]]: - from alicebot_api.vnext_label_repair import INPUT_SELECTS_V3 +_postgres_tables = load_postgres_label_tables - tables: dict[str, list[dict[str, object]]] = {} - for table, statement in INPUT_SELECTS_V3.items(): - cursor = conn.execute(statement) - names = [column[0] for column in cursor.description] - tables[table] = [row if isinstance(row, dict) else dict(zip(names, row)) for row in cursor.fetchall()] - return tables + +def _lock_repair_rows(store, changes) -> None: + """Take exactly the planned row locks in the relabel table order.""" + for table in ("generated_artifacts", "projects", "open_loops", "memories"): + ids = [row_id for changed_table, _user, row_id, *_ in changes if changed_table == table] + if ids: + locked = store.conn.execute( + f"SELECT id FROM {table} WHERE id = ANY(%s::uuid[]) ORDER BY id FOR UPDATE", (ids,) + ).fetchall() + require_changed(int(len(locked) == len(ids)), table, "planned rows") def _run_vnext_labels_check(ctx: CLIContext, args: object) -> str: del args try: - with _vnext_store_context(ctx) as store: - conn = store.conn - if not conn.in_transaction: - conn.execute("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + with user_read_snapshot_connection(ctx.database_url, ctx.user_id) as conn: below, unverified = classify_stored_labels(_postgres_tables(conn)) except DerivedDomainRepairError as exc: print(f"labels check failed: {exc}") @@ -45,6 +46,7 @@ def _run_vnext_labels_repair(ctx: CLIContext, args: object) -> str: store.lock_graph_mutation() acquire_exclusive_label_lock(store) changes = plan_label_repairs(_postgres_tables(store.conn)) + _lock_repair_rows(store, changes) # The SQLite writer is not used here. Postgres repair applies the # same plan through label-only updates inside this transaction. applied = 0 @@ -54,25 +56,22 @@ def _run_vnext_labels_repair(ctx: CLIContext, args: object) -> str: from alicebot_api.vnext_derived_labels import labels_raised_payload from alicebot_api.vnext_event_log import build_event_log_record - metadata = dict(node.get("metadata_json") or {}) - metadata["project_scope"] = list(new["project_scope"]) - metadata["project_floor"] = list(new["project_floor"]) + metadata = {"project_scope": list(new["project_scope"]), "project_floor": list(new["project_floor"])} + project_sql = ", project_id = %s" if table in {"memories", "open_loops"} else "" + project_guard = " AND project_id IS NOT DISTINCT FROM %s" if project_sql else "" + params = [new["domain"], new["sensitivity"], json.dumps(metadata)] + if project_sql: + params.append(label_project_id(new["project_scope"])) + params.extend([user, row_id, previous["domain"], previous["sensitivity"], json.dumps(node.get("metadata_json") or {})]) + if project_sql: + params.append(node.get("project_id")) cursor = store.conn.execute( - f"UPDATE {table} SET domain = %s, sensitivity = %s, metadata_json = %s::jsonb " - "WHERE user_id = %s::uuid AND id = %s::uuid " - "AND domain = %s AND sensitivity = %s", - ( - new["domain"], - new["sensitivity"], - json.dumps(metadata), - user, - row_id, - previous["domain"], - previous["sensitivity"], - ), + f"UPDATE {table} SET domain = %s, sensitivity = %s, metadata_json = metadata_json || %s::jsonb " + f"{project_sql} WHERE user_id = %s::uuid AND id = %s::uuid " + f"AND domain = %s AND sensitivity = %s AND metadata_json = %s::jsonb{project_guard}", + tuple(params), ) - if cursor.rowcount != 1: - continue + require_changed(cursor.rowcount, table, row_id) target = { "memories": "memory", "open_loops": "open_loop", diff --git a/apps/api/src/alicebot_api/db.py b/apps/api/src/alicebot_api/db.py index 11c1c0953..e677a463c 100644 --- a/apps/api/src/alicebot_api/db.py +++ b/apps/api/src/alicebot_api/db.py @@ -68,6 +68,20 @@ def direct_user_connection(database_url: str, user_id: UUID) -> Iterator[UserCon yield conn +@contextmanager +def user_read_snapshot_connection(database_url: str, user_id: UUID) -> Iterator[UserConnection]: + """One repeatable read, read-only snapshot with transaction-local RLS. + + Set the transaction mode before the first query, including the identity + query, and clear that identity when the transaction ends. + """ + with psycopg.connect(database_url, row_factory=dict_row) as conn: + with conn.transaction(): + conn.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") + set_current_user(conn, user_id) + yield conn + + def _new_connection_pool(database_url: str) -> ConnectionPool[UserConnection]: pool = cast( ConnectionPool[UserConnection], @@ -143,6 +157,7 @@ def pooled_user_connection(pool: ConnectionPoolLike, user_id: UUID) -> Iterator[ "set_current_user", "set_current_user_account", "user_connection", + "user_read_snapshot_connection", ] diff --git a/apps/api/src/alicebot_api/label_commands.py b/apps/api/src/alicebot_api/label_commands.py index 708cad473..f6d523716 100644 --- a/apps/api/src/alicebot_api/label_commands.py +++ b/apps/api/src/alicebot_api/label_commands.py @@ -68,7 +68,7 @@ def run_labels(args) -> int: return 1 if below or unverified or raised_on_next_open else 0 try: with sqlite_user_connection(db, args.user_id) as conn: - relabel_labels_sqlite(conn) + relabel_labels_sqlite(conn, explicit=True) except DerivedDomainRepairError as exc: print(f"labels repair failed: {exc}") return 2 diff --git a/apps/api/src/alicebot_api/vault_doctor.py b/apps/api/src/alicebot_api/vault_doctor.py index 488bb0aba..6c3f61bd1 100644 --- a/apps/api/src/alicebot_api/vault_doctor.py +++ b/apps/api/src/alicebot_api/vault_doctor.py @@ -111,10 +111,16 @@ def compile_local_vault_doctor( (uid, CANDIDATE_STATUS), ) missing_vector_line = _missing_vector_line(store) - from alicebot_api.vnext_label_repair import label_gap_counts + from alicebot_api.vnext_label_repair import LabelCheckUnavailable, label_gap_counts - below, unverified = label_gap_counts(store) - label_line = f"derived labels: {below} below their inputs, {unverified} unverified" + try: + below, unverified = label_gap_counts(store) + label_line = f"derived labels: {below} below their inputs, {unverified} unverified" + labels_available = True + except LabelCheckUnavailable: + below, unverified = 0, 0 + labels_available = False + label_line = "derived labels: unavailable; run labels check" flagged_ids = _flagged_source_ids(store) superseded_count = count_prunable_sources(store) try: diff --git a/apps/api/src/alicebot_api/vnext_doctor.py b/apps/api/src/alicebot_api/vnext_doctor.py index fce33a0d3..e4c14b65b 100644 --- a/apps/api/src/alicebot_api/vnext_doctor.py +++ b/apps/api/src/alicebot_api/vnext_doctor.py @@ -316,14 +316,20 @@ def _content_checks(self, checks: list[DoctorCheck]) -> None: }, ) - from alicebot_api.vnext_label_repair import label_gap_counts + from alicebot_api.vnext_label_repair import LabelCheckUnavailable, label_gap_counts - below, unverified = label_gap_counts(self.store) - label_line = f"derived labels: {below} below their inputs, {unverified} unverified" + try: + below, unverified = label_gap_counts(self.store) + label_line = f"derived labels: {below} below their inputs, {unverified} unverified" + labels_available = True + except LabelCheckUnavailable: + below, unverified = 0, 0 + labels_available = False + label_line = "derived labels: unavailable; run labels check" self._check( checks, name="derived_labels", - ok=below == 0 and unverified == 0, + ok=labels_available and below == 0 and unverified == 0, severity="warning", message_ok=label_line, message_fail=label_line, diff --git a/apps/api/src/alicebot_api/vnext_label_repair.py b/apps/api/src/alicebot_api/vnext_label_repair.py index 8e4b6b910..2db1b4d59 100644 --- a/apps/api/src/alicebot_api/vnext_label_repair.py +++ b/apps/api/src/alicebot_api/vnext_label_repair.py @@ -40,9 +40,9 @@ "beliefs": "SELECT id, user_id, memory_id FROM beliefs", } _UPDATES = { - "memories": "UPDATE memories SET domain = ?, sensitivity = ?, metadata_json = ? WHERE user_id = ? AND id = ?", + "memories": "UPDATE memories SET domain = ?, sensitivity = ?, metadata_json = json_patch(metadata_json, ?), project_id = ? WHERE user_id = ? AND id = ? AND domain = ? AND sensitivity = ? AND metadata_json = ? AND project_id IS ?", "open_loops": ( - "UPDATE open_loops SET domain = ?, sensitivity = ?, metadata_json = ? WHERE user_id = ? AND id = ?" + "UPDATE open_loops SET domain = ?, sensitivity = ?, metadata_json = json_patch(metadata_json, ?), project_id = ? WHERE user_id = ? AND id = ? AND domain = ? AND sensitivity = ? AND metadata_json = ? AND project_id IS ?" ), } @@ -84,6 +84,7 @@ def plan_label_repairs( for row in rows: node = dict(row) node["kind"] = kind + node["_stored_metadata"] = row.get("metadata_json") node["metadata_json"] = _json_object(row.get("metadata_json")) if isinstance(row.get("value"), str): node["value"] = _json_object(row.get("value")) @@ -112,6 +113,18 @@ def plan_label_repairs( return changes +def label_project_id(scope: Sequence[object]) -> str | None: + """Mirror a single UUID scope; named and global scopes use metadata only.""" + from uuid import UUID + + if len(scope) != 1: + return None + try: + return str(UUID(str(scope[0]))) + except ValueError: + return None + + def _load_tables(conn) -> dict[str, list[dict[str, object]]]: available = { row[0] if not isinstance(row, dict) else row["name"] @@ -131,8 +144,8 @@ def _stamped(conn) -> bool: return conn.execute("SELECT value FROM alice_schema_state WHERE key = ?", (REPAIR_STATE_KEY,)).fetchone() is not None -def relabel_labels_sqlite(conn, *, restoring: bool = False) -> None: - """Raise stored derived labels once, or again on a restore. +def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = False) -> None: + """Raise stored derived labels once, or always for a restore or owner repair. When no transaction is open this begins one and reads the state key inside it. A caller that already has a transaction keeps it. @@ -143,18 +156,19 @@ def relabel_labels_sqlite(conn, *, restoring: bool = False) -> None: conn.execute("BEGIN IMMEDIATE") owns = True try: - if not restoring and _stamped(conn): + if not restoring and not explicit and _stamped(conn): if owns: conn.commit() return changes = plan_label_repairs(_load_tables(conn)) - for table, user, stored, _previous, new, node in changes: - metadata = dict(node.get("metadata_json") or {}) - metadata["project_scope"] = list(new["project_scope"]) - metadata["project_floor"] = list(new["project_floor"]) + for table, user, stored, previous, new, node in changes: + metadata = {"project_scope": list(new["project_scope"]), "project_floor": list(new["project_floor"])} + raw_metadata = node.get("_stored_metadata") changed = conn.execute( _UPDATES[table], - (new["domain"], new["sensitivity"], json.dumps(metadata), user, stored), + (new["domain"], new["sensitivity"], json.dumps(metadata), + label_project_id(new["project_scope"]), user, stored, + previous["domain"], previous["sensitivity"], raw_metadata, node.get("project_id")), ).rowcount require_changed(changed, table, stored) for table, user, stored, previous, new, _node in changes: @@ -205,6 +219,7 @@ def classify_stored_labels( for row in rows: node = dict(row) node["kind"] = kind + node["_stored_metadata"] = row.get("metadata_json") node["metadata_json"] = _json_object(row.get("metadata_json")) if isinstance(row.get("value"), str): node["value"] = _json_object(row.get("value")) @@ -258,23 +273,44 @@ def format_label_check( return "\n".join(lines) +class LabelCheckUnavailable(RuntimeError): + """The label planner could not read this store; no count is available.""" + + +def load_postgres_label_tables(conn) -> dict[str, list[dict[str, object]]]: + """Read every label input using PostgreSQL's current RLS identity.""" + + tables = {} + for table, statement in INPUT_SELECTS_V3.items(): + cursor = conn.execute(statement) + names = [column[0] for column in cursor.description] + tables[table] = [row if isinstance(row, dict) else dict(zip(names, row)) for row in cursor.fetchall()] + return tables + + def label_gap_counts(store: object) -> tuple[int, int]: - """How many derived rows are below their inputs, and how many are unverified. + """Counts from the real store, or an explicit unavailable result. - A store that cannot be read returns zeros so a doctor does not fail closed. + PostgreSQL reads use a savepoint so an unavailable table does not poison + the doctor's surrounding application transaction. """ conn = getattr(store, "conn", None) module = type(conn).__module__ if conn is not None else "" - if conn is None or not (module.startswith("sqlite3") or module.startswith("psycopg")): - return (0, 0) try: - tables = _load_tables(conn) + if module.startswith("sqlite3"): + tables = _load_tables(conn) + elif module.startswith("psycopg"): + with conn.transaction(): + tables = load_postgres_label_tables(conn) + else: + raise LabelCheckUnavailable("derived label counts are unavailable for this store") below, unverified = classify_stored_labels(tables) - except Exception: - return (0, 0) - unverified_count = sum(len(ids) for ids in unverified.values()) - return (len(below), unverified_count) + except LabelCheckUnavailable: + raise + except Exception as exc: + raise LabelCheckUnavailable("derived label counts could not be read") from exc + return len(below), sum(len(ids) for ids in unverified.values()) def recorded_sqlite_label_repairs(conn, user_id: str) -> dict[tuple[str, str, str], set[str]]: @@ -330,6 +366,8 @@ def recorded_sqlite_label_repairs(conn, user_id: str) -> dict[tuple[str, str, st __all__ = [ "INPUT_SELECTS_V3", + "LabelCheckUnavailable", + "load_postgres_label_tables", "REPAIR_STATE_KEY", "DerivedDomainRepairError", "classify_stored_labels", From df31b34c45bf3b9f0f4961661a0d16733847e780 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:27:44 +0200 Subject: [PATCH 129/270] Use ordered repair locks and retain the CLI compatibility contract --- apps/api/src/alicebot_api/cli/__init__.py | 3 + apps/api/src/alicebot_api/cli/labels.py | 59 ++++++++++++------ apps/api/src/alicebot_api/vault_doctor.py | 4 +- apps/api/src/alicebot_api/vnext_doctor.py | 25 +++----- .../src/alicebot_api/vnext_label_repair.py | 61 ++++++++++++------- 5 files changed, 94 insertions(+), 58 deletions(-) diff --git a/apps/api/src/alicebot_api/cli/__init__.py b/apps/api/src/alicebot_api/cli/__init__.py index b75de90b8..a8c313536 100644 --- a/apps/api/src/alicebot_api/cli/__init__.py +++ b/apps/api/src/alicebot_api/cli/__init__.py @@ -50,6 +50,9 @@ for _carrier in _CARRIER_MODULES: globals().pop(_carrier.__name__.rsplit(".", maxsplit=1)[-1], None) +# The owner label commands do not extend the compatibility facade. +globals().pop("labels", None) + _PUBLIC_NAME_ORDER = "annotations argparse Iterator Sequence contextmanager redirect_stderr dataclass UTC datetime StringIO json logging os Path sys tempfile time TypedDict cast URLError Request urlopen UUID uuid4 psycopg format_artifact_detail_output format_capture_output format_continuity_brief_output format_contradiction_case_detail_output format_contradiction_case_list_output format_contradiction_sync_output format_explain_output format_lifecycle_detail_output format_lifecycle_list_output format_memory_operation_candidates_output format_memory_operation_commit_output format_memory_operations_output format_open_loops_output format_recall_output format_resume_output format_review_apply_output format_review_detail_output format_review_queue_output format_status_output format_task_brief_comparison_output format_task_brief_output format_temporal_explain_output format_temporal_state_output format_temporal_timeline_output format_trust_signals_output format_trusted_fact_pattern_explain_output format_trusted_fact_pattern_list_output format_trusted_fact_playbook_explain_output format_trusted_fact_playbook_list_output Settings get_runtime_settings get_settings ContinuityCaptureValidationError capture_continuity_input ContinuityBriefValidationError compile_continuity_brief ContinuityEvidenceNotFoundError build_continuity_explain get_continuity_artifact_detail ContinuityContradictionNotFoundError ContinuityContradictionValidationError get_contradiction_case list_contradiction_cases resolve_contradiction_case sync_contradictions MemoryMutationValidationError commit_memory_operations generate_memory_operation_candidates list_memory_operation_candidates list_memory_operations default_continuity_promotable default_continuity_searchable ContinuityLifecycleNotFoundError ContinuityLifecycleValidationError get_continuity_lifecycle_state list_continuity_lifecycle_state ContinuityOpenLoopValidationError compile_continuity_open_loop_dashboard get_thread_health_dashboard ContinuityRecallValidationError query_continuity_recall ContinuityResumptionValidationError compile_continuity_resumption_brief ContinuityReviewNotFoundError ContinuityReviewValidationError apply_continuity_correction get_continuity_review_detail list_continuity_review_queue list_trust_signals get_memory_hygiene_dashboard_summary CONTINUITY_CAPTURE_EXPLICIT_SIGNALS CONTINUITY_CORRECTION_ACTIONS CONTRADICTION_RESOLUTION_ACTIONS CONTINUITY_BRIEF_TYPE_ORDER DEFAULT_CONTINUITY_CAPTURE_LIMIT DEFAULT_CONTINUITY_BRIEF_CONFLICT_LIMIT DEFAULT_CONTINUITY_BRIEF_RELEVANT_FACT_LIMIT DEFAULT_CONTINUITY_BRIEF_TIMELINE_LIMIT DEFAULT_CONTINUITY_LIFECYCLE_LIMIT DEFAULT_CONTINUITY_OPEN_LOOP_LIMIT DEFAULT_CONTINUITY_RECALL_LIMIT DEFAULT_CONTINUITY_RESUMPTION_OPEN_LOOP_LIMIT DEFAULT_CONTINUITY_RESUMPTION_RECENT_CHANGES_LIMIT DEFAULT_CONTINUITY_REVIEW_LIMIT DEFAULT_TEMPORAL_TIMELINE_LIMIT DEFAULT_TASK_BRIEF_TOKEN_BUDGET DEFAULT_TRUSTED_FACT_PROMOTION_LIMIT MAX_CONTINUITY_REVIEW_LIMIT MAX_CONTINUITY_OPEN_LOOP_LIMIT MAX_CONTINUITY_RECALL_LIMIT MAX_CONTINUITY_BRIEF_CONFLICT_LIMIT MAX_CONTINUITY_BRIEF_RELEVANT_FACT_LIMIT MAX_CONTINUITY_BRIEF_TIMELINE_LIMIT MAX_CONTINUITY_LIFECYCLE_LIMIT MAX_CONTINUITY_RESUMPTION_OPEN_LOOP_LIMIT MAX_CONTINUITY_RESUMPTION_RECENT_CHANGES_LIMIT MAX_TASK_BRIEF_TOKEN_BUDGET MAX_TEMPORAL_TIMELINE_LIMIT MAX_TRUSTED_FACT_PROMOTION_LIMIT ContradictionCaseListQueryInput ContradictionResolveInput ContradictionSyncInput ContinuityCaptureCreateInput ContinuityBriefRequestInput ContinuityCorrectionInput ContinuityLifecycleQueryInput ContinuityOpenLoopDashboardQueryInput ContinuityRecallQueryInput ContinuityResumptionBriefRequestInput ContinuityReviewQueueQueryInput MemoryOperationCommitInput MemoryOperationGenerateInput MemoryOperationListInput TaskBriefCompileRequestInput TemporalExplainQueryInput TemporalStateAtQueryInput TemporalTimelineQueryInput TrustSignalListQueryInput TrustedFactPatternListQueryInput TrustedFactPlaybookListQueryInput TaskBriefNotFoundError TaskBriefValidationError compare_task_briefs compile_and_persist_task_brief get_persisted_task_brief ping_database user_connection get_public_eval_run list_public_eval_runs list_public_eval_suites run_public_evals write_public_eval_report get_retrieval_evaluation_summary ContinuityStore ContinuityJsonObject legacy_surfaces_enabled TemporalStateValidationError get_temporal_explain get_temporal_state_at get_temporal_timeline TrustedFactPromotionNotFoundError get_trusted_fact_pattern get_trusted_fact_playbook list_trusted_fact_patterns list_trusted_fact_playbooks PERMISSION_PROFILES AgentIdentity PolicyDecision agent_metadata append_policy_events ensure_policy_allowed evaluate_agent_policy summarize_agent_policy_telemetry AgentKeyValidationError create_agent_key dispatch_vnext_artifact_review VNextCaptureService VNextCaptureValidationError BrainArtifactRequest VNextBrainService VNextBrainValidationError ConnectionFinderRequest VNextConnectionService VNextConnectionValidationError VNextConnectorService VNextConnectorValidationError list_connector_definitions load_connector_items_from_file scan_local_folder ContextTreeRequest VNextContextTreeService VNextContextTreeStore ContradictionFinderRequest VNextContradictionService VNextContradictionValidationError VNextDogfoodingService LOCAL_VNEXT_FRONTEND_ORIGINS VNextDoctorService local_live_cors_status VNEXT_EVAL_SUITE_ORDER run_vnext_evals write_vnext_benchmark_corpus write_vnext_eval_report ProjectAutomationRequest VNextProjectService VNextProjectValidationError QueueTaskRequest VNextQueueService VNextQueueValidationError JsonObject BUDGET_STRATEGIES CONTEXT_DEPTHS VNextRetrievalRequest VNextRetrievalService VNextRetrievalStore VNextRetrievalValidationError SchedulerRunRequest VNextSchedulerService VNextSchedulerStore VNextSchedulerValidationError WORKFLOW_TYPES default_schedule DEFAULT_LOG_FILE DEFAULT_PID_FILE DEFAULT_STATUS_FILE SchedulerRuntimeConfig daemon_status run_due_workflows_durable run_foreground_daemon run_now_durable start_background_daemon stop_daemon DeferredMemoryEmbedding EMBEDDINGS_API_KEY_ENV EMBEDDINGS_BASE_URL_ENV EMBEDDINGS_MODEL_ENV EMBEDDING_SIGNATURE_VERSION MAX_EMBEDDINGS_BATCH_SIZE endpoint_fingerprint get_embedding_provider memory_embedding_text persist_deferred_memory_embeddings_best_effort append_event json_safe redact_memory_flow VNextMemoryCommitService VNextMemoryCommitValidationError memory_commit_request_from_payload InMemorySecretProvider PostgresVNextStore DEFAULT_CLI_USER_ID DEFAULT_VNEXT_SENSITIVITY_ALLOWED MAINTENANCE_REPORT_PATH_ENV DEFAULT_MAINTENANCE_REPORT_PATH DEFAULT_VNEXT_DEMO_DATASET_PATH REVIEW_STATUS_CHOICES DEMO_SECRET_MARKERS logger EvalGateFailure EmbeddingBackfillFailure PartialCommandFailure CLIContext ModelGenerationKwargs build_parser main".split() _public_values = {name: globals().pop(name) for name in _PUBLIC_NAME_ORDER} globals().update(_public_values) diff --git a/apps/api/src/alicebot_api/cli/labels.py b/apps/api/src/alicebot_api/cli/labels.py index fc3b92b37..93f290062 100644 --- a/apps/api/src/alicebot_api/cli/labels.py +++ b/apps/api/src/alicebot_api/cli/labels.py @@ -2,27 +2,24 @@ from __future__ import annotations +from psycopg import sql + from alicebot_api.cli.shared import CLIContext, _vnext_store_context from alicebot_api.db import user_read_snapshot_connection from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError, require_changed -from alicebot_api.vnext_label_repair import classify_stored_labels, format_label_check, plan_label_repairs, load_postgres_label_tables, label_project_id -from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock +from alicebot_api.vnext_label_repair import ( + classify_stored_labels, + format_label_check, + plan_label_repairs, + load_postgres_label_tables, + label_project_id, +) +from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock, lock_settled_label_rows _postgres_tables = load_postgres_label_tables -def _lock_repair_rows(store, changes) -> None: - """Take exactly the planned row locks in the relabel table order.""" - for table in ("generated_artifacts", "projects", "open_loops", "memories"): - ids = [row_id for changed_table, _user, row_id, *_ in changes if changed_table == table] - if ids: - locked = store.conn.execute( - f"SELECT id FROM {table} WHERE id = ANY(%s::uuid[]) ORDER BY id FOR UPDATE", (ids,) - ).fetchall() - require_changed(int(len(locked) == len(ids)), table, "planned rows") - - def _run_vnext_labels_check(ctx: CLIContext, args: object) -> str: del args try: @@ -46,7 +43,21 @@ def _run_vnext_labels_repair(ctx: CLIContext, args: object) -> str: store.lock_graph_mutation() acquire_exclusive_label_lock(store) changes = plan_label_repairs(_postgres_tables(store.conn)) - _lock_repair_rows(store, changes) + lock_settled_label_rows( + store, + [ + { + "kind": { + "generated_artifacts": "artifact", + "projects": "project", + "open_loops": "open_loop", + "memories": "memory", + }[table], + "id": row_id, + } + for table, _user, row_id, *_ in changes + ], + ) # The SQLite writer is not used here. Postgres repair applies the # same plan through label-only updates inside this transaction. applied = 0 @@ -59,16 +70,26 @@ def _run_vnext_labels_repair(ctx: CLIContext, args: object) -> str: metadata = {"project_scope": list(new["project_scope"]), "project_floor": list(new["project_floor"])} project_sql = ", project_id = %s" if table in {"memories", "open_loops"} else "" project_guard = " AND project_id IS NOT DISTINCT FROM %s" if project_sql else "" - params = [new["domain"], new["sensitivity"], json.dumps(metadata)] + params: list[object] = [new["domain"], new["sensitivity"], json.dumps(metadata)] if project_sql: params.append(label_project_id(new["project_scope"])) - params.extend([user, row_id, previous["domain"], previous["sensitivity"], json.dumps(node.get("metadata_json") or {})]) + params.extend( + [ + user, + row_id, + previous["domain"], + previous["sensitivity"], + json.dumps(node.get("metadata_json") or {}), + ] + ) if project_sql: params.append(node.get("project_id")) cursor = store.conn.execute( - f"UPDATE {table} SET domain = %s, sensitivity = %s, metadata_json = metadata_json || %s::jsonb " - f"{project_sql} WHERE user_id = %s::uuid AND id = %s::uuid " - f"AND domain = %s AND sensitivity = %s AND metadata_json = %s::jsonb{project_guard}", + sql.SQL( + "UPDATE {} SET domain = %s, sensitivity = %s, metadata_json = metadata_json || %s::jsonb " + "{} WHERE user_id = %s::uuid AND id = %s::uuid " + "AND domain = %s AND sensitivity = %s AND metadata_json = %s::jsonb{}" + ).format(sql.Identifier(table), sql.SQL(project_sql), sql.SQL(project_guard)), tuple(params), ) require_changed(cursor.rowcount, table, row_id) diff --git a/apps/api/src/alicebot_api/vault_doctor.py b/apps/api/src/alicebot_api/vault_doctor.py index 6c3f61bd1..9442b562a 100644 --- a/apps/api/src/alicebot_api/vault_doctor.py +++ b/apps/api/src/alicebot_api/vault_doctor.py @@ -90,9 +90,7 @@ def compile_local_vault_doctor( """Render the vault census for the acting local user.""" if COMMITTED_MEMORY_STATUSES != ("active", "accepted"): - raise RuntimeError( - "committed-fact COUNT SQL is written for active and accepted only" - ) + raise RuntimeError("committed-fact COUNT SQL is written for active and accepted only") resolved = Path(db_path).expanduser().resolve() with sqlite_user_connection(resolved, user_id) as connection: diff --git a/apps/api/src/alicebot_api/vnext_doctor.py b/apps/api/src/alicebot_api/vnext_doctor.py index e4c14b65b..5eda25f00 100644 --- a/apps/api/src/alicebot_api/vnext_doctor.py +++ b/apps/api/src/alicebot_api/vnext_doctor.py @@ -22,9 +22,7 @@ # The workspace dashboard runs this doctor on every load. Stop and say so # rather than scanning the rest of a large source table in that request. _FLAGGED_SOURCE_SCAN_LIMIT = 10_000 -LOCAL_VNEXT_CORS_RECOMMENDED_FIX = ( - "CORS_ALLOWED_ORIGINS=http://127.0.0.1:3000,http://localhost:3000" -) +LOCAL_VNEXT_CORS_RECOMMENDED_FIX = "CORS_ALLOWED_ORIGINS=http://127.0.0.1:3000,http://localhost:3000" PGVECTOR_MINIMUM_VERSION = (0, 8, 0) PGVECTOR_MINIMUM_VERSION_TEXT = ".".join(str(part) for part in PGVECTOR_MINIMUM_VERSION) @@ -159,10 +157,7 @@ def run(self, *, fix_safe: bool = False, ci: bool = False, include_content_diagn f"pgvector {pgvector_version} satisfies the required " f">= {PGVECTOR_MINIMUM_VERSION_TEXT} runtime contract." ), - message_fail=( - "pgvector is missing, unparseable, or older than " - f"{PGVECTOR_MINIMUM_VERSION_TEXT}." - ), + message_fail=(f"pgvector is missing, unparseable, or older than {PGVECTOR_MINIMUM_VERSION_TEXT}."), recommended_fix=( "Install pgvector >= " f"{PGVECTOR_MINIMUM_VERSION_TEXT}, run ALTER EXTENSION vector UPDATE, " @@ -220,7 +215,9 @@ def run(self, *, fix_safe: bool = False, ci: bool = False, include_content_diagn ) scheduler = daemon_status() - scheduler_known = not bool(scheduler.get("stopped")) or "pid file" not in str(scheduler.get("message", "")).casefold() + scheduler_known = ( + not bool(scheduler.get("stopped")) or "pid file" not in str(scheduler.get("message", "")).casefold() + ) self._check( checks, name="scheduler_daemon", @@ -232,7 +229,9 @@ def run(self, *, fix_safe: bool = False, ci: bool = False, include_content_diagn details=cast(JsonObject, scheduler), ) - health = VNextConnectorService(cast(Any, self.store), secret_provider=self.secret_provider).connector_health_all() + health = VNextConnectorService( + cast(Any, self.store), secret_provider=self.secret_provider + ).connector_health_all() failing_connectors = [] for item in cast(list[JsonObject], health.get("items", [])): failed_value = item.get("items_failed", 0) @@ -294,9 +293,7 @@ def _content_checks(self, checks: list[DoctorCheck]) -> None: flagged_ids, stopped_early = _flagged_source_scan(self.store) remedy = _flagged_source_remedy(self.store) if flagged_ids: - message = ( - f"{len(flagged_ids)} stored sources carry credential material. {remedy}" - ) + message = f"{len(flagged_ids)} stored sources carry credential material. {remedy}" else: message = "No stored source carries credential material." if stopped_early: @@ -411,9 +408,7 @@ def local_live_cors_status( ) -> JsonObject: merged_env = _merged_local_env(os.environ if env is None else env, cwd or Path.cwd()) frontend_api_base_url = ( - merged_env.get("NEXT_PUBLIC_ALICEBOT_API_BASE_URL") - or merged_env.get("ALICEBOT_API_BASE_URL") - or "" + merged_env.get("NEXT_PUBLIC_ALICEBOT_API_BASE_URL") or merged_env.get("ALICEBOT_API_BASE_URL") or "" ).strip() frontend_user_id = ( merged_env.get("NEXT_PUBLIC_ALICEBOT_USER_ID") or merged_env.get("ALICEBOT_USER_ID") or "" diff --git a/apps/api/src/alicebot_api/vnext_label_repair.py b/apps/api/src/alicebot_api/vnext_label_repair.py index 2db1b4d59..3dad92c6d 100644 --- a/apps/api/src/alicebot_api/vnext_label_repair.py +++ b/apps/api/src/alicebot_api/vnext_label_repair.py @@ -8,12 +8,23 @@ import json from collections.abc import Mapping, Sequence +from typing import TypedDict from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError, require_changed from alicebot_api.vnext_derived_labels import labels_raised_payload, settle_labels from alicebot_api.vnext_event_log import build_event_log_record from alicebot_api.vnext_project_scope import project_scope_identity + +class LabelParts(TypedDict): + domain: str + sensitivity: str + project_scope: list[str] + project_floor: list[str] + + +LabelRepair = tuple[str, str, str, LabelParts, LabelParts, dict[str, object]] + REPAIR_STATE_KEY = "derived_labels_v3" _TABLE_KIND = { "sources": "source", @@ -26,12 +37,9 @@ _WRITABLE = frozenset({"memories", "open_loops", "generated_artifacts", "projects"}) INPUT_SELECTS_V3 = { "sources": "SELECT id, user_id, domain, sensitivity, metadata_json, deleted_at FROM sources", - "memories": ( - "SELECT id, user_id, domain, sensitivity, metadata_json, value, project_id, deleted_at FROM memories" - ), + "memories": ("SELECT id, user_id, domain, sensitivity, metadata_json, value, project_id, deleted_at FROM memories"), "open_loops": ( - "SELECT id, user_id, domain, sensitivity, metadata_json, project_id, source_id, memory_id " - "FROM open_loops" + "SELECT id, user_id, domain, sensitivity, metadata_json, project_id, source_id, memory_id FROM open_loops" ), "generated_artifacts": ( "SELECT id, user_id, domain, sensitivity, metadata_json, artifact_type FROM generated_artifacts" @@ -70,7 +78,7 @@ def _same(previous: Mapping[str, object], new: Mapping[str, object]) -> bool: def plan_label_repairs( tables: Mapping[str, Sequence[Mapping[str, object]]], -) -> list[tuple[str, str, str, dict[str, object], dict[str, object], dict[str, object]]]: +) -> list[LabelRepair]: """Label changes for derived rows whose stored label is below the inputs. A malformed record makes its row unverified and is not rewritten. A cycle @@ -91,17 +99,17 @@ def plan_label_repairs( nodes.append(node) index.append((table, node)) settled = settle_labels(nodes, on_cycle="raise") - changes = [] + changes: list[LabelRepair] = [] for (table, node), label in zip(index, settled.rows, strict=True): if not label.derived or label.unverified or table not in _WRITABLE: continue - previous = { + previous: LabelParts = { "domain": str(node.get("domain") or "unknown"), "sensitivity": str(node.get("sensitivity") or "unknown"), "project_scope": list(label.stored_scope), "project_floor": list(label.stored_floor), } - new = { + new: LabelParts = { "domain": label.domain, "sensitivity": label.sensitivity, "project_scope": list(label.project_scope), @@ -141,7 +149,9 @@ def _load_tables(conn) -> dict[str, list[dict[str, object]]]: def _stamped(conn) -> bool: - return conn.execute("SELECT value FROM alice_schema_state WHERE key = ?", (REPAIR_STATE_KEY,)).fetchone() is not None + return ( + conn.execute("SELECT value FROM alice_schema_state WHERE key = ?", (REPAIR_STATE_KEY,)).fetchone() is not None + ) def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = False) -> None: @@ -166,9 +176,18 @@ def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = Fal raw_metadata = node.get("_stored_metadata") changed = conn.execute( _UPDATES[table], - (new["domain"], new["sensitivity"], json.dumps(metadata), - label_project_id(new["project_scope"]), user, stored, - previous["domain"], previous["sensitivity"], raw_metadata, node.get("project_id")), + ( + new["domain"], + new["sensitivity"], + json.dumps(metadata), + label_project_id(new["project_scope"]), + user, + stored, + previous["domain"], + previous["sensitivity"], + raw_metadata, + node.get("project_id"), + ), ).rowcount require_changed(changed, table, stored) for table, user, stored, previous, new, _node in changes: @@ -206,7 +225,7 @@ def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = Fal def classify_stored_labels( tables: Mapping[str, Sequence[Mapping[str, object]]], -) -> tuple[list[tuple[str, str, str, dict[str, object], dict[str, object], dict[str, object]]], dict[str, list[str]]]: +) -> tuple[list[LabelRepair], dict[str, list[str]]]: """Rows below their inputs, and unverified ids grouped by reason. A cycle is reported as unverified instead of raising. ``labels check`` uses this. @@ -226,7 +245,7 @@ def classify_stored_labels( nodes.append(node) index.append((table, node)) settled = settle_labels(nodes, on_cycle="unverified") - below = [] + below: list[LabelRepair] = [] unverified: dict[str, list[str]] = {} for (table, node), label in zip(index, settled.rows, strict=True): if not label.derived: @@ -237,13 +256,13 @@ def classify_stored_labels( continue if table not in _WRITABLE: continue - previous = { + previous: LabelParts = { "domain": str(node.get("domain") or "unknown"), "sensitivity": str(node.get("sensitivity") or "unknown"), "project_scope": list(label.stored_scope), "project_floor": list(label.stored_floor), } - new = { + new: LabelParts = { "domain": label.domain, "sensitivity": label.sensitivity, "project_scope": list(label.project_scope), @@ -296,7 +315,9 @@ def label_gap_counts(store: object) -> tuple[int, int]: """ conn = getattr(store, "conn", None) - module = type(conn).__module__ if conn is not None else "" + if conn is None: + raise LabelCheckUnavailable("derived label counts are unavailable for this store") + module = type(conn).__module__ try: if module.startswith("sqlite3"): tables = _load_tables(conn) @@ -350,9 +371,7 @@ def recorded_sqlite_label_repairs(conn, user_id: str) -> dict[tuple[str, str, st if not isinstance(previous, Mapping): continue repairs.setdefault((table, row_id, "domain"), set()).add(str(previous.get("domain") or "unknown")) - repairs.setdefault((table, row_id, "sensitivity"), set()).add( - str(previous.get("sensitivity") or "unknown") - ) + repairs.setdefault((table, row_id, "sensitivity"), set()).add(str(previous.get("sensitivity") or "unknown")) scope = previous.get("project_scope") floor = previous.get("project_floor") repairs.setdefault((table, row_id, "project_scope"), set()).add( From d547df4eccb5b52401f19e2a12ccf4a2e3366f51 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:27:44 +0200 Subject: [PATCH 130/270] Prove label upgrade restore and interruption safety through real readers --- .../test_derived_labels_migration_postgres.py | 456 ++++++++++++++++++ tests/unit/test_cli_package_split.py | 8 +- tests/unit/test_derived_domain_mutations.py | 6 + tests/unit/test_sqlite_derived_labels_v3.py | 429 ++++++++++++++++ 4 files changed, 897 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_derived_labels_migration_postgres.py create mode 100644 tests/unit/test_sqlite_derived_labels_v3.py diff --git a/tests/integration/test_derived_labels_migration_postgres.py b/tests/integration/test_derived_labels_migration_postgres.py new file mode 100644 index 000000000..0c9152a5e --- /dev/null +++ b/tests/integration/test_derived_labels_migration_postgres.py @@ -0,0 +1,456 @@ +"""Owner migration and application repair acceptance with real restricted readers.""" + +from __future__ import annotations + +from contextlib import contextmanager +import json +import threading +from types import SimpleNamespace +from uuid import uuid4 + +from alembic import command, op +import psycopg +import pytest + +import alicebot_api.main as main_module +from alicebot_api import vnext_label_repair as repair +from alicebot_api.cli import labels +from alicebot_api.config import Settings +from alicebot_api.db import close_connection_pools, user_connection, user_read_snapshot_connection +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError +from alicebot_api.migrations import make_alembic_config +from alicebot_api.routers import vnext_retrieval as retrieval_router +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_doctor import VNextDoctorService +from alicebot_api.vnext_derived_labels import with_derived_from +from alicebot_api.vnext_label_guard import LabelGuard +from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.test_vnext_omitted_domains_api import invoke_request +from tests.integration.conftest import _create_role_separated_database, _drop_database +from urllib.parse import urlsplit + +TABLES = ("sources", "memories", "open_loops", "generated_artifacts", "beliefs", "event_log", "projects") +SENTINEL = "Violet private migration sentinel" + + +@contextmanager +def owner_bracket(url): + with psycopg.connect(url) as conn: + for table in TABLES: + conn.execute(f"ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY") + yield conn + for table in TABLES: + conn.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + + +def assert_roles_and_force(urls): + for url in (urls["admin"], urls["app"]): + with psycopg.connect(url) as conn: + assert conn.execute( + "SELECT rolsuper, rolbypassrls FROM pg_roles WHERE rolname=current_user" + ).fetchone() == (False, False) + with psycopg.connect(urls["admin"]) as conn: + assert all( + row[0] + for row in conn.execute("SELECT relforcerowsecurity FROM pg_class WHERE relname=ANY(%s)", (list(TABLES),)) + ) + + +def seed_stale(urls, user=None): + user = user or uuid4() + with without_insert_floor(), user_connection(urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"fixture-{user}@example.invalid", "Synthetic fixture") + store = PostgresVNextStore(conn) + project = store.create_project({"slug": "alpha", "name": "Alpha", "domain": "project", "sensitivity": "public"}) + source = store.create_source( + { + "source_type": "note", + "title": "Synthetic private input", + "content_hash": "sha256:" + uuid4().hex, + "domain": "health", + "sensitivity": "confidential", + "metadata_json": {}, + } + ) + copies = [] + for i in range(3): + copies.append( + store.create_memory( + { + "memory_key": f"derived.{i}", + "canonical_text": SENTINEL, + "domain": "unknown", + "sensitivity": "public", + "status": "active", + "project_id": str(project["id"]), + "metadata_json": {"source_id": str(source["id"]), "project_scope": [str(project["id"])]}, + } + ) + ) + loop = store.create_open_loop( + { + "title": SENTINEL, + "domain": "unknown", + "sensitivity": "public", + "source_id": str(source["id"]), + "project_id": str(project["id"]), + "metadata_json": { + "discovered_by": "vnext_daily_brief", + "source_id": str(source["id"]), + "project_scope": [str(project["id"])], + }, + }, + actor_type="user", + ) + belief = store.create_belief( + {"memory_id": str(copies[0]["id"]), "claim": "Synthetic belief", "status": "active", "confidence": 0.8}, + actor_type="user", + ) + reports = [] + for key, metadata in ( + ("source", {"input_summary": {"source_ids": [str(source["id"])]}}), + ("memory", {"input_summary": {"memory_ids": [str(copies[0]["id"])]}}), + ("loop", {"input_summary": {"open_loop_ids": [str(loop["id"])]}}), + ("belief", {"belief_ids": [str(belief["id"])]}), + ): + reports.append( + store.create_artifact( + { + "artifact_type": "contradiction_report" if key == "belief" else "daily_brief", + "title": f"Synthetic {key} report", + "content_markdown": SENTINEL, + "status": "needs_review", + "domain": "unknown", + "sensitivity": "public", + "metadata_json": {**metadata, "project_scope": [str(project["id"])]}, + } + ) + ) + # Raw state matches a pre-floor accepted project update while keeping its real recorded parents. + conn.execute( + "UPDATE projects SET metadata_json=%s::jsonb WHERE id=%s", + (json.dumps(with_derived_from({}, {"memories": [copies[0]], "artifacts": [reports[0]]})), project["id"]), + ) + return user, { + "memories": [str(r["id"]) for r in copies], + "open_loops": [str(loop["id"])], + "generated_artifacts": [str(r["id"]) for r in reports], + "projects": [str(project["id"])], + } + + +def stored(urls, user, targets): + with user_connection(urls["app"], user) as conn: + rows = { + table: conn.execute( + f"SELECT domain, sensitivity, metadata_json{', project_id' if table in {'memories', 'open_loops'} else ''} FROM {table} WHERE id=ANY(%s::uuid[]) ORDER BY id", + (ids,), + ).fetchall() + for table, ids in targets.items() + } + events = conn.execute( + "SELECT target_id, payload_json FROM event_log WHERE event_type LIKE '%%.labels_raised' ORDER BY id" + ).fetchall() + return rows, events + + +def assert_repaired(urls, user, targets): + rows, events = stored(urls, user, targets) + for table, values in rows.items(): + assert len(values) == len(targets[table]) + for row in values: + assert (row["domain"], row["sensitivity"]) == ("health", "confidential"), (table, row) + assert row["metadata_json"]["project_scope"] == [], (table, row) + assert row["metadata_json"]["project_floor"] == [], (table, row) + if "project_id" in row: + assert row["project_id"] is None + assert len(events) == sum(len(ids) for ids in targets.values()) + assert SENTINEL not in json.dumps(events, default=str) + return rows, events + + +def restricted_reads(urls, user, targets, monkeypatch, *, guard_off=False): + with user_connection(urls["app"], user) as conn: + _, raw = create_agent_key( + PostgresVNextStore(conn), user_id=user, agent_id="migration-reader", permission_profile="read_only_agent" + ) + with monkeypatch.context() as patch: + for module in (main_module, retrieval_router): + patch.setattr(module, "get_settings", lambda: Settings(database_url=urls["app"])) + patch.setenv("ALICE_AGENT_API_KEY", raw) + patch.setenv("ALICE_MCP_FULL_TOOLS", "1") + patch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + if guard_off: + patch.setattr(LabelGuard, "effective_row", lambda self, kind, row: row) + status, pack = invoke_request( + "POST", + "/v0/vnext/context-packs", + authorization=f"Bearer {raw}", + payload={"user_id": str(user), "query": "Violet", "scope": {}, "options": {"include_sources": True}}, + ) + assert status == 201, pack + assert SENTINEL not in json.dumps(pack), (guard_off, pack) + context = MCPRuntimeContext(database_url=urls["app"], user_id=user) + result = call_mcp_tool(context, name="alice_recall", arguments={"query": "Violet"}) + assert SENTINEL not in json.dumps(result, default=str) + for row_id in targets["memories"]: + with pytest.raises(MCPToolError, match="requested explanation is unavailable"): + call_mcp_tool(context, name="alice_explain", arguments={"memory_id": row_id}) + + +def cli_context(urls, user): + return SimpleNamespace(database_url=urls["app"], user_id=user) + + +@pytest.mark.parametrize("revision", ("20260721_0094", "20261004_0095")) +def test_upgrade_repairs_all_labels_as_nobypassrls_owner(database_urls, monkeypatch, revision): + command.upgrade(make_alembic_config(database_urls["admin"]), revision) + user, targets = seed_stale(database_urls) + command.upgrade(make_alembic_config(database_urls["admin"]), "head") + assert_roles_and_force(database_urls) + assert_repaired(database_urls, user, targets) + restricted_reads(database_urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(database_urls, user, targets, monkeypatch) + + +def test_head_restored_rows_are_guarded_then_fixed_by_owner_command(migrated_database_urls, monkeypatch): + urls = migrated_database_urls + user, targets = seed_stale(urls) + assert stored(urls, user, targets)[0]["memories"][0]["sensitivity"] == "public" + restricted_reads(urls, user, targets, monkeypatch) + assert labels._run_vnext_labels_repair(cli_context(urls, user), None).startswith("labels repair updated ") + assert_repaired(urls, user, targets) + restricted_reads(urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(urls, user, targets, monkeypatch) + assert labels._run_vnext_labels_check(cli_context(urls, user), None) == "below_inputs 0" + + +def test_labels_check_is_one_read_only_repeatable_snapshot(migrated_database_urls, monkeypatch): + urls = migrated_database_urls + user, targets = seed_stale(urls) + original = labels._postgres_tables + + def inspect_snapshot(conn): + assert conn.execute("SHOW transaction_isolation").fetchone()["transaction_isolation"] == "repeatable read" + assert conn.execute("SHOW transaction_read_only").fetchone()["transaction_read_only"] == "on" + before = conn.execute("SELECT count(*) AS count FROM memories").fetchone()["count"] + with user_connection(urls["app"], user) as other: + other.execute("UPDATE memories SET sensitivity='regulated' WHERE id=%s", (targets["memories"][0],)) + rows = original(conn) + assert len(rows["memories"]) == before + assert all(row["sensitivity"] == "public" for row in rows["memories"]) + with pytest.raises(psycopg.errors.ReadOnlySqlTransaction): + with conn.transaction(): + conn.execute("UPDATE memories SET sensitivity='public'") + return rows + + monkeypatch.setattr(labels, "_postgres_tables", inspect_snapshot) + with pytest.raises(SystemExit) as caught: + labels._run_vnext_labels_check(cli_context(urls, user), None) + assert caught.value.code == 1 + with user_read_snapshot_connection(urls["app"], user) as conn: + assert conn.execute("SELECT app.current_user_id() AS id").fetchone()["id"] == user + with psycopg.connect(urls["app"]) as conn: + assert conn.execute("SELECT app.current_user_id()").fetchone()[0] is None + + +def test_repair_compare_and_set_failure_after_writes_rolls_everything_back(migrated_database_urls, monkeypatch): + urls = migrated_database_urls + user, targets = seed_stale(urls) + before = stored(urls, user, targets) + original = labels.require_changed + writes = [] + + def fail_third(count, table, row_id): + if row_id != "planned rows": + writes.append((count, table, row_id)) + if len(writes) == 3: + count = 0 + return original(count, table, row_id) + + with monkeypatch.context() as patch: + patch.setattr(labels, "require_changed", fail_third) + with pytest.raises(SystemExit) as caught: + labels._run_vnext_labels_repair(cli_context(urls, user), None) + assert caught.value.code == 2 and len(writes) == 3 + assert stored(urls, user, targets) == before + labels._run_vnext_labels_repair(cli_context(urls, user), None) + assert_repaired(urls, user, targets) + restricted_reads(urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(urls, user, targets, monkeypatch) + + +def test_doctor_counts_real_postgres_rows_and_failure_stays_warning(migrated_database_urls, monkeypatch): + urls = migrated_database_urls + user, targets = seed_stale(urls) + with user_connection(urls["app"], user) as conn: + store = PostgresVNextStore(conn) + below, unverified = repair.label_gap_counts(store) + assert below == sum(map(len, targets.values())) and unverified == 0 + + def broken_loader(conn): + conn.execute("SELECT * FROM synthetic_missing_label_table") + + monkeypatch.setattr(repair, "load_postgres_label_tables", broken_loader) + with pytest.raises(repair.LabelCheckUnavailable): + repair.label_gap_counts(store) + assert conn.execute("SELECT 1 AS ok").fetchone()["ok"] == 1 + result = VNextDoctorService(store).run(ci=True) + check = next(row for row in result["checks"] if row["name"] == "derived_labels") + assert check["severity"] == "warning" and check["status"] == "fail" + assert "unavailable" in check["message"] and "0 below" not in check["message"] + assert conn.execute("SELECT 1 AS ok").fetchone()["ok"] == 1 + + +def test_interrupted_migration_rolls_back_and_restored_backup_repeats(database_urls, monkeypatch): + urls = database_urls + config = make_alembic_config(urls["admin"]) + command.upgrade(config, "20261004_0095") + user, targets = seed_stale(urls) + before = stored(urls, user, targets) + backup_name = "alicebot_backup_" + uuid4().hex[:12] + restored_name = "alicebot_restored_" + uuid4().hex[:12] + close_connection_pools() + _create_role_separated_database(backup_name, template=urlsplit(urls["admin"]).path.lstrip("/")) + try: + original = repair.plan_label_repairs + + def interrupted_plan(tables): + # Alembic iterates this sequence and really writes its first three rows + # before the iterator raises; labels, events and FORCE stay atomic. + changes = original(tables) + for index, change in enumerate(changes): + if index == 3: + raise RuntimeError("injected after three migration writes") + yield change + + with monkeypatch.context() as patch: + patch.setattr(repair, "plan_label_repairs", interrupted_plan) + with pytest.raises(RuntimeError, match="three migration writes"): + command.upgrade(config, "head") + assert stored(urls, user, targets) == before + assert_roles_and_force(urls) + with psycopg.connect(urls["admin"]) as conn: + assert conn.execute("SELECT version_num FROM alembic_version").fetchone()[0] == "20261004_0095" + command.upgrade(config, "head") + assert_repaired(urls, user, targets) + restricted_reads(urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(urls, user, targets, monkeypatch) + close_connection_pools() + restored_urls = _create_role_separated_database(restored_name, template=backup_name) + try: + assert stored(restored_urls, user, targets) == before + command.upgrade(make_alembic_config(restored_urls["admin"]), "head") + assert_repaired(restored_urls, user, targets) + assert_roles_and_force(restored_urls) + restricted_reads(restored_urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(restored_urls, user, targets, monkeypatch) + finally: + close_connection_pools() + _drop_database(restored_name) + finally: + _drop_database(backup_name) + + +@pytest.mark.parametrize("table", TABLES) +def test_each_force_bracket_guard_is_behaviorally_required(database_urls, monkeypatch, table): + urls = database_urls + command.upgrade(make_alembic_config(urls["admin"]), "20261004_0095") + user, targets = seed_stale(urls) + original = op.execute + + def omit_one(statement, *args, **kwargs): + if str(statement) == f"ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY": + return None + return original(statement, *args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(op, "execute", omit_one) + try: + command.upgrade(make_alembic_config(urls["admin"]), "head") + except Exception as exc: + assert table == "event_log", (table, exc) + else: + with pytest.raises(AssertionError): + assert_repaired(urls, user, targets) + # Return every intentional mutant to a safe settled state and a real reader. + command.downgrade(make_alembic_config(urls["admin"]), "20261004_0095") + command.upgrade(make_alembic_config(urls["admin"]), "head") + assert_repaired(urls, user, targets) + assert_roles_and_force(urls) + restricted_reads(urls, user, targets, monkeypatch) + + +def test_downgrade_to_0095_and_upgrade_is_idempotent(database_urls, monkeypatch): + urls = database_urls + config = make_alembic_config(urls["admin"]) + command.upgrade(config, "20261004_0095") + user, targets = seed_stale(urls) + command.upgrade(config, "head") + before = assert_repaired(urls, user, targets) + command.downgrade(config, "20261004_0095") + command.upgrade(config, "head") + assert stored(urls, user, targets) == before + restricted_reads(urls, user, targets, monkeypatch) + + +def test_labels_repair_cannot_overwrite_a_relabel(migrated_database_urls, monkeypatch): + from concurrent.futures import ThreadPoolExecutor + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock, held_label_locks + + urls = migrated_database_urls + user, targets = seed_stale(urls) + planned, release, relabel_started = threading.Event(), threading.Event(), threading.Event() + original = labels._postgres_tables + + def pause_snapshot(conn): + assert held_label_locks(PostgresVNextStore(conn)) == (True, True, True) + rows = original(conn) + planned.set() + assert release.wait(10) + return rows + + monkeypatch.setattr(labels, "_postgres_tables", pause_snapshot) + + def raise_source(): + with user_connection(urls["app"], user) as conn: + store = PostgresVNextStore(conn) + source_id = str(conn.execute("SELECT id FROM sources").fetchone()["id"]) + relabel_started.set() + store.lock_graph_mutation() + acquire_exclusive_label_lock(store) + store.update_source(source_id=source_id, patch={"sensitivity": "regulated"}, actor_type="user") + + with ThreadPoolExecutor(max_workers=2) as executor: + repaired = executor.submit(labels._run_vnext_labels_repair, cli_context(urls, user), None) + assert planned.wait(10) + relabelled = executor.submit(raise_source) + assert relabel_started.wait(10) + try: + # The second transaction has really reached its lock request. + import time + + deadline = time.monotonic() + 5 + waiting = False + while time.monotonic() < deadline: + with psycopg.connect(urls["admin"]) as conn: + waiting = bool( + conn.execute( + "SELECT count(*) FROM pg_locks WHERE locktype='advisory' AND NOT granted AND database=(SELECT oid FROM pg_database WHERE datname=current_database())" + ).fetchone()[0] + ) + if waiting: + break + time.sleep(0.01) + assert waiting and not relabelled.done() + finally: + release.set() + repaired.result(timeout=10) + relabelled.result(timeout=10) + rows, _events = stored(urls, user, targets) + assert all(row["sensitivity"] == "regulated" for values in rows.values() for row in values) + restricted_reads(urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(urls, user, targets, monkeypatch) diff --git a/tests/unit/test_cli_package_split.py b/tests/unit/test_cli_package_split.py index 14875781a..2cc8a95d3 100644 --- a/tests/unit/test_cli_package_split.py +++ b/tests/unit/test_cli_package_split.py @@ -35,6 +35,7 @@ "errors.py", "evals.py", "memories.py", + "labels.py", "models.py", "parser.py", "runner.py", @@ -49,11 +50,13 @@ # and again after adding it to `vnext sources capture-file`, `vnext connectors # browser-clipper capture` and `vnext agents ingest-output` (three more # actions, no new command). +# The owner label check and repair commands add three parsers, four actions, +# two leaves and two handlers under both flag states. # Both keys are updated together: leaving one at its old value is how the # other half silently rots. EXPECTED_PARSER_RECEIPTS = { - False: (159, 735, 122, 118), - True: (163, 768, 125, 121), + False: (162, 739, 124, 120), + True: (166, 772, 127, 123), } EXPECTED_PUBLIC_NAME_COUNT = 270 EXPECTED_PUBLIC_NAMES_SHA256 = "8d97ffb088d5d8dea239c81589e9c109b81f7dc50b916d7bfb593ce13acae5fa" @@ -68,6 +71,7 @@ "errors", "evals", "memories", + "labels", "models", "parser", "runner", diff --git a/tests/unit/test_derived_domain_mutations.py b/tests/unit/test_derived_domain_mutations.py index aa28d4819..0e73a7ae5 100644 --- a/tests/unit/test_derived_domain_mutations.py +++ b/tests/unit/test_derived_domain_mutations.py @@ -78,6 +78,12 @@ def fresh(check): def restore(directory, patch): + # This harness protects the frozen v2 repair. Its v3 successor must not + # mask a removed v2 guard; v3 has its own staged-restore mutation cases. + from alicebot_api import sqlite_schema, vnext_label_repair + patch.setattr(sqlite_schema, "_relabel_derived_labels", lambda conn: None) + patch.setattr(vnext_label_repair, "relabel_labels_sqlite", lambda conn, **kwargs: None) + patch.setattr(vnext_label_repair, "recorded_sqlite_label_repairs", lambda conn, user: {}) review.test_restore_repairs_derived_rows_before_publication(directory, patch, True) diff --git a/tests/unit/test_sqlite_derived_labels_v3.py b/tests/unit/test_sqlite_derived_labels_v3.py new file mode 100644 index 000000000..b07162b93 --- /dev/null +++ b/tests/unit/test_sqlite_derived_labels_v3.py @@ -0,0 +1,429 @@ +"""Stored labels, staged restore and crash recovery through real key readers.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import select +import shutil +import sqlite3 +import subprocess +import sys +from uuid import UUID + +import pytest + +from alicebot_api import sqlite_schema, vnext_label_repair as repair +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError +from alicebot_api.onramp import bootstrap_database, main as onramp_main, sqlite_url_for_path +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_label_guard import LabelGuard +from alicebot_api.vnext_label_writes import without_insert_floor +from tests.unit.test_derived_domain_fence import USER + +SENTINEL = "Violet confidential recovery sentinel" + + +def old_vault(path, monkeypatch, *, copies=1, cross_project=False, domain="health"): + with monkeypatch.context() as patch: + patch.setattr(sqlite_schema, "_relabel_derived_domains", lambda conn: None) + patch.setattr(sqlite_schema, "_relabel_derived_labels", lambda conn: None) + bootstrap_database(path, user_id=USER, user_email="fixture@example.invalid") + with without_insert_floor(), sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source( + { + "source_type": "note", + "title": "Synthetic input", + "content_hash": "sha256:violet", + "domain": domain, + "sensitivity": "internal" if cross_project else "confidential", + "metadata_json": {} if cross_project else {"project_scope": ["beta"]}, + } + ) + ids = [] + for index in range(copies): + row = store.create_memory( + { + "memory_key": f"copy.{index}", + "canonical_text": SENTINEL, + "status": "active", + "domain": "project" if cross_project else "unknown", + "sensitivity": "public", + "project_id": "alpha", + "metadata_json": {"source_id": source["id"], "project_scope": ["alpha"]}, + } + ) + ids.append(str(row["id"])) + if cross_project: + a = store.create_memory( + { + "memory_key": "alpha.input", + "canonical_text": "Violet alpha input", + "status": "active", + "domain": "project", + "sensitivity": "internal", + "metadata_json": {"project_scope": ["alpha"]}, + } + ) + b = store.create_memory( + { + "memory_key": "beta.input", + "canonical_text": "Violet beta input", + "status": "active", + "domain": "project", + "sensitivity": "internal", + "metadata_json": {"project_scope": ["beta"]}, + } + ) + for key, marker in ( + ("rollup", {"rollup": {"member_ids": [a["id"], b["id"]]}}), + ("consolidation", {"consolidation": {"cluster_member_ids": [a["id"], b["id"]]}}), + ): + row = store.create_memory( + { + "memory_key": key, + "canonical_text": SENTINEL, + "status": "active", + "domain": "project", + "sensitivity": "internal", + "value": {"rollup": {"member_ids": [a["id"], b["id"]]}} if key == "rollup" else {}, + "metadata_json": { + **marker, + "candidate_kind": "memory_rollup" if key == "rollup" else "memory_consolidation", + "project_scope": ["alpha", "beta"], + }, + } + ) + ids.append(str(row["id"])) + return ids + + +def export_old(path, backup, monkeypatch): + with monkeypatch.context() as patch: + patch.setattr(sqlite_schema, "_relabel_derived_domains", lambda conn: None) + patch.setattr(sqlite_schema, "_relabel_derived_labels", lambda conn: None) + assert onramp_main(["export", "--db", str(path), "--user-id", USER, "--out", str(backup)]) == 0 + + +def read_stored_columns(path, ids): + with sqlite3.connect(path) as conn: + rows = [ + conn.execute( + "SELECT domain, sensitivity, project_id, metadata_json FROM memories WHERE id=?", (row_id,) + ).fetchone() + for row_id in ids + ] + events = conn.execute("SELECT payload_json FROM event_log WHERE event_type='memory.labels_raised'").fetchall() + state = conn.execute("SELECT value FROM alice_schema_state WHERE key=?", (repair.REPAIR_STATE_KEY,)).fetchone() + return [(row[0], row[1], row[2], json.loads(row[3])) for row in rows], events, state + + +def restricted_reads(path, ids, monkeypatch, *, guard_off=False, project=None): + with sqlite_user_connection(path, USER) as conn: + _, raw = create_agent_key( + SQLiteVNextStore(conn, USER), + user_id=USER, + agent_id="recovery-reader", + permission_profile="read_only_agent", + project_scope=project, + ) + with monkeypatch.context() as patch: + patch.setenv("ALICE_AGENT_API_KEY", raw) + patch.setenv("ALICE_MCP_FULL_TOOLS", "1") + patch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + if guard_off: + patch.setattr(LabelGuard, "effective_row", lambda self, kind, row: row) + context = MCPRuntimeContext(database_url=sqlite_url_for_path(path), user_id=UUID(USER)) + for name, args in (("alice_recall", {"query": "Violet"}), ("alice_context_pack", {"query": "Violet"})): + result = call_mcp_tool(context, name=name, arguments=args) + assert SENTINEL not in json.dumps(result, default=str), (name, guard_off, result) + assert not any(row_id in json.dumps(result, default=str) for row_id in ids) + for row_id in ids: + with pytest.raises(MCPToolError, match="requested explanation is unavailable"): + call_mcp_tool(context, name="alice_explain", arguments={"memory_id": row_id}) + + +@pytest.mark.parametrize("upgraded", (False, True), ids=("fresh", "already_v3")) +def test_old_backup_is_repaired_before_publication(tmp_path, monkeypatch, upgraded): + old, target, backup = tmp_path / "old.db", tmp_path / "target.db", tmp_path / "old.jsonl" + ids = old_vault(old, monkeypatch) + export_old(old, backup, monkeypatch) + assert read_stored_columns(old, ids)[0][0][1] == "public" + if upgraded: + bootstrap_database(target, user_id=USER, user_email="fixture@example.invalid") + assert read_stored_columns(target, [])[2] is not None + assert onramp_main(["import", "--db", str(target), "--user-id", USER, "--in", str(backup)]) == 0 + rows, events, state = read_stored_columns(target, ids) + assert rows[0][:3] == ("health", "confidential", None) + assert rows[0][3]["project_scope"] == [] + assert rows[0][3]["project_floor"] == ["beta"] + assert len(events) == 1 and state is not None + assert SENTINEL not in json.dumps(events) + restricted_reads(target, ids, monkeypatch, guard_off=True) + restricted_reads(target, ids, monkeypatch) + + +def test_old_backup_with_cross_project_aggregates(tmp_path, monkeypatch): + old, target, backup = tmp_path / "old.db", tmp_path / "target.db", tmp_path / "old.jsonl" + ids = old_vault(old, monkeypatch, cross_project=True, domain="project") + export_old(old, backup, monkeypatch) + assert onramp_main(["import", "--db", str(target), "--user-id", USER, "--in", str(backup)]) == 0 + rows, events, _ = read_stored_columns(target, ids) + assert all(row[2] is None and row[3]["project_scope"] == [] for row in rows) + assert rows[0][3]["project_floor"] == [] + assert all(row[3]["project_floor"] == ["alpha", "beta"] for row in rows[1:]) + assert len(events) == 3 + restricted_reads(target, ids, monkeypatch, guard_off=True, project="alpha") + restricted_reads(target, ids, monkeypatch, project="alpha") + + +def test_explicit_repair_ignores_completion_stamp(tmp_path, monkeypatch): + path = tmp_path / "stamped.db" + ids = old_vault(path, monkeypatch, domain="project") + with sqlite3.connect(path) as conn: + conn.execute("INSERT OR REPLACE INTO alice_schema_state VALUES (?, '1')", (repair.REPAIR_STATE_KEY,)) + assert onramp_main(["labels", "repair", "--db", str(path), "--user-id", USER]) == 0 + rows, events, _ = read_stored_columns(path, ids) + assert rows[0][1] == "confidential", "explicit repair must revisit a stamped vault" + assert len(events) == 1 + restricted_reads(path, ids, monkeypatch, guard_off=True) + restricted_reads(path, ids, monkeypatch) + + +def test_physical_copy_is_repaired_on_first_open(tmp_path, monkeypatch): + old, target = tmp_path / "old.db", tmp_path / "copied.db" + ids = old_vault(old, monkeypatch, domain="project") + shutil.copy2(old, target) + assert read_stored_columns(target, ids)[0][0][1] == "public" + with sqlite_user_connection(target, USER): + pass + assert read_stored_columns(target, ids)[0][0][1] == "confidential" + restricted_reads(target, ids, monkeypatch, guard_off=True) + restricted_reads(target, ids, monkeypatch) + + +def snapshot(path): + with sqlite3.connect(path) as conn: + return { + table: conn.execute(f"SELECT * FROM {table} ORDER BY 1").fetchall() + for table in ("memories", "sources", "event_log", "alice_schema_state") + } + + +def assert_recovered(path, ids, monkeypatch): + with sqlite_user_connection(path, USER): + pass + rows, _, state = read_stored_columns(path, ids) + assert all(row[1] == "confidential" for row in rows) and state is not None + restricted_reads(path, ids, monkeypatch, guard_off=True) + restricted_reads(path, ids, monkeypatch) + + +def test_interrupted_sqlite_repair_rolls_back_and_backup_repeats(tmp_path, monkeypatch): + path, backup = tmp_path / "interrupted.db", tmp_path / "preupgrade.db" + ids = old_vault(path, monkeypatch, copies=3, domain="project") + shutil.copy2(path, backup) + before = snapshot(path) + original = repair.require_changed + calls = [] + + def fail_after_writes(*args): + original(*args) + calls.append(args) + if len(calls) == 2: + raise RuntimeError("injected after second label write") + + with sqlite3.connect(path) as conn, monkeypatch.context() as patch: + patch.setattr(repair, "require_changed", fail_after_writes) + with pytest.raises(RuntimeError, match="after second"): + repair.relabel_labels_sqlite(conn, restoring=True) + assert len(calls) == 2 and snapshot(path) == before + assert_recovered(path, ids, monkeypatch) + repeated = tmp_path / "restored_preupgrade.db" + shutil.copy2(backup, repeated) + assert snapshot(repeated) == before + assert_recovered(repeated, ids, monkeypatch) + + +def test_killed_sqlite_repair_rolls_back_and_backup_repeats(tmp_path, monkeypatch): + path, backup = tmp_path / "killed.db", tmp_path / "preupgrade.db" + ids = old_vault(path, monkeypatch, copies=3, domain="project") + shutil.copy2(path, backup) + before = snapshot(path) + program = """ +import sqlite3, sys, time +from alicebot_api import vnext_label_repair as repair +original = repair.require_changed +calls = 0 +def pause(*args): + global calls + original(*args) + calls += 1 + if calls == 2: + print('second write reached', flush=True) + time.sleep(30) +repair.require_changed = pause +with sqlite3.connect(sys.argv[1]) as conn: + repair.relabel_labels_sqlite(conn, restoring=True) +""" + process = subprocess.Popen( + [sys.executable, "-c", program, str(path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=os.environ.copy(), + ) + try: + assert select.select([process.stdout], [], [], 10)[0], "child never reached its writes" + assert process.stdout.readline().strip() == "second write reached" + process.kill() + process.wait(timeout=10) + assert snapshot(path) == before + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + assert_recovered(path, ids, monkeypatch) + repeated = tmp_path / "restored_preupgrade.db" + shutil.copy2(backup, repeated) + assert snapshot(repeated) == before + assert_recovered(repeated, ids, monkeypatch) + + +def test_repeating_skip_composes_v2_v3_legacy_scope_and_open_loop(tmp_path, monkeypatch): + old, target, backup = tmp_path / "old.db", tmp_path / "target.db", tmp_path / "old.jsonl" + ids = old_vault(old, monkeypatch) + with monkeypatch.context() as patch: + patch.setattr(sqlite_schema, "_relabel_derived_domains", lambda conn: None) + patch.setattr(sqlite_schema, "_relabel_derived_labels", lambda conn: None) + with without_insert_floor(), sqlite_user_connection(old, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source_id = conn.execute("SELECT id FROM sources").fetchone()["id"] + legacy = store.create_memory( + { + "memory_key": "legacy.project", + "canonical_text": SENTINEL, + "domain": "unknown", + "sensitivity": "public", + "status": "active", + "project_id": "alpha", + "metadata_json": {"source_id": source_id}, + } + ) + loop = store.create_open_loop( + { + "title": SENTINEL, + "domain": "unknown", + "sensitivity": "public", + "source_id": source_id, + "metadata_json": {"source_id": source_id, "discovered_by": "vnext_daily_brief"}, + } + ) + ids.append(str(legacy["id"])) + v2_input = store.create_memory( + { + "memory_key": "v2.input", + "canonical_text": "Violet private original", + "domain": "health", + "sensitivity": "confidential", + "status": "active", + } + ) + aggregate = store.create_memory( + { + "memory_key": "v2.aggregate", + "canonical_text": SENTINEL, + "domain": "unknown", + "sensitivity": "public", + "status": "active", + "metadata_json": {"consolidation": {"cluster_member_ids": [v2_input["id"]]}}, + } + ) + ids.append(str(aggregate["id"])) + export_old(old, backup, monkeypatch) + argv = ["import", "--db", str(target), "--user-id", USER, "--in", str(backup)] + assert onramp_main(argv) == 0 + before = snapshot(target) + with sqlite3.connect(target) as conn: + assert conn.execute("SELECT domain,sensitivity FROM open_loops WHERE id=?", (loop["id"],)).fetchone() == ( + "health", + "confidential", + ) + assert ( + conn.execute("SELECT count(*) FROM event_log WHERE event_type='memory.domain_relabelled'").fetchone()[0] + >= 1 + ) + assert onramp_main([*argv, "--mode", "skip"]) == 0 + assert snapshot(target) == before, "repeat skip must change no stored row, event, or stamp" + restricted_reads(target, ids, monkeypatch, guard_off=True) + restricted_reads(target, ids, monkeypatch) + + +def test_repeating_skip_accepts_a_later_supersede_label_raise(tmp_path, monkeypatch): + old, target, backup = tmp_path / "old.db", tmp_path / "target.db", tmp_path / "memory_backup.jsonl" + ids = old_vault(old, monkeypatch, domain="project") + with monkeypatch.context() as patch: + patch.setattr(sqlite_schema, "_relabel_derived_domains", lambda conn: None) + patch.setattr(sqlite_schema, "_relabel_derived_labels", lambda conn: None) + with sqlite_user_connection(old, USER) as conn: + source = dict(conn.execute("SELECT * FROM sources").fetchone()) + source["source_type"], source["connector_name"], source["raw_path"] = ( + "markdown", + "markdown_folder", + "fixture.md", + ) + conn.execute( + "UPDATE sources SET source_type=?, connector_name=?, raw_path=? WHERE id=?", + ("markdown", "markdown_folder", "fixture.md", source["id"]), + ) + export_old(old, backup, monkeypatch) + # Old unversioned memory-only backups are valid. The source already lives + # in this destination and is intentionally outside this incremental file. + records = [json.loads(line) for line in backup.read_text().splitlines()] + backup.write_text("\n".join(json.dumps(row) for row in records if row["record_type"] == "memory") + "\n") + bootstrap_database(target, user_id=USER, user_email="fixture@example.invalid") + with sqlite_user_connection(target, USER) as conn: + source["metadata_json"] = json.loads(source["metadata_json"]) + SQLiteVNextStore(conn, USER).create_source(source) + argv = ["import", "--db", str(target), "--user-id", USER, "--in", str(backup)] + assert onramp_main(argv) == 0 + with sqlite_user_connection(target, USER) as conn: + store = SQLiteVNextStore(conn, USER) + replacement = store.create_source( + {**source, "id": None, "dedupe_key": None, "content_hash": "sha256:replacement", "sensitivity": "regulated"} + ) + store.supersede_source(str(source["id"]), superseded_by=str(replacement["id"])) + assert store.get_memory(ids[0])["sensitivity"] == "regulated" + payloads = [ + json.loads(row["payload_json"]) + for row in conn.execute( + "SELECT payload_json FROM event_log WHERE event_type='memory.labels_raised' AND target_id=?", (ids[0],) + ) + ] + assert any(row["cause"] == "input_relabelled" for row in payloads) + with sqlite_user_connection(target, USER): + pass + before = snapshot(target) + assert onramp_main([*argv, "--mode", "skip"]) == 0 + assert snapshot(target) == before + restricted_reads(target, ids, monkeypatch, guard_off=True) + restricted_reads(target, ids, monkeypatch) + + +def test_unrepairable_open_keeps_guarded_read_available(tmp_path, monkeypatch, caplog): + path = tmp_path / "unrepairable.db" + ids = old_vault(path, monkeypatch, domain="project") + + def fail(conn, **kwargs): + raise repair.DerivedDomainRepairError("synthetic cycle did not settle") + + monkeypatch.setattr(repair, "relabel_labels_sqlite", fail) + with sqlite_user_connection(path, USER): + pass + assert read_stored_columns(path, ids)[2] is None + assert "alice-memory: label repair did not run" in caplog.text + restricted_reads(path, ids, monkeypatch) From 4fc733e7855a7bb2e87f5b6ae0050130223f7007 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:00:27 +0200 Subject: [PATCH 131/270] Check ordered repair statements and diagnose stale dependent reports --- .../test_derived_labels_migration_postgres.py | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_derived_labels_migration_postgres.py b/tests/integration/test_derived_labels_migration_postgres.py index 0c9152a5e..3f24a8a19 100644 --- a/tests/integration/test_derived_labels_migration_postgres.py +++ b/tests/integration/test_derived_labels_migration_postgres.py @@ -451,6 +451,47 @@ def raise_source(): repaired.result(timeout=10) relabelled.result(timeout=10) rows, _events = stored(urls, user, targets) - assert all(row["sensitivity"] == "regulated" for values in rows.values() for row in values) + assert all(row["sensitivity"] == "regulated" for values in rows.values() for row in values), [ + (table, row["metadata_json"], row["sensitivity"]) + for table, values in rows.items() + for row in values + if row["sensitivity"] != "regulated" + ] + restricted_reads(urls, user, targets, monkeypatch, guard_off=True) + restricted_reads(urls, user, targets, monkeypatch) + + +def test_repair_takes_ordered_row_locks_and_preserves_nonlabel_fields(migrated_database_urls, monkeypatch): + urls = migrated_database_urls + user, targets = seed_stale(urls) + with user_connection(urls["app"], user) as conn: + before = conn.execute("SELECT * FROM memories ORDER BY id").fetchall() + original = psycopg.Cursor.execute + locks = [] + + def record(cursor, query, *args, **kwargs): + text = query.as_string(cursor.connection) if hasattr(query, "as_string") else str(query) + normalized = " ".join(text.split()) + if normalized.startswith("SELECT id FROM ") and "ORDER BY id FOR UPDATE" in normalized: + locks.append(normalized.split()[3]) + return original(cursor, query, *args, **kwargs) + + monkeypatch.setattr(psycopg.Cursor, "execute", record) + labels._run_vnext_labels_repair(cli_context(urls, user), None) + assert locks == ["generated_artifacts", "projects", "open_loops", "memories"] + with user_connection(urls["app"], user) as conn: + after = conn.execute("SELECT * FROM memories ORDER BY id").fetchall() + for previous, new in zip(before, after, strict=True): + assert { + key: value + for key, value in previous.items() + if key not in {"domain", "sensitivity", "metadata_json", "project_id"} + } == { + key: value + for key, value in new.items() + if key not in {"domain", "sensitivity", "metadata_json", "project_id"} + } + assert new["metadata_json"]["source_id"] == previous["metadata_json"]["source_id"] + assert_repaired(urls, user, targets) restricted_reads(urls, user, targets, monkeypatch, guard_off=True) restricted_reads(urls, user, targets, monkeypatch) From 82d8480f72de4fa2f7f28f1a945689e9143d0d70 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:19:42 +0200 Subject: [PATCH 132/270] Report the changed row count from explicit SQLite label repair --- apps/api/src/alicebot_api/label_commands.py | 4 ++-- apps/api/src/alicebot_api/vnext_label_repair.py | 5 +++-- tests/unit/test_sqlite_derived_labels_v3.py | 8 ++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/api/src/alicebot_api/label_commands.py b/apps/api/src/alicebot_api/label_commands.py index f6d523716..c780585d4 100644 --- a/apps/api/src/alicebot_api/label_commands.py +++ b/apps/api/src/alicebot_api/label_commands.py @@ -68,9 +68,9 @@ def run_labels(args) -> int: return 1 if below or unverified or raised_on_next_open else 0 try: with sqlite_user_connection(db, args.user_id) as conn: - relabel_labels_sqlite(conn, explicit=True) + changed = relabel_labels_sqlite(conn, explicit=True) except DerivedDomainRepairError as exc: print(f"labels repair failed: {exc}") return 2 - print("labels repair finished") + print(f"labels repair updated {changed}") return 0 diff --git a/apps/api/src/alicebot_api/vnext_label_repair.py b/apps/api/src/alicebot_api/vnext_label_repair.py index 3dad92c6d..e3163aaa4 100644 --- a/apps/api/src/alicebot_api/vnext_label_repair.py +++ b/apps/api/src/alicebot_api/vnext_label_repair.py @@ -154,7 +154,7 @@ def _stamped(conn) -> bool: ) -def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = False) -> None: +def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = False) -> int: """Raise stored derived labels once, or always for a restore or owner repair. When no transaction is open this begins one and reads the state key inside @@ -169,7 +169,7 @@ def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = Fal if not restoring and not explicit and _stamped(conn): if owns: conn.commit() - return + return 0 changes = plan_label_repairs(_load_tables(conn)) for table, user, stored, previous, new, node in changes: metadata = {"project_scope": list(new["project_scope"]), "project_floor": list(new["project_floor"])} @@ -217,6 +217,7 @@ def relabel_labels_sqlite(conn, *, restoring: bool = False, explicit: bool = Fal conn.execute("INSERT OR REPLACE INTO alice_schema_state (key, value) VALUES (?, ?)", (REPAIR_STATE_KEY, "1")) if owns: conn.commit() + return len(changes) except Exception: if owns: conn.rollback() diff --git a/tests/unit/test_sqlite_derived_labels_v3.py b/tests/unit/test_sqlite_derived_labels_v3.py index b07162b93..f7856e023 100644 --- a/tests/unit/test_sqlite_derived_labels_v3.py +++ b/tests/unit/test_sqlite_derived_labels_v3.py @@ -181,15 +181,19 @@ def test_old_backup_with_cross_project_aggregates(tmp_path, monkeypatch): restricted_reads(target, ids, monkeypatch, project="alpha") -def test_explicit_repair_ignores_completion_stamp(tmp_path, monkeypatch): +def test_explicit_repair_ignores_completion_stamp(tmp_path, monkeypatch, capsys): path = tmp_path / "stamped.db" ids = old_vault(path, monkeypatch, domain="project") with sqlite3.connect(path) as conn: conn.execute("INSERT OR REPLACE INTO alice_schema_state VALUES (?, '1')", (repair.REPAIR_STATE_KEY,)) assert onramp_main(["labels", "repair", "--db", str(path), "--user-id", USER]) == 0 - rows, events, _ = read_stored_columns(path, ids) + assert "labels repair updated 1" in capsys.readouterr().out + rows, events, state = read_stored_columns(path, ids) assert rows[0][1] == "confidential", "explicit repair must revisit a stamped vault" assert len(events) == 1 + assert onramp_main(["labels", "repair", "--db", str(path), "--user-id", USER]) == 0 + assert "labels repair updated 0" in capsys.readouterr().out + assert read_stored_columns(path, ids) == (rows, events, state) restricted_reads(path, ids, monkeypatch, guard_off=True) restricted_reads(path, ids, monkeypatch) From 94b8e09527b34ede5d3496722976c98b585e2aa1 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:29:41 +0200 Subject: [PATCH 133/270] Use the shared label reader in upgrade evidence --- scripts/run_phase5_ops_evidence.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/scripts/run_phase5_ops_evidence.py b/scripts/run_phase5_ops_evidence.py index bf0d9f01f..843cb968d 100755 --- a/scripts/run_phase5_ops_evidence.py +++ b/scripts/run_phase5_ops_evidence.py @@ -1095,19 +1095,10 @@ def _verify_derived_labels(admin_url: str) -> None: """The migrations must leave no derived row below its inputs or unverified.""" from alicebot_api.db import direct_user_connection - from alicebot_api.vnext_label_repair import classify_stored_labels + from alicebot_api.vnext_label_repair import classify_stored_labels, load_postgres_label_tables with direct_user_connection(admin_url, USER_ID) as conn: - from alicebot_api.vnext_label_repair import INPUT_SELECTS_V3 - - tables: dict[str, list[dict[str, object]]] = {} - for table, statement in INPUT_SELECTS_V3.items(): - cursor = conn.execute(statement) - names = [column[0] for column in cursor.description] - tables[table] = [ - row if isinstance(row, dict) else dict(zip(names, row)) for row in cursor.fetchall() - ] - below, unverified = classify_stored_labels(tables) + below, unverified = classify_stored_labels(load_postgres_label_tables(conn)) if below or any(unverified.values()): raise EvidenceError("derived_labels_not_repaired") From 0b77d21030c5714692e66209f8142289dd4b3b8e Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:14:57 +0200 Subject: [PATCH 134/270] Keep legacy repair proof isolated under separated database roles --- tests/integration/test_derived_domain_postgres.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_derived_domain_postgres.py b/tests/integration/test_derived_domain_postgres.py index 3c68bc2dc..b248e0aca 100644 --- a/tests/integration/test_derived_domain_postgres.py +++ b/tests/integration/test_derived_domain_postgres.py @@ -1,5 +1,6 @@ """Restricted-domain generation and migration on the role-separated database.""" +import os from uuid import uuid4 from urllib.parse import urlsplit, urlunsplit @@ -183,8 +184,11 @@ def test_postgres_repair_as_documented_nobypassrls_owner(database_urls, monkeypa role_url = urlunsplit( parsed._replace(netloc=f"{role}:fixture-role-password@{parsed.hostname}:{parsed.port or 5432}") ) - with psycopg.connect(database_urls["admin"], autocommit=True) as admin: - owner = admin.execute("SELECT current_user").fetchone()[0] + lifecycle = urlsplit(os.getenv("DATABASE_LIFECYCLE_URL", database_urls["admin"])) + lifecycle_url = urlunsplit(lifecycle._replace(path=parsed.path)) + with psycopg.connect(lifecycle_url, autocommit=True) as admin: + owner = parsed.username + assert owner is not None admin.execute( sql.SQL("CREATE ROLE {} LOGIN NOSUPERUSER NOBYPASSRLS PASSWORD {}").format( sql.Identifier(role), sql.Literal("fixture-role-password") @@ -210,10 +214,11 @@ def injected_failure(*args): with monkeypatch.context() as patch: patch.setattr(repair, "relabel_event", injected_failure) with pytest.raises(RuntimeError, match="injected audit failure"): - command.upgrade(make_alembic_config(role_url), "head") + command.upgrade(make_alembic_config(role_url), "20261004_0095") assert admin.execute("SELECT version_num FROM alembic_version").fetchone()[0] == "20260721_0094" else: - command.upgrade(make_alembic_config(role_url), "head") + # Keep this v2 guard proof independent of the later v3 repair. + command.upgrade(make_alembic_config(role_url), "20261004_0095") assert all( row[0] for row in admin.execute( From 4cc29cafb55a15df4be0702ca2c9f919e3f3d297 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:36:13 +0200 Subject: [PATCH 135/270] Prove full owner doctor counts and filtered content omission --- tests/unit/test_derived_labels_real_keys.py | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/unit/test_derived_labels_real_keys.py b/tests/unit/test_derived_labels_real_keys.py index 991454b09..88da4a5e1 100644 --- a/tests/unit/test_derived_labels_real_keys.py +++ b/tests/unit/test_derived_labels_real_keys.py @@ -84,3 +84,28 @@ def test_sqlite_real_key_core_doors(tmp_path, monkeypatch, reader): if not admitted: assert str(row["title"]) not in rendered + +def test_full_owner_doctor_keeps_true_counts_and_filtered_view_skips_content(tmp_path): + user_id = uuid4() + path = tmp_path / "doctor.sqlite3" + bootstrap_database(path, user_id=str(user_id), user_email="synthetic@example.invalid") + with sqlite_user_connection(path, user_id) as conn: + store = SQLiteVNextStore(conn, user_id) + # SQLite has no provider/connector doctor protocol. Keep those synthetic + # operations fixed while the real connection supplies content diagnostics. + diagnostic_store = DoctorStore() + diagnostic_store.conn = store.conn + diagnostic_store.list_sources = lambda **kwargs: [row for batch in store.iter_label_rows("source") for row in batch] + service = VNextDoctorService(diagnostic_store, secret_provider=InMemorySecretProvider(), env={}, cwd=tmp_path) + before = service.run(include_content_diagnostics=False) + rows = seed_read_rows(store) + full = service.run() + scoped = service.run(include_content_diagnostics=False) + derived = next(check for check in full["checks"] if check["name"] == "derived_labels") + assert derived["message"] == "derived labels: 1 below their inputs, 1 unverified" + skipped = [check for check in scoped["checks"] if check["name"] in {"derived_labels", "flagged_sources"}] + assert len(skipped) == 2 + assert all(check["status"] == "skipped" and check["details"] == {"scope": "filtered_workspace", "evaluated": False} for check in skipped) + assert all(str(row["id"]) not in json.dumps(scoped) for row in rows.values()) + for field in ("status", "blocking_failure_count", "warning_count", "recommended_fixes"): + assert scoped[field] == before[field] From b841bb5781f8957d706d6fbb60fe2b2c7f0b5155 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Mon, 5 Oct 2026 23:33:15 +0200 Subject: [PATCH 136/270] docs: clarify complete counts and derived label recovery --- CHANGELOG.md | 10 ++- docs/alpha/backup-and-restore.md | 11 ++- docs/alpha/doctor.md | 7 ++ docs/alpha/mcp-tools.md | 8 +-- tests/unit/test_derived_domain_docs.py | 4 +- tests/unit/test_derived_labels_docs.py | 92 +++++++++++++++++++++++++- 6 files changed, 117 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4d5a1541..40e6d72f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,14 +3,12 @@ ## Unreleased - Unreleased (on main, not in v0.20.0): the derived-row pages state the three-part label, who reads an unverified row, and that the five operator screens apply the caller's sensitivity ceiling. The v0.20.0 notes omitted both limits. No migration is required. -- Unreleased (on main, not in v0.20.0): PostgreSQL migration `20261005_0096` raises a stored derived row to the labels of its inputs, and `alicebot vnext labels` and `alice-memory labels` check and repair the same rows. The doctors print how many derived rows are below their inputs or unverified, as a warning. v0.20.0 left the stored label where it was written. -- Unreleased (on main, not in v0.20.0): opening a SQLite vault raises a derived row whose stored label is below its inputs, and a restore does the same before it publishes. A repair that cannot finish leaves the vault open and the completion key unstamped. v0.20.0 left the stored label where it was written. -- Unreleased (on main, not in v0.20.0): recall, context packs, resume, recent decisions, the session brief, the review queue, open-loop lists, and the workspace, dashboard, context tree, and dogfooding lists drop a loaded row whose effective label is outside the request. The artifact list, the source trace, the project list, the dashboard project row, and the belief state route apply the caller's sensitivity ceiling, including titles, ids, and counts. Resume, recent decisions, the session brief, audit, and explain apply the target row's current effective label, so an excluded key does not learn of a label change. v0.20.0 returned those rows from the label stored on them. No migration is required. -- Unreleased (on main, not in v0.20.0): a daily brief, connection report, contradiction report, consolidation, roll-up, project update, staleness sweep, and open-loop review drop an input whose effective label is outside the request. v0.20.0 kept a public copy of a confidential input in the report text. No migration is required. +- Unreleased (on main, not in v0.20.0): PostgreSQL migration `20261005_0096` raises a stored derived row to the labels of its inputs, and SQLite does the same on the next open and before a restore is published. The open pass can leave a vault open with its completion key unstamped after a failed repair; explicit repair and restore refuse the whole change on failure. `alicebot vnext labels` and `alice-memory labels` check and repair the same rows, including rows added after an earlier repair. The doctors print how many derived rows are below their inputs or unverified, as a warning, and report failed checks as unavailable. v0.20.0 left the stored label where it was written. Migration `20261005_0096` must run before serving requests as the NOSUPERUSER NOBYPASSRLS table owner. +- Unreleased (on main, not in v0.20.0): recall, context packs, resume, recent decisions, the session brief, the review queue, open-loop lists, and the workspace, dashboard, context tree, and dogfooding lists drop a loaded row whose effective label is outside the request. The artifact list, the source trace, the project list, the dashboard project row, and the belief state route apply the caller's sensitivity ceiling, including titles, ids, and counts. Resume, recent decisions, the session brief, audit, and explain apply the target row's current effective label, so an excluded key does not learn of a label change. The producer input readers also drop an input whose effective label is outside the request before it appears in report text. v0.20.0 returned rows by their stored labels and could copy a public row with confidential inputs into a new report. No migration is required. - Unreleased (on main, not in v0.20.0): an exact read of a derived artifact, a write that cites a derived memory, explain, memory review, redaction, and an open-loop update use the labels of that row's inputs. v0.20.0 used the label stored on the row, so a public copy of a confidential input could be read. A row whose inputs cannot be checked is refused to a locked key. The owner and an unbound admin still read the stored row. No migration is required. - Unreleased (on main, not in v0.20.0): a locked key's consolidation, roll-up, staleness sweep and open-loop review keep only inputs whose scope and floor are both inside the binding. The overlap check stays, and the exact test runs after it. Consolidation and roll-ups compare the group scope (scope united with floor), so a card stored with an empty scope and a floor of two projects is still found and can be accepted. The operator artifact list, given a project, matches that floor as well as the scope. Each new report and aggregate records `derived_from` for the rows it used. A project view that also asks for global rows keeps a row with no Alice project id only when every Alice project id in its floor is in that view. v0.20.0 selected by overlap of the scope alone and wrote no `derived_from`. No migration is required. -- Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. Raising a source or a memory walks the rows derived from it and raises those too. A candidate open loop and a generated artifact take the same insert floor. Replacing a SQLite source with a stricter one raises the memories that cite it before they are retired. Moving a source to another project answers with how many derived rows a project-bound key would lose, and waits for confirm_label_hide before it writes. A relabel that cannot finish answers 409, and one that waits more than 3 seconds for the label lock answers 503. No migration is required. -- Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. The function is what the write path calls. A read that does not go through that path still behaves as it does in v0.20.0. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. +- Unreleased (on main, not in v0.20.0): a memory copied from another row is stored at least as strict as that row. v0.20.0 kept the label the copy was given at insert, so a later reader could see a health or confidential input through a copy stored as `unknown` and `public`. The insert now re-reads the inputs inside the write and raises the domain, the sensitivity, and the project floor. A later metadata write cannot drop the marker that makes the row derived, or put back an older project scope or project floor. Raising a source or a memory walks the rows derived from it and raises those too. A candidate open loop and a generated artifact take the same insert floor. Replacing a SQLite source with a stricter one raises the memories that cite it before they are retired. Moving a source to another project previews how many derived rows a project-bound key would lose and requires `confirm_label_hide` when that count is nonzero. The owner or an unbound admin recovers fresh candidates through `POST /v0/vnext/sources/{source_id}/regenerate`, then reruns the normal report generation route; existing provenance is kept. A relabel that cannot finish answers 409 with a named cause, and one that waits more than 3 seconds for the label lock answers 503 with `Retry-After: 2` and nothing changed. No migration is required. +- Unreleased (on main, not in v0.20.0): the label of a derived row (its domain, its sensitivity, and the projects it requires) is now decided by one pure function. v0.20.0 kept the label a derived row was given when it was made. This pure kernel is shared by the later write, read and repair paths; it does not rewrite stored rows on its own. A locked key is refused when a caller passes a project floor that is not inside the key's binding. No migration is required. - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` now scrub an open loop that names the source only in its metadata: the id, or `source:`, as the text under `source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references` or `selected_source_ids` at any depth. That is the rule the open-loop lookup of a source already used. They blanked only loops whose `source_id` column held the id, so a loop with an empty column kept its title, description and metadata, and they reached a new export. The delete preview now counts the same loops as the receipt. The rule reads the id only as stored or as `source:`: a loop that names the source in any spelling other than those two keeps its text, although the memory rule reads several other spellings. No migration is required. - Unreleased (on main, not in v0.20.0): the SQLite derived-label repair now updates and records each row under the id it is stored with (SQLite keeps capitals, missing hyphens, braces and `urn:uuid:` as written) and matches rows by the normalised id only to read the graph. If two stored spellings of one id exist, each is compared with the label its own recorded inputs give it, so neither is left at its old label while the other is relabelled, and neither is lowered. An update that changes no row stops the repair with `DerivedDomainRepairError` before any event or the completion stamp is written, on open and on restore; migration `20261004_0095` refuses the same way. Before, a derived memory stored under such an id kept its label while its relabel event and the stamp were written, and the pair survived export and import. A PostgreSQL uuid column was not affected. Correction (2026-10-05): the v3 pass raises such a row on the next open, including a vault the earlier repair already stamped. No migration is required. - Unreleased (on main, not in v0.20.0): v0.20.0 could store a consolidation report with a sensitivity below the memories it prints. The report took its sensitivity from the near-duplicate clusters only, so a run whose proposals were roll-up cards had no cluster to take it from and was stored as `unknown`. A key whose ceiling is below those memories (a `trusted_local_agent` key for confidential ones, a `read_only_agent` key for private ones) could then read the card topic and the member ids through `GET /v0/vnext/artifacts/{id}`. The report now takes its domain and its sensitivity over every row it names: the cluster members, the members of every proposed roll-up group, the members of the groups that a skip line names by key (`topic:...`, `entity:...`, `semantic:cluster-`), the pending, accepted, expired or held roll-up cards it names by id, and the sources that the `source_refs` of its cluster members name, which the report still prints. The open-loop review is labelled over the sources whose ids it prints as well as over its loops. The run digest of both reports covers those sources, so a source that was reclassified makes a new report instead of returning the earlier one. The first run after the upgrade over loops that link a source, or over cluster members that cite one, makes one new report, and a run that names no source keeps the digest it had. A report whose inputs are all unrestricted is now stored with the label of those inputs, `internal` where it was `unknown`, and every permission profile reads the two the same way. Correction (2026-10-05): a report stored by v0.20.0 does not keep a sensitivity below its inputs, because migration `20261005_0096` and the SQLite v3 pass repair sensitivity. The other report producers (daily brief, weekly synthesis, connection and contradiction reports, project updates, staleness reports) were checked and already label over every row they print. Memories and sources are counted apart when the label is taken, because an id is unique only within its own table: a source that shares a memory's id can no longer replace that memory's label. No migration is required. diff --git a/docs/alpha/backup-and-restore.md b/docs/alpha/backup-and-restore.md index 85ae5d99d..4e93d5e41 100644 --- a/docs/alpha/backup-and-restore.md +++ b/docs/alpha/backup-and-restore.md @@ -313,7 +313,11 @@ counts as a recorded input is under [Derived row domains](mcp-tools.md#derived-r Unreleased (on main, not in v0.20.0): `alice-memory labels check` prints how many derived rows are below their inputs or unverified, and `alice-memory labels repair` -raises the rows that are below their inputs. Run check after a restore. +raises the rows that are below their inputs. Check reads a private snapshot and +does not upgrade the live vault. Explicit repair checks every time, including +after the open pass has stamped completion, under `BEGIN IMMEDIATE`; a failure +rolls back the whole repair. The open pass remains a one-time upgrade, and a +restore always repairs its staged copy before publication. Run check after a restore. This command restores a SQLite database. It is not a PostgreSQL import. @@ -466,7 +470,10 @@ inside its own transaction and turns it back on before it commits. A failure, including derived rows in a cycle whose labels do not settle within a bounded number of changes or an update that changes no row, rolls the relabels, their audit events and the FORCE change back together. The downgrade keeps the repaired -labels. +labels. `alicebot vnext labels check` uses one `REPEATABLE READ READ ONLY` +snapshot, set before the acting user's row-security identity. Explicit repair +takes the supersession lock, then the exclusive label lock, then ordered row +locks; an update that changes no row rolls back the whole repair. ## Upgrade checkpoint diff --git a/docs/alpha/doctor.md b/docs/alpha/doctor.md index 85dd6a853..fd254873f 100644 --- a/docs/alpha/doctor.md +++ b/docs/alpha/doctor.md @@ -23,6 +23,13 @@ Expected success: - warnings include a recommended fix - no secret value appears in output +Unreleased (on main, not in v0.20.0): the doctors report `derived labels: N below +their inputs, M unverified` as a warning. If the label check cannot read the +store, they report `derived labels: unavailable; run labels check`, also as a +warning. A failed read is never reported as zero rows. Run the store's `labels +check` command to inspect the cause, then `labels repair` for stale labels; +missing inputs need restoring or regeneration. + Common fixes: ```bash diff --git a/docs/alpha/mcp-tools.md b/docs/alpha/mcp-tools.md index 6f2685221..036aa7933 100644 --- a/docs/alpha/mcp-tools.md +++ b/docs/alpha/mcp-tools.md @@ -1083,10 +1083,10 @@ Unreleased (on main, not in v0.20.0): an until-only request also checks the uppe Unreleased (on main, not in v0.20.0): a derived memory or report keeps the most frequent restricted input label, with alphabetical ties, the highest sensitivity, and every project of every input, and none of those is lowered. An explicit request domain cannot override it. With no restricted inputs, each producer retains its prior selection. This covers briefs, weekly synthesis and its candidates, roll-ups, consolidation, connection and contradiction reports, staleness reports, open-loop reviews, project updates, promoted copies of reports, memories extracted from a source, the candidate open loops found in one, and the state a project update copies onto a project. Consolidation reports take their domain and their sensitivity over every row they name: the cluster members, the roll-up inputs, the members of the groups that a skip line names by key, and the roll-up cards they name by id. The report keeps printing the `source_refs` it copies from its cluster members, and its label also covers the sources they name, archived ones included. Open-loop reviews do the same over the sources whose ids they print. The run digest of both covers those sources, so a source that was reclassified makes a new report. A report whose inputs are all unrestricted carries the label of those inputs, so `internal` where it was `unknown`, and every profile reads the two alike. New staleness reports also inherit the highest sensitivity of the memories whose titles they include. -Unreleased (on main, not in v0.20.0): the stored-row repair resolves recorded input IDs within the same user and labels each derived row after the rows it reads, so a chain of any length settles with one read of each row, including promoted artifact copies identified by `value.kind`, `value.artifact_id` or `metadata_json.source_artifact_id`. Rows that record each other as inputs in a cycle are read again until their labels settle. Already restricted labels can change to the settled restricted label; they never become unrestricted. A cycle whose labels do not settle within a bounded number of changes aborts the migration or the restore instead of publishing intermediate labels, with an error that names up to five of the rows that kept changing and says to remove their circular input references or restore an earlier backup. The open pass does not abort the vault. A relabel and the rows that follow it commit together, at any depth, or the whole relabel is refused and nothing changes. Labels only rise. Regenerating the row is how a looser input is taken. An owner edit that would lower a derived row is held at its inputs, and the answer names `label_floor_applied`. One `labels_raised` event records the labels and carries no text. A relabel waits at most 3 s for a running write and then answers "try again". A redacted row has no inputs and is left alone. Text is never used to guess an input. A derived row with no resolvable record is not repaired but is unverified for readers. +Unreleased (on main, not in v0.20.0): the stored-row repair resolves recorded input IDs within the same user and labels each derived row after the rows it reads, so a chain of any length settles with one read of each row, including promoted artifact copies identified by `value.kind`, `value.artifact_id` or `metadata_json.source_artifact_id`. Rows that record each other as inputs in a cycle are read again until their labels settle. Already restricted labels can change to the settled restricted label; they never become unrestricted. A cycle whose labels do not settle within a bounded number of changes aborts the migration or the restore instead of publishing intermediate labels, with an error that names up to five of the rows that kept changing and says to remove their circular input references or restore an earlier backup. The open pass does not abort the vault. A relabel and the rows that follow it commit together, at any depth, or the whole relabel is refused and nothing changes. Labels only rise. Regenerating the row is how a looser input is taken. An owner edit that would lower a derived row is held at its inputs, and the answer names `label_floor_applied`. One `labels_raised` event records the labels and carries no text. A relabel waits at most 3 s for a running write and then answers "try again" with HTTP 503 and `Retry-After: 2`, with nothing changed. A whole refusal answers HTTP 409 with the cause `propagation_bound`, `row_changed`, `dependency_cycle`, `lock_order` or `database_error`, without input text or ids. A redacted row has no inputs and is left alone. Text is never used to guess an input. A derived row with no resolvable record is not repaired but is unverified for readers. -Unreleased (on main, not in v0.20.0): the repair runs when SQLite upgrades a vault, while `alice-memory import` stages a restore, in PostgreSQL migration `20261004_0095` (domain) and in PostgreSQL migration `20261005_0096` (domain, sensitivity, project scope and floor), and through `labels check` and `labels repair`. The open pass does not stop a vault from opening and leaves a row it could not repair to the read check. Each path, and what a restore can and cannot repair, is in [Backup and restore](backup-and-restore.md). +Unreleased (on main, not in v0.20.0): the repair runs when SQLite upgrades a vault, while `alice-memory import` stages a restore, in PostgreSQL migration `20261004_0095` (domain) and in PostgreSQL migration `20261005_0096` (domain, sensitivity, project scope and floor), and through `labels check` and `labels repair`. The open pass does not stop a vault from opening and leaves a row it could not repair to the read check. Explicit repair checks rows even after the open pass has stamped completion; every restore repairs its staged copy again. A failed explicit repair or restore rolls back the whole change. Each path, and what a restore can and cannot repair, is in [Backup and restore](backup-and-restore.md). -Unreleased (on main, not in v0.20.0): owner, trusted and admin callers that explicitly filter by an unrelated domain stop receiving a derived row after its label changes from `unknown` to a restricted domain; `unknown` previously matched every domain filter. New weekly candidate memories also store `input_summary`, which explain returns. These are intentional differences for unrestricted readers. Domain fields and explain policy-event labels/hashes reflect the new label; additive metadata changes context-pack token estimates. Repair audit events are additional history. The repair does not add missing historical input summaries. Correction (2026-10-05): sensitivity and project floor are repaired by migration `20261005_0096` and the SQLite v3 pass. Every reader checks a derived row, a belief and a project row against its inputs as they are now. A row whose recorded inputs cannot be found is unverified, so only the owner and an unbound admin key read it. Checking those inputs adds a read of the recorded graph. `derived_from` and `project_floor` add to pack token estimates. The artifact list, the source trace, the project list, the project row of the dashboard and the belief state route apply the caller's sensitivity ceiling, including titles, ids and counts. Counts on those screens and on the workspace and dashboards are over the rows the caller may read. Domain and project restrictions on those screens stay as they are. +Unreleased (on main, not in v0.20.0): owner, trusted and admin callers that explicitly filter by an unrelated domain stop receiving a derived row after its label changes from `unknown` to a restricted domain; `unknown` previously matched every domain filter. New weekly candidate memories also store `input_summary`, which explain returns. These are intentional differences for unrestricted readers. Domain fields and explain policy-event labels/hashes reflect the new label; additive metadata changes context-pack token estimates. Repair audit events are additional history. The repair does not add missing historical input summaries. Correction (2026-10-05): sensitivity and project floor are repaired by migration `20261005_0096` and the SQLite v3 pass. Every reader checks a derived row, a belief and a project row against its inputs as they are now. A row whose recorded inputs cannot be found is unverified, so only the owner and an unbound admin key read it. Checking those inputs adds a read of the recorded graph. `derived_from` and `project_floor` add to pack token estimates. The artifact list, the source trace, the project list, the project row of the dashboard and the belief state route apply the caller's sensitivity ceiling, including titles, ids and counts. Counts on those screens and on the workspace and dashboards are over the rows the caller may read. Each total checks the complete counted set before pagination, including rows excluded from the displayed page. Domain and project restrictions on those screens stay as they are. -Unreleased (on main, not in v0.20.0): a new derived row and a stored one take every project of every input, and a project is not dropped. A restricted key's own reports are built only from inputs that key may read. The owner's cross-project briefs stay. A report with an input that has no project is global and keeps a floor of every input's project, and it is not readable by every project. Correction (2026-10-05): the text that stood here said a reader matching one scope could potentially read a summary of other scopes. In v0.20.0 and on main before this change the reach was wider than a summary: titles and ids of other projects' artifacts, titles and ids of rows with no project, and the full text of memories with no project, in a report a bound key could read with 200 although its direct read of those rows was refused; promotion made the report recallable. +Unreleased (on main, not in v0.20.0): a new derived row and a stored one take every project of every input, and a project is not dropped. A restricted key's own reports are built only from inputs that key may read. The owner's cross-project briefs stay. A report with an input that has no project is global and keeps a floor of every input's project, and it is not readable by every project. Moving a source first returns `derived_rows_hidden_from_project_keys` and writes nothing when the count is nonzero until `confirm_label_hide` is true. On PostgreSQL, the keyless local owner or an unbound admin can call `POST /v0/vnext/sources/{source_id}/regenerate` with `user_id` to create fresh candidate memories and open loops from all stored chunks under the source's current labels, scope and provenance. HTTP 201 returns `memory_ids`, `open_loop_ids`, `memory_count` and `open_loop_count`; trusted and project-bound keys are refused. Old rows and their provenance stay intact and strict. Rerun a report's normal generation route to rebuild the report from the regenerated inputs. Correction (2026-10-05): the text that stood here said a reader matching one scope could potentially read a summary of other scopes. In v0.20.0 and on main before this change the reach was wider than a summary: titles and ids of other projects' artifacts, titles and ids of rows with no project, and the full text of memories with no project, in a report a bound key could read with 200 although its direct read of those rows was refused; promotion made the report recallable. diff --git a/tests/unit/test_derived_domain_docs.py b/tests/unit/test_derived_domain_docs.py index 2740feddb..619832b0f 100644 --- a/tests/unit/test_derived_domain_docs.py +++ b/tests/unit/test_derived_domain_docs.py @@ -68,7 +68,7 @@ def test_the_limitations_page_has_one_short_bullet_that_links_to_the_explanation Mutations, each one alone: delete ``keeps the label its inputs had when it was made``, ``follows its inputs when they are relabelled``, ``read only by the owner and an unbound admin key``, ``sensitivity - ceiling`` or the link from the bullet; point the link at ``#derived-rows``; add a ``## Derived`` heading + ceiling``, ``counting rows the caller may read`` or the link from the bullet; point the link at ``#derived-rows``; add a ``## Derived`` heading with a paragraph to the page. """ @@ -77,6 +77,7 @@ def test_the_limitations_page_has_one_short_bullet_that_links_to_the_explanation assert "follows its inputs when they are relabelled" in bullet assert "read only by the owner and an unbound admin key" in bullet assert "sensitivity ceiling" in bullet + assert "counting rows the caller may read" in bullet assert bullet.endswith(f"See [{HEADING}]({ANCHOR})") assert "\n## Derived" not in LIMITATIONS.read_text(encoding="utf-8") @@ -152,6 +153,7 @@ def test_the_mcp_tools_section_states_the_label_rule_the_repair_and_the_changes_ assert "apply no label" not in readers assert "every project of every input" in scope + assert "A restricted key's own reports are built only from inputs that key may read" in scope assert "not readable by every project" in scope assert "could potentially read a summary of other scopes" in scope diff --git a/tests/unit/test_derived_labels_docs.py b/tests/unit/test_derived_labels_docs.py index ca0c09b16..50f538065 100644 --- a/tests/unit/test_derived_labels_docs.py +++ b/tests/unit/test_derived_labels_docs.py @@ -16,8 +16,6 @@ import importlib.util from pathlib import Path -from alicebot_api.vnext_label_repair import REPAIR_STATE_KEY - ROOT = Path(__file__).resolve().parents[2] MARK = "Unreleased (on main, not in v0.20.0):" MIGRATION = ROOT / "apps/api/alembic/versions/20261005_0096_derived_label_floor.py" @@ -37,6 +35,8 @@ def _text(path: str) -> str: def test_the_pages_name_the_v3_repair_and_the_seven_table_migration() -> None: """Docs name the state key, the revision and every table the migration brackets, including projects.""" + from alicebot_api.vnext_label_repair import REPAIR_STATE_KEY + migration = _migration() tables = [item.split()[2] for item in migration._RELAX_RLS] assert migration.revision == "20261005_0096" @@ -60,6 +60,7 @@ def test_the_pages_name_the_v3_repair_and_the_seven_table_migration() -> None: assert "at most 3 s" in tools assert "apply the caller's sensitivity ceiling" in tools assert "rows the caller may read" in tools + assert "Each total checks the complete counted set before pagination" in tools assert "apply no label" not in tools assert f"`{migration.revision}`" in tools assert REPAIR_STATE_KEY not in tools @@ -95,3 +96,90 @@ def test_the_pages_name_the_v3_repair_and_the_seven_table_migration() -> None: assert "Such a row is unverified." in changelog assert "the five operator screens apply the caller's sensitivity ceiling" in changelog assert "No migration is required." in changelog.split("## Unreleased", 1)[1].split("\n- ", 2)[1] + + +def test_each_derived_label_change_has_one_complete_changelog_entry() -> None: + """One entry covers each PR, including both stores and all list and producer input doors. + + Mutations: split the repair entry by store; split producer input filtering from the list-door entry; + remove the migration role or its before-serving requirement; remove a no-migration ending. + """ + + entries = [line.removeprefix("- ") for line in _text("CHANGELOG.md").splitlines() if line.startswith("- ")] + starts = ( + "the derived-row pages", + "PostgreSQL migration `20261005_0096`", + "recall, context packs", + "an exact read", + "a locked key's consolidation", + "a memory copied from another row", + "the label of a derived row", + ) + selected: dict[str, str] = {} + for start in starts: + matches = [entry for entry in entries if entry.startswith(f"{MARK} {start}")] + assert len(matches) == 1, (start, len(matches)) + selected[start] = matches[0] + assert not any(entry.startswith(f"{MARK} opening a SQLite vault") for entry in entries) + assert not any(entry.startswith(f"{MARK} a daily brief, connection report") for entry in entries) + repair = selected[starts[1]] + assert "SQLite" in repair + assert "NOSUPERUSER NOBYPASSRLS" in repair + assert repair.endswith( + "Migration `20261005_0096` must run before serving requests as the NOSUPERUSER NOBYPASSRLS table owner." + ) + assert "producer input readers" in selected[starts[2]] + for start, entry in selected.items(): + assert entry.startswith(MARK) + assert "v0.20.0" in entry + if start != starts[1]: + assert entry.endswith("No migration is required."), start + + +def test_repair_docs_distinguish_open_restore_and_explicit_commands() -> None: + """Repair after completion and failed reads have different operator outcomes from a gated open. + + Mutations: gate explicit repair by completion; make restore one-time; turn a failed explicit repair + into partial success; remove the read-only snapshot contract; replace an unavailable doctor check + with a zero count. + """ + + backup = " ".join(_text("docs/alpha/backup-and-restore.md").split()) + assert "Explicit repair checks every time, including after the open pass has stamped completion" in backup + assert "under `BEGIN IMMEDIATE`; a failure rolls back the whole repair" in backup + assert "The open pass remains a one-time upgrade" in backup + assert "a restore always repairs its staged copy before publication" in backup + assert "Check reads a private snapshot and does not upgrade the live vault" in backup + assert "one `REPEATABLE READ READ ONLY` snapshot, set before the acting user's row-security identity" in backup + assert "an update that changes no row rolls back the whole repair" in backup + tools = _text("docs/alpha/mcp-tools.md") + assert "Explicit repair checks rows even after the open pass has stamped completion" in tools + assert "every restore repairs its staged copy again" in tools + assert "A failed explicit repair or restore rolls back the whole change" in tools + doctor = " ".join(_text("docs/alpha/doctor.md").split()) + assert f"{MARK} the doctors report" in doctor + assert "`derived labels: unavailable; run labels check`" in doctor + assert "A failed read is never reported as zero rows" in doctor + + +def test_source_move_docs_name_the_proved_recovery_and_failure_contract() -> None: + """A source move names a tested recovery route, confirmation and whole-refusal causes. + + Mutations: remove confirmation, the recovery route, current provenance, the report regeneration + step, a failure cause or the retry header. A source move must not suggest that old rows are loosened. + """ + + tools = _text("docs/alpha/mcp-tools.md") + assert "`derived_rows_hidden_from_project_keys`" in tools + assert "writes nothing when the count is nonzero until `confirm_label_hide` is true" in tools + assert "On PostgreSQL, the keyless local owner or an unbound admin" in tools + assert "`POST /v0/vnext/sources/{source_id}/regenerate` with `user_id`" in tools + assert "from all stored chunks under the source's current labels, scope and provenance" in tools + assert "Old rows and their provenance stay intact and strict" in tools + assert "Rerun a report's normal generation route" in tools + assert "HTTP 201 returns `memory_ids`, `open_loop_ids`, `memory_count` and `open_loop_count`" in tools + assert "trusted and project-bound keys are refused" in tools + assert "HTTP 503 and `Retry-After: 2`, with nothing changed" in tools + assert "HTTP 409 with the cause" in tools + for cause in ("propagation_bound", "row_changed", "dependency_cycle", "lock_order", "database_error"): + assert f"`{cause}`" in tools From 7fc8227947ca987350f35b9ba6e3960b3f884503 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:23:30 +0200 Subject: [PATCH 137/270] docs: explain filtered workspace diagnostics --- docs/alpha/mcp-tools.md | 2 +- tests/unit/test_derived_domain_docs.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/alpha/mcp-tools.md b/docs/alpha/mcp-tools.md index 036aa7933..69a92bc92 100644 --- a/docs/alpha/mcp-tools.md +++ b/docs/alpha/mcp-tools.md @@ -1087,6 +1087,6 @@ Unreleased (on main, not in v0.20.0): the stored-row repair resolves recorded in Unreleased (on main, not in v0.20.0): the repair runs when SQLite upgrades a vault, while `alice-memory import` stages a restore, in PostgreSQL migration `20261004_0095` (domain) and in PostgreSQL migration `20261005_0096` (domain, sensitivity, project scope and floor), and through `labels check` and `labels repair`. The open pass does not stop a vault from opening and leaves a row it could not repair to the read check. Explicit repair checks rows even after the open pass has stamped completion; every restore repairs its staged copy again. A failed explicit repair or restore rolls back the whole change. Each path, and what a restore can and cannot repair, is in [Backup and restore](backup-and-restore.md). -Unreleased (on main, not in v0.20.0): owner, trusted and admin callers that explicitly filter by an unrelated domain stop receiving a derived row after its label changes from `unknown` to a restricted domain; `unknown` previously matched every domain filter. New weekly candidate memories also store `input_summary`, which explain returns. These are intentional differences for unrestricted readers. Domain fields and explain policy-event labels/hashes reflect the new label; additive metadata changes context-pack token estimates. Repair audit events are additional history. The repair does not add missing historical input summaries. Correction (2026-10-05): sensitivity and project floor are repaired by migration `20261005_0096` and the SQLite v3 pass. Every reader checks a derived row, a belief and a project row against its inputs as they are now. A row whose recorded inputs cannot be found is unverified, so only the owner and an unbound admin key read it. Checking those inputs adds a read of the recorded graph. `derived_from` and `project_floor` add to pack token estimates. The artifact list, the source trace, the project list, the project row of the dashboard and the belief state route apply the caller's sensitivity ceiling, including titles, ids and counts. Counts on those screens and on the workspace and dashboards are over the rows the caller may read. Each total checks the complete counted set before pagination, including rows excluded from the displayed page. Domain and project restrictions on those screens stay as they are. +Unreleased (on main, not in v0.20.0): owner, trusted and admin callers that explicitly filter by an unrelated domain stop receiving a derived row after its label changes from `unknown` to a restricted domain; `unknown` previously matched every domain filter. New weekly candidate memories also store `input_summary`, which explain returns. These are intentional differences for unrestricted readers. Domain fields and explain policy-event labels/hashes reflect the new label; additive metadata changes context-pack token estimates. Repair audit events are additional history. The repair does not add missing historical input summaries. Correction (2026-10-05): sensitivity and project floor are repaired by migration `20261005_0096` and the SQLite v3 pass. Every reader checks a derived row, a belief and a project row against its inputs as they are now. A row whose recorded inputs cannot be found is unverified, so only the owner and an unbound admin key read it. Checking those inputs adds a read of the recorded graph. `derived_from` and `project_floor` add to pack token estimates. The artifact list, the source trace, the project list, the project row of the dashboard and the belief state route apply the caller's sensitivity ceiling, including titles, ids and counts. Counts on those screens and on the workspace and dashboards are over the rows the caller may read. Each total checks the complete counted set before pagination, including rows excluded from the displayed page. The filtered workspace skips content diagnostics; run doctor for the full derived-label and flagged-source report. Domain and project restrictions on those screens stay as they are. Unreleased (on main, not in v0.20.0): a new derived row and a stored one take every project of every input, and a project is not dropped. A restricted key's own reports are built only from inputs that key may read. The owner's cross-project briefs stay. A report with an input that has no project is global and keeps a floor of every input's project, and it is not readable by every project. Moving a source first returns `derived_rows_hidden_from_project_keys` and writes nothing when the count is nonzero until `confirm_label_hide` is true. On PostgreSQL, the keyless local owner or an unbound admin can call `POST /v0/vnext/sources/{source_id}/regenerate` with `user_id` to create fresh candidate memories and open loops from all stored chunks under the source's current labels, scope and provenance. HTTP 201 returns `memory_ids`, `open_loop_ids`, `memory_count` and `open_loop_count`; trusted and project-bound keys are refused. Old rows and their provenance stay intact and strict. Rerun a report's normal generation route to rebuild the report from the regenerated inputs. Correction (2026-10-05): the text that stood here said a reader matching one scope could potentially read a summary of other scopes. In v0.20.0 and on main before this change the reach was wider than a summary: titles and ids of other projects' artifacts, titles and ids of rows with no project, and the full text of memories with no project, in a report a bound key could read with 200 although its direct read of those rows was refused; promotion made the report recallable. diff --git a/tests/unit/test_derived_domain_docs.py b/tests/unit/test_derived_domain_docs.py index 619832b0f..2924d9e5a 100644 --- a/tests/unit/test_derived_domain_docs.py +++ b/tests/unit/test_derived_domain_docs.py @@ -150,6 +150,8 @@ def test_the_mcp_tools_section_states_the_label_rule_the_repair_and_the_changes_ assert "or repair historical sensitivity values" not in readers assert "apply the caller's sensitivity ceiling" in readers assert "rows the caller may read" in readers + assert "The filtered workspace skips content diagnostics" in readers + assert "run doctor for the full derived-label and flagged-source report" in readers assert "apply no label" not in readers assert "every project of every input" in scope From 845246870f27def737682cd8296723f1d457c1cf Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:31:28 +0200 Subject: [PATCH 138/270] Require each producer input kind in the sentinel grid --- .../test_derived_labels_producers_postgres.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/integration/test_derived_labels_producers_postgres.py b/tests/integration/test_derived_labels_producers_postgres.py index 9a1dbc79b..832a28d20 100644 --- a/tests/integration/test_derived_labels_producers_postgres.py +++ b/tests/integration/test_derived_labels_producers_postgres.py @@ -133,6 +133,20 @@ def test_a_bound_key_generates_from_admitted_inputs_only(migrated_database_urls, conn.execute("UPDATE memories SET valid_to='2026-10-04T12:00:00Z'") artifact, body = generate(producer, user_id, alpha, key) assert_canonical_printed_inputs(artifact, rows) + expected_kinds = { + "daily": ("sources", "memories", "open_loops", "artifacts"), + "weekly": ("sources", "memories", "open_loops", "artifacts"), + "connections": ("sources", "memories"), + "contradictions": ("sources", "beliefs"), + "open_loop_review": ("open_loops",), + "project_update": ("sources", "memories"), + "consolidation": ("memories", "artifacts"), + "staleness": ("memories",), + } + record = artifact["metadata_json"]["derived_from"] + for kind in expected_kinds[producer]: + assert any(str(row["id"]) in record[kind] for label, row_kind, row in rows + if label == "alpha" and row_kind == kind), (producer, "missing positive input kind", kind, record) artifact_id = str(artifact["id"]) surfaces = [body] with user_connection(app_url, user_id) as conn: From 83ed832f1187460da9841f27a6bedaf143164881 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:29:48 +0200 Subject: [PATCH 139/270] Cover each lexical read branch and bound admin review controls --- tests/unit/test_label_guard_exact_doors.py | 12 ++++++++++++ tests/unit/test_list_door_readers.py | 18 ++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_label_guard_exact_doors.py b/tests/unit/test_label_guard_exact_doors.py index bc255ceab..506c37132 100644 --- a/tests/unit/test_label_guard_exact_doors.py +++ b/tests/unit/test_label_guard_exact_doors.py @@ -195,6 +195,18 @@ def test_a_memory_review_decision_uses_the_input_label() -> None: assert decision.decision == "blocked" +def test_bound_admin_review_uses_the_effective_input_project_floor() -> None: + from alicebot_api.vnext_memory_commit import VNextMemoryCommitService + + store = _LabelStore() + store.source["metadata_json"]["project_scope"] = ["prj_" + "b" * 16] + identity = _locked_admin() + service = VNextMemoryCommitService(store) + assert service._write_policy_decision(identity=identity, action="memory.review", memory=store.memory).decision == "blocked" + store.source["metadata_json"]["project_scope"] = [ALPHA] + assert service._write_policy_decision(identity=identity, action="memory.review", memory=store.memory).decision != "blocked" + + def test_an_open_loop_update_uses_the_input_label() -> None: from alicebot_api.vnext_memory_commit import VNextMemoryCommitService diff --git a/tests/unit/test_list_door_readers.py b/tests/unit/test_list_door_readers.py index 442d11b79..723374f97 100644 --- a/tests/unit/test_list_door_readers.py +++ b/tests/unit/test_list_door_readers.py @@ -4,6 +4,7 @@ from contextlib import contextmanager from uuid import UUID +import pytest from alicebot_api.mcp.retrieval import _resume_event_honours_policy_fence from alicebot_api.routers import vnext_review @@ -125,16 +126,29 @@ def _identity(profile: str) -> AgentIdentity: return AgentIdentity(agent_id="reader", agent_type="unknown", permission_profile=profile) -def test_fts_stage_drops_a_public_copy_of_a_confidential_source() -> None: +@pytest.mark.parametrize("stage", ("strict", "or_fallback", "legacy")) +def test_fts_stage_drops_a_public_copy_of_a_confidential_source(stage) -> None: store = _LabelStore() + if stage == "or_fallback": + store.search_memories_fts = lambda **kwargs: [store.memory] if kwargs.get("match_any") else [] + elif stage == "legacy": + store.search_memories_fts = None + store.search_memories = lambda **kwargs: [store.memory] rows, _status = VNextRetrievalService(store)._memory_fts_rows( - query="sentinel", + query="sentinel fact" if stage == "or_fallback" else "sentinel", domains=["project"], sensitivity_allowed=["public", "internal", "private", "unknown"], limit=10, ) assert rows == [] assert SECRET not in str(rows) + read_rows = store.read_label_rows + store.read_label_rows = lambda kind, ids: [{**_source(), "domain": "project", "sensitivity": "public"}] if kind == "source" else read_rows(kind, ids) + visible, _status = VNextRetrievalService(store)._memory_fts_rows( + query="sentinel fact" if stage == "or_fallback" else "sentinel", domains=["project"], + sensitivity_allowed=["public", "internal", "private", "unknown"], limit=10, + ) + assert [row["id"] for row in visible] == [MEMORY_ID] def test_operator_screens_hide_a_confidential_row_from_a_trusted_key() -> None: From e61e673e55ec57cc246d7a35fad22e60a8b2d492 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:37:26 +0200 Subject: [PATCH 140/270] Reuse capture label grants within the current transaction --- apps/api/src/alicebot_api/vnext_capture.py | 147 +++++++++-------- .../src/alicebot_api/vnext_label_writes.py | 70 +++++++- apps/api/src/alicebot_api/vnext_store.py | 3 + .../test_capture_label_batch_postgres.py | 155 ++++++++++++++++++ 4 files changed, 301 insertions(+), 74 deletions(-) create mode 100644 tests/integration/test_capture_label_batch_postgres.py diff --git a/apps/api/src/alicebot_api/vnext_capture.py b/apps/api/src/alicebot_api/vnext_capture.py index 8b86ba37e..250a414f1 100644 --- a/apps/api/src/alicebot_api/vnext_capture.py +++ b/apps/api/src/alicebot_api/vnext_capture.py @@ -1545,84 +1545,87 @@ def _capture_source(self, source_input: SourceCaptureInput) -> CaptureResult: sensitivity=source_input.sensitivity, ) memory_rows: list[JsonObject] = [] - for candidate in candidates: - # Speaker provenance is only stamped when a role was derived, - # so provenance-free captures keep byte-identical metadata. - provenance_metadata: JsonObject = ( - { - "provenance_role": candidate.provenance_role, - "assertion_class": candidate.assertion_class, - } - if candidate.provenance_role is not None - else {} - ) - memory = self.store.create_memory( - { - "memory_key": _memory_key( - content_hash=content_hash, - candidate=candidate, - domain=source_input.domain, - sensitivity=source_input.sensitivity, - ), - "value": { - "text": candidate.text, + from alicebot_api.vnext_label_writes import label_write_batch + + with label_write_batch(self.store): + for candidate in candidates: + # Speaker provenance is only stamped when a role was derived, + # so provenance-free captures keep byte-identical metadata. + provenance_metadata: JsonObject = ( + { + "provenance_role": candidate.provenance_role, + "assertion_class": candidate.assertion_class, + } + if candidate.provenance_role is not None + else {} + ) + memory = self.store.create_memory( + { + "memory_key": _memory_key( + content_hash=content_hash, + candidate=candidate, + domain=source_input.domain, + sensitivity=source_input.sensitivity, + ), + "value": { + "text": candidate.text, + "source_id": source_id, + "source_chunk_id": candidate.source_chunk_id, + }, + "status": "candidate", + "source_event_ids": [source_id, candidate.source_chunk_id], + "memory_type": candidate.memory_type, + "confidence": candidate.confidence, + "title": _truncate(candidate.text, max_length=120), + "canonical_text": candidate.text, + "summary": _truncate(candidate.text, max_length=280), + "domain": source_input.domain, + "sensitivity": source_input.sensitivity, + "project_id": project_scope[0] if len(project_scope) == 1 else None, + "created_by_agent_id": self.actor_id if self.actor_type == "agent" else None, + "run_id": self.run_id if self.actor_type == "agent" else None, + "metadata_json": { + "source_id": source_id, + "source_chunk_id": candidate.source_chunk_id, + "source_chunk_index": candidate.source_chunk_index, + "extraction_rule": candidate.extraction_rule, + "capture_content_hash": content_hash, + **provenance_metadata, + **project_scope_metadata, + "generated_by": self.actor_type, + "agent_identity": self.agent_identity, + "agent_id": self.actor_id if self.actor_type == "agent" else None, + "agent_run_id": self.run_id if self.actor_type == "agent" else None, + "trace_id": self.trace_id, + "policy_decision": self.policy_decision, + }, + }, + actor_type=self.actor_type, + ) + memory_rows.append(memory) + self.store.create_provenance_link( + { + "target_type": "memory", + "target_id": str(memory["id"]), "source_id": source_id, "source_chunk_id": candidate.source_chunk_id, + "quote": candidate.text, + "evidence_role": "quoted_from", + "confidence": candidate.confidence, }, - "status": "candidate", - "source_event_ids": [source_id, candidate.source_chunk_id], - "memory_type": candidate.memory_type, - "confidence": candidate.confidence, - "title": _truncate(candidate.text, max_length=120), - "canonical_text": candidate.text, - "summary": _truncate(candidate.text, max_length=280), - "domain": source_input.domain, - "sensitivity": source_input.sensitivity, - "project_id": project_scope[0] if len(project_scope) == 1 else None, - "created_by_agent_id": self.actor_id if self.actor_type == "agent" else None, - "run_id": self.run_id if self.actor_type == "agent" else None, - "metadata_json": { + actor_type=self.actor_type, + ) + self._log_event( + event_type="memory.candidate_created", + target_type="memory", + target_id=str(memory["id"]), + payload={ "source_id": source_id, "source_chunk_id": candidate.source_chunk_id, - "source_chunk_index": candidate.source_chunk_index, - "extraction_rule": candidate.extraction_rule, - "capture_content_hash": content_hash, - **provenance_metadata, - **project_scope_metadata, - "generated_by": self.actor_type, - "agent_identity": self.agent_identity, - "agent_id": self.actor_id if self.actor_type == "agent" else None, - "agent_run_id": self.run_id if self.actor_type == "agent" else None, - "trace_id": self.trace_id, - "policy_decision": self.policy_decision, + "memory_type": candidate.memory_type, + "confidence": candidate.confidence, }, - }, - actor_type=self.actor_type, - ) - memory_rows.append(memory) - self.store.create_provenance_link( - { - "target_type": "memory", - "target_id": str(memory["id"]), - "source_id": source_id, - "source_chunk_id": candidate.source_chunk_id, - "quote": candidate.text, - "evidence_role": "quoted_from", - "confidence": candidate.confidence, - }, - actor_type=self.actor_type, - ) - self._log_event( - event_type="memory.candidate_created", - target_type="memory", - target_id=str(memory["id"]), - payload={ - "source_id": source_id, - "source_chunk_id": candidate.source_chunk_id, - "memory_type": candidate.memory_type, - "confidence": candidate.confidence, - }, - ) + ) # Capture writes candidate memories only, and recall cannot return a # candidate, so no text is sent to the embeddings endpoint here: the diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 9265da97e..fd9cedb2b 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -10,8 +10,9 @@ import re from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager +from contextvars import ContextVar from functools import wraps -from dataclasses import replace +from dataclasses import dataclass, replace from typing import Any from alicebot_api.vnext_agent_control import RESTRICTED_DOMAINS @@ -59,6 +60,67 @@ class LabelLockOrderError(RuntimeError): REFUSED_DETAIL = "the label change could not be applied to every dependent row; nothing was changed" +@dataclass +class _LabelWriteBatch: + conn: Any + transaction_id: str + rollback_counter: int + shared_lock: bool = True + + @property + def key(self) -> tuple[str, int]: + return self.transaction_id, self.rollback_counter + + +_LABEL_WRITE_BATCH: ContextVar[_LabelWriteBatch | None] = ContextVar("label_write_batch", default=None) + + +def label_savepoint_rolled_back(store: Any) -> None: + """Invalidate a batch grant whenever the store rolls a savepoint back.""" + + conn = store.conn + counter = int(getattr(conn, "_alice_label_rollback_counter", 0)) + 1 + conn._alice_label_rollback_counter = counter + + +def _current_write_batch(store: Any) -> _LabelWriteBatch | None: + batch = _LABEL_WRITE_BATCH.get() + if batch is None or batch.conn is not getattr(store, "conn", None) or not _in_transaction(store): + return None + counter = int(getattr(batch.conn, "_alice_label_rollback_counter", 0)) + if counter != batch.rollback_counter: + batch.rollback_counter = counter + batch.shared_lock = False + return batch + + +@contextmanager +def label_write_batch(store: Any) -> Iterator[None]: + """Reuse one shared grant for capture's bounded candidate writes. + + The managed transaction forbids a commit inside this scope. Its actual + Postgres transaction id and the store's savepoint rollback counter key the + grant. No input labels are cached, and strict mode always takes a live lock. + """ + + conn = getattr(store, "conn", None) + if conn is None or _sqlite(store) or not callable(getattr(conn, "transaction", None)): + yield + return + with conn.transaction(): + with conn.cursor() as cur: + cur.execute("SELECT pg_current_xact_id()::text AS transaction_id") + row = cur.fetchone() + transaction_id = str(row["transaction_id"] if isinstance(row, Mapping) else row[0]) + store.lock_label_writes(exclusive=False) + batch = _LabelWriteBatch(conn, transaction_id, int(getattr(conn, "_alice_label_rollback_counter", 0))) + token = _LABEL_WRITE_BATCH.set(batch) + try: + yield + finally: + _LABEL_WRITE_BATCH.reset(token) + + def takes_label_lock(fn: Any) -> Any: """Take the shared label lock before a store method writes or row-locks a label table.""" @@ -66,7 +128,11 @@ def takes_label_lock(fn: Any) -> Any: def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: lock = getattr(self, "lock_label_writes", None) if callable(lock): - lock(exclusive=False) + batch = _current_write_batch(self) + if STRICT_LOCK_ORDER or batch is None or not batch.shared_lock: + lock(exclusive=False) + if batch is not None: + batch.shared_lock = True return fn(self, *args, **kwargs) wrapper.__takes_label_lock__ = True # type: ignore[attr-defined] diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 7ee1a2f93..19943d528 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -3874,6 +3874,9 @@ def savepoint(self) -> Iterator[None]: try: yield except BaseException: + from alicebot_api.vnext_label_writes import label_savepoint_rolled_back + + label_savepoint_rolled_back(self) try: self.conn.execute(f"ROLLBACK TO SAVEPOINT {name}") self.conn.execute(f"RELEASE SAVEPOINT {name}") diff --git a/tests/integration/test_capture_label_batch_postgres.py b/tests/integration/test_capture_label_batch_postgres.py new file mode 100644 index 000000000..5ed06cfd9 --- /dev/null +++ b/tests/integration/test_capture_label_batch_postgres.py @@ -0,0 +1,155 @@ +"""A capture batch reuses only a live transaction's shared advisory grant.""" +from uuid import uuid4 + +import psycopg +import pytest +from psycopg.rows import dict_row + +from alicebot_api import vnext_label_writes as writes +from alicebot_api.db import set_current_user, user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_capture import VNextCaptureService +from alicebot_api.vnext_store import PostgresVNextStore + + +def _store(url): + user = uuid4() + return user, user_connection(url, user) + + +def _user(conn, user): + ContinuityStore(conn).create_user(user, f"batch-{user}@example.test", "Synthetic") + + +def _memory(store, key, source=None): + return store.create_memory({"memory_key": key, "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])} if source else {}}) + + +def test_capture_reuses_only_the_shared_grant_in_production_mode(migrated_database_urls, monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", False) + user, context = _store(migrated_database_urls["app"]) + with context as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + count = 0 + real_lock = store.lock_label_writes + def counted_lock(*, exclusive=False): + nonlocal count + count += 1 + return real_lock(exclusive=exclusive) + monkeypatch.setattr(store, "lock_label_writes", counted_lock) + result = VNextCaptureService(store).capture_text("\n".join(f"Decision: Synthetic item {i} is assigned to synthetic route {i}." for i in range(200)), domain="project", sensitivity="public") + assert result.candidate_memory_count == 200 + # Source creation precedes the batch, and its candidate writes share + # one further grant. All insert floors still read their current inputs. + assert count == 2 + assert writes._current_write_batch(store) is None + assert writes.held_label_locks(store)[1] is True + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 200 + + +def test_batch_grant_is_keyed_by_transaction_and_savepoint_rollback(migrated_database_urls, monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", False) + user, context = _store(migrated_database_urls["app"]) + with context as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + calls = [] + real_lock = store.lock_label_writes + def counted_lock(*, exclusive=False): + calls.append(exclusive) + return real_lock(exclusive=exclusive) + monkeypatch.setattr(store, "lock_label_writes", counted_lock) + with writes.label_write_batch(store): + frame = writes._current_write_batch(store) + before = frame.key + assert before[0] == str(conn.execute("SELECT pg_current_xact_id()::text AS id").fetchone()["id"]) + try: + with store.savepoint(): + _memory(store, "rolled-back") + raise ValueError("Synthetic rollback") + except ValueError: + pass + _memory(store, "after-rollback") + after = writes._current_write_batch(store).key + assert after[0] == before[0] and after[1] == before[1] + 1 + assert calls == [False, False] + assert writes.held_label_locks(store)[1] is True + assert conn.execute("SELECT memory_key FROM memories").fetchone()["memory_key"] == "after-rollback" + + +def test_a_failed_batch_leaves_no_grant_memo_or_rows(migrated_database_urls, monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", False) + user, context = _store(migrated_database_urls["app"]) + with context as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + with pytest.raises(ValueError): + with store.savepoint(): + with writes.label_write_batch(store): + _memory(store, "failed") + assert writes.held_label_locks(store)[1] is True + raise ValueError("Synthetic failure") + assert writes._current_write_batch(store) is None + assert writes.held_label_locks(store)[1] is False + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 0 + _memory(store, "fresh") + assert writes.held_label_locks(store)[1] is True + + +def test_strict_mode_does_not_reuse_the_batch_grant(migrated_database_urls, monkeypatch): + user, context = _store(migrated_database_urls["app"]) + with context as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + count = 0 + real_lock = store.lock_label_writes + def counted_lock(*, exclusive=False): + nonlocal count + count += 1 + return real_lock(exclusive=exclusive) + monkeypatch.setattr(store, "lock_label_writes", counted_lock) + assert writes.STRICT_LOCK_ORDER is True + with writes.label_write_batch(store): + _memory(store, "first") + _memory(store, "second") + assert writes.held_label_locks(store)[1] is True + assert count == 3 + + +def test_batch_has_no_stale_input_label_cache_after_a_same_transaction_raise(migrated_database_urls): + user, context = _store(migrated_database_urls["app"]) + with context as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + source = store.create_source({"source_type": "note", "title": "Synthetic", "content_hash": str(user), "domain": "project", "sensitivity": "public"}) + with writes.label_write_batch(store): + first = _memory(store, "before-raise", source) + assert first["sensitivity"] == "public" + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) + second = _memory(store, "after-raise", source) + assert second["sensitivity"] == "confidential" + + +def test_one_store_reacquires_a_batch_grant_in_a_new_transaction(migrated_database_urls, monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", False) + user = uuid4() + keys = [] + with psycopg.connect(migrated_database_urls["app"], row_factory=dict_row) as conn: + store = PostgresVNextStore(conn) + for index in range(2): + set_current_user(conn, user) + if index == 0: + _user(conn, user) + with writes.label_write_batch(store): + keys.append(writes._current_write_batch(store).key) + with pytest.raises(psycopg.ProgrammingError): + conn.commit() + _memory(store, str(index)) + assert writes.held_label_locks(store)[1] is True + assert writes._current_write_batch(store) is None + conn.commit() + assert conn.info.transaction_status == psycopg.pq.TransactionStatus.IDLE + assert keys[0][0] != keys[1][0] From e1a090db4cfa64bd0c1d436b312afdfecc8a4e04 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:38:51 +0200 Subject: [PATCH 141/270] Seed legacy promotion fixture in graph before label lock order --- tests/integration/test_derived_domain_postgres.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_derived_domain_postgres.py b/tests/integration/test_derived_domain_postgres.py index b248e0aca..87ba9ed76 100644 --- a/tests/integration/test_derived_domain_postgres.py +++ b/tests/integration/test_derived_domain_postgres.py @@ -302,6 +302,8 @@ def test_promoted_artifact_uuid_alias_repaired(database_urls): with without_insert_floor(), user_connection(database_urls["app"], user) as conn: ContinuityStore(conn).create_user(user, "alias@example.invalid", "Alias fixture") store = PostgresVNextStore(conn) + # Promotion takes the graph lock before any label-table writes. + store.lock_graph_mutation() memory = store.create_memory({"memory_key": "health", "canonical_text": "Private observation", "domain": "health", "sensitivity": "public", "status": "active"}) artifact = store.create_artifact({"artifact_type": "daily_brief", "title": "Fixture brief", From 4232ada0b8617bfd098ae90d92307d1ce0af0d26 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:39:55 +0200 Subject: [PATCH 142/270] Revert "Reuse capture label grants within the current transaction" This reverts commit e61e673e55ec57cc246d7a35fad22e60a8b2d492. --- apps/api/src/alicebot_api/vnext_capture.py | 147 ++++++++--------- .../src/alicebot_api/vnext_label_writes.py | 70 +------- apps/api/src/alicebot_api/vnext_store.py | 3 - .../test_capture_label_batch_postgres.py | 155 ------------------ 4 files changed, 74 insertions(+), 301 deletions(-) delete mode 100644 tests/integration/test_capture_label_batch_postgres.py diff --git a/apps/api/src/alicebot_api/vnext_capture.py b/apps/api/src/alicebot_api/vnext_capture.py index 250a414f1..8b86ba37e 100644 --- a/apps/api/src/alicebot_api/vnext_capture.py +++ b/apps/api/src/alicebot_api/vnext_capture.py @@ -1545,87 +1545,84 @@ def _capture_source(self, source_input: SourceCaptureInput) -> CaptureResult: sensitivity=source_input.sensitivity, ) memory_rows: list[JsonObject] = [] - from alicebot_api.vnext_label_writes import label_write_batch - - with label_write_batch(self.store): - for candidate in candidates: - # Speaker provenance is only stamped when a role was derived, - # so provenance-free captures keep byte-identical metadata. - provenance_metadata: JsonObject = ( - { - "provenance_role": candidate.provenance_role, - "assertion_class": candidate.assertion_class, - } - if candidate.provenance_role is not None - else {} - ) - memory = self.store.create_memory( - { - "memory_key": _memory_key( - content_hash=content_hash, - candidate=candidate, - domain=source_input.domain, - sensitivity=source_input.sensitivity, - ), - "value": { - "text": candidate.text, - "source_id": source_id, - "source_chunk_id": candidate.source_chunk_id, - }, - "status": "candidate", - "source_event_ids": [source_id, candidate.source_chunk_id], - "memory_type": candidate.memory_type, - "confidence": candidate.confidence, - "title": _truncate(candidate.text, max_length=120), - "canonical_text": candidate.text, - "summary": _truncate(candidate.text, max_length=280), - "domain": source_input.domain, - "sensitivity": source_input.sensitivity, - "project_id": project_scope[0] if len(project_scope) == 1 else None, - "created_by_agent_id": self.actor_id if self.actor_type == "agent" else None, - "run_id": self.run_id if self.actor_type == "agent" else None, - "metadata_json": { - "source_id": source_id, - "source_chunk_id": candidate.source_chunk_id, - "source_chunk_index": candidate.source_chunk_index, - "extraction_rule": candidate.extraction_rule, - "capture_content_hash": content_hash, - **provenance_metadata, - **project_scope_metadata, - "generated_by": self.actor_type, - "agent_identity": self.agent_identity, - "agent_id": self.actor_id if self.actor_type == "agent" else None, - "agent_run_id": self.run_id if self.actor_type == "agent" else None, - "trace_id": self.trace_id, - "policy_decision": self.policy_decision, - }, - }, - actor_type=self.actor_type, - ) - memory_rows.append(memory) - self.store.create_provenance_link( - { - "target_type": "memory", - "target_id": str(memory["id"]), + for candidate in candidates: + # Speaker provenance is only stamped when a role was derived, + # so provenance-free captures keep byte-identical metadata. + provenance_metadata: JsonObject = ( + { + "provenance_role": candidate.provenance_role, + "assertion_class": candidate.assertion_class, + } + if candidate.provenance_role is not None + else {} + ) + memory = self.store.create_memory( + { + "memory_key": _memory_key( + content_hash=content_hash, + candidate=candidate, + domain=source_input.domain, + sensitivity=source_input.sensitivity, + ), + "value": { + "text": candidate.text, "source_id": source_id, "source_chunk_id": candidate.source_chunk_id, - "quote": candidate.text, - "evidence_role": "quoted_from", - "confidence": candidate.confidence, }, - actor_type=self.actor_type, - ) - self._log_event( - event_type="memory.candidate_created", - target_type="memory", - target_id=str(memory["id"]), - payload={ + "status": "candidate", + "source_event_ids": [source_id, candidate.source_chunk_id], + "memory_type": candidate.memory_type, + "confidence": candidate.confidence, + "title": _truncate(candidate.text, max_length=120), + "canonical_text": candidate.text, + "summary": _truncate(candidate.text, max_length=280), + "domain": source_input.domain, + "sensitivity": source_input.sensitivity, + "project_id": project_scope[0] if len(project_scope) == 1 else None, + "created_by_agent_id": self.actor_id if self.actor_type == "agent" else None, + "run_id": self.run_id if self.actor_type == "agent" else None, + "metadata_json": { "source_id": source_id, "source_chunk_id": candidate.source_chunk_id, - "memory_type": candidate.memory_type, - "confidence": candidate.confidence, + "source_chunk_index": candidate.source_chunk_index, + "extraction_rule": candidate.extraction_rule, + "capture_content_hash": content_hash, + **provenance_metadata, + **project_scope_metadata, + "generated_by": self.actor_type, + "agent_identity": self.agent_identity, + "agent_id": self.actor_id if self.actor_type == "agent" else None, + "agent_run_id": self.run_id if self.actor_type == "agent" else None, + "trace_id": self.trace_id, + "policy_decision": self.policy_decision, }, - ) + }, + actor_type=self.actor_type, + ) + memory_rows.append(memory) + self.store.create_provenance_link( + { + "target_type": "memory", + "target_id": str(memory["id"]), + "source_id": source_id, + "source_chunk_id": candidate.source_chunk_id, + "quote": candidate.text, + "evidence_role": "quoted_from", + "confidence": candidate.confidence, + }, + actor_type=self.actor_type, + ) + self._log_event( + event_type="memory.candidate_created", + target_type="memory", + target_id=str(memory["id"]), + payload={ + "source_id": source_id, + "source_chunk_id": candidate.source_chunk_id, + "memory_type": candidate.memory_type, + "confidence": candidate.confidence, + }, + ) # Capture writes candidate memories only, and recall cannot return a # candidate, so no text is sent to the embeddings endpoint here: the diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index fd9cedb2b..9265da97e 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -10,9 +10,8 @@ import re from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager -from contextvars import ContextVar from functools import wraps -from dataclasses import dataclass, replace +from dataclasses import replace from typing import Any from alicebot_api.vnext_agent_control import RESTRICTED_DOMAINS @@ -60,67 +59,6 @@ class LabelLockOrderError(RuntimeError): REFUSED_DETAIL = "the label change could not be applied to every dependent row; nothing was changed" -@dataclass -class _LabelWriteBatch: - conn: Any - transaction_id: str - rollback_counter: int - shared_lock: bool = True - - @property - def key(self) -> tuple[str, int]: - return self.transaction_id, self.rollback_counter - - -_LABEL_WRITE_BATCH: ContextVar[_LabelWriteBatch | None] = ContextVar("label_write_batch", default=None) - - -def label_savepoint_rolled_back(store: Any) -> None: - """Invalidate a batch grant whenever the store rolls a savepoint back.""" - - conn = store.conn - counter = int(getattr(conn, "_alice_label_rollback_counter", 0)) + 1 - conn._alice_label_rollback_counter = counter - - -def _current_write_batch(store: Any) -> _LabelWriteBatch | None: - batch = _LABEL_WRITE_BATCH.get() - if batch is None or batch.conn is not getattr(store, "conn", None) or not _in_transaction(store): - return None - counter = int(getattr(batch.conn, "_alice_label_rollback_counter", 0)) - if counter != batch.rollback_counter: - batch.rollback_counter = counter - batch.shared_lock = False - return batch - - -@contextmanager -def label_write_batch(store: Any) -> Iterator[None]: - """Reuse one shared grant for capture's bounded candidate writes. - - The managed transaction forbids a commit inside this scope. Its actual - Postgres transaction id and the store's savepoint rollback counter key the - grant. No input labels are cached, and strict mode always takes a live lock. - """ - - conn = getattr(store, "conn", None) - if conn is None or _sqlite(store) or not callable(getattr(conn, "transaction", None)): - yield - return - with conn.transaction(): - with conn.cursor() as cur: - cur.execute("SELECT pg_current_xact_id()::text AS transaction_id") - row = cur.fetchone() - transaction_id = str(row["transaction_id"] if isinstance(row, Mapping) else row[0]) - store.lock_label_writes(exclusive=False) - batch = _LabelWriteBatch(conn, transaction_id, int(getattr(conn, "_alice_label_rollback_counter", 0))) - token = _LABEL_WRITE_BATCH.set(batch) - try: - yield - finally: - _LABEL_WRITE_BATCH.reset(token) - - def takes_label_lock(fn: Any) -> Any: """Take the shared label lock before a store method writes or row-locks a label table.""" @@ -128,11 +66,7 @@ def takes_label_lock(fn: Any) -> Any: def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: lock = getattr(self, "lock_label_writes", None) if callable(lock): - batch = _current_write_batch(self) - if STRICT_LOCK_ORDER or batch is None or not batch.shared_lock: - lock(exclusive=False) - if batch is not None: - batch.shared_lock = True + lock(exclusive=False) return fn(self, *args, **kwargs) wrapper.__takes_label_lock__ = True # type: ignore[attr-defined] diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 19943d528..7ee1a2f93 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -3874,9 +3874,6 @@ def savepoint(self) -> Iterator[None]: try: yield except BaseException: - from alicebot_api.vnext_label_writes import label_savepoint_rolled_back - - label_savepoint_rolled_back(self) try: self.conn.execute(f"ROLLBACK TO SAVEPOINT {name}") self.conn.execute(f"RELEASE SAVEPOINT {name}") diff --git a/tests/integration/test_capture_label_batch_postgres.py b/tests/integration/test_capture_label_batch_postgres.py deleted file mode 100644 index 5ed06cfd9..000000000 --- a/tests/integration/test_capture_label_batch_postgres.py +++ /dev/null @@ -1,155 +0,0 @@ -"""A capture batch reuses only a live transaction's shared advisory grant.""" -from uuid import uuid4 - -import psycopg -import pytest -from psycopg.rows import dict_row - -from alicebot_api import vnext_label_writes as writes -from alicebot_api.db import set_current_user, user_connection -from alicebot_api.store import ContinuityStore -from alicebot_api.vnext_capture import VNextCaptureService -from alicebot_api.vnext_store import PostgresVNextStore - - -def _store(url): - user = uuid4() - return user, user_connection(url, user) - - -def _user(conn, user): - ContinuityStore(conn).create_user(user, f"batch-{user}@example.test", "Synthetic") - - -def _memory(store, key, source=None): - return store.create_memory({"memory_key": key, "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])} if source else {}}) - - -def test_capture_reuses_only_the_shared_grant_in_production_mode(migrated_database_urls, monkeypatch): - monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", False) - user, context = _store(migrated_database_urls["app"]) - with context as conn: - _user(conn, user) - store = PostgresVNextStore(conn) - count = 0 - real_lock = store.lock_label_writes - def counted_lock(*, exclusive=False): - nonlocal count - count += 1 - return real_lock(exclusive=exclusive) - monkeypatch.setattr(store, "lock_label_writes", counted_lock) - result = VNextCaptureService(store).capture_text("\n".join(f"Decision: Synthetic item {i} is assigned to synthetic route {i}." for i in range(200)), domain="project", sensitivity="public") - assert result.candidate_memory_count == 200 - # Source creation precedes the batch, and its candidate writes share - # one further grant. All insert floors still read their current inputs. - assert count == 2 - assert writes._current_write_batch(store) is None - assert writes.held_label_locks(store)[1] is True - assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 200 - - -def test_batch_grant_is_keyed_by_transaction_and_savepoint_rollback(migrated_database_urls, monkeypatch): - monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", False) - user, context = _store(migrated_database_urls["app"]) - with context as conn: - _user(conn, user) - store = PostgresVNextStore(conn) - calls = [] - real_lock = store.lock_label_writes - def counted_lock(*, exclusive=False): - calls.append(exclusive) - return real_lock(exclusive=exclusive) - monkeypatch.setattr(store, "lock_label_writes", counted_lock) - with writes.label_write_batch(store): - frame = writes._current_write_batch(store) - before = frame.key - assert before[0] == str(conn.execute("SELECT pg_current_xact_id()::text AS id").fetchone()["id"]) - try: - with store.savepoint(): - _memory(store, "rolled-back") - raise ValueError("Synthetic rollback") - except ValueError: - pass - _memory(store, "after-rollback") - after = writes._current_write_batch(store).key - assert after[0] == before[0] and after[1] == before[1] + 1 - assert calls == [False, False] - assert writes.held_label_locks(store)[1] is True - assert conn.execute("SELECT memory_key FROM memories").fetchone()["memory_key"] == "after-rollback" - - -def test_a_failed_batch_leaves_no_grant_memo_or_rows(migrated_database_urls, monkeypatch): - monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", False) - user, context = _store(migrated_database_urls["app"]) - with context as conn: - _user(conn, user) - store = PostgresVNextStore(conn) - with pytest.raises(ValueError): - with store.savepoint(): - with writes.label_write_batch(store): - _memory(store, "failed") - assert writes.held_label_locks(store)[1] is True - raise ValueError("Synthetic failure") - assert writes._current_write_batch(store) is None - assert writes.held_label_locks(store)[1] is False - assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 0 - _memory(store, "fresh") - assert writes.held_label_locks(store)[1] is True - - -def test_strict_mode_does_not_reuse_the_batch_grant(migrated_database_urls, monkeypatch): - user, context = _store(migrated_database_urls["app"]) - with context as conn: - _user(conn, user) - store = PostgresVNextStore(conn) - count = 0 - real_lock = store.lock_label_writes - def counted_lock(*, exclusive=False): - nonlocal count - count += 1 - return real_lock(exclusive=exclusive) - monkeypatch.setattr(store, "lock_label_writes", counted_lock) - assert writes.STRICT_LOCK_ORDER is True - with writes.label_write_batch(store): - _memory(store, "first") - _memory(store, "second") - assert writes.held_label_locks(store)[1] is True - assert count == 3 - - -def test_batch_has_no_stale_input_label_cache_after_a_same_transaction_raise(migrated_database_urls): - user, context = _store(migrated_database_urls["app"]) - with context as conn: - _user(conn, user) - store = PostgresVNextStore(conn) - store.lock_graph_mutation() - store.lock_label_writes(exclusive=True) - source = store.create_source({"source_type": "note", "title": "Synthetic", "content_hash": str(user), "domain": "project", "sensitivity": "public"}) - with writes.label_write_batch(store): - first = _memory(store, "before-raise", source) - assert first["sensitivity"] == "public" - store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) - second = _memory(store, "after-raise", source) - assert second["sensitivity"] == "confidential" - - -def test_one_store_reacquires_a_batch_grant_in_a_new_transaction(migrated_database_urls, monkeypatch): - monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", False) - user = uuid4() - keys = [] - with psycopg.connect(migrated_database_urls["app"], row_factory=dict_row) as conn: - store = PostgresVNextStore(conn) - for index in range(2): - set_current_user(conn, user) - if index == 0: - _user(conn, user) - with writes.label_write_batch(store): - keys.append(writes._current_write_batch(store).key) - with pytest.raises(psycopg.ProgrammingError): - conn.commit() - _memory(store, str(index)) - assert writes.held_label_locks(store)[1] is True - assert writes._current_write_batch(store) is None - conn.commit() - assert conn.info.transaction_status == psycopg.pq.TransactionStatus.IDLE - assert keys[0][0] != keys[1][0] From 1d5cf903738298ac52cdfe22ce977698e97b1941 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:38:39 +0200 Subject: [PATCH 143/270] Normalize direct loop references before derived read admission --- apps/api/src/alicebot_api/vnext_store.py | 2 +- .../vnext_stores/postgres/columns.py | 4 +- ...derived_labels_read_acceptance_postgres.py | 43 ++++++++++++++++++- 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 7ee1a2f93..4449fb169 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -543,7 +543,7 @@ def iter_label_rows(self, kind: str, *, batch_size: int = 200) -> Iterator[list[ if kind == "memory": extra = ", status, value, project_id, source_event_ids, deleted_at" elif kind == "open_loop": - extra = ", status, project_id, source_id, memory_id" + extra = ", status, project_id, source_id::text AS source_id, memory_id::text AS memory_id" elif kind == "artifact": extra = ", status, artifact_type" elif kind == "project": diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/columns.py b/apps/api/src/alicebot_api/vnext_stores/postgres/columns.py index f97ff71d9..f5ca74801 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/columns.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/columns.py @@ -172,7 +172,7 @@ OPEN_LOOP_COLUMNS = """ id, user_id, - memory_id, + memory_id::text AS memory_id, title, status, opened_at, @@ -185,7 +185,7 @@ priority, project_id, person_id, - source_id, + source_id::text AS source_id, closed_at, domain, sensitivity, diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py index 699272e96..beaeb4f87 100644 --- a/tests/integration/test_derived_labels_read_acceptance_postgres.py +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -15,8 +15,11 @@ from alicebot_api.routers import vnext_review, vnext_retrieval, vnext_projects, vnext_memories, workspaces from alicebot_api.store import ContinuityStore from alicebot_api.vnext_agent_control import ALL_SENSITIVITY +from alicebot_api.vnext_agent_keys import resolve_agent_identity from alicebot_api.vnext_event_log import append_event +from alicebot_api.vnext_label_guard import LabelGuard from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_source_fence import SourceReadFence from alicebot_api.vnext_store import PostgresVNextStore from tests.unit.test_derived_labels_real_keys import READERS, expected_read, real_reader_key, seed_read_rows @@ -72,7 +75,8 @@ def test_all_five_operator_screens_with_real_keys(migrated_database_urls, monkey store = PostgresVNextStore(conn) source = store.create_source({"source_type": "note", "title": "Cedar hidden source", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential"}) public_source = store.create_source({"source_type": "note", "title": "Public source", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public"}) - memory = store.create_memory({"memory_key": "belief", "canonical_text": "Cedar hidden belief", "status": "active", "domain": "project", "sensitivity": "confidential"}) + with without_insert_floor(): + memory = store.create_memory({"memory_key": "belief", "canonical_text": "Cedar hidden belief", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])}}) artifact = store.create_artifact({"artifact_type": "daily_brief", "title": "Cedar hidden artifact", "content_markdown": "Cedar hidden artifact", "domain": "project", "sensitivity": "confidential", "metadata_json": {"derived_from": {"v": 1, "sources": [str(public_source["id"])], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}}, "source_refs": [str(public_source["id"])]}}) project = store.create_project({"name": "Cedar hidden project", "slug": "cedar-hidden", "current_state": "Cedar hidden state", "domain": "project", "sensitivity": "confidential"}) belief_id = uuid4() @@ -249,3 +253,40 @@ async def send(message): assert "below their inputs" not in check["message"] for field in ("status", "warning_count", "blocking_failure_count", "recommended_fixes"): assert diagnostic[field] == before[1]["doctor"][field] + + +@pytest.mark.parametrize("reader", ("owner", "admin", "trusted")) +@pytest.mark.parametrize("source_sensitivity", ("public", "confidential")) +def test_direct_column_loop_references_reach_real_read_guard(migrated_database_urls, monkeypatch, reader, source_sensitivity): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + admitted = reader != "trusted" or source_sensitivity == "public" + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Direct parent", "content_hash": str(uuid4()), "domain": "project", "sensitivity": source_sensitivity}) + memory = store.create_memory({"memory_key": "direct-loop-parent", "canonical_text": "Direct memory", "domain": "project", "sensitivity": "public"}) + with without_insert_floor(): + loop = store.create_open_loop({"title": "Direct loop sentinel", "source_id": str(source["id"]), "memory_id": str(memory["id"]), "domain": "project", "sensitivity": "public", "metadata_json": {"discovered_by": "vnext_daily_capture"}}) + key = real_reader_key(store, user_id, reader) + identity = resolve_agent_identity(store, user_id=user_id, raw_key=key, payload={}) + row = store.get_open_loop(str(loop["id"])) + fence = SourceReadFence.for_identity(identity) + guard = LabelGuard.for_fence(store, fence) + judged = guard.effective_row("open_loop", row) + assert judged is not None + assert (judged["sensitivity"] == source_sensitivity) or not guard.active + assert fence.admits_memory(judged) is admitted + for projection in (row, store.read_label_rows("open_loop", [str(loop["id"])])[0], next(store.iter_label_rows("open_loop"))[0]): + assert projection["source_id"] == str(source["id"]) + assert projection["memory_id"] == str(memory["id"]) + count_guard = LabelGuard.for_filters(store, (), ("public", "internal", "private", "unknown")) + assert count_guard.readable_status_counts("open_loop").get("open", 0) == int(source_sensitivity == "public") + if key: + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + result = call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name="alice_open_loops", arguments={"sensitivity_allowed": list(ALL_SENSITIVITY)}) + rendered = json.dumps(result, default=str) + assert (str(loop["id"]) in rendered) is admitted + assert ("Direct loop sentinel" in rendered) is admitted From 525ac56fbece24276b74000fc5dd2394176fb753 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:38:51 +0200 Subject: [PATCH 144/270] Seed legacy promotion fixture in graph before label lock order --- tests/integration/test_derived_domain_postgres.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_derived_domain_postgres.py b/tests/integration/test_derived_domain_postgres.py index b248e0aca..87ba9ed76 100644 --- a/tests/integration/test_derived_domain_postgres.py +++ b/tests/integration/test_derived_domain_postgres.py @@ -302,6 +302,8 @@ def test_promoted_artifact_uuid_alias_repaired(database_urls): with without_insert_floor(), user_connection(database_urls["app"], user) as conn: ContinuityStore(conn).create_user(user, "alias@example.invalid", "Alias fixture") store = PostgresVNextStore(conn) + # Promotion takes the graph lock before any label-table writes. + store.lock_graph_mutation() memory = store.create_memory({"memory_key": "health", "canonical_text": "Private observation", "domain": "health", "sensitivity": "public", "status": "active"}) artifact = store.create_artifact({"artifact_type": "daily_brief", "title": "Fixture brief", From 20fbb1aae3d7c7b0e8a4cc88e2c79c87078d26c9 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:40:44 +0200 Subject: [PATCH 145/270] Pin normalized PostgreSQL loop dependency columns --- tests/unit/test_store_graph_open_loops_split.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index 0e0249075..7f695dbb0 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -187,7 +187,8 @@ "c333a0dacf8733a16fb86f3acc1bbf61bd25fa66ce96cc3bc25805c4a85d9203" ), "BELIEF_COLUMNS": "32bac57e9e38fead1af29b9324777978f3b03bfe20d5306695fae98079d82dc7", - "OPEN_LOOP_COLUMNS": "651275ee48d37e13228bf339ac4f260a50333fc7b86197ab16573cc099f912bf", + # Reviewed: normalize the two direct dependency UUID columns to text. + "OPEN_LOOP_COLUMNS": "44970b53460b10e1e414a9c1c7905f15ea7ecfed4570b1fcd5becf65952b3c64", }, SQLITE_COLUMNS_PATH: { "GRAPH_EDGE_COLUMNS": "587b88564c446c03420441371a180e11618ea6bf192e5e20d2ad5d426ce890f2", From 0f6d53fc61cefa12a1e016c32135bc301885a628 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:41:58 +0200 Subject: [PATCH 146/270] Retain the reviewed PostgreSQL column source receipt --- tests/unit/test_store_graph_open_loops_split.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index 7f695dbb0..44c28de60 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -133,7 +133,7 @@ # Re-minted for the combined floor-aware partition read and canonical source-reference batch lookup. # Previous receipt 9a2634be... SQLITE_CARRIER_PATH: "d80d53bc64e2395f9480eb6b06338f337c44b811b0eb86974b4dbc22308a88d0", - POSTGRES_COLUMNS_PATH: "5b0d972a55abf8590ce14394a37fd71b9b88ba7ab3de82d61efc1bddfc022b71", + POSTGRES_COLUMNS_PATH: "1a782bf3eb87f68f67434508baab0f82d49cb40a0c547cde49bbc972e2f1d182", SQLITE_COLUMNS_PATH: "be81b8628d0831d3d02b280b5455fb02333db5740ebef8d85d58024384ae6556", } EXPECTED_METHOD_AST_MANIFESTS = { From 4fe175a9237fa5cb2b8ccc8defcd3a2c2eac2d76 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:27:11 +0200 Subject: [PATCH 147/270] Follow canonical encoded references during label propagation --- .../src/alicebot_api/vnext_label_writes.py | 18 +++- apps/api/src/alicebot_api/vnext_store.py | 2 +- ...est_encoded_label_dependencies_postgres.py | 95 +++++++++++++++++++ tests/unit/test_encoded_label_dependencies.py | 76 +++++++++++++++ 4 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 tests/integration/test_encoded_label_dependencies_postgres.py create mode 100644 tests/unit/test_encoded_label_dependencies.py diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 04dba3302..9265da97e 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -366,11 +366,14 @@ def _sqlite_dependants(store: Any, table: str, kind: str, compacts: Sequence[str if table == "memories": extra = ", value, project_id, NULL AS source_id, NULL AS memory_id" text_clause, _ = _like_clause("metadata_json", len(compacts), qmark=True) + # A nested JSON string may encode every character of an id. Keep all + # escaped candidates for the canonical dependency parser below. + text_clause += " OR instr(coalesce(metadata_json, ''), char(92)) > 0" params: list[object] = [store.user_id, *[f"%{item}%" for item in compacts]] value_sql = "" if with_value: value_clause, _ = _like_clause("value", len(compacts), qmark=True) - value_sql = f" OR {value_clause}" + value_sql = f" OR {value_clause} OR instr(coalesce(value, ''), char(92)) > 0" params.extend(f"%{item}%" for item in compacts) column_sql = "" if table == "open_loops": @@ -396,23 +399,30 @@ def _postgres_dependants(store: Any, table: str, kind: str, compacts: Sequence[s if table == "memories": extra = ", value, project_id" elif table == "open_loops": - extra = ", NULL::jsonb AS value, project_id, source_id, memory_id" + extra = ", NULL::jsonb AS value, project_id, source_id::text AS source_id, memory_id::text AS memory_id" elif table == "generated_artifacts": extra = ", NULL::jsonb AS value, artifact_type" else: extra = ", NULL::jsonb AS value" text_clause, _ = _like_clause("metadata_json::text", len(compacts), qmark=False) + text_clause += " OR strpos(coalesce(metadata_json::text, ''), chr(92)) > 0" params: list[object] = [f"%{item}%" for item in compacts] value_sql = "" if with_value: value_clause, _ = _like_clause("value::text", len(compacts), qmark=False) - value_sql = f" OR {value_clause}" + value_sql = f" OR {value_clause} OR strpos(coalesce(value::text, ''), chr(92)) > 0" params.extend(f"%{item}%" for item in compacts) + column_sql = "" + if table == "open_loops": + for column in ("source_id", "memory_id"): + clause, _ = _like_clause(f"{column}::text", len(compacts), qmark=False) + column_sql += f" OR {clause}" + params.extend(f"%{item}%" for item in compacts) rows = store._fetch_all( f""" SELECT id::text AS id, user_id::text AS user_id, domain, sensitivity, metadata_json{extra} FROM {table} - WHERE ({text_clause}{value_sql}) + WHERE ({text_clause}{value_sql}{column_sql}) """, # nosec B608 # internal literal table/columns; every external value is bound tuple(params), ) diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index f172b2e6a..2a6a4e209 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -513,7 +513,7 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: if table == "memories": extra = ", value, project_id" elif table == "open_loops": - extra = ", project_id, source_id, memory_id" + extra = ", project_id, source_id::text AS source_id, memory_id::text AS memory_id" elif table == "beliefs": extra = ", memory_id" elif table == "generated_artifacts": diff --git a/tests/integration/test_encoded_label_dependencies_postgres.py b/tests/integration/test_encoded_label_dependencies_postgres.py new file mode 100644 index 000000000..cd7c5f701 --- /dev/null +++ b/tests/integration/test_encoded_label_dependencies_postgres.py @@ -0,0 +1,95 @@ +"""Actual PostgreSQL encoded dependency and confirmation controls.""" +import json +from uuid import UUID, uuid4 + +import pytest +from psycopg.types.json import Jsonb + +from alicebot_api.config import Settings +from alicebot_api.db import user_connection +from alicebot_api.routers import vnext_memories as router +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_derived_labels import dependencies_of +from alicebot_api.vnext_label_writes import walk_dependants +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.test_source_move_label_preview_postgres import ALPHA, BETA, _snapshot +from tests.unit.test_encoded_label_dependencies import encoded_object + + +def test_encoded_source_move_previews_without_writes_then_confirms(migrated_database_urls, monkeypatch): + url = migrated_database_urls["app"] + user = uuid4() + with user_connection(url, user) as conn: + ContinuityStore(conn).create_user(user, f"encoded-{user}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Synthetic", "content_hash": str(user), "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA]}}) + reference = encoded_object("source_id", str(source["id"])) + copy = store.create_memory({"memory_key": "encoded", "canonical_text": "Synthetic", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": reference, "project_scope": [ALPHA]}}) + unrelated = store.create_memory({"memory_key": "unrelated", "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"note": encoded_object("source_id", str(uuid4()))}}) + assert ("source", str(source["id"])) in dependencies_of("memory", copy) + assert {str(row["id"]) for row in walk_dependants(store, [str(source["id"])])} == {str(copy["id"])} + monkeypatch.setattr(router, "get_settings", lambda: Settings(database_url=url)) + before = _snapshot(url, user, source) + request = router.VNextSourceReviewRequest(user_id=user, action="assign_project", project_id=BETA, sensitivity="confidential") + preview = router.review_vnext_source(UUID(str(source["id"])), request) + payload = json.loads(preview.body) + assert preview.status_code == 200 and payload["preview"] is True + assert payload["derived_rows_hidden_from_project_keys"] == 1 + assert payload["confirm_required"] is True + assert _snapshot(url, user, source) == before + confirmed = router.review_vnext_source(UUID(str(source["id"])), request.model_copy(update={"confirm_label_hide": True})) + assert confirmed.status_code == 200 + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + stored = store.get_memory(str(copy["id"])) + assert stored["sensitivity"] == "confidential" + assert stored["metadata_json"]["source_id"] == reference + assert BETA in stored["metadata_json"]["project_floor"] + assert store.get_memory(str(unrelated["id"]))["sensitivity"] == "public" + assert _snapshot(url, user, source)["provenance_links"] == before["provenance_links"] + + +@pytest.mark.parametrize("key,kind", [("source_id", "source"), ("memory_id", "memory"), ("artifact_id", "artifact"), ("belief_ids", "belief")]) +def test_encoded_metadata_references_keep_the_canonical_reverse_edge(migrated_database_urls, key, kind): + user = uuid4() + root_id = str(uuid4()) + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"encoded-keys-{user}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + row = store.create_memory({"memory_key": "encoded-key", "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public"}) + metadata = {key: [root_id] if key.endswith("ids") else root_id, "consolidation": {}} + raw = json.dumps(metadata) + encoded = "".join("\\u%04x" % ord(char) if char.isalnum() or char == "_" else char for char in raw) + conn.execute("UPDATE memories SET metadata_json = %s::jsonb WHERE id = %s", (encoded, row["id"])) + stored = store.get_memory(str(row["id"])) + assert (kind, root_id) in dependencies_of("memory", stored) + assert {str(item["id"]) for item in walk_dependants(store, [root_id])} == {str(row["id"])} + + +@pytest.mark.parametrize("key,kind", [("source_id", "source"), ("artifact_id", "artifact")]) +def test_encoded_value_object_keeps_its_canonical_reverse_edge(migrated_database_urls, key, kind): + user = uuid4() + root_id = str(uuid4()) + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"encoded-value-{user}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + row = store.create_memory({"memory_key": "encoded-value", "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public"}) + conn.execute("UPDATE memories SET value = %s, metadata_json = %s WHERE id = %s", (Jsonb(encoded_object(key, root_id)), Jsonb({"consolidation": {}}), row["id"])) + stored = store.get_memory(str(row["id"])) + assert (kind, root_id) in dependencies_of("memory", stored) + assert {str(item["id"]) for item in walk_dependants(store, [root_id])} == {str(row["id"])} + + +def test_a_candidate_open_loop_direct_source_column_is_a_reverse_edge(migrated_database_urls): + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"loop-edge-{user}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + source = store.create_source({"source_type": "note", "title": "Synthetic", "content_hash": str(user), "domain": "project", "sensitivity": "public"}) + loop = store.create_open_loop({"title": "Synthetic", "loop_type": "task", "source_id": str(source["id"]), "domain": "project", "sensitivity": "public", "metadata_json": {"discovered_by": "synthetic"}}) + assert {str(row["id"]) for row in walk_dependants(store, [str(source["id"])])} == {str(loop["id"])} + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) + assert store.get_open_loop(str(loop["id"]))["sensitivity"] == "confidential" diff --git a/tests/unit/test_encoded_label_dependencies.py b/tests/unit/test_encoded_label_dependencies.py new file mode 100644 index 000000000..fb94635ff --- /dev/null +++ b/tests/unit/test_encoded_label_dependencies.py @@ -0,0 +1,76 @@ +"""The reverse lookup remains a superset of canonical encoded references.""" +import json +from uuid import uuid4 + +import pytest + +from alicebot_api.onramp import bootstrap_database +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_derived_labels import dependencies_of +from alicebot_api.vnext_label_writes import count_rows_hidden_by_scope_move, walk_dependants +from tests.unit.test_derived_domain_fence import USER + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +def encoded_object(key, row_id): + escaped = "".join("\\u%04x" % ord(character) for character in row_id) + return '{"' + key + '":"' + escaped + '"}' + + +def test_an_encoded_source_is_previewed_and_raised_without_losing_its_reference(tmp_path): + path = tmp_path / "encoded.sqlite3" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.test") + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source({"source_type": "note", "title": "Synthetic", "content_hash": "encoded", "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA]}}) + reference = encoded_object("source_id", str(source["id"])) + copy = store.create_memory({"memory_key": "encoded", "canonical_text": "Synthetic", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": reference, "project_scope": [ALPHA]}}) + unrelated = store.create_memory({"memory_key": "unrelated", "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"note": encoded_object("source_id", str(uuid4()))}}) + before = {table: list(conn.execute(f"SELECT * FROM {table} ORDER BY id")) for table in ("sources", "memories", "event_log", "provenance_links")} + assert ("source", str(source["id"])) in dependencies_of("memory", copy) + assert {str(row["id"]) for row in walk_dependants(store, [str(source["id"])])} == {str(copy["id"])} + assert count_rows_hidden_by_scope_move(store, source, [BETA]) == 1 + assert {table: list(conn.execute(f"SELECT * FROM {table} ORDER BY id")) for table in before} == before + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential", "metadata_json": {"project_scope": [BETA]}}) + stored = store.get_memory(str(copy["id"])) + assert stored["sensitivity"] == "confidential" + assert BETA in stored["metadata_json"]["project_floor"] + assert stored["metadata_json"]["source_id"] == reference + assert store.get_memory(str(unrelated["id"]))["sensitivity"] == "public" + + +@pytest.mark.parametrize("key,kind", [("source_id", "source"), ("memory_id", "memory"), ("artifact_id", "artifact"), ("belief_ids", "belief")]) +def test_unicode_encoded_metadata_keys_and_ids_are_not_cut_before_the_parser(tmp_path, key, kind): + path = tmp_path / "encoded-keys.sqlite3" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.test") + row_id = str(uuid4()) + value = [row_id] if key.endswith("ids") else row_id + metadata = json.dumps({key: value, "consolidation": {}}) + encoded = "".join("\\u%04x" % ord(character) if character.isalnum() or character == "_" else character for character in metadata) + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + row = store.create_memory({"memory_key": "raw-encoded", "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public"}) + conn.execute("UPDATE memories SET metadata_json = ? WHERE id = ?", (encoded, row["id"])) + stored = store.get_memory(str(row["id"])) + assert (kind, row_id) in dependencies_of("memory", stored) + assert {str(item["id"]) for item in walk_dependants(store, [row_id])} == {str(row["id"])} + + +@pytest.mark.parametrize("key,kind", [("source_id", "source"), ("artifact_id", "artifact")]) +def test_encoded_value_object_keeps_the_same_reverse_edge(tmp_path, key, kind): + path = tmp_path / "encoded-value.sqlite3" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.test") + row_id = str(uuid4()) + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + row = store.create_memory({"memory_key": "encoded-value", "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public"}) + # The dependency lives in the encoded value alone. A marker without + # the root id states the row class without making SQL's id term pass. + metadata = {"consolidation": {}} + conn.execute("UPDATE memories SET value = ?, metadata_json = ? WHERE id = ?", (json.dumps(encoded_object(key, row_id)), json.dumps(metadata), row["id"])) + stored = store.get_memory(str(row["id"])) + assert (kind, row_id) in dependencies_of("memory", stored) + assert {str(item["id"]) for item in walk_dependants(store, [row_id])} == {str(row["id"])} + From 2fec285a2a95c8c91d59a46351fc180bff290c13 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:31:28 +0200 Subject: [PATCH 148/270] Require each producer input kind in the sentinel grid --- .../test_derived_labels_producers_postgres.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/integration/test_derived_labels_producers_postgres.py b/tests/integration/test_derived_labels_producers_postgres.py index 9a1dbc79b..832a28d20 100644 --- a/tests/integration/test_derived_labels_producers_postgres.py +++ b/tests/integration/test_derived_labels_producers_postgres.py @@ -133,6 +133,20 @@ def test_a_bound_key_generates_from_admitted_inputs_only(migrated_database_urls, conn.execute("UPDATE memories SET valid_to='2026-10-04T12:00:00Z'") artifact, body = generate(producer, user_id, alpha, key) assert_canonical_printed_inputs(artifact, rows) + expected_kinds = { + "daily": ("sources", "memories", "open_loops", "artifacts"), + "weekly": ("sources", "memories", "open_loops", "artifacts"), + "connections": ("sources", "memories"), + "contradictions": ("sources", "beliefs"), + "open_loop_review": ("open_loops",), + "project_update": ("sources", "memories"), + "consolidation": ("memories", "artifacts"), + "staleness": ("memories",), + } + record = artifact["metadata_json"]["derived_from"] + for kind in expected_kinds[producer]: + assert any(str(row["id"]) in record[kind] for label, row_kind, row in rows + if label == "alpha" and row_kind == kind), (producer, "missing positive input kind", kind, record) artifact_id = str(artifact["id"]) surfaces = [body] with user_connection(app_url, user_id) as conn: From 04b4c6ae9e425c36b15f760b2b6eba717eaf8dc4 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:29:48 +0200 Subject: [PATCH 149/270] Cover each lexical read branch and bound admin review controls --- tests/unit/test_label_guard_exact_doors.py | 12 ++++++++++++ tests/unit/test_list_door_readers.py | 18 ++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_label_guard_exact_doors.py b/tests/unit/test_label_guard_exact_doors.py index bc255ceab..506c37132 100644 --- a/tests/unit/test_label_guard_exact_doors.py +++ b/tests/unit/test_label_guard_exact_doors.py @@ -195,6 +195,18 @@ def test_a_memory_review_decision_uses_the_input_label() -> None: assert decision.decision == "blocked" +def test_bound_admin_review_uses_the_effective_input_project_floor() -> None: + from alicebot_api.vnext_memory_commit import VNextMemoryCommitService + + store = _LabelStore() + store.source["metadata_json"]["project_scope"] = ["prj_" + "b" * 16] + identity = _locked_admin() + service = VNextMemoryCommitService(store) + assert service._write_policy_decision(identity=identity, action="memory.review", memory=store.memory).decision == "blocked" + store.source["metadata_json"]["project_scope"] = [ALPHA] + assert service._write_policy_decision(identity=identity, action="memory.review", memory=store.memory).decision != "blocked" + + def test_an_open_loop_update_uses_the_input_label() -> None: from alicebot_api.vnext_memory_commit import VNextMemoryCommitService diff --git a/tests/unit/test_list_door_readers.py b/tests/unit/test_list_door_readers.py index 442d11b79..723374f97 100644 --- a/tests/unit/test_list_door_readers.py +++ b/tests/unit/test_list_door_readers.py @@ -4,6 +4,7 @@ from contextlib import contextmanager from uuid import UUID +import pytest from alicebot_api.mcp.retrieval import _resume_event_honours_policy_fence from alicebot_api.routers import vnext_review @@ -125,16 +126,29 @@ def _identity(profile: str) -> AgentIdentity: return AgentIdentity(agent_id="reader", agent_type="unknown", permission_profile=profile) -def test_fts_stage_drops_a_public_copy_of_a_confidential_source() -> None: +@pytest.mark.parametrize("stage", ("strict", "or_fallback", "legacy")) +def test_fts_stage_drops_a_public_copy_of_a_confidential_source(stage) -> None: store = _LabelStore() + if stage == "or_fallback": + store.search_memories_fts = lambda **kwargs: [store.memory] if kwargs.get("match_any") else [] + elif stage == "legacy": + store.search_memories_fts = None + store.search_memories = lambda **kwargs: [store.memory] rows, _status = VNextRetrievalService(store)._memory_fts_rows( - query="sentinel", + query="sentinel fact" if stage == "or_fallback" else "sentinel", domains=["project"], sensitivity_allowed=["public", "internal", "private", "unknown"], limit=10, ) assert rows == [] assert SECRET not in str(rows) + read_rows = store.read_label_rows + store.read_label_rows = lambda kind, ids: [{**_source(), "domain": "project", "sensitivity": "public"}] if kind == "source" else read_rows(kind, ids) + visible, _status = VNextRetrievalService(store)._memory_fts_rows( + query="sentinel fact" if stage == "or_fallback" else "sentinel", domains=["project"], + sensitivity_allowed=["public", "internal", "private", "unknown"], limit=10, + ) + assert [row["id"] for row in visible] == [MEMORY_ID] def test_operator_screens_hide_a_confidential_row_from_a_trusted_key() -> None: From daca4a1f8e74407ca2a1be7f749382b1a7958abb Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:43:03 +0200 Subject: [PATCH 150/270] Normalize PostgreSQL loop dependencies in complete readable counts --- apps/api/src/alicebot_api/vnext_store.py | 2 +- ...derived_labels_read_acceptance_postgres.py | 43 ++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 7ee1a2f93..4449fb169 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -543,7 +543,7 @@ def iter_label_rows(self, kind: str, *, batch_size: int = 200) -> Iterator[list[ if kind == "memory": extra = ", status, value, project_id, source_event_ids, deleted_at" elif kind == "open_loop": - extra = ", status, project_id, source_id, memory_id" + extra = ", status, project_id, source_id::text AS source_id, memory_id::text AS memory_id" elif kind == "artifact": extra = ", status, artifact_type" elif kind == "project": diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py index 699272e96..beaeb4f87 100644 --- a/tests/integration/test_derived_labels_read_acceptance_postgres.py +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -15,8 +15,11 @@ from alicebot_api.routers import vnext_review, vnext_retrieval, vnext_projects, vnext_memories, workspaces from alicebot_api.store import ContinuityStore from alicebot_api.vnext_agent_control import ALL_SENSITIVITY +from alicebot_api.vnext_agent_keys import resolve_agent_identity from alicebot_api.vnext_event_log import append_event +from alicebot_api.vnext_label_guard import LabelGuard from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_source_fence import SourceReadFence from alicebot_api.vnext_store import PostgresVNextStore from tests.unit.test_derived_labels_real_keys import READERS, expected_read, real_reader_key, seed_read_rows @@ -72,7 +75,8 @@ def test_all_five_operator_screens_with_real_keys(migrated_database_urls, monkey store = PostgresVNextStore(conn) source = store.create_source({"source_type": "note", "title": "Cedar hidden source", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential"}) public_source = store.create_source({"source_type": "note", "title": "Public source", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public"}) - memory = store.create_memory({"memory_key": "belief", "canonical_text": "Cedar hidden belief", "status": "active", "domain": "project", "sensitivity": "confidential"}) + with without_insert_floor(): + memory = store.create_memory({"memory_key": "belief", "canonical_text": "Cedar hidden belief", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])}}) artifact = store.create_artifact({"artifact_type": "daily_brief", "title": "Cedar hidden artifact", "content_markdown": "Cedar hidden artifact", "domain": "project", "sensitivity": "confidential", "metadata_json": {"derived_from": {"v": 1, "sources": [str(public_source["id"])], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}}, "source_refs": [str(public_source["id"])]}}) project = store.create_project({"name": "Cedar hidden project", "slug": "cedar-hidden", "current_state": "Cedar hidden state", "domain": "project", "sensitivity": "confidential"}) belief_id = uuid4() @@ -249,3 +253,40 @@ async def send(message): assert "below their inputs" not in check["message"] for field in ("status", "warning_count", "blocking_failure_count", "recommended_fixes"): assert diagnostic[field] == before[1]["doctor"][field] + + +@pytest.mark.parametrize("reader", ("owner", "admin", "trusted")) +@pytest.mark.parametrize("source_sensitivity", ("public", "confidential")) +def test_direct_column_loop_references_reach_real_read_guard(migrated_database_urls, monkeypatch, reader, source_sensitivity): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + admitted = reader != "trusted" or source_sensitivity == "public" + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Direct parent", "content_hash": str(uuid4()), "domain": "project", "sensitivity": source_sensitivity}) + memory = store.create_memory({"memory_key": "direct-loop-parent", "canonical_text": "Direct memory", "domain": "project", "sensitivity": "public"}) + with without_insert_floor(): + loop = store.create_open_loop({"title": "Direct loop sentinel", "source_id": str(source["id"]), "memory_id": str(memory["id"]), "domain": "project", "sensitivity": "public", "metadata_json": {"discovered_by": "vnext_daily_capture"}}) + key = real_reader_key(store, user_id, reader) + identity = resolve_agent_identity(store, user_id=user_id, raw_key=key, payload={}) + row = store.get_open_loop(str(loop["id"])) + fence = SourceReadFence.for_identity(identity) + guard = LabelGuard.for_fence(store, fence) + judged = guard.effective_row("open_loop", row) + assert judged is not None + assert (judged["sensitivity"] == source_sensitivity) or not guard.active + assert fence.admits_memory(judged) is admitted + for projection in (row, store.read_label_rows("open_loop", [str(loop["id"])])[0], next(store.iter_label_rows("open_loop"))[0]): + assert projection["source_id"] == str(source["id"]) + assert projection["memory_id"] == str(memory["id"]) + count_guard = LabelGuard.for_filters(store, (), ("public", "internal", "private", "unknown")) + assert count_guard.readable_status_counts("open_loop").get("open", 0) == int(source_sensitivity == "public") + if key: + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + result = call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name="alice_open_loops", arguments={"sensitivity_allowed": list(ALL_SENSITIVITY)}) + rendered = json.dumps(result, default=str) + assert (str(loop["id"]) in rendered) is admitted + assert ("Direct loop sentinel" in rendered) is admitted From 84778ae91f377576b0438563b0afd647c0a79e7f Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:43:44 +0200 Subject: [PATCH 151/270] Normalize native PostgreSQL loop references at exact read boundaries --- apps/api/src/alicebot_api/vnext_stores/postgres/columns.py | 4 ++-- tests/unit/test_store_graph_open_loops_split.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/columns.py b/apps/api/src/alicebot_api/vnext_stores/postgres/columns.py index f97ff71d9..f5ca74801 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/columns.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/columns.py @@ -172,7 +172,7 @@ OPEN_LOOP_COLUMNS = """ id, user_id, - memory_id, + memory_id::text AS memory_id, title, status, opened_at, @@ -185,7 +185,7 @@ priority, project_id, person_id, - source_id, + source_id::text AS source_id, closed_at, domain, sensitivity, diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index 91cea70ad..bd68d493d 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -134,7 +134,7 @@ # Re-minted so the open-loop partition read passes the floor identity. # Previous receipt 9a2634be... SQLITE_CARRIER_PATH: "a050eda266e928f9289730a941f6f10d8ad048c6eb4b288fa57e8121e717b83c", - POSTGRES_COLUMNS_PATH: "5b0d972a55abf8590ce14394a37fd71b9b88ba7ab3de82d61efc1bddfc022b71", + POSTGRES_COLUMNS_PATH: "1a782bf3eb87f68f67434508baab0f82d49cb40a0c547cde49bbc972e2f1d182", SQLITE_COLUMNS_PATH: "be81b8628d0831d3d02b280b5455fb02333db5740ebef8d85d58024384ae6556", } EXPECTED_METHOD_AST_MANIFESTS = { @@ -188,7 +188,8 @@ "c333a0dacf8733a16fb86f3acc1bbf61bd25fa66ce96cc3bc25805c4a85d9203" ), "BELIEF_COLUMNS": "32bac57e9e38fead1af29b9324777978f3b03bfe20d5306695fae98079d82dc7", - "OPEN_LOOP_COLUMNS": "651275ee48d37e13228bf339ac4f260a50333fc7b86197ab16573cc099f912bf", + # Reviewed: normalize the two direct dependency UUID columns to text. + "OPEN_LOOP_COLUMNS": "44970b53460b10e1e414a9c1c7905f15ea7ecfed4570b1fcd5becf65952b3c64", }, SQLITE_COLUMNS_PATH: { "GRAPH_EDGE_COLUMNS": "587b88564c446c03420441371a180e11618ea6bf192e5e20d2ad5d426ce890f2", From 17630064fccd0bba3948b36511c6cf07e387792e Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:43:13 +0200 Subject: [PATCH 152/270] Normalize reverse dependency candidates once per column --- .../src/alicebot_api/vnext_label_writes.py | 23 +++- .../test_label_reverse_search_postgres.py | 128 ++++++++++++++++++ 2 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 tests/integration/test_label_reverse_search_postgres.py diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 9265da97e..ca5ca7cdc 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -404,20 +404,31 @@ def _postgres_dependants(store: Any, table: str, kind: str, compacts: Sequence[s extra = ", NULL::jsonb AS value, artifact_type" else: extra = ", NULL::jsonb AS value" - text_clause, _ = _like_clause("metadata_json::text", len(compacts), qmark=False) + # Normalize each column once per row, rather than once per frontier id. + # This remains only a superset lookup; the canonical parser selects exact edges. + pattern = "(?:" + "|".join(re.escape(item) for item in compacts) + ")" + + def candidate_clause(column: str) -> str: + return ( + "replace(replace(replace(replace(lower(coalesce(" + + column + + ",'')),'-',''),'{',''),'}',''),' ','') ~ %s" + ) + + text_clause = candidate_clause("metadata_json::text") text_clause += " OR strpos(coalesce(metadata_json::text, ''), chr(92)) > 0" - params: list[object] = [f"%{item}%" for item in compacts] + params: list[object] = [pattern] value_sql = "" if with_value: - value_clause, _ = _like_clause("value::text", len(compacts), qmark=False) + value_clause = candidate_clause("value::text") value_sql = f" OR {value_clause} OR strpos(coalesce(value::text, ''), chr(92)) > 0" - params.extend(f"%{item}%" for item in compacts) + params.append(pattern) column_sql = "" if table == "open_loops": for column in ("source_id", "memory_id"): - clause, _ = _like_clause(f"{column}::text", len(compacts), qmark=False) + clause = candidate_clause(f"{column}::text") column_sql += f" OR {clause}" - params.extend(f"%{item}%" for item in compacts) + params.append(pattern) rows = store._fetch_all( f""" SELECT id::text AS id, user_id::text AS user_id, domain, sensitivity, metadata_json{extra} diff --git a/tests/integration/test_label_reverse_search_postgres.py b/tests/integration/test_label_reverse_search_postgres.py new file mode 100644 index 000000000..f68ebd541 --- /dev/null +++ b/tests/integration/test_label_reverse_search_postgres.py @@ -0,0 +1,128 @@ +"""The faster reverse search preserves the previous superset and exact closure.""" + +from uuid import uuid4 + +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_label_writes import _compact_id, _like_clause, _postgres_dependants, walk_dependants +from alicebot_api.vnext_store import PostgresVNextStore +from tests.unit.test_encoded_label_dependencies import encoded_object + + +def test_the_regex_candidate_search_matches_the_previous_scan_for_aliases_and_encoded_objects(migrated_database_urls): + user = uuid4() + roots = [str(uuid4()) for _ in range(3)] + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"reverse-{user}@example.invalid", "Reverse") + store = PostgresVNextStore(conn) + expected = set() + for index, root in enumerate(roots): + for variant, reference in enumerate( + (root, root.upper(), root.replace("-", ""), "{" + root.upper() + "}", encoded_object("source_id", root)) + ): + row = store.create_memory( + { + "memory_key": f"alias.{index}.{variant}", + "canonical_text": "Synthetic alias", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"source_id": reference}, + } + ) + expected.add(str(row["id"])) + decoy = store.create_memory( + { + "memory_key": "decoy", + "canonical_text": "Synthetic decoy", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"note": roots[0]}, + } + ) + unrelated = store.create_memory( + { + "memory_key": "unrelated", + "canonical_text": "Synthetic unrelated", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"note": encoded_object("source_id", str(uuid4()))}, + } + ) + compacts = [_compact_id(root) for root in roots] + text, _ = _like_clause("metadata_json::text", len(compacts), qmark=False) + value, _ = _like_clause("value::text", len(compacts), qmark=False) + previous = conn.execute( + f"SELECT id::text AS id FROM memories WHERE ({text}) OR ({value}) " + "OR strpos(metadata_json::text,chr(92))>0 OR strpos(value::text,chr(92))>0", + [*(f"%{item}%" for item in compacts), *(f"%{item}%" for item in compacts)], + ).fetchall() + current = _postgres_dependants(store, "memories", "memory", compacts, with_value=True) + assert {row["id"] for row in current} == {row["id"] for row in previous} + assert {row["id"] for row in walk_dependants(store, roots)} == expected + assert str(decoy["id"]) not in expected + assert str(unrelated["id"]) not in expected + + +def test_multiple_frontier_batches_keep_memory_chains_and_both_direct_loop_columns(migrated_database_urls): + user = uuid4() + root = str(uuid4()) + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"frontier-{user}@example.invalid", "Frontier") + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + store.create_source( + { + "id": root, + "source_type": "note", + "title": "Synthetic source", + "content_hash": str(user), + "domain": "project", + "sensitivity": "public", + } + ) + roots = [str(uuid4()) for _ in range(220)] + conn.execute( + """INSERT INTO memories(id,user_id,memory_key,value,source_event_ids,canonical_text, + status,domain,sensitivity,metadata_json) + SELECT id,app.current_user_id(),'frontier.'||id,'{}'::jsonb,'[]'::jsonb,'Synthetic frontier', + 'active','project','public', + jsonb_build_object('source_id',%s::text) FROM unnest(%s::uuid[]) id""", + (root, roots), + ) + descendants = [] + for index in (0, 199, 219): + row = store.create_memory( + { + "memory_key": f"descendant.{index}", + "canonical_text": "Synthetic descendant", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"derived_from": {"v": 1, "memories": [roots[index]], "counts": {"memories": 1}}}, + } + ) + descendants.append(str(row["id"])) + direct_source = store.create_open_loop( + { + "title": "Synthetic source loop", + "source_id": root, + "domain": "project", + "sensitivity": "public", + "metadata_json": {"discovered_by": "synthetic"}, + } + ) + direct_memory = store.create_open_loop( + { + "title": "Synthetic memory loop", + "source_id": root, + "memory_id": roots[-1], + "domain": "project", + "sensitivity": "public", + "metadata_json": {"discovered_by": "synthetic"}, + } + ) + expected = {*roots, *descendants, str(direct_source["id"]), str(direct_memory["id"])} + assert {row["id"] for row in walk_dependants(store, [root])} == expected + memory_candidates = _postgres_dependants( + store, "open_loops", "open_loop", [_compact_id(roots[-1])], with_value=False + ) + assert str(direct_memory["id"]) in {row["id"] for row in memory_candidates} From 480d92fbe13ac02fe16760cc5f14889fa6fa2220 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:46:33 +0200 Subject: [PATCH 153/270] Reuse source label inputs during bounded capture writes --- apps/api/src/alicebot_api/vnext_capture.py | 147 ++++++++-------- .../src/alicebot_api/vnext_label_writes.py | 81 ++++++++- apps/api/src/alicebot_api/vnext_store.py | 3 + .../test_capture_label_batch_postgres.py | 159 ++++++++++++++++++ 4 files changed, 316 insertions(+), 74 deletions(-) create mode 100644 tests/integration/test_capture_label_batch_postgres.py diff --git a/apps/api/src/alicebot_api/vnext_capture.py b/apps/api/src/alicebot_api/vnext_capture.py index 8b86ba37e..e7ee46558 100644 --- a/apps/api/src/alicebot_api/vnext_capture.py +++ b/apps/api/src/alicebot_api/vnext_capture.py @@ -1545,84 +1545,87 @@ def _capture_source(self, source_input: SourceCaptureInput) -> CaptureResult: sensitivity=source_input.sensitivity, ) memory_rows: list[JsonObject] = [] - for candidate in candidates: - # Speaker provenance is only stamped when a role was derived, - # so provenance-free captures keep byte-identical metadata. - provenance_metadata: JsonObject = ( - { - "provenance_role": candidate.provenance_role, - "assertion_class": candidate.assertion_class, - } - if candidate.provenance_role is not None - else {} - ) - memory = self.store.create_memory( - { - "memory_key": _memory_key( - content_hash=content_hash, - candidate=candidate, - domain=source_input.domain, - sensitivity=source_input.sensitivity, - ), - "value": { - "text": candidate.text, + from alicebot_api.vnext_label_writes import capture_label_inputs + + with capture_label_inputs(self.store, source_id): + for candidate in candidates: + # Speaker provenance is only stamped when a role was derived, + # so provenance-free captures keep byte-identical metadata. + provenance_metadata: JsonObject = ( + { + "provenance_role": candidate.provenance_role, + "assertion_class": candidate.assertion_class, + } + if candidate.provenance_role is not None + else {} + ) + memory = self.store.create_memory( + { + "memory_key": _memory_key( + content_hash=content_hash, + candidate=candidate, + domain=source_input.domain, + sensitivity=source_input.sensitivity, + ), + "value": { + "text": candidate.text, + "source_id": source_id, + "source_chunk_id": candidate.source_chunk_id, + }, + "status": "candidate", + "source_event_ids": [source_id, candidate.source_chunk_id], + "memory_type": candidate.memory_type, + "confidence": candidate.confidence, + "title": _truncate(candidate.text, max_length=120), + "canonical_text": candidate.text, + "summary": _truncate(candidate.text, max_length=280), + "domain": source_input.domain, + "sensitivity": source_input.sensitivity, + "project_id": project_scope[0] if len(project_scope) == 1 else None, + "created_by_agent_id": self.actor_id if self.actor_type == "agent" else None, + "run_id": self.run_id if self.actor_type == "agent" else None, + "metadata_json": { + "source_id": source_id, + "source_chunk_id": candidate.source_chunk_id, + "source_chunk_index": candidate.source_chunk_index, + "extraction_rule": candidate.extraction_rule, + "capture_content_hash": content_hash, + **provenance_metadata, + **project_scope_metadata, + "generated_by": self.actor_type, + "agent_identity": self.agent_identity, + "agent_id": self.actor_id if self.actor_type == "agent" else None, + "agent_run_id": self.run_id if self.actor_type == "agent" else None, + "trace_id": self.trace_id, + "policy_decision": self.policy_decision, + }, + }, + actor_type=self.actor_type, + ) + memory_rows.append(memory) + self.store.create_provenance_link( + { + "target_type": "memory", + "target_id": str(memory["id"]), "source_id": source_id, "source_chunk_id": candidate.source_chunk_id, + "quote": candidate.text, + "evidence_role": "quoted_from", + "confidence": candidate.confidence, }, - "status": "candidate", - "source_event_ids": [source_id, candidate.source_chunk_id], - "memory_type": candidate.memory_type, - "confidence": candidate.confidence, - "title": _truncate(candidate.text, max_length=120), - "canonical_text": candidate.text, - "summary": _truncate(candidate.text, max_length=280), - "domain": source_input.domain, - "sensitivity": source_input.sensitivity, - "project_id": project_scope[0] if len(project_scope) == 1 else None, - "created_by_agent_id": self.actor_id if self.actor_type == "agent" else None, - "run_id": self.run_id if self.actor_type == "agent" else None, - "metadata_json": { + actor_type=self.actor_type, + ) + self._log_event( + event_type="memory.candidate_created", + target_type="memory", + target_id=str(memory["id"]), + payload={ "source_id": source_id, "source_chunk_id": candidate.source_chunk_id, - "source_chunk_index": candidate.source_chunk_index, - "extraction_rule": candidate.extraction_rule, - "capture_content_hash": content_hash, - **provenance_metadata, - **project_scope_metadata, - "generated_by": self.actor_type, - "agent_identity": self.agent_identity, - "agent_id": self.actor_id if self.actor_type == "agent" else None, - "agent_run_id": self.run_id if self.actor_type == "agent" else None, - "trace_id": self.trace_id, - "policy_decision": self.policy_decision, + "memory_type": candidate.memory_type, + "confidence": candidate.confidence, }, - }, - actor_type=self.actor_type, - ) - memory_rows.append(memory) - self.store.create_provenance_link( - { - "target_type": "memory", - "target_id": str(memory["id"]), - "source_id": source_id, - "source_chunk_id": candidate.source_chunk_id, - "quote": candidate.text, - "evidence_role": "quoted_from", - "confidence": candidate.confidence, - }, - actor_type=self.actor_type, - ) - self._log_event( - event_type="memory.candidate_created", - target_type="memory", - target_id=str(memory["id"]), - payload={ - "source_id": source_id, - "source_chunk_id": candidate.source_chunk_id, - "memory_type": candidate.memory_type, - "confidence": candidate.confidence, - }, - ) + ) # Capture writes candidate memories only, and recall cannot return a # candidate, so no text is sent to the embeddings endpoint here: the diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index ca5ca7cdc..28c54c159 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -10,8 +10,9 @@ import re from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager +from contextvars import ContextVar from functools import wraps -from dataclasses import replace +from dataclasses import dataclass, field, replace from typing import Any from alicebot_api.vnext_agent_control import RESTRICTED_DOMAINS @@ -59,6 +60,73 @@ class LabelLockOrderError(RuntimeError): REFUSED_DETAIL = "the label change could not be applied to every dependent row; nothing was changed" +@dataclass +class _CaptureLabelInputs: + conn: Any + transaction_id: str + rollback_counter: int + source_id: str + rows: dict[tuple[str, str], list[dict[str, object]]] = field(default_factory=dict) + + @property + def key(self) -> tuple[str, int]: + return self.transaction_id, self.rollback_counter + + +_CAPTURE_LABEL_INPUTS: ContextVar[_CaptureLabelInputs | None] = ContextVar("capture_label_inputs", default=None) + + +def invalidate_capture_label_inputs(store: Any) -> None: + batch = _CAPTURE_LABEL_INPUTS.get() + if batch is not None and batch.conn is getattr(store, "conn", None): + batch.rows.clear() + + +def label_savepoint_rolled_back(store: Any) -> None: + """Invalidate capture inputs whenever the store rolls a savepoint back.""" + + conn = store.conn + conn._alice_label_rollback_counter = int(getattr(conn, "_alice_label_rollback_counter", 0)) + 1 + invalidate_capture_label_inputs(store) + + +def _current_capture_inputs(store: Any) -> _CaptureLabelInputs | None: + batch = _CAPTURE_LABEL_INPUTS.get() + if batch is None or batch.conn is not getattr(store, "conn", None) or not _in_transaction(store): + return None + counter = int(getattr(batch.conn, "_alice_label_rollback_counter", 0)) + if counter != batch.rollback_counter: + batch.rows.clear() + batch.rollback_counter = counter + return batch + + +@contextmanager +def capture_label_inputs(store: Any, source_id: str) -> Iterator[None]: + """Reuse only the new source's label row during capture's candidate loop. + + A managed transaction fixes the actual transaction id for this synchronous + loop. Savepoint rollback and label updates invalidate its input rows. Every + writer still executes its advisory lock, and no grant is memoized. + """ + + conn = getattr(store, "conn", None) + if conn is None or _sqlite(store) or not callable(getattr(conn, "transaction", None)): + yield + return + with conn.transaction(): + with conn.cursor() as cur: + cur.execute("SELECT pg_current_xact_id()::text AS transaction_id") + row = cur.fetchone() + transaction_id = str(row["transaction_id"] if isinstance(row, Mapping) else row[0]) + batch = _CaptureLabelInputs(conn, transaction_id, int(getattr(conn, "_alice_label_rollback_counter", 0)), identifier(source_id)) + token = _CAPTURE_LABEL_INPUTS.set(batch) + try: + yield + finally: + _CAPTURE_LABEL_INPUTS.reset(token) + + def takes_label_lock(fn: Any) -> Any: """Take the shared label lock before a store method writes or row-locks a label table.""" @@ -177,6 +245,7 @@ def prepare_label_patch( ) -> JsonObject: """Check a proposed label change before its UPDATE or FOR UPDATE statement.""" + invalidate_capture_label_inputs(store) proposed = dict(before or {}) proposed.update({key: value for key, value in patch.items() if value is not None}) proposed["kind"] = kind @@ -220,7 +289,14 @@ def apply_insert_floor(store: Any, kind: str, payload: Mapping[str, object]) -> own["kind"] = kind own["id"] = own_id own["user_id"] = user_id - nodes, exceeded = collect_label_rows(store, [own], max_nodes=PROPAGATION_BOUND) + batch = _current_capture_inputs(store) + cache = batch.rows if batch is not None else None + nodes, exceeded = collect_label_rows(store, [own], max_nodes=PROPAGATION_BOUND, cache=cache) + if batch is not None: + # Keep only this capture's source. Other dependencies are read afresh. + for key in list(batch.rows): + if key != ("source", batch.source_id): + del batch.rows[key] if exceeded: raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") if not user_id: @@ -601,6 +677,7 @@ def write_settled_label( ) -> None: """Label-only update. A statement that changes no row refuses the whole relabel.""" + invalidate_capture_label_inputs(store) table = {"memory": "memories", "open_loop": "open_loops", "artifact": "generated_artifacts", "project": "projects"}[kind] blob = json.dumps({key: metadata[key] for key in ("project_scope", "project_floor") if key in metadata}) if _sqlite(store): diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 4449fb169..a7d3671e1 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -3874,6 +3874,9 @@ def savepoint(self) -> Iterator[None]: try: yield except BaseException: + from alicebot_api.vnext_label_writes import label_savepoint_rolled_back + + label_savepoint_rolled_back(self) try: self.conn.execute(f"ROLLBACK TO SAVEPOINT {name}") self.conn.execute(f"RELEASE SAVEPOINT {name}") diff --git a/tests/integration/test_capture_label_batch_postgres.py b/tests/integration/test_capture_label_batch_postgres.py new file mode 100644 index 000000000..e7a6d8549 --- /dev/null +++ b/tests/integration/test_capture_label_batch_postgres.py @@ -0,0 +1,159 @@ +"""Capture reuses source inputs while every writer takes a live label lock.""" +from uuid import uuid4 + +import psycopg +import pytest +from psycopg.rows import dict_row + +from alicebot_api import vnext_label_writes as writes +from alicebot_api.db import set_current_user, user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_capture import VNextCaptureService +from alicebot_api.vnext_store import PostgresVNextStore + + +def _user(conn, user): + ContinuityStore(conn).create_user(user, f"batch-{user}@example.test", "Synthetic") + + +def _source(store, user): + return store.create_source({"source_type": "note", "title": "Synthetic", "content_hash": str(user), "domain": "project", "sensitivity": "public"}) + + +def _memory(store, key, source): + return store.create_memory({"memory_key": key, "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])}}) + + +def test_capture_reuses_only_source_inputs_and_keeps_every_writer_lock(migrated_database_urls, monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", False) + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + calls, reads, identities = [], [], [] + real_lock, real_reader, real_memory = store.lock_label_writes, store.read_label_rows, store.create_memory + def lock(*, exclusive=False): + calls.append(exclusive) + return real_lock(exclusive=exclusive) + def reader(kind, ids): + reads.append((kind, ids)) + return real_reader(kind, ids) + def memory(payload, **kwargs): + identities.append(conn.execute("SELECT pg_current_xact_id()::text AS tx, app.current_user_id()::text AS tenant").fetchone()) + frame = writes._current_capture_inputs(store) + assert frame.key[0] == identities[-1]["tx"] + return real_memory(payload, **kwargs) + monkeypatch.setattr(store, "lock_label_writes", lock) + monkeypatch.setattr(store, "read_label_rows", reader) + monkeypatch.setattr(store, "create_memory", memory) + result = VNextCaptureService(store).capture_text("\n".join(f"Decision: Synthetic item {i} is assigned to synthetic route {i}." for i in range(200)), domain="project", sensitivity="public") + assert result.candidate_memory_count == 200 + assert len(calls) == 401 # Source, 200 memories, 200 provenance links. + assert len(reads) == 1 and reads[0][0] == "source" + assert len({(row["tx"], row["tenant"]) for row in identities}) == 1 + assert identities[0]["tenant"] == str(user) + assert writes._current_capture_inputs(store) is None + assert writes.held_label_locks(store)[1] is True + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 200 + + +def test_source_input_memo_is_keyed_by_transaction_and_rollback_counter(migrated_database_urls): + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + source = _source(store, user) + with writes.capture_label_inputs(store, str(source["id"])): + _memory(store, "before", source) + frame = writes._current_capture_inputs(store) + before = frame.key + assert frame.rows + with pytest.raises(ValueError): + with store.savepoint(): + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) + assert _memory(store, "rolled-back", source)["sensitivity"] == "confidential" + raise ValueError("Synthetic rollback") + assert _memory(store, "after", source)["sensitivity"] == "public" + after = writes._current_capture_inputs(store).key + assert after == (before[0], before[1] + 1) + assert writes.held_label_locks(store)[1] is True + assert {row["memory_key"] for row in conn.execute("SELECT memory_key FROM memories").fetchall()} == {"before", "after"} + + +def test_failed_native_savepoint_batch_leaves_no_memo_or_rows(migrated_database_urls): + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + with pytest.raises(ValueError): + with conn.transaction(): + source = _source(store, user) + with writes.capture_label_inputs(store, str(source["id"])): + _memory(store, "failed", source) + assert writes.held_label_locks(store)[1] is True + raise ValueError("Synthetic failure") + assert writes._current_capture_inputs(store) is None + assert writes.held_label_locks(store)[1] is False + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 0 + source = _source(store, user) + _memory(store, "fresh", source) + assert writes.held_label_locks(store)[1] is True + + +def test_source_raise_invalidates_inputs_in_the_same_transaction(migrated_database_urls, monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", False) + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + source = _source(store, user) + with writes.capture_label_inputs(store, str(source["id"])): + assert _memory(store, "before", source)["sensitivity"] == "public" + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) + assert writes._current_capture_inputs(store).rows == {} + assert _memory(store, "after", source)["sensitivity"] == "confidential" + + +def test_strict_mode_still_locks_every_writer_with_source_input_reuse(migrated_database_urls, monkeypatch): + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + source = _source(store, user) + count = 0 + real_lock = store.lock_label_writes + def lock(*, exclusive=False): + nonlocal count + count += 1 + return real_lock(exclusive=exclusive) + monkeypatch.setattr(store, "lock_label_writes", lock) + assert writes.STRICT_LOCK_ORDER is True + with writes.capture_label_inputs(store, str(source["id"])): + _memory(store, "first", source) + _memory(store, "second", source) + assert writes.held_label_locks(store)[1] is True + assert count == 2 + + +def test_one_store_starts_fresh_source_inputs_in_each_transaction(migrated_database_urls): + user = uuid4() + keys = [] + with psycopg.connect(migrated_database_urls["app"], row_factory=dict_row) as conn: + store = PostgresVNextStore(conn) + for index in range(2): + set_current_user(conn, user) + if index == 0: + _user(conn, user) + source = _source(store, uuid4()) + with writes.capture_label_inputs(store, str(source["id"])): + keys.append(writes._current_capture_inputs(store).key) + with pytest.raises(psycopg.ProgrammingError): + conn.commit() + _memory(store, str(index), source) + assert writes._current_capture_inputs(store) is None + conn.commit() + assert keys[0][0] != keys[1][0] From 314104dedcf4122fe544a177f7914859306b792d Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:43:13 +0200 Subject: [PATCH 154/270] Normalize reverse dependency candidates once per column --- .../src/alicebot_api/vnext_label_writes.py | 23 +++- .../test_label_reverse_search_postgres.py | 128 ++++++++++++++++++ 2 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 tests/integration/test_label_reverse_search_postgres.py diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 9265da97e..ca5ca7cdc 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -404,20 +404,31 @@ def _postgres_dependants(store: Any, table: str, kind: str, compacts: Sequence[s extra = ", NULL::jsonb AS value, artifact_type" else: extra = ", NULL::jsonb AS value" - text_clause, _ = _like_clause("metadata_json::text", len(compacts), qmark=False) + # Normalize each column once per row, rather than once per frontier id. + # This remains only a superset lookup; the canonical parser selects exact edges. + pattern = "(?:" + "|".join(re.escape(item) for item in compacts) + ")" + + def candidate_clause(column: str) -> str: + return ( + "replace(replace(replace(replace(lower(coalesce(" + + column + + ",'')),'-',''),'{',''),'}',''),' ','') ~ %s" + ) + + text_clause = candidate_clause("metadata_json::text") text_clause += " OR strpos(coalesce(metadata_json::text, ''), chr(92)) > 0" - params: list[object] = [f"%{item}%" for item in compacts] + params: list[object] = [pattern] value_sql = "" if with_value: - value_clause, _ = _like_clause("value::text", len(compacts), qmark=False) + value_clause = candidate_clause("value::text") value_sql = f" OR {value_clause} OR strpos(coalesce(value::text, ''), chr(92)) > 0" - params.extend(f"%{item}%" for item in compacts) + params.append(pattern) column_sql = "" if table == "open_loops": for column in ("source_id", "memory_id"): - clause, _ = _like_clause(f"{column}::text", len(compacts), qmark=False) + clause = candidate_clause(f"{column}::text") column_sql += f" OR {clause}" - params.extend(f"%{item}%" for item in compacts) + params.append(pattern) rows = store._fetch_all( f""" SELECT id::text AS id, user_id::text AS user_id, domain, sensitivity, metadata_json{extra} diff --git a/tests/integration/test_label_reverse_search_postgres.py b/tests/integration/test_label_reverse_search_postgres.py new file mode 100644 index 000000000..f68ebd541 --- /dev/null +++ b/tests/integration/test_label_reverse_search_postgres.py @@ -0,0 +1,128 @@ +"""The faster reverse search preserves the previous superset and exact closure.""" + +from uuid import uuid4 + +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_label_writes import _compact_id, _like_clause, _postgres_dependants, walk_dependants +from alicebot_api.vnext_store import PostgresVNextStore +from tests.unit.test_encoded_label_dependencies import encoded_object + + +def test_the_regex_candidate_search_matches_the_previous_scan_for_aliases_and_encoded_objects(migrated_database_urls): + user = uuid4() + roots = [str(uuid4()) for _ in range(3)] + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"reverse-{user}@example.invalid", "Reverse") + store = PostgresVNextStore(conn) + expected = set() + for index, root in enumerate(roots): + for variant, reference in enumerate( + (root, root.upper(), root.replace("-", ""), "{" + root.upper() + "}", encoded_object("source_id", root)) + ): + row = store.create_memory( + { + "memory_key": f"alias.{index}.{variant}", + "canonical_text": "Synthetic alias", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"source_id": reference}, + } + ) + expected.add(str(row["id"])) + decoy = store.create_memory( + { + "memory_key": "decoy", + "canonical_text": "Synthetic decoy", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"note": roots[0]}, + } + ) + unrelated = store.create_memory( + { + "memory_key": "unrelated", + "canonical_text": "Synthetic unrelated", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"note": encoded_object("source_id", str(uuid4()))}, + } + ) + compacts = [_compact_id(root) for root in roots] + text, _ = _like_clause("metadata_json::text", len(compacts), qmark=False) + value, _ = _like_clause("value::text", len(compacts), qmark=False) + previous = conn.execute( + f"SELECT id::text AS id FROM memories WHERE ({text}) OR ({value}) " + "OR strpos(metadata_json::text,chr(92))>0 OR strpos(value::text,chr(92))>0", + [*(f"%{item}%" for item in compacts), *(f"%{item}%" for item in compacts)], + ).fetchall() + current = _postgres_dependants(store, "memories", "memory", compacts, with_value=True) + assert {row["id"] for row in current} == {row["id"] for row in previous} + assert {row["id"] for row in walk_dependants(store, roots)} == expected + assert str(decoy["id"]) not in expected + assert str(unrelated["id"]) not in expected + + +def test_multiple_frontier_batches_keep_memory_chains_and_both_direct_loop_columns(migrated_database_urls): + user = uuid4() + root = str(uuid4()) + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"frontier-{user}@example.invalid", "Frontier") + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + store.create_source( + { + "id": root, + "source_type": "note", + "title": "Synthetic source", + "content_hash": str(user), + "domain": "project", + "sensitivity": "public", + } + ) + roots = [str(uuid4()) for _ in range(220)] + conn.execute( + """INSERT INTO memories(id,user_id,memory_key,value,source_event_ids,canonical_text, + status,domain,sensitivity,metadata_json) + SELECT id,app.current_user_id(),'frontier.'||id,'{}'::jsonb,'[]'::jsonb,'Synthetic frontier', + 'active','project','public', + jsonb_build_object('source_id',%s::text) FROM unnest(%s::uuid[]) id""", + (root, roots), + ) + descendants = [] + for index in (0, 199, 219): + row = store.create_memory( + { + "memory_key": f"descendant.{index}", + "canonical_text": "Synthetic descendant", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"derived_from": {"v": 1, "memories": [roots[index]], "counts": {"memories": 1}}}, + } + ) + descendants.append(str(row["id"])) + direct_source = store.create_open_loop( + { + "title": "Synthetic source loop", + "source_id": root, + "domain": "project", + "sensitivity": "public", + "metadata_json": {"discovered_by": "synthetic"}, + } + ) + direct_memory = store.create_open_loop( + { + "title": "Synthetic memory loop", + "source_id": root, + "memory_id": roots[-1], + "domain": "project", + "sensitivity": "public", + "metadata_json": {"discovered_by": "synthetic"}, + } + ) + expected = {*roots, *descendants, str(direct_source["id"]), str(direct_memory["id"])} + assert {row["id"] for row in walk_dependants(store, [root])} == expected + memory_candidates = _postgres_dependants( + store, "open_loops", "open_loop", [_compact_id(roots[-1])], with_value=False + ) + assert str(direct_memory["id"]) in {row["id"] for row in memory_candidates} From a6a79dd7d6e4b4b182c0fa31d8d82e0af41603fd Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:46:33 +0200 Subject: [PATCH 155/270] Reuse source label inputs during bounded capture writes --- apps/api/src/alicebot_api/vnext_capture.py | 147 ++++++++-------- .../src/alicebot_api/vnext_label_writes.py | 81 ++++++++- apps/api/src/alicebot_api/vnext_store.py | 3 + .../test_capture_label_batch_postgres.py | 159 ++++++++++++++++++ 4 files changed, 316 insertions(+), 74 deletions(-) create mode 100644 tests/integration/test_capture_label_batch_postgres.py diff --git a/apps/api/src/alicebot_api/vnext_capture.py b/apps/api/src/alicebot_api/vnext_capture.py index 301341ccc..7a4503c27 100644 --- a/apps/api/src/alicebot_api/vnext_capture.py +++ b/apps/api/src/alicebot_api/vnext_capture.py @@ -1542,84 +1542,87 @@ def _capture_source(self, source_input: SourceCaptureInput) -> CaptureResult: sensitivity=source_input.sensitivity, ) memory_rows: list[JsonObject] = [] - for candidate in candidates: - # Speaker provenance is only stamped when a role was derived, - # so provenance-free captures keep byte-identical metadata. - provenance_metadata: JsonObject = ( - { - "provenance_role": candidate.provenance_role, - "assertion_class": candidate.assertion_class, - } - if candidate.provenance_role is not None - else {} - ) - memory = self.store.create_memory( - { - "memory_key": _memory_key( - content_hash=content_hash, - candidate=candidate, - domain=source_input.domain, - sensitivity=source_input.sensitivity, - ), - "value": { - "text": candidate.text, + from alicebot_api.vnext_label_writes import capture_label_inputs + + with capture_label_inputs(self.store, source_id): + for candidate in candidates: + # Speaker provenance is only stamped when a role was derived, + # so provenance-free captures keep byte-identical metadata. + provenance_metadata: JsonObject = ( + { + "provenance_role": candidate.provenance_role, + "assertion_class": candidate.assertion_class, + } + if candidate.provenance_role is not None + else {} + ) + memory = self.store.create_memory( + { + "memory_key": _memory_key( + content_hash=content_hash, + candidate=candidate, + domain=source_input.domain, + sensitivity=source_input.sensitivity, + ), + "value": { + "text": candidate.text, + "source_id": source_id, + "source_chunk_id": candidate.source_chunk_id, + }, + "status": "candidate", + "source_event_ids": [source_id, candidate.source_chunk_id], + "memory_type": candidate.memory_type, + "confidence": candidate.confidence, + "title": _truncate(candidate.text, max_length=120), + "canonical_text": candidate.text, + "summary": _truncate(candidate.text, max_length=280), + "domain": source_input.domain, + "sensitivity": source_input.sensitivity, + "project_id": project_scope[0] if len(project_scope) == 1 else None, + "created_by_agent_id": self.actor_id if self.actor_type == "agent" else None, + "run_id": self.run_id if self.actor_type == "agent" else None, + "metadata_json": { + "source_id": source_id, + "source_chunk_id": candidate.source_chunk_id, + "source_chunk_index": candidate.source_chunk_index, + "extraction_rule": candidate.extraction_rule, + "capture_content_hash": content_hash, + **provenance_metadata, + **project_scope_metadata, + "generated_by": self.actor_type, + "agent_identity": self.agent_identity, + "agent_id": self.actor_id if self.actor_type == "agent" else None, + "agent_run_id": self.run_id if self.actor_type == "agent" else None, + "trace_id": self.trace_id, + "policy_decision": self.policy_decision, + }, + }, + actor_type=self.actor_type, + ) + memory_rows.append(memory) + self.store.create_provenance_link( + { + "target_type": "memory", + "target_id": str(memory["id"]), "source_id": source_id, "source_chunk_id": candidate.source_chunk_id, + "quote": candidate.text, + "evidence_role": "quoted_from", + "confidence": candidate.confidence, }, - "status": "candidate", - "source_event_ids": [source_id, candidate.source_chunk_id], - "memory_type": candidate.memory_type, - "confidence": candidate.confidence, - "title": _truncate(candidate.text, max_length=120), - "canonical_text": candidate.text, - "summary": _truncate(candidate.text, max_length=280), - "domain": source_input.domain, - "sensitivity": source_input.sensitivity, - "project_id": project_scope[0] if len(project_scope) == 1 else None, - "created_by_agent_id": self.actor_id if self.actor_type == "agent" else None, - "run_id": self.run_id if self.actor_type == "agent" else None, - "metadata_json": { + actor_type=self.actor_type, + ) + self._log_event( + event_type="memory.candidate_created", + target_type="memory", + target_id=str(memory["id"]), + payload={ "source_id": source_id, "source_chunk_id": candidate.source_chunk_id, - "source_chunk_index": candidate.source_chunk_index, - "extraction_rule": candidate.extraction_rule, - "capture_content_hash": content_hash, - **provenance_metadata, - **project_scope_metadata, - "generated_by": self.actor_type, - "agent_identity": self.agent_identity, - "agent_id": self.actor_id if self.actor_type == "agent" else None, - "agent_run_id": self.run_id if self.actor_type == "agent" else None, - "trace_id": self.trace_id, - "policy_decision": self.policy_decision, + "memory_type": candidate.memory_type, + "confidence": candidate.confidence, }, - }, - actor_type=self.actor_type, - ) - memory_rows.append(memory) - self.store.create_provenance_link( - { - "target_type": "memory", - "target_id": str(memory["id"]), - "source_id": source_id, - "source_chunk_id": candidate.source_chunk_id, - "quote": candidate.text, - "evidence_role": "quoted_from", - "confidence": candidate.confidence, - }, - actor_type=self.actor_type, - ) - self._log_event( - event_type="memory.candidate_created", - target_type="memory", - target_id=str(memory["id"]), - payload={ - "source_id": source_id, - "source_chunk_id": candidate.source_chunk_id, - "memory_type": candidate.memory_type, - "confidence": candidate.confidence, - }, - ) + ) # Capture writes candidate memories only, and recall cannot return a # candidate, so no text is sent to the embeddings endpoint here: the diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index ca5ca7cdc..28c54c159 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -10,8 +10,9 @@ import re from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager +from contextvars import ContextVar from functools import wraps -from dataclasses import replace +from dataclasses import dataclass, field, replace from typing import Any from alicebot_api.vnext_agent_control import RESTRICTED_DOMAINS @@ -59,6 +60,73 @@ class LabelLockOrderError(RuntimeError): REFUSED_DETAIL = "the label change could not be applied to every dependent row; nothing was changed" +@dataclass +class _CaptureLabelInputs: + conn: Any + transaction_id: str + rollback_counter: int + source_id: str + rows: dict[tuple[str, str], list[dict[str, object]]] = field(default_factory=dict) + + @property + def key(self) -> tuple[str, int]: + return self.transaction_id, self.rollback_counter + + +_CAPTURE_LABEL_INPUTS: ContextVar[_CaptureLabelInputs | None] = ContextVar("capture_label_inputs", default=None) + + +def invalidate_capture_label_inputs(store: Any) -> None: + batch = _CAPTURE_LABEL_INPUTS.get() + if batch is not None and batch.conn is getattr(store, "conn", None): + batch.rows.clear() + + +def label_savepoint_rolled_back(store: Any) -> None: + """Invalidate capture inputs whenever the store rolls a savepoint back.""" + + conn = store.conn + conn._alice_label_rollback_counter = int(getattr(conn, "_alice_label_rollback_counter", 0)) + 1 + invalidate_capture_label_inputs(store) + + +def _current_capture_inputs(store: Any) -> _CaptureLabelInputs | None: + batch = _CAPTURE_LABEL_INPUTS.get() + if batch is None or batch.conn is not getattr(store, "conn", None) or not _in_transaction(store): + return None + counter = int(getattr(batch.conn, "_alice_label_rollback_counter", 0)) + if counter != batch.rollback_counter: + batch.rows.clear() + batch.rollback_counter = counter + return batch + + +@contextmanager +def capture_label_inputs(store: Any, source_id: str) -> Iterator[None]: + """Reuse only the new source's label row during capture's candidate loop. + + A managed transaction fixes the actual transaction id for this synchronous + loop. Savepoint rollback and label updates invalidate its input rows. Every + writer still executes its advisory lock, and no grant is memoized. + """ + + conn = getattr(store, "conn", None) + if conn is None or _sqlite(store) or not callable(getattr(conn, "transaction", None)): + yield + return + with conn.transaction(): + with conn.cursor() as cur: + cur.execute("SELECT pg_current_xact_id()::text AS transaction_id") + row = cur.fetchone() + transaction_id = str(row["transaction_id"] if isinstance(row, Mapping) else row[0]) + batch = _CaptureLabelInputs(conn, transaction_id, int(getattr(conn, "_alice_label_rollback_counter", 0)), identifier(source_id)) + token = _CAPTURE_LABEL_INPUTS.set(batch) + try: + yield + finally: + _CAPTURE_LABEL_INPUTS.reset(token) + + def takes_label_lock(fn: Any) -> Any: """Take the shared label lock before a store method writes or row-locks a label table.""" @@ -177,6 +245,7 @@ def prepare_label_patch( ) -> JsonObject: """Check a proposed label change before its UPDATE or FOR UPDATE statement.""" + invalidate_capture_label_inputs(store) proposed = dict(before or {}) proposed.update({key: value for key, value in patch.items() if value is not None}) proposed["kind"] = kind @@ -220,7 +289,14 @@ def apply_insert_floor(store: Any, kind: str, payload: Mapping[str, object]) -> own["kind"] = kind own["id"] = own_id own["user_id"] = user_id - nodes, exceeded = collect_label_rows(store, [own], max_nodes=PROPAGATION_BOUND) + batch = _current_capture_inputs(store) + cache = batch.rows if batch is not None else None + nodes, exceeded = collect_label_rows(store, [own], max_nodes=PROPAGATION_BOUND, cache=cache) + if batch is not None: + # Keep only this capture's source. Other dependencies are read afresh. + for key in list(batch.rows): + if key != ("source", batch.source_id): + del batch.rows[key] if exceeded: raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") if not user_id: @@ -601,6 +677,7 @@ def write_settled_label( ) -> None: """Label-only update. A statement that changes no row refuses the whole relabel.""" + invalidate_capture_label_inputs(store) table = {"memory": "memories", "open_loop": "open_loops", "artifact": "generated_artifacts", "project": "projects"}[kind] blob = json.dumps({key: metadata[key] for key in ("project_scope", "project_floor") if key in metadata}) if _sqlite(store): diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 2a6a4e209..c34354a85 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -3802,6 +3802,9 @@ def savepoint(self) -> Iterator[None]: try: yield except BaseException: + from alicebot_api.vnext_label_writes import label_savepoint_rolled_back + + label_savepoint_rolled_back(self) try: self.conn.execute(f"ROLLBACK TO SAVEPOINT {name}") self.conn.execute(f"RELEASE SAVEPOINT {name}") diff --git a/tests/integration/test_capture_label_batch_postgres.py b/tests/integration/test_capture_label_batch_postgres.py new file mode 100644 index 000000000..e7a6d8549 --- /dev/null +++ b/tests/integration/test_capture_label_batch_postgres.py @@ -0,0 +1,159 @@ +"""Capture reuses source inputs while every writer takes a live label lock.""" +from uuid import uuid4 + +import psycopg +import pytest +from psycopg.rows import dict_row + +from alicebot_api import vnext_label_writes as writes +from alicebot_api.db import set_current_user, user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_capture import VNextCaptureService +from alicebot_api.vnext_store import PostgresVNextStore + + +def _user(conn, user): + ContinuityStore(conn).create_user(user, f"batch-{user}@example.test", "Synthetic") + + +def _source(store, user): + return store.create_source({"source_type": "note", "title": "Synthetic", "content_hash": str(user), "domain": "project", "sensitivity": "public"}) + + +def _memory(store, key, source): + return store.create_memory({"memory_key": key, "canonical_text": "Synthetic", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])}}) + + +def test_capture_reuses_only_source_inputs_and_keeps_every_writer_lock(migrated_database_urls, monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", False) + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + calls, reads, identities = [], [], [] + real_lock, real_reader, real_memory = store.lock_label_writes, store.read_label_rows, store.create_memory + def lock(*, exclusive=False): + calls.append(exclusive) + return real_lock(exclusive=exclusive) + def reader(kind, ids): + reads.append((kind, ids)) + return real_reader(kind, ids) + def memory(payload, **kwargs): + identities.append(conn.execute("SELECT pg_current_xact_id()::text AS tx, app.current_user_id()::text AS tenant").fetchone()) + frame = writes._current_capture_inputs(store) + assert frame.key[0] == identities[-1]["tx"] + return real_memory(payload, **kwargs) + monkeypatch.setattr(store, "lock_label_writes", lock) + monkeypatch.setattr(store, "read_label_rows", reader) + monkeypatch.setattr(store, "create_memory", memory) + result = VNextCaptureService(store).capture_text("\n".join(f"Decision: Synthetic item {i} is assigned to synthetic route {i}." for i in range(200)), domain="project", sensitivity="public") + assert result.candidate_memory_count == 200 + assert len(calls) == 401 # Source, 200 memories, 200 provenance links. + assert len(reads) == 1 and reads[0][0] == "source" + assert len({(row["tx"], row["tenant"]) for row in identities}) == 1 + assert identities[0]["tenant"] == str(user) + assert writes._current_capture_inputs(store) is None + assert writes.held_label_locks(store)[1] is True + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 200 + + +def test_source_input_memo_is_keyed_by_transaction_and_rollback_counter(migrated_database_urls): + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + source = _source(store, user) + with writes.capture_label_inputs(store, str(source["id"])): + _memory(store, "before", source) + frame = writes._current_capture_inputs(store) + before = frame.key + assert frame.rows + with pytest.raises(ValueError): + with store.savepoint(): + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) + assert _memory(store, "rolled-back", source)["sensitivity"] == "confidential" + raise ValueError("Synthetic rollback") + assert _memory(store, "after", source)["sensitivity"] == "public" + after = writes._current_capture_inputs(store).key + assert after == (before[0], before[1] + 1) + assert writes.held_label_locks(store)[1] is True + assert {row["memory_key"] for row in conn.execute("SELECT memory_key FROM memories").fetchall()} == {"before", "after"} + + +def test_failed_native_savepoint_batch_leaves_no_memo_or_rows(migrated_database_urls): + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + with pytest.raises(ValueError): + with conn.transaction(): + source = _source(store, user) + with writes.capture_label_inputs(store, str(source["id"])): + _memory(store, "failed", source) + assert writes.held_label_locks(store)[1] is True + raise ValueError("Synthetic failure") + assert writes._current_capture_inputs(store) is None + assert writes.held_label_locks(store)[1] is False + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 0 + source = _source(store, user) + _memory(store, "fresh", source) + assert writes.held_label_locks(store)[1] is True + + +def test_source_raise_invalidates_inputs_in_the_same_transaction(migrated_database_urls, monkeypatch): + monkeypatch.setattr(writes, "STRICT_LOCK_ORDER", False) + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + source = _source(store, user) + with writes.capture_label_inputs(store, str(source["id"])): + assert _memory(store, "before", source)["sensitivity"] == "public" + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) + assert writes._current_capture_inputs(store).rows == {} + assert _memory(store, "after", source)["sensitivity"] == "confidential" + + +def test_strict_mode_still_locks_every_writer_with_source_input_reuse(migrated_database_urls, monkeypatch): + user = uuid4() + with user_connection(migrated_database_urls["app"], user) as conn: + _user(conn, user) + store = PostgresVNextStore(conn) + source = _source(store, user) + count = 0 + real_lock = store.lock_label_writes + def lock(*, exclusive=False): + nonlocal count + count += 1 + return real_lock(exclusive=exclusive) + monkeypatch.setattr(store, "lock_label_writes", lock) + assert writes.STRICT_LOCK_ORDER is True + with writes.capture_label_inputs(store, str(source["id"])): + _memory(store, "first", source) + _memory(store, "second", source) + assert writes.held_label_locks(store)[1] is True + assert count == 2 + + +def test_one_store_starts_fresh_source_inputs_in_each_transaction(migrated_database_urls): + user = uuid4() + keys = [] + with psycopg.connect(migrated_database_urls["app"], row_factory=dict_row) as conn: + store = PostgresVNextStore(conn) + for index in range(2): + set_current_user(conn, user) + if index == 0: + _user(conn, user) + source = _source(store, uuid4()) + with writes.capture_label_inputs(store, str(source["id"])): + keys.append(writes._current_capture_inputs(store).key) + with pytest.raises(psycopg.ProgrammingError): + conn.commit() + _memory(store, str(index), source) + assert writes._current_capture_inputs(store) is None + conn.commit() + assert keys[0][0] != keys[1][0] From f97004e8b692498288abd83074b0e564705db9d0 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:48:26 +0200 Subject: [PATCH 156/270] Bound large PostgreSQL reverse frontier query cost --- .../src/alicebot_api/vnext_label_writes.py | 12 +++-- .../test_label_reverse_search_postgres.py | 50 ++++++++++++++++++- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 28c54c159..0efd18e34 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -482,7 +482,12 @@ def _postgres_dependants(store: Any, table: str, kind: str, compacts: Sequence[s extra = ", NULL::jsonb AS value" # Normalize each column once per row, rather than once per frontier id. # This remains only a superset lookup; the canonical parser selects exact edges. - pattern = "(?:" + "|".join(re.escape(item) for item in compacts) + ")" + if len(compacts) >= 32 and all(re.fullmatch(r"[0-9a-f]{32}", item) for item in compacts): + # A constant pattern avoids constructing a large automaton for UUID + # frontiers. More candidates are safe because exact edges are parsed below. + pattern = r"[0-9a-f]{32}" + else: + pattern = "(?:" + "|".join(re.escape(item) for item in compacts) + ")" def candidate_clause(column: str) -> str: return ( @@ -538,8 +543,9 @@ def walk_dependants(store: Any, roots: Sequence[str]) -> list[dict[str, object]] while pending: if len(seen) > PROPAGATION_BOUND: raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") - batch = pending[:200] - pending = pending[200:] + batch_size = 200 if _sqlite(store) else 2000 + batch = pending[:batch_size] + pending = pending[batch_size:] belief_aliases = getattr(store, "list_belief_ids_for_memories", None) if callable(belief_aliases): for belief_id in belief_aliases(batch): diff --git a/tests/integration/test_label_reverse_search_postgres.py b/tests/integration/test_label_reverse_search_postgres.py index f68ebd541..ef62b33d3 100644 --- a/tests/integration/test_label_reverse_search_postgres.py +++ b/tests/integration/test_label_reverse_search_postgres.py @@ -80,7 +80,7 @@ def test_multiple_frontier_batches_keep_memory_chains_and_both_direct_loop_colum "sensitivity": "public", } ) - roots = [str(uuid4()) for _ in range(220)] + roots = [str(uuid4()) for _ in range(2200)] conn.execute( """INSERT INTO memories(id,user_id,memory_key,value,source_event_ids,canonical_text, status,domain,sensitivity,metadata_json) @@ -90,7 +90,7 @@ def test_multiple_frontier_batches_keep_memory_chains_and_both_direct_loop_colum (root, roots), ) descendants = [] - for index in (0, 199, 219): + for index in (0, 1999, 2199): row = store.create_memory( { "memory_key": f"descendant.{index}", @@ -126,3 +126,49 @@ def test_multiple_frontier_batches_keep_memory_chains_and_both_direct_loop_colum store, "open_loops", "open_loop", [_compact_id(roots[-1])], with_value=False ) assert str(direct_memory["id"]) in {row["id"] for row in memory_candidates} + + +def test_large_uuid_frontiers_use_a_broader_superset_but_only_canonical_edges_enter_the_closure(migrated_database_urls): + user = uuid4() + roots = [str(uuid4()) for _ in range(32)] + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"broad-{user}@example.invalid", "Broad") + store = PostgresVNextStore(conn) + expected = set() + for index, reference in enumerate((roots[0], roots[-1].upper(), encoded_object("source_id", roots[1]))): + row = store.create_memory( + { + "memory_key": f"broad.{index}", + "canonical_text": "Synthetic dependency", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"source_id": reference}, + } + ) + expected.add(str(row["id"])) + unrelated = store.create_memory( + { + "memory_key": "unrelated.edge", + "canonical_text": "Synthetic unrelated", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"source_id": str(uuid4())}, + } + ) + note = store.create_memory( + { + "memory_key": "unrelated.note", + "canonical_text": "Synthetic unrelated note", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"note": roots[0]}, + } + ) + candidates = _postgres_dependants( + store, "memories", "memory", [_compact_id(root) for root in roots], with_value=True + ) + candidate_ids = {str(row["id"]) for row in candidates} + assert expected <= candidate_ids + assert str(unrelated["id"]) in candidate_ids + assert str(note["id"]) in candidate_ids + assert {str(row["id"]) for row in walk_dependants(store, roots)} == expected From 5122bb1ff570dc4a1cc98cb8c7cafe6279b539cf Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:48:14 +0200 Subject: [PATCH 157/270] Prove native PostgreSQL owner edit clamp controls --- .../test_label_floor_ancestry_postgres.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/integration/test_label_floor_ancestry_postgres.py b/tests/integration/test_label_floor_ancestry_postgres.py index 2a16690f1..882999373 100644 --- a/tests/integration/test_label_floor_ancestry_postgres.py +++ b/tests/integration/test_label_floor_ancestry_postgres.py @@ -95,3 +95,38 @@ def test_a_relabel_traverses_the_belief_backing_memory(migrated_database_urls): store = PostgresVNextStore(conn) assert store.get_memory(str(copy["id"]))["sensitivity"] == "regulated" assert store.get_artifact(str(report["id"]))["sensitivity"] == "regulated" + + +def test_postgres_owner_edit_is_clamped_in_response_event_and_storage(migrated_database_urls, monkeypatch): + import json + from uuid import UUID + from alicebot_api.config import Settings + from alicebot_api.routers import vnext_memories as router + from alicebot_api.vnext_label_writes import without_insert_floor + + user_id = uuid4() + url = migrated_database_urls["app"] + with user_connection(url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"owner-clamp-{user_id}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Synthetic private input", "content_hash": str(user_id), "domain": "health", "sensitivity": "confidential"}) + with without_insert_floor(): + memory = store.create_memory({"memory_key": "stale-copy", "canonical_text": "Synthetic private observation", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])}}) + assert memory["sensitivity"] == "public" + monkeypatch.setattr(router, "get_settings", lambda: Settings(database_url=url)) + response = router.review_vnext_memory(UUID(str(memory["id"])), router.VNextMemoryReviewRequest(user_id=user_id, action="edit", domain="project", sensitivity="public"), authorization=None) + assert response.status_code == 200 + payload = json.loads(response.body) + assert payload["label_floor_applied"] is True + assert payload["memory"]["domain"] == "health" + assert payload["memory"]["sensitivity"] == "confidential" + with user_connection(url, user_id) as conn: + store = PostgresVNextStore(conn) + stored = store.get_memory(str(memory["id"])) + assert stored["domain"] == "health" and stored["sensitivity"] == "confidential" + assert stored["metadata_json"]["source_id"] == str(source["id"]) + event = conn.execute("SELECT payload_json FROM event_log WHERE event_type='memory.labels_raised' AND target_id=%s", (str(memory["id"]),)).fetchone() + assert event["payload_json"]["cause"] == "floor_clamped" + encoded = json.dumps(event["payload_json"]) + assert "Synthetic private input" not in encoded + assert "Synthetic private observation" not in encoded From 68946a42e979833a39c300ed80c734d8f7288552 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:49:45 +0200 Subject: [PATCH 158/270] Align broad unit fixtures with derived label contracts --- tests/unit/test_legacy_gated_router_split.py | 14 +++-- tests/unit/test_main.py | 8 +-- tests/unit/test_mcp.py | 47 +++++++++++++++ tests/unit/test_providers_router_split.py | 5 +- tests/unit/test_saved_provenance_reader.py | 1 + tests/unit/test_source_refs_read_fence.py | 3 + tests/unit/test_surface_gates.py | 6 +- tests/unit/test_vnext_main.py | 24 +++++++- tests/unit/test_vnext_retrieval.py | 53 +++++++++++++---- tests/unit/test_vnext_store.py | 62 +++++++++++++------- tests/unit/test_workspaces_router_split.py | 34 +++++++---- 11 files changed, 197 insertions(+), 60 deletions(-) diff --git a/tests/unit/test_legacy_gated_router_split.py b/tests/unit/test_legacy_gated_router_split.py index a19305c3d..4def45c37 100644 --- a/tests/unit/test_legacy_gated_router_split.py +++ b/tests/unit/test_legacy_gated_router_split.py @@ -240,8 +240,10 @@ EXPECTED_OWNED_SUPPORT_AST_SHA256 = "8bbbcb0cf52c4bd1da5fce4e50878d8a62599be59ee25e09dcd905c94950d406" EXPECTED_FULL_SUPPORT_AST_SHA256 = "07acb5deafb700e040c459969610e8877e86e62b04d4e9d86493fe314cb02868" EXPECTED_GATED_OPERATION_SHA256 = "55490460fd78990a6872ebf22c6cfd927f7d0a5548b6df90adde8261ede8da65" -EXPECTED_DEFAULT_DEEP_ROUTE_SHA256 = "a865b2264c911ed1b055853777b850e070bd594d79fafb167df7b915b7c3c9ce" -EXPECTED_LEGACY_DEEP_ROUTE_SHA256 = "6743a8b03d649328341eab462aeea0271f997f14c5c558a666b5aff92f297d79" + +# Source regeneration adds one default route before source review. +EXPECTED_DEFAULT_DEEP_ROUTE_SHA256 = "9d9e83c35818387a13c719d455c88f4dcb4d139bf5f4dd5888bc6ff3723077a7" +EXPECTED_LEGACY_DEEP_ROUTE_SHA256 = "5ba780b5ec62584b844cf1fcab8fc35b703f4494c1486db870588ab68a1f57a6" EXPECTED_INTEGRATION_PATCH_COUNTS = { "tests/integration/test_approval_api.py": 8, @@ -667,9 +669,9 @@ def test_main_preserves_frozen_flag_policy_and_five_mount_seams() -> None: def test_flagged_surface_preserves_deep_order_ids_and_import_timing() -> None: default = _isolated_surface_manifest(None) assert default == { - "operation_count": 183, + "operation_count": 184, "legacy_count": 0, - "deep_count": 187, + "deep_count": 188, "deep_digest": EXPECTED_DEFAULT_DEEP_ROUTE_SHA256, "gated_count": 0, "gated_digest": hashlib.sha256(b"[]").hexdigest(), @@ -679,9 +681,9 @@ def test_flagged_surface_preserves_deep_order_ids_and_import_timing() -> None: } legacy = _isolated_surface_manifest("1") assert legacy == { - "operation_count": 232, + "operation_count": 233, "legacy_count": 49, - "deep_count": 236, + "deep_count": 237, "deep_digest": EXPECTED_LEGACY_DEEP_ROUTE_SHA256, "gated_count": 49, "gated_digest": EXPECTED_GATED_OPERATION_SHA256, diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 43c657341..6c8783108 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -312,7 +312,7 @@ def test_openapi_has_concrete_success_contracts_and_accurate_statuses() -> None: } operations = list(operations_by_key.values()) - assert len(operations) == 183 + assert len(operations) == 184 assert all(operation.get("tags") for operation in operations) assert all(operation.get("description") for operation in operations) assert all("default" in operation["responses"] for operation in operations) @@ -323,7 +323,7 @@ def test_openapi_has_concrete_success_contracts_and_accurate_statuses() -> None: if status.startswith("2") for json_body in [response.get("content", {}).get("application/json", {})] ] - assert len(success_schemas) == 187 + assert len(success_schemas) == 188 assert "APIJsonDocument" not in components assert all(document.get("$ref", "").startswith("#/components/schemas/") for document in success_schemas) resolved_success_schemas = [components[document["$ref"].rsplit("/", 1)[-1]] for document in success_schemas] @@ -347,7 +347,7 @@ def test_openapi_has_concrete_success_contracts_and_accurate_statuses() -> None: assert exact_keys | set(operation_registry) == set(operations_by_key), coverage_report assert exact_keys.isdisjoint(operation_registry), coverage_report assert len(exact_keys) == 42 - assert len(operation_registry) == 141 + assert len(operation_registry) == 142 assert 0 < len(polymorphic_operations) <= 3 assert set(polymorphic_operations) <= set(operation_registry) assert all(reason.strip() for reason in polymorphic_operations.values()) @@ -645,7 +645,7 @@ def test_openapi_helper_backed_contracts_track_authoritative_response_types() -> def test_openapi_store_row_contracts_track_authoritative_column_sets() -> None: def column_fields(columns: str) -> set[str]: - return {column.strip() for column in columns.split(",") if column.strip()} + return {column.strip().rsplit(" AS ", 1)[-1] for column in columns.split(",") if column.strip()} row_contracts = { ("GET", "/v0/vnext/sources/{source_id}"): SOURCE_COLUMNS, diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index ff52aa588..f73db46df 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -1278,8 +1278,48 @@ def _ascii_query_fold(value: str) -> str: return value.translate(_ASCII_QUERY_CASE_TRANSLATION) +class FakeLabelCursor: + """SQL guard responses from the fake store's current lock state.""" + + def __init__(self, store) -> None: + self.store = store + self.query = "" + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def execute(self, query, params=None): + assert any(marker in query for marker in ( + "FROM pg_locks", "current_setting('lock_timeout')", "SET LOCAL lock_timeout", "set_config('lock_timeout'", + )), query + self.query = query + + def fetchone(self): + if "FROM pg_locks" in self.query: + return {"graph": self.store.graph_locked, "labels": self.store.labels_locked, + "exclusive": self.store.labels_exclusive} + if "current_setting('lock_timeout')" in self.query: + return {"lock_timeout": "0"} + raise AssertionError(self.query) + + +class FakeLabelConnection: + def __init__(self, store) -> None: + self.store = store + + def cursor(self): + return FakeLabelCursor(self.store) + + class FakeVNextMCPStore: def __init__(self) -> None: + self.graph_locked = False + self.labels_locked = False + self.labels_exclusive = False + self.conn = FakeLabelConnection(self) self.events: list[dict[str, object]] = [] self.sources: list[dict[str, object]] = [] self.chunks: list[dict[str, object]] = [] @@ -1318,6 +1358,13 @@ def __init__(self) -> None: } } + def lock_graph_mutation(self) -> None: + self.graph_locked = True + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + self.labels_locked = True + self.labels_exclusive |= exclusive + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: # The fake search methods expose fixed persisted rows as well as writes. collections = { diff --git a/tests/unit/test_providers_router_split.py b/tests/unit/test_providers_router_split.py index a8b88d978..04a176b8d 100644 --- a/tests/unit/test_providers_router_split.py +++ b/tests/unit/test_providers_router_split.py @@ -117,7 +117,10 @@ # lone_surrogates.py and main.py only registers it, so it adds no definition # here. Earlier re-pin (2026-09-26): _rewrite_user_id_json_body writes the # rewritten JSON into request._body before call_next. -EXPECTED_CARRIER_AST_SHA256 = "ab3fc6d61cb81a1b9c1a6573adc8e1e297cbbcf01e230effd4a0824dee2d8e2b" + +# Re-pin 2026-10-06: source regeneration is a new protected write route in the +# central vNext route policy; the app carrier keeps the same definitions. +EXPECTED_CARRIER_AST_SHA256 = "fcd6d722e2d6c3be28b138449555e703f6930168f52b8e94b41dacbcb937b605" EXPECTED_ROUTE_NAME_MANIFEST_SHA256 = "1a438538e16120361f92d30375cc94679d598fe4b78ba5a58a7d8a4dda6af83c" EXPECTED_OPERATION_MANIFEST_SHA256 = "8b79ceaf996b8c51b5bb2f3f38a8c19a4e33796955d8b8f7a66e7ac01ea1732d" EXPECTED_IMPORT_MANIFEST_SHA256 = "17484ccdd460e42e2ad5c82a8ca867664694feaf871118c410a134996a532358" diff --git a/tests/unit/test_saved_provenance_reader.py b/tests/unit/test_saved_provenance_reader.py index 446ca83b1..5fb12c853 100644 --- a/tests/unit/test_saved_provenance_reader.py +++ b/tests/unit/test_saved_provenance_reader.py @@ -730,6 +730,7 @@ def test_review_by_id_reads_saved_provenance_only_through_the_reader() -> None: ("apps/api/src/alicebot_api/mcp/memories.py", "_handle_alice_vnext_commit_memory"): "passes the argument to commit", ("apps/api/src/alicebot_api/cli/memories.py", "_run_vnext_memory_commit"): "passes the argument to commit", ("apps/api/src/alicebot_api/vnext_capture.py", "_capture_source"): "link quote = the candidate's own text", + ("apps/api/src/alicebot_api/vnext_source_regeneration.py", "regenerate_source_inputs"): "fresh candidate quote; SavedProvenanceReader and row readers apply the source fence", ("apps/api/src/alicebot_api/vnext_connectors.py", "ingest_agent_output"): "link quote = the item title", ("apps/api/src/alicebot_api/vnext_retrieval.py", "_supporting_evidence"): "reads the link, fenced by admits_link", ("apps/api/src/alicebot_api/onramp.py", "_apply_import_quarantine"): "replaces a quote with a placeholder", diff --git a/tests/unit/test_source_refs_read_fence.py b/tests/unit/test_source_refs_read_fence.py index 7ffe35cdc..11815f8d2 100644 --- a/tests/unit/test_source_refs_read_fence.py +++ b/tests/unit/test_source_refs_read_fence.py @@ -1509,6 +1509,8 @@ def _provenance_link_call_sites() -> set[tuple[str, str]]: # The source was captured or ingested by this very call, so the caller named no id. ("vnext_capture.py", "_capture_source"), ("vnext_connectors.py", "ingest_agent_output"), + # Regeneration reads the authorized source and its chunks before making new rows. + ("vnext_source_regeneration.py", "regenerate_source_inputs"), } _NAMED_SOURCE_SITES = { # The caller named the id. Each is behind ``resolve_attachable_sources`` and the caller's ``SourceReadFence``. @@ -1702,6 +1704,7 @@ def test_attachable_sources_are_built_only_by_the_resolver() -> None: ("memory.py", "_create_open_loop_for_memory"), ("vnext_brain.py", "_create_candidate_open_loops"), ("vnext_projects.py", "extract_open_loops"), + ("vnext_source_regeneration.py", "regenerate_source_inputs"), ("vnext_scheduler.py", "_publish_mutation"), ("vnext_stores/postgres/graph_open_loops.py", "upsert_open_loop_by_automation_digest"), ("vnext_stores/sqlite/graph_open_loops.py", "upsert_open_loop_by_automation_digest"), diff --git a/tests/unit/test_surface_gates.py b/tests/unit/test_surface_gates.py index e23a12886..92eab0b86 100644 --- a/tests/unit/test_surface_gates.py +++ b/tests/unit/test_surface_gates.py @@ -94,7 +94,7 @@ def _isolated_proxy_execution_posture(flag_value: str | None) -> dict[str, objec @pytest.mark.parametrize("flag_value", [None, "", "0", "true", "yes", "on", "01", " 1"]) def test_http_legacy_surface_gate_fails_closed_for_every_non_exact_value(flag_value: str | None) -> None: assert _isolated_http_inventory(flag_value) == { - "count": 183, + "count": 184, "legacy_count": 0, "removed_count": 0, "runtime_invoke_count": 1, @@ -103,7 +103,7 @@ def test_http_legacy_surface_gate_fails_closed_for_every_non_exact_value(flag_va def test_http_legacy_surface_gate_mounts_exact_inventory_only_for_one() -> None: assert _isolated_http_inventory("1") == { - "count": 232, + "count": 233, "legacy_count": 49, "removed_count": 0, "runtime_invoke_count": 1, @@ -161,7 +161,7 @@ def inventory(): text=True, ) - expected_inventory = {"count": 183, "legacy_count": 0} + expected_inventory = {"count": 184, "legacy_count": 0} assert json.loads(completed.stdout) == { "before": expected_inventory, "after": expected_inventory, diff --git a/tests/unit/test_vnext_main.py b/tests/unit/test_vnext_main.py index 377306289..7676ebad7 100644 --- a/tests/unit/test_vnext_main.py +++ b/tests/unit/test_vnext_main.py @@ -54,18 +54,28 @@ def __init__(self, _conn) -> None: self.agent_api_keys: list[dict[str, object]] = [] self.browser_clip_capabilities: dict[str, dict[str, object]] = {} self.revisions: list[dict[str, object]] = [] + self.graph_locked = False + self.labels_locked = False + self.labels_exclusive = False def lock_graph_mutation(self) -> None: - return None + self.graph_locked = True def lock_label_writes(self, *, exclusive: bool = False) -> None: - return None + self.labels_locked = True + self.labels_exclusive |= exclusive def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: collection = {"source": self.sources.values(), "memory": self.memories, "open_loop": self.open_loops, "artifact": self.artifacts.values(), "belief": self.beliefs.values(), "project": self.projects.values()}.get(kind, []) return [dict(row) for row in collection if str(row.get("id")) in ids] + def iter_label_rows(self, kind: str): + collections = {"source": self.sources.values(), "memory": self.memories, + "open_loop": self.open_loops, "artifact": self.artifacts.values(), + "belief": self.beliefs.values(), "project": self.projects.values()} + yield [dict(row) for row in collections[kind]] + def _fetch_all(self, query: str, _params: tuple[object, ...]) -> list[dict[str, object]]: # The label dependant walker performs its exact canonical reference filter after this prefilter. for table, kind in (("memories", "memory"), ("open_loops", "open_loop"), ("generated_artifacts", "artifact"), ("projects", "project")): @@ -900,6 +910,10 @@ def list_connector_states(self) -> list[dict[str, object]]: def _install_fake_vnext_store(monkeypatch, store: FakeVNextStore) -> None: from alicebot_api import vnext_label_writes monkeypatch.setattr(vnext_label_writes, "acquire_exclusive_label_lock", lambda target: target.lock_label_writes(exclusive=True)) + monkeypatch.setattr( + vnext_label_writes, "held_label_locks", + lambda target: (target.graph_locked, target.labels_locked, target.labels_exclusive), + ) @contextmanager def fake_user_connection(database_url, current_user_id): assert database_url == "postgresql://db" @@ -1219,7 +1233,7 @@ def test_vnext_route_inventory_fails_closed_without_route_local_policy() -> None } assert not (main_module._VNEXT_ROUTE_LOCAL_POLICY & main_module._VNEXT_CENTRAL_OPERATOR_ROUTES) assert (main_module._VNEXT_ROUTE_LOCAL_POLICY | main_module._VNEXT_CENTRAL_OPERATOR_ROUTES) == registered - assert len(registered) == 71 + assert len(registered) == 72 project_bound = main_module.AgentIdentity( agent_id="project-reader", @@ -1368,6 +1382,7 @@ def test_vnext_memories_router_partitions_preserve_global_route_sequence() -> No vnext_memories_router.source_review_router, [ ("GET", "/v0/vnext/sources/{source_id}"), + ("POST", "/v0/vnext/sources/{source_id}/regenerate"), ("POST", "/v0/vnext/sources/{source_id}/review"), ], ), @@ -4466,6 +4481,9 @@ def test_dogfooding_dashboard_and_insight_feedback_api(monkeypatch) -> None: "sensitivity": "private", } ) + from alicebot_api.vnext_derived_labels import stamp_derived_from + + stamp_derived_from(artifact, {}) store.create_artifact_quality_rating( { "artifact_id": artifact["id"], diff --git a/tests/unit/test_vnext_retrieval.py b/tests/unit/test_vnext_retrieval.py index 70454a4a5..b2ccaebff 100644 --- a/tests/unit/test_vnext_retrieval.py +++ b/tests/unit/test_vnext_retrieval.py @@ -99,8 +99,10 @@ def __init__( entities: list[dict[str, object]] | None = None, edges: list[dict[str, object]] | None = None, source_chunks: list[dict[str, object]] | None = None, + stored_memories: list[dict[str, object]] | None = None, ) -> None: self.memories = memories + self.stored_memories = stored_memories or [] self.sources = sources self.open_loops = open_loops or [] self.provenance_links = provenance_links or [] @@ -121,6 +123,19 @@ def __init__( self.vector_limits: list[int] = [] self.memory_bulk_reads = 0 + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + # Complete persisted ancestry, including rows outside the FTS match list. + collections = { + "memory": [*self.memories, *self.stored_memories, *(self.vector_memories or [])], + "source": self.sources, "open_loop": self.open_loops, + "belief": self.beliefs or [], + } + from alicebot_api.vnext_derived_labels import identifier + + wanted = {identifier(item) for item in ids} + found = {str(row.get("id")): row for row in collections.get(kind, [])} + return [dict(row) for row in found.values() if identifier(row.get("id")) in wanted] + def append_event(self, event: dict[str, object]) -> dict[str, object]: self.events.append(event) return event @@ -649,7 +664,7 @@ def test_context_pack_degrades_to_fts_when_query_embedding_fails() -> None: assert pack["trace"]["stages"]["vector"]["candidate_count"] == 0 -def test_context_pack_filters_sensitive_memories_and_records_trace_exclusion() -> None: +def test_context_pack_filters_sensitive_memories_before_recording_trace() -> None: store = InMemoryVNextRetrievalStore( memories=[ _memory_row( @@ -680,8 +695,9 @@ def test_context_pack_filters_sensitive_memories_and_records_trace_exclusion() - ) assert [memory["id"] for memory in pack["relevant_memories"]] == ["memory-public"] - assert "sensitive_items_filtered" in pack["warnings"] - assert pack["trace"]["excluded_counts"] == {"sensitivity_filtered": 1} + assert pack["warnings"] == [] + assert pack["trace"]["excluded_counts"] == {} + assert "memory-secret" not in json.dumps(pack["trace"]) assert [record["target_id"] for record in pack["trace"]["selected"]] == ["memory-public"] @@ -1120,7 +1136,10 @@ def test_strict_count_candidate_statistic_without_selected_rollup_stays_trace_on ) for index in range(1, 4) ], - sources=[], + sources=[ + {"id": f"source-{index}", "source_type": "note", "domain": "project", + "sensitivity": "private"} for index in range(1, 4) + ], ) pack = VNextRetrievalService(store).compile_context_pack( @@ -1958,14 +1977,12 @@ def _capture_currency_source_lookup(rows, *, source_lookup): source_fence=SourceReadFence.unfenced() ) - assert source_resolver_results == [None] + assert source_resolver_results == [] + assert pack["relevant_memories"] == [] assert pack["sources"] == [] assert pack["supporting_evidence"] == [] assert "derived_values" not in pack assert all("event_time" not in memory for memory in pack["relevant_memories"]) - first = pack["relevant_memories"][0] - assert "source_created_at" not in first - assert first["metadata_json"] == {"project_id": "alicebot", "source_refs": []} assert "source-hermes" not in json.dumps(pack, sort_keys=True) @@ -2670,6 +2687,7 @@ def test_context_pack_populates_contradicting_evidence_from_active_beliefs() -> ) ], sources=[], + stored_memories=[_memory_row("memory-belief", "The deployment pipeline is ready for production launch.")], beliefs=[ { "id": "belief-1", @@ -3447,7 +3465,10 @@ def test_contradictions_first_lets_contradictions_survive_a_budget_that_drops_th } def compile_with(strategy: str, max_tokens: int | None) -> dict[str, object]: - store = InMemoryVNextRetrievalStore(memories=[dict(memory)], sources=[], beliefs=[dict(belief)]) + store = InMemoryVNextRetrievalStore( + memories=[dict(memory)], sources=[], beliefs=[dict(belief)], + stored_memories=[_memory_row("memory-belief", str(belief["claim"]))], + ) return VNextRetrievalService(store).compile_context_pack( VNextRetrievalRequest( query="deployment pipeline production launch", @@ -3687,6 +3708,7 @@ def build_store() -> InMemoryVNextRetrievalStore: return InMemoryVNextRetrievalStore( memories=[_memory_row("memory-1", "The deployment pipeline is not ready for production launch.")], sources=[], + stored_memories=[_memory_row("memory-belief", "The deployment pipeline is ready for production launch.")], beliefs=[ { "id": "belief-1", @@ -4494,7 +4516,12 @@ def build_store() -> InMemoryVNextRetrievalStore: for index in range(1, 25) ] swim = _memory_row("memory-swim", "Swimming laps review.", metadata_json={"source_id": "src-swim"}) - return InMemoryVNextRetrievalStore(memories=[*fillers, swim], sources=[]) + return InMemoryVNextRetrievalStore( + memories=[*fillers, swim], + sources=[{"id": source_id, "source_type": "note", "domain": "project", + "sensitivity": "private"} + for source_id in ["src-shared", *[f"src-{index:02d}" for index in range(3, 25)], "src-swim"]], + ) control_store = build_store() with monkeypatch.context() as patch: @@ -4729,7 +4756,11 @@ def test_naturally_selected_unrelated_rollup_does_not_turn_trace_count_into_answ ["actual-member-1", "actual-member-2", "actual-member-3"], ) pack = VNextRetrievalService( - InMemoryVNextRetrievalStore(memories=[*unrelated, card], sources=[]) + InMemoryVNextRetrievalStore( + memories=[*unrelated, card], sources=[], + stored_memories=[_memory_row(f"actual-member-{index}", f"A different activity {index}.") + for index in range(1, 4)], + ) ).compile_context_pack( VNextRetrievalRequest( query="How many times did I host board game night?", diff --git a/tests/unit/test_vnext_store.py b/tests/unit/test_vnext_store.py index de7109f4d..3ff885712 100644 --- a/tests/unit/test_vnext_store.py +++ b/tests/unit/test_vnext_store.py @@ -27,6 +27,10 @@ def __init__( self.executed: list[tuple[str, tuple[object, ...] | None]] = [] self.fetchone_results = list(fetchone_results) self.fetchall_result = fetchall_result or [] + self.current_query = "" + self.graph_locked = False + self.labels_locked = False + self.labels_exclusive = False def __enter__(self) -> "RecordingCursor": return self @@ -38,13 +42,28 @@ def execute(self, query: str, params: tuple[object, ...] | None = None) -> None: if params is not None: assert query.count("%s") == len(params) self.executed.append((query, params)) + self.current_query = query + if "pg_advisory_xact_lock" in query: + if "vnext_supersession" in query: + self.graph_locked = True + elif "vnext_labels" in query: + self.labels_locked = True + self.labels_exclusive |= "pg_advisory_xact_lock_shared" not in query @property def statements(self) -> list[tuple[str, tuple[object, ...] | None]]: """Business SQL, with transaction guards retained separately in executed.""" - return [(query, params) for query, params in self.executed if "pg_advisory_xact_lock" not in query] + return [(query, params) for query, params in self.executed if not any(marker in query for marker in ( + "pg_advisory_xact_lock", "FROM pg_locks", "current_setting('lock_timeout')", + "SET LOCAL lock_timeout", "set_config('lock_timeout'", + ))] def fetchone(self) -> dict[str, Any] | None: + if "current_setting('lock_timeout')" in self.current_query: + return {"lock_timeout": "0"} + if "FROM pg_locks" in self.current_query: + return {"graph": self.graph_locked, "labels": self.labels_locked, + "exclusive": self.labels_exclusive} if not self.fetchone_results: return None return self.fetchone_results.pop(0) @@ -96,15 +115,12 @@ def test_source_crud_and_chunks_write_audit_events() -> None: {"id": source_id}, _event_row(source_id), {"id": source_id}, - { - "id": source_id, - "content_hash": "sha256:abc", - "dedupe_key": "capture-md5:legacy", - "domain": "project", - "sensitivity": "private", - "metadata_json": {"path": "docs/spec.md"}, - }, - {"id": source_id}, + # The unlocked label pre-read precedes the source row lock. + {"id": source_id, "content_hash": "sha256:abc", "dedupe_key": "capture-md5:legacy", + "domain": "project", "sensitivity": "private", "metadata_json": {"path": "docs/spec.md"}}, + {"id": source_id, "content_hash": "sha256:abc", "dedupe_key": "capture-md5:legacy", + "domain": "project", "sensitivity": "private", "metadata_json": {"path": "docs/spec.md"}}, + {"id": source_id, "domain": "project", "sensitivity": "private", "metadata_json": {"rev": 2}}, _event_row(source_id), {"id": source_id}, _event_row(source_id), @@ -293,7 +309,8 @@ def test_update_source_recomputes_postgres_dedupe_key_with_the_same_statement() } cursor = RecordingCursor( fetchone_results=[ - current, + current, # label pre-read + current, # locked capture identity None, {**current, "domain": "professional"}, _event_row(source_id), @@ -339,7 +356,7 @@ def test_update_source_postgres_collision_fails_before_mutation_event() -> None: "sensitivity": "private", "metadata_json": {"raw_text": raw_text, "project_scope": ["Alpha"]}, } - cursor = RecordingCursor(fetchone_results=[current, {"id": str(uuid4())}]) + cursor = RecordingCursor(fetchone_results=[current, current, {"id": str(uuid4())}]) store = PostgresVNextStore(RecordingConnection(cursor)) with pytest.raises(ContinuityStoreInvariantError, match="already belongs"): @@ -364,7 +381,8 @@ def test_update_source_postgres_releases_key_when_changed_identity_has_no_raw_te } cursor = RecordingCursor( fetchone_results=[ - current, + current, # label pre-read + current, # locked capture identity {**current, "dedupe_key": None, "metadata_json": {"project_scope": ["Beta"]}}, _event_row(source_id), ] @@ -1409,6 +1427,7 @@ def test_project_people_belief_and_open_loop_methods_write_audit_events() -> Non {"id": project_id}, _event_row(project_id), {"id": project_id}, + {"id": project_id}, # label pre-read before the update {"id": project_id}, _event_row(project_id), {"id": person_id}, @@ -1426,6 +1445,7 @@ def test_project_people_belief_and_open_loop_methods_write_audit_events() -> Non {"id": loop_id}, {"id": loop_id}, _event_row(loop_id), + {"id": loop_id}, # label pre-read before the final update {"id": loop_id}, _event_row(loop_id), ] @@ -1464,14 +1484,14 @@ def test_project_people_belief_and_open_loop_methods_write_audit_events() -> Non None, 3, ) - assert "UPDATE projects" in cursor.statements[4][0] - assert "INSERT INTO people" in cursor.statements[6][0] - assert "UPDATE people" in cursor.statements[9][0] - assert "INSERT INTO beliefs" in cursor.statements[11][0] - assert "UPDATE beliefs" in cursor.statements[14][0] - assert "INSERT INTO open_loops" in cursor.statements[16][0] - assert "UPDATE open_loops" in cursor.statements[19][0] - assert "UPDATE open_loops" in cursor.statements[21][0] + assert "UPDATE projects" in cursor.statements[5][0] + assert "INSERT INTO people" in cursor.statements[7][0] + assert "UPDATE people" in cursor.statements[10][0] + assert "INSERT INTO beliefs" in cursor.statements[12][0] + assert "UPDATE beliefs" in cursor.statements[15][0] + assert "INSERT INTO open_loops" in cursor.statements[17][0] + assert "UPDATE open_loops" in cursor.statements[20][0] + assert "UPDATE open_loops" in cursor.statements[23][0] def test_get_artifact_for_update_locks_the_persisted_authorization_target() -> None: diff --git a/tests/unit/test_workspaces_router_split.py b/tests/unit/test_workspaces_router_split.py index 6b8422b91..9a5b4230b 100644 --- a/tests/unit/test_workspaces_router_split.py +++ b/tests/unit/test_workspaces_router_split.py @@ -82,11 +82,14 @@ # The later typing repair adds Sequence and annotates _workspace_rows. # Two local helpers admit rows and check event targets; the payload uses them # before returning lists/counts. Route bodies, mounts and middleware are unchanged. -EXPECTED_ROUTE_AST_SHA256 = "fb8925ebcda058598b0c6d5e7eca83abe18606a0126bdb6cde0fd1c22444a795" -EXPECTED_SUPPORT_AST_SHA256 = "0e5708e69de1dd1358ca91709d4a60effeddcd262fc8beddb19ba490cab3b0f1" + +# Re-pin 2026-10-06: workspace reads authenticate the protected identity and +# admit rows through effective labels before totals or dashboard disclosure. +EXPECTED_ROUTE_AST_SHA256 = "b99f1435de67d9499819acb9ed7ed61b588a3fb0ff037e782eeca070b39af742" +EXPECTED_SUPPORT_AST_SHA256 = "8700f0ab3b8a87f9863a3e95132e834f87bf82725c6e8d33602345bc3253bce1" EXPECTED_ROUTE_NAME_MANIFEST_SHA256 = "225c57c08bd8314156c56352dd1c53ffed3f556ce285c666dd6fca125115d0b4" EXPECTED_OPERATION_MANIFEST_SHA256 = "c320979b62d7ee8de244fe38bde5bf3761a4f9d76f76bf3cd8576c30fce9857e" -EXPECTED_IMPORT_MANIFEST_SHA256 = "4a9df193d620b33578f1e42760b118a4267d3e1c8ff01ad92bee1f1bba1c1e9f" +EXPECTED_IMPORT_MANIFEST_SHA256 = "d8887934cacc6a4e5d52f080dae3c2c0d0cdf23d1518a4060a885a8c7f209c0c" EXPECTED_CARRIER_NAMES_SHA256 = "2c109fc234a05dd8f44e4c34bee49e797fbb5e49e92413391541a7e504da328b" # Re-pinned 2026-10-02 (DB-005, legacy /v0 routes). One definition changed, # found by a per-definition AST diff against the previous pin: @@ -126,15 +129,18 @@ # lone_surrogates.py and main.py only registers it, so it adds no definition # here. Earlier re-pin (2026-09-26): _rewrite_user_id_json_body writes the # rewritten JSON into request._body before call_next. -EXPECTED_CARRIER_AST_SHA256 = "ab3fc6d61cb81a1b9c1a6573adc8e1e297cbbcf01e230effd4a0824dee2d8e2b" + +# Re-pin 2026-10-06: source regeneration is a new protected write route in the +# central vNext route policy; the app carrier keeps the same definitions. +EXPECTED_CARRIER_AST_SHA256 = "fcd6d722e2d6c3be28b138449555e703f6930168f52b8e94b41dacbcb937b605" EXPECTED_ROUTE_NODE_SHA256 = { - "get_vnext_workspace": "6c2151bf38b1b1311f016c00d14394afc7077a6ea219f7ce3dcfd9b701474ae7", + "get_vnext_workspace": "52c12b20d7bb33759f8dafa2249b2d775b54666130402c0c75045e9ad57ed587", "bootstrap_v1_workspace": "07b1fe2a4cd03a5ba69abe76e258a457e85e92b0bfba592520ee02d01d759c4b", "get_v1_workspace_bootstrap_status": "2849d7126ee37b6e3ffd9ebe84b2a8e719eb0f811da750a29f7e0a0798305faa", } EXPECTED_SUPPORT_NODE_SHA256 = { "_vnext_status_counts": "0bf0ed228a14bd648a9d18fcd5f99ebf8c585bd29f4b5e81e1df17fe0201fd15", - "_vnext_workspace_payload": "4b6e4a3d59d16538b0e859c425ede21207e9c80f11ab31c3af2ce4d604e14edf", + "_vnext_workspace_payload": "83bc100509fc21e950702cba190ac8d94f33efe7774df673809d49daa6864136", "_workspace_rows": "070bdfbd1eae10608bd8208b08367e1c0ea10e2064f03ad5a84121a190ed4cf0", "_workspace_event_visible": "8343c060909326a5cb69fa6f671ac62f160630ecf989f04d78e792ea74c0ea90", } @@ -483,7 +489,7 @@ def test_workspace_import_direction_pruning_timing_and_runtime_identities_are_ex main_definitions = _top_level_definitions(main_tree) router_imports = _import_manifest(router_tree) - assert len(router_imports) == 32 + assert len(router_imports) == 38 assert hashlib.sha256(json.dumps(router_imports, separators=(",", ":")).encode()).hexdigest() == ( EXPECTED_IMPORT_MANIFEST_SHA256 ) @@ -512,6 +518,12 @@ def test_workspace_import_direction_pruning_timing_and_runtime_identities_are_ex assert MAIN_PRUNED_BINDINGS.isdisjoint(main_imports) assert MAIN_PRUNED_BINDINGS <= router_import_bindings assert main_imports & router_import_bindings == { + "AgentIdentity", + "AgentKeyAuthenticationError", + "Header", + "_vnext_agent_auth_error_response", + "agent_key_from_authorization", + "resolve_protected_agent_identity", "JSONResponse", "PostgresVNextStore", "Request", @@ -524,9 +536,9 @@ def test_workspace_import_direction_pruning_timing_and_runtime_identities_are_ex main_loads = { node.id for node in ast.walk(main_tree) if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) } - # /v1 agent-key authentication resolves the bound user in main, so no - # shared binding is a re-export-only import any more. - assert (main_imports & router_import_bindings) - main_loads == set() + # Workspace auth now consumes Header; the carrier retains its historical + # Header import while the other shared bindings remain runtime dependencies. + assert (main_imports & router_import_bindings) - main_loads == {"Header"} provider_module_imports = [ node @@ -593,7 +605,7 @@ def test_workspace_routes_preserve_mount_order_origins_and_operation_ids() -> No for method in sorted(getattr(route, "methods", None) or set()) if method in {"GET", "POST", "PUT", "PATCH", "DELETE"} ] - expected_indices = (84, 224, 225) if main_module.LEGACY_SURFACES_ENABLED else (38, 175, 176) + expected_indices = (84, 225, 226) if main_module.LEGACY_SURFACES_ENABLED else (38, 176, 177) assert all(effective_pairs.count((method, path)) == 1 for method, path, _name in EXPECTED_ROUTE_MANIFEST) observed_indices = tuple( effective_pairs.index((method, path)) From 97576fcf5c94191ab4192c2cddda65f1dc7f1a2c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:48:26 +0200 Subject: [PATCH 159/270] Bound large PostgreSQL reverse frontier query cost --- .../src/alicebot_api/vnext_label_writes.py | 12 +++-- .../test_label_reverse_search_postgres.py | 50 ++++++++++++++++++- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 28c54c159..0efd18e34 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -482,7 +482,12 @@ def _postgres_dependants(store: Any, table: str, kind: str, compacts: Sequence[s extra = ", NULL::jsonb AS value" # Normalize each column once per row, rather than once per frontier id. # This remains only a superset lookup; the canonical parser selects exact edges. - pattern = "(?:" + "|".join(re.escape(item) for item in compacts) + ")" + if len(compacts) >= 32 and all(re.fullmatch(r"[0-9a-f]{32}", item) for item in compacts): + # A constant pattern avoids constructing a large automaton for UUID + # frontiers. More candidates are safe because exact edges are parsed below. + pattern = r"[0-9a-f]{32}" + else: + pattern = "(?:" + "|".join(re.escape(item) for item in compacts) + ")" def candidate_clause(column: str) -> str: return ( @@ -538,8 +543,9 @@ def walk_dependants(store: Any, roots: Sequence[str]) -> list[dict[str, object]] while pending: if len(seen) > PROPAGATION_BOUND: raise LabelPropagationTooLarge(f"label propagation stopped after {PROPAGATION_BOUND} rows") - batch = pending[:200] - pending = pending[200:] + batch_size = 200 if _sqlite(store) else 2000 + batch = pending[:batch_size] + pending = pending[batch_size:] belief_aliases = getattr(store, "list_belief_ids_for_memories", None) if callable(belief_aliases): for belief_id in belief_aliases(batch): diff --git a/tests/integration/test_label_reverse_search_postgres.py b/tests/integration/test_label_reverse_search_postgres.py index f68ebd541..ef62b33d3 100644 --- a/tests/integration/test_label_reverse_search_postgres.py +++ b/tests/integration/test_label_reverse_search_postgres.py @@ -80,7 +80,7 @@ def test_multiple_frontier_batches_keep_memory_chains_and_both_direct_loop_colum "sensitivity": "public", } ) - roots = [str(uuid4()) for _ in range(220)] + roots = [str(uuid4()) for _ in range(2200)] conn.execute( """INSERT INTO memories(id,user_id,memory_key,value,source_event_ids,canonical_text, status,domain,sensitivity,metadata_json) @@ -90,7 +90,7 @@ def test_multiple_frontier_batches_keep_memory_chains_and_both_direct_loop_colum (root, roots), ) descendants = [] - for index in (0, 199, 219): + for index in (0, 1999, 2199): row = store.create_memory( { "memory_key": f"descendant.{index}", @@ -126,3 +126,49 @@ def test_multiple_frontier_batches_keep_memory_chains_and_both_direct_loop_colum store, "open_loops", "open_loop", [_compact_id(roots[-1])], with_value=False ) assert str(direct_memory["id"]) in {row["id"] for row in memory_candidates} + + +def test_large_uuid_frontiers_use_a_broader_superset_but_only_canonical_edges_enter_the_closure(migrated_database_urls): + user = uuid4() + roots = [str(uuid4()) for _ in range(32)] + with user_connection(migrated_database_urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"broad-{user}@example.invalid", "Broad") + store = PostgresVNextStore(conn) + expected = set() + for index, reference in enumerate((roots[0], roots[-1].upper(), encoded_object("source_id", roots[1]))): + row = store.create_memory( + { + "memory_key": f"broad.{index}", + "canonical_text": "Synthetic dependency", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"source_id": reference}, + } + ) + expected.add(str(row["id"])) + unrelated = store.create_memory( + { + "memory_key": "unrelated.edge", + "canonical_text": "Synthetic unrelated", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"source_id": str(uuid4())}, + } + ) + note = store.create_memory( + { + "memory_key": "unrelated.note", + "canonical_text": "Synthetic unrelated note", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"note": roots[0]}, + } + ) + candidates = _postgres_dependants( + store, "memories", "memory", [_compact_id(root) for root in roots], with_value=True + ) + candidate_ids = {str(row["id"]) for row in candidates} + assert expected <= candidate_ids + assert str(unrelated["id"]) in candidate_ids + assert str(note["id"]) in candidate_ids + assert {str(row["id"]) for row in walk_dependants(store, roots)} == expected From 12a851dfb9c30d99b64963593d1abd9150595fe9 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:48:14 +0200 Subject: [PATCH 160/270] Prove native PostgreSQL owner edit clamp controls --- .../test_label_floor_ancestry_postgres.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/integration/test_label_floor_ancestry_postgres.py b/tests/integration/test_label_floor_ancestry_postgres.py index 2a16690f1..882999373 100644 --- a/tests/integration/test_label_floor_ancestry_postgres.py +++ b/tests/integration/test_label_floor_ancestry_postgres.py @@ -95,3 +95,38 @@ def test_a_relabel_traverses_the_belief_backing_memory(migrated_database_urls): store = PostgresVNextStore(conn) assert store.get_memory(str(copy["id"]))["sensitivity"] == "regulated" assert store.get_artifact(str(report["id"]))["sensitivity"] == "regulated" + + +def test_postgres_owner_edit_is_clamped_in_response_event_and_storage(migrated_database_urls, monkeypatch): + import json + from uuid import UUID + from alicebot_api.config import Settings + from alicebot_api.routers import vnext_memories as router + from alicebot_api.vnext_label_writes import without_insert_floor + + user_id = uuid4() + url = migrated_database_urls["app"] + with user_connection(url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"owner-clamp-{user_id}@example.test", "Synthetic") + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Synthetic private input", "content_hash": str(user_id), "domain": "health", "sensitivity": "confidential"}) + with without_insert_floor(): + memory = store.create_memory({"memory_key": "stale-copy", "canonical_text": "Synthetic private observation", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"])}}) + assert memory["sensitivity"] == "public" + monkeypatch.setattr(router, "get_settings", lambda: Settings(database_url=url)) + response = router.review_vnext_memory(UUID(str(memory["id"])), router.VNextMemoryReviewRequest(user_id=user_id, action="edit", domain="project", sensitivity="public"), authorization=None) + assert response.status_code == 200 + payload = json.loads(response.body) + assert payload["label_floor_applied"] is True + assert payload["memory"]["domain"] == "health" + assert payload["memory"]["sensitivity"] == "confidential" + with user_connection(url, user_id) as conn: + store = PostgresVNextStore(conn) + stored = store.get_memory(str(memory["id"])) + assert stored["domain"] == "health" and stored["sensitivity"] == "confidential" + assert stored["metadata_json"]["source_id"] == str(source["id"]) + event = conn.execute("SELECT payload_json FROM event_log WHERE event_type='memory.labels_raised' AND target_id=%s", (str(memory["id"]),)).fetchone() + assert event["payload_json"]["cause"] == "floor_clamped" + encoded = json.dumps(event["payload_json"]) + assert "Synthetic private input" not in encoded + assert "Synthetic private observation" not in encoded From bcd042973797e07675a11cccf4c77325a68e854a Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:54:36 +0200 Subject: [PATCH 161/270] Judge loop memory references through their effective labels --- .../vnext_open_loop_references.py | 8 ++++++- tests/unit/test_derived_labels_real_keys.py | 23 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/apps/api/src/alicebot_api/vnext_open_loop_references.py b/apps/api/src/alicebot_api/vnext_open_loop_references.py index 0dfe79d33..ea06a3650 100644 --- a/apps/api/src/alicebot_api/vnext_open_loop_references.py +++ b/apps/api/src/alicebot_api/vnext_open_loop_references.py @@ -139,7 +139,13 @@ def withhold_unreadable_references( source_rows = _rows_by_id(store, sorted(source_ids | metadata_ids), bulk="get_sources_by_ids", single="get_source") memory_rows = _rows_by_id(store, sorted(memory_ids | metadata_ids), bulk="get_memories_by_ids", single="get_memory") admitted_sources = frozenset(key for key, row in source_rows.items() if fence.admits(row)) - admitted_memories = frozenset(key for key, row in memory_rows.items() if fence.admits_memory(row)) + from alicebot_api.vnext_label_guard import LabelGuard + + guard = LabelGuard.for_fence(store, fence) + admitted_memories = frozenset( + key for key, row in memory_rows.items() + if isinstance(effective := guard.effective_row("memory", row), Mapping) and fence.admits_memory(effective) + ) refused = (set(source_rows) - admitted_sources) | (set(memory_rows) - admitted_memories) # Every id of a row the fence refuses, and every id at a reference position that names no admitted row (a refused, # a deleted, a removed or a missing one), is withheld at every position of every row of the response, in every diff --git a/tests/unit/test_derived_labels_real_keys.py b/tests/unit/test_derived_labels_real_keys.py index 88da4a5e1..f0f57b943 100644 --- a/tests/unit/test_derived_labels_real_keys.py +++ b/tests/unit/test_derived_labels_real_keys.py @@ -85,6 +85,29 @@ def test_sqlite_real_key_core_doors(tmp_path, monkeypatch, reader): assert str(row["title"]) not in rendered +@pytest.mark.parametrize("reader", READERS) +def test_original_loop_holds_back_stale_backing_memory_reference(tmp_path, monkeypatch, reader): + user_id = uuid4() + path = tmp_path / "loop-refs.sqlite3" + bootstrap_database(path, user_id=str(user_id), user_email="synthetic@example.invalid") + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.setenv("ALICE_PROJECT_SCOPING", "off") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + with sqlite_user_connection(path, user_id) as conn: + store = SQLiteVNextStore(conn, user_id) + memory = seed_read_rows(store)["verified_confidential"] + loop = store.create_open_loop({"title": "Visible original loop", "memory_id": str(memory["id"]), "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA]}}) + key = real_reader_key(store, user_id, reader) + if key: + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + result = call_mcp_tool(MCPRuntimeContext(database_url=sqlite_url_for_path(path), user_id=user_id), name="alice_open_loops", arguments={"sensitivity_allowed": list(ALL_SENSITIVITY)}) + item = next(row for row in result["items"] if str(row["id"]) == str(loop["id"])) + admitted = expected_read(reader, "verified_confidential") + assert item["memory_id"] == (str(memory["id"]) if admitted else None) + assert (str(memory["id"]) in json.dumps(result, default=str)) is admitted + + def test_full_owner_doctor_keeps_true_counts_and_filtered_view_skips_content(tmp_path): user_id = uuid4() path = tmp_path / "doctor.sqlite3" From 50406b8e0b19da39e23c9fc54c190fe57c14a6f6 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:55:16 +0200 Subject: [PATCH 162/270] Pipeline live capture locks before ordered inserts --- .../src/alicebot_api/vnext_label_writes.py | 5 ++- .../test_capture_label_batch_postgres.py | 43 ++++++++++++++++++- .../test_label_floor_ancestry_postgres.py | 21 ++++++--- 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 0efd18e34..68f905528 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -122,7 +122,10 @@ def capture_label_inputs(store: Any, source_id: str) -> Iterator[None]: batch = _CaptureLabelInputs(conn, transaction_id, int(getattr(conn, "_alice_label_rollback_counter", 0)), identifier(source_id)) token = _CAPTURE_LABEL_INPUTS.set(batch) try: - yield + # Every lock statement remains live and precedes its INSERT on the + # server. Fetching the INSERT result synchronizes the ordered queue. + with conn.pipeline(): + yield finally: _CAPTURE_LABEL_INPUTS.reset(token) diff --git a/tests/integration/test_capture_label_batch_postgres.py b/tests/integration/test_capture_label_batch_postgres.py index e7a6d8549..49f7410aa 100644 --- a/tests/integration/test_capture_label_batch_postgres.py +++ b/tests/integration/test_capture_label_batch_postgres.py @@ -114,7 +114,6 @@ def test_source_raise_invalidates_inputs_in_the_same_transaction(migrated_databa with writes.capture_label_inputs(store, str(source["id"])): assert _memory(store, "before", source)["sensitivity"] == "public" store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) - assert writes._current_capture_inputs(store).rows == {} assert _memory(store, "after", source)["sensitivity"] == "confidential" @@ -157,3 +156,45 @@ def test_one_store_starts_fresh_source_inputs_in_each_transaction(migrated_datab assert writes._current_capture_inputs(store) is None conn.commit() assert keys[0][0] != keys[1][0] + + +def test_pipeline_writer_waits_for_live_lock_and_reads_committed_source(migrated_database_urls): + from concurrent.futures import ThreadPoolExecutor + from threading import Event + from time import monotonic, sleep + + user = uuid4() + url = migrated_database_urls["app"] + with user_connection(url, user) as conn: + _user(conn, user) + source = _source(PostgresVNextStore(conn), user) + queued = Event() + def writer(): + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + real_lock = store.lock_label_writes + def lock(*, exclusive=False): + real_lock(exclusive=exclusive) + queued.set() + store.lock_label_writes = lock + with writes.capture_label_inputs(store, str(source["id"])): + return _memory(store, "waited", source) + with ThreadPoolExecutor(max_workers=1) as pool: + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) + future = pool.submit(writer) + assert queued.wait(3) + deadline = monotonic() + 3 + while monotonic() < deadline: + waiting = conn.execute("SELECT count(*) AS n FROM pg_locks WHERE locktype='advisory' AND NOT granted AND classid=(hashtext('vnext_labels')::bigint & 4294967295)::oid AND objid=(hashtext(app.current_user_id()::text)::bigint & 4294967295)::oid").fetchone()["n"] + if waiting: + break + sleep(0.01) + assert waiting == 1 + assert future.done() is False + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 0 + result = future.result(timeout=5) + assert result["sensitivity"] == "confidential" diff --git a/tests/integration/test_label_floor_ancestry_postgres.py b/tests/integration/test_label_floor_ancestry_postgres.py index 882999373..ce42fae48 100644 --- a/tests/integration/test_label_floor_ancestry_postgres.py +++ b/tests/integration/test_label_floor_ancestry_postgres.py @@ -117,16 +117,27 @@ def test_postgres_owner_edit_is_clamped_in_response_event_and_storage(migrated_d response = router.review_vnext_memory(UUID(str(memory["id"])), router.VNextMemoryReviewRequest(user_id=user_id, action="edit", domain="project", sensitivity="public"), authorization=None) assert response.status_code == 200 payload = json.loads(response.body) - assert payload["label_floor_applied"] is True - assert payload["memory"]["domain"] == "health" - assert payload["memory"]["sensitivity"] == "confidential" with user_connection(url, user_id) as conn: store = PostgresVNextStore(conn) stored = store.get_memory(str(memory["id"])) - assert stored["domain"] == "health" and stored["sensitivity"] == "confidential" assert stored["metadata_json"]["source_id"] == str(source["id"]) event = conn.execute("SELECT payload_json FROM event_log WHERE event_type='memory.labels_raised' AND target_id=%s", (str(memory["id"]),)).fetchone() - assert event["payload_json"]["cause"] == "floor_clamped" + observed = { + "response_flag": payload.get("label_floor_applied", False), + "response_domain": payload["memory"]["domain"], + "response_sensitivity": payload["memory"]["sensitivity"], + "stored_domain": stored["domain"], + "stored_sensitivity": stored["sensitivity"], + "event_cause": event["payload_json"].get("cause") if event else None, + } + assert observed == { + "response_flag": True, + "response_domain": "health", + "response_sensitivity": "confidential", + "stored_domain": "health", + "stored_sensitivity": "confidential", + "event_cause": "floor_clamped", + } encoded = json.dumps(event["payload_json"]) assert "Synthetic private input" not in encoded assert "Synthetic private observation" not in encoded From 98f2b6f510c48c6004efa0409cc6b3322ca46b63 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:53:05 +0200 Subject: [PATCH 163/270] Measure committed derived-label work in disposable PostgreSQL fixtures --- scripts/measure_derived_label_budgets.py | 211 +++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 scripts/measure_derived_label_budgets.py diff --git a/scripts/measure_derived_label_budgets.py b/scripts/measure_derived_label_budgets.py new file mode 100644 index 000000000..02ab97704 --- /dev/null +++ b/scripts/measure_derived_label_budgets.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Measure actual capture and relabel work in fresh disposable PostgreSQL databases.""" + +from __future__ import annotations + +import argparse +import cProfile +import json +import os +from pathlib import Path +import pstats +import statistics +import time +from uuid import uuid4 + +from alembic import command + +from alicebot_api.db import user_connection +from alicebot_api.migrations import make_alembic_config +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_capture import SourceCaptureInput, VNextCaptureService +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import _create_role_separated_database, _drop_database + + +def capture(url, repetitions): + samples = [] + for repeat in range(repetitions + 1): + user_id = uuid4() + with user_connection(url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"capture-{user_id}@example.invalid", "Capture") + store = PostgresVNextStore(conn) + text = "\n".join( + f"Decision: Synthetic item {index} is assigned to synthetic route {index}." for index in range(200) + ) + start = time.perf_counter() + profile = cProfile.Profile() if os.getenv("ALICE_BUDGET_PROFILE") else None + if profile: + profile.enable() + result = VNextCaptureService(store).capture_source( + SourceCaptureInput( + source_type="manual_text", + title="Synthetic capture", + raw_text=text, + domain="project", + sensitivity="public", + ) + ) + elapsed = time.perf_counter() - start + if profile: + profile.disable() + profile.dump_stats(os.environ["ALICE_BUDGET_PROFILE"]) + pstats.Stats(profile).sort_stats("cumulative").print_stats(45) + assert result.candidate_memory_count == 200, result.to_record() + with user_connection(url, user_id) as conn: + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 200 + if repeat: + samples.append(elapsed) + return { + "facts": 200, + "warmup_runs": 1, + "repetitions": repetitions, + "seconds": samples, + "median_seconds": statistics.median(samples), + "timing_includes_commit": True, + "fixture_validation_timed": False, + } + + +def relabel(url, repetitions, dependants, *, state=None, prepare=False): + from alicebot_api.vnext_derived_labels import with_derived_from + + user_id = uuid4() if state is None else state["user_id"] + if state is None: + with user_connection(url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"relabel-{user_id}@example.invalid", "Relabel") + store = PostgresVNextStore(conn) + source = store.create_source( + { + "source_type": "manual_text", + "title": "Synthetic input", + "content_hash": str(uuid4()), + "domain": "project", + "sensitivity": "public", + } + ) + source_id = str(source["id"]) + conn.execute( + """INSERT INTO memories(id,user_id,memory_key,value,title,canonical_text,status,domain,sensitivity,source_event_ids,metadata_json) + SELECT gen_random_uuid(),app.current_user_id(),'scale.'||n,'{}'::jsonb,'Synthetic row '||n,'Synthetic scale row '||n, + 'active','project','public','[]'::jsonb,CASE WHEN n<=%s THEN jsonb_build_object('source_id',%s::text) ELSE '{}'::jsonb END + FROM generate_series(1,50000) n""", + (dependants, source_id), + ) + record = with_derived_from({"workflow": "daily_brief"}, {"sources": [source]}) + conn.execute( + """INSERT INTO generated_artifacts(id,user_id,artifact_type,title,content_markdown,status,domain,sensitivity,generated_by,metadata_json) + SELECT gen_random_uuid(),app.current_user_id(),'daily_brief','Synthetic artifact '||n,'Synthetic scale artifact '||n, + 'needs_review','project','public','system',CASE WHEN n<=%s THEN %s::jsonb ELSE %s::jsonb END + FROM generate_series(1,2000) n""", + (dependants, json.dumps(record), json.dumps(with_derived_from({"workflow": "daily_brief"}, {}))), + ) + else: + source_id = state["source_id"] + if prepare: + return {"user_id": str(user_id), "source_id": source_id, "dependants": dependants} + samples = [] + # Commit is timed. Restore only this synthetic fixture outside each timed operation. + for repeat in range(repetitions + 1): + with user_connection(url, user_id) as conn: + store = PostgresVNextStore(conn) + before_events = conn.execute( + "SELECT count(*) AS n FROM event_log WHERE event_type LIKE '%%.labels_raised'" + ).fetchone()["n"] + start = time.perf_counter() + profile = cProfile.Profile() if os.getenv("ALICE_BUDGET_PROFILE") else None + if profile: + profile.enable() + store.update_source(source_id=source_id, patch={"sensitivity": "confidential"}) + elapsed = time.perf_counter() - start + if profile: + profile.disable() + profile.dump_stats(os.environ["ALICE_BUDGET_PROFILE"]) + pstats.Stats(profile).sort_stats("cumulative").print_stats(45) + with user_connection(url, user_id) as conn: + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 50000 + assert conn.execute("SELECT count(*) AS n FROM generated_artifacts").fetchone()["n"] == 2000 + assert ( + conn.execute("SELECT count(*) AS n FROM memories WHERE sensitivity='confidential'").fetchone()["n"] + == dependants + ) + assert ( + conn.execute( + "SELECT count(*) AS n FROM generated_artifacts WHERE sensitivity='confidential'" + ).fetchone()["n"] + == dependants + ) + assert ( + conn.execute("SELECT sensitivity FROM sources WHERE id=%s", (source_id,)).fetchone()["sensitivity"] + == "confidential" + ) + assert ( + conn.execute("SELECT count(*) AS n FROM event_log WHERE event_type LIKE '%%.labels_raised'").fetchone()[ + "n" + ] + - before_events + == 2 * dependants + ) + conn.execute("UPDATE sources SET sensitivity='public' WHERE id=%s", (source_id,)) + conn.execute("UPDATE memories SET sensitivity='public' WHERE sensitivity='confidential'") + conn.execute("UPDATE generated_artifacts SET sensitivity='public' WHERE sensitivity='confidential'") + if repeat: + samples.append(elapsed) + return { + "sources": 1, + "memories": 50000, + "artifacts": 2000, + "dependent_memories": dependants, + "dependent_artifacts": dependants, + "warmup_runs": 1, + "repetitions": repetitions, + "seconds": samples, + "median_seconds": statistics.median(samples), + "budget_seconds": 3, + "label_events_per_run": 2 * dependants, + "timing_includes_commit": True, + "fixture_validation_and_reset_timed": False, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=("capture", "relabel"), required=True) + parser.add_argument("--repetitions", type=int, default=5) + parser.add_argument("--dependants", type=int, default=100) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--prepare", type=Path) + parser.add_argument("--prepared", type=Path) + args = parser.parse_args() + prepared = json.loads(args.prepared.read_text()) if args.prepared else None + name = prepared["database_name"] if prepared else "alice_label_perf_" + uuid4().hex[:12] + urls = prepared["urls"] if prepared else _create_role_separated_database(name) + retained = False + try: + if prepared is None: + command.upgrade(make_alembic_config(urls["admin"]), "head") + if args.prepare: + state = relabel(urls["app"], 0, args.dependants, prepare=True) if args.mode == "relabel" else None + args.prepare.write_text(json.dumps({"database_name": name, "urls": urls, "state": state}, indent=2) + "\n") + retained = True + print("Synthetic scale fixture prepared") + return + record = ( + capture(urls["app"], args.repetitions) + if args.mode == "capture" + else relabel( + urls["app"], + args.repetitions, + prepared["state"]["dependants"] if prepared else args.dependants, + state=prepared["state"] if prepared else None, + ) + ) + args.output.write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps(record)) + finally: + if not retained: + _drop_database(name) + + +if __name__ == "__main__": + main() From e5bd6fc4b08c5c573f511aec78dc92754149a737 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:58:09 +0200 Subject: [PATCH 164/270] Align write-stage unit fixtures and route inventories --- tests/unit/test_legacy_gated_router_split.py | 14 +++-- tests/unit/test_main.py | 6 +- tests/unit/test_mcp.py | 47 +++++++++++++++ tests/unit/test_providers_router_split.py | 5 +- tests/unit/test_saved_provenance_reader.py | 1 + tests/unit/test_source_refs_read_fence.py | 3 + tests/unit/test_surface_gates.py | 6 +- tests/unit/test_vnext_main.py | 34 ++++++++++- tests/unit/test_vnext_store.py | 63 +++++++++++++------- tests/unit/test_workspaces_router_split.py | 6 +- 10 files changed, 146 insertions(+), 39 deletions(-) diff --git a/tests/unit/test_legacy_gated_router_split.py b/tests/unit/test_legacy_gated_router_split.py index a19305c3d..4def45c37 100644 --- a/tests/unit/test_legacy_gated_router_split.py +++ b/tests/unit/test_legacy_gated_router_split.py @@ -240,8 +240,10 @@ EXPECTED_OWNED_SUPPORT_AST_SHA256 = "8bbbcb0cf52c4bd1da5fce4e50878d8a62599be59ee25e09dcd905c94950d406" EXPECTED_FULL_SUPPORT_AST_SHA256 = "07acb5deafb700e040c459969610e8877e86e62b04d4e9d86493fe314cb02868" EXPECTED_GATED_OPERATION_SHA256 = "55490460fd78990a6872ebf22c6cfd927f7d0a5548b6df90adde8261ede8da65" -EXPECTED_DEFAULT_DEEP_ROUTE_SHA256 = "a865b2264c911ed1b055853777b850e070bd594d79fafb167df7b915b7c3c9ce" -EXPECTED_LEGACY_DEEP_ROUTE_SHA256 = "6743a8b03d649328341eab462aeea0271f997f14c5c558a666b5aff92f297d79" + +# Source regeneration adds one default route before source review. +EXPECTED_DEFAULT_DEEP_ROUTE_SHA256 = "9d9e83c35818387a13c719d455c88f4dcb4d139bf5f4dd5888bc6ff3723077a7" +EXPECTED_LEGACY_DEEP_ROUTE_SHA256 = "5ba780b5ec62584b844cf1fcab8fc35b703f4494c1486db870588ab68a1f57a6" EXPECTED_INTEGRATION_PATCH_COUNTS = { "tests/integration/test_approval_api.py": 8, @@ -667,9 +669,9 @@ def test_main_preserves_frozen_flag_policy_and_five_mount_seams() -> None: def test_flagged_surface_preserves_deep_order_ids_and_import_timing() -> None: default = _isolated_surface_manifest(None) assert default == { - "operation_count": 183, + "operation_count": 184, "legacy_count": 0, - "deep_count": 187, + "deep_count": 188, "deep_digest": EXPECTED_DEFAULT_DEEP_ROUTE_SHA256, "gated_count": 0, "gated_digest": hashlib.sha256(b"[]").hexdigest(), @@ -679,9 +681,9 @@ def test_flagged_surface_preserves_deep_order_ids_and_import_timing() -> None: } legacy = _isolated_surface_manifest("1") assert legacy == { - "operation_count": 232, + "operation_count": 233, "legacy_count": 49, - "deep_count": 236, + "deep_count": 237, "deep_digest": EXPECTED_LEGACY_DEEP_ROUTE_SHA256, "gated_count": 49, "gated_digest": EXPECTED_GATED_OPERATION_SHA256, diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 43c657341..563513a30 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -312,7 +312,7 @@ def test_openapi_has_concrete_success_contracts_and_accurate_statuses() -> None: } operations = list(operations_by_key.values()) - assert len(operations) == 183 + assert len(operations) == 184 assert all(operation.get("tags") for operation in operations) assert all(operation.get("description") for operation in operations) assert all("default" in operation["responses"] for operation in operations) @@ -323,7 +323,7 @@ def test_openapi_has_concrete_success_contracts_and_accurate_statuses() -> None: if status.startswith("2") for json_body in [response.get("content", {}).get("application/json", {})] ] - assert len(success_schemas) == 187 + assert len(success_schemas) == 188 assert "APIJsonDocument" not in components assert all(document.get("$ref", "").startswith("#/components/schemas/") for document in success_schemas) resolved_success_schemas = [components[document["$ref"].rsplit("/", 1)[-1]] for document in success_schemas] @@ -347,7 +347,7 @@ def test_openapi_has_concrete_success_contracts_and_accurate_statuses() -> None: assert exact_keys | set(operation_registry) == set(operations_by_key), coverage_report assert exact_keys.isdisjoint(operation_registry), coverage_report assert len(exact_keys) == 42 - assert len(operation_registry) == 141 + assert len(operation_registry) == 142 assert 0 < len(polymorphic_operations) <= 3 assert set(polymorphic_operations) <= set(operation_registry) assert all(reason.strip() for reason in polymorphic_operations.values()) diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index 8561365d0..e9dd310e4 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -1277,8 +1277,48 @@ def _ascii_query_fold(value: str) -> str: return value.translate(_ASCII_QUERY_CASE_TRANSLATION) +class FakeLabelCursor: + """SQL guard responses from the fake store's current lock state.""" + + def __init__(self, store) -> None: + self.store = store + self.query = "" + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def execute(self, query, params=None): + assert any(marker in query for marker in ( + "FROM pg_locks", "current_setting('lock_timeout')", "SET LOCAL lock_timeout", "set_config('lock_timeout'", + )), query + self.query = query + + def fetchone(self): + if "FROM pg_locks" in self.query: + return {"graph": self.store.graph_locked, "labels": self.store.labels_locked, + "exclusive": self.store.labels_exclusive} + if "current_setting('lock_timeout')" in self.query: + return {"lock_timeout": "0"} + raise AssertionError(self.query) + + +class FakeLabelConnection: + def __init__(self, store) -> None: + self.store = store + + def cursor(self): + return FakeLabelCursor(self.store) + + class FakeVNextMCPStore: def __init__(self) -> None: + self.graph_locked = False + self.labels_locked = False + self.labels_exclusive = False + self.conn = FakeLabelConnection(self) self.events: list[dict[str, object]] = [] self.sources: list[dict[str, object]] = [] self.chunks: list[dict[str, object]] = [] @@ -1317,6 +1357,13 @@ def __init__(self) -> None: } } + def lock_graph_mutation(self) -> None: + self.graph_locked = True + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + self.labels_locked = True + self.labels_exclusive |= exclusive + @staticmethod def _is_live(row: dict[str, object]) -> bool: return row.get("deleted_at") is None diff --git a/tests/unit/test_providers_router_split.py b/tests/unit/test_providers_router_split.py index a8b88d978..04a176b8d 100644 --- a/tests/unit/test_providers_router_split.py +++ b/tests/unit/test_providers_router_split.py @@ -117,7 +117,10 @@ # lone_surrogates.py and main.py only registers it, so it adds no definition # here. Earlier re-pin (2026-09-26): _rewrite_user_id_json_body writes the # rewritten JSON into request._body before call_next. -EXPECTED_CARRIER_AST_SHA256 = "ab3fc6d61cb81a1b9c1a6573adc8e1e297cbbcf01e230effd4a0824dee2d8e2b" + +# Re-pin 2026-10-06: source regeneration is a new protected write route in the +# central vNext route policy; the app carrier keeps the same definitions. +EXPECTED_CARRIER_AST_SHA256 = "fcd6d722e2d6c3be28b138449555e703f6930168f52b8e94b41dacbcb937b605" EXPECTED_ROUTE_NAME_MANIFEST_SHA256 = "1a438538e16120361f92d30375cc94679d598fe4b78ba5a58a7d8a4dda6af83c" EXPECTED_OPERATION_MANIFEST_SHA256 = "8b79ceaf996b8c51b5bb2f3f38a8c19a4e33796955d8b8f7a66e7ac01ea1732d" EXPECTED_IMPORT_MANIFEST_SHA256 = "17484ccdd460e42e2ad5c82a8ca867664694feaf871118c410a134996a532358" diff --git a/tests/unit/test_saved_provenance_reader.py b/tests/unit/test_saved_provenance_reader.py index 446ca83b1..5fb12c853 100644 --- a/tests/unit/test_saved_provenance_reader.py +++ b/tests/unit/test_saved_provenance_reader.py @@ -730,6 +730,7 @@ def test_review_by_id_reads_saved_provenance_only_through_the_reader() -> None: ("apps/api/src/alicebot_api/mcp/memories.py", "_handle_alice_vnext_commit_memory"): "passes the argument to commit", ("apps/api/src/alicebot_api/cli/memories.py", "_run_vnext_memory_commit"): "passes the argument to commit", ("apps/api/src/alicebot_api/vnext_capture.py", "_capture_source"): "link quote = the candidate's own text", + ("apps/api/src/alicebot_api/vnext_source_regeneration.py", "regenerate_source_inputs"): "fresh candidate quote; SavedProvenanceReader and row readers apply the source fence", ("apps/api/src/alicebot_api/vnext_connectors.py", "ingest_agent_output"): "link quote = the item title", ("apps/api/src/alicebot_api/vnext_retrieval.py", "_supporting_evidence"): "reads the link, fenced by admits_link", ("apps/api/src/alicebot_api/onramp.py", "_apply_import_quarantine"): "replaces a quote with a placeholder", diff --git a/tests/unit/test_source_refs_read_fence.py b/tests/unit/test_source_refs_read_fence.py index 7ffe35cdc..11815f8d2 100644 --- a/tests/unit/test_source_refs_read_fence.py +++ b/tests/unit/test_source_refs_read_fence.py @@ -1509,6 +1509,8 @@ def _provenance_link_call_sites() -> set[tuple[str, str]]: # The source was captured or ingested by this very call, so the caller named no id. ("vnext_capture.py", "_capture_source"), ("vnext_connectors.py", "ingest_agent_output"), + # Regeneration reads the authorized source and its chunks before making new rows. + ("vnext_source_regeneration.py", "regenerate_source_inputs"), } _NAMED_SOURCE_SITES = { # The caller named the id. Each is behind ``resolve_attachable_sources`` and the caller's ``SourceReadFence``. @@ -1702,6 +1704,7 @@ def test_attachable_sources_are_built_only_by_the_resolver() -> None: ("memory.py", "_create_open_loop_for_memory"), ("vnext_brain.py", "_create_candidate_open_loops"), ("vnext_projects.py", "extract_open_loops"), + ("vnext_source_regeneration.py", "regenerate_source_inputs"), ("vnext_scheduler.py", "_publish_mutation"), ("vnext_stores/postgres/graph_open_loops.py", "upsert_open_loop_by_automation_digest"), ("vnext_stores/sqlite/graph_open_loops.py", "upsert_open_loop_by_automation_digest"), diff --git a/tests/unit/test_surface_gates.py b/tests/unit/test_surface_gates.py index e23a12886..92eab0b86 100644 --- a/tests/unit/test_surface_gates.py +++ b/tests/unit/test_surface_gates.py @@ -94,7 +94,7 @@ def _isolated_proxy_execution_posture(flag_value: str | None) -> dict[str, objec @pytest.mark.parametrize("flag_value", [None, "", "0", "true", "yes", "on", "01", " 1"]) def test_http_legacy_surface_gate_fails_closed_for_every_non_exact_value(flag_value: str | None) -> None: assert _isolated_http_inventory(flag_value) == { - "count": 183, + "count": 184, "legacy_count": 0, "removed_count": 0, "runtime_invoke_count": 1, @@ -103,7 +103,7 @@ def test_http_legacy_surface_gate_fails_closed_for_every_non_exact_value(flag_va def test_http_legacy_surface_gate_mounts_exact_inventory_only_for_one() -> None: assert _isolated_http_inventory("1") == { - "count": 232, + "count": 233, "legacy_count": 49, "removed_count": 0, "runtime_invoke_count": 1, @@ -161,7 +161,7 @@ def inventory(): text=True, ) - expected_inventory = {"count": 183, "legacy_count": 0} + expected_inventory = {"count": 184, "legacy_count": 0} assert json.loads(completed.stdout) == { "before": expected_inventory, "after": expected_inventory, diff --git a/tests/unit/test_vnext_main.py b/tests/unit/test_vnext_main.py index 8af8173c1..90335d3dd 100644 --- a/tests/unit/test_vnext_main.py +++ b/tests/unit/test_vnext_main.py @@ -54,6 +54,29 @@ def __init__(self, _conn) -> None: self.agent_api_keys: list[dict[str, object]] = [] self.browser_clip_capabilities: dict[str, dict[str, object]] = {} self.revisions: list[dict[str, object]] = [] + self.graph_locked = False + self.labels_locked = False + self.labels_exclusive = False + + def lock_graph_mutation(self) -> None: + self.graph_locked = True + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + self.labels_locked = True + self.labels_exclusive |= exclusive + + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + collection = {"source": self.sources.values(), "memory": self.memories, "open_loop": self.open_loops, + "artifact": self.artifacts.values(), "belief": self.beliefs.values(), "project": self.projects.values()}.get(kind, []) + return [dict(row) for row in collection if str(row.get("id")) in ids] + + def _fetch_all(self, query: str, _params: tuple[object, ...]) -> list[dict[str, object]]: + # The label dependant walker performs its exact canonical reference filter after this prefilter. + for table, kind in (("memories", "memory"), ("open_loops", "open_loop"), ("generated_artifacts", "artifact"), ("projects", "project")): + if f"FROM {table}" in query: + collection = {"memory": self.memories, "open_loop": self.open_loops, "artifact": self.artifacts.values(), "project": self.projects.values()}[kind] + return [{**row, "kind": kind} for row in collection] + raise AssertionError(query) def create_browser_clip_capability( self, @@ -879,6 +902,12 @@ def list_connector_states(self) -> list[dict[str, object]]: def _install_fake_vnext_store(monkeypatch, store: FakeVNextStore) -> None: + from alicebot_api import vnext_label_writes + monkeypatch.setattr(vnext_label_writes, "acquire_exclusive_label_lock", lambda target: target.lock_label_writes(exclusive=True)) + monkeypatch.setattr( + vnext_label_writes, "held_label_locks", + lambda target: (target.graph_locked, target.labels_locked, target.labels_exclusive), + ) @contextmanager def fake_user_connection(database_url, current_user_id): assert database_url == "postgresql://db" @@ -1198,7 +1227,7 @@ def test_vnext_route_inventory_fails_closed_without_route_local_policy() -> None } assert not (main_module._VNEXT_ROUTE_LOCAL_POLICY & main_module._VNEXT_CENTRAL_OPERATOR_ROUTES) assert (main_module._VNEXT_ROUTE_LOCAL_POLICY | main_module._VNEXT_CENTRAL_OPERATOR_ROUTES) == registered - assert len(registered) == 71 + assert len(registered) == 72 project_bound = main_module.AgentIdentity( agent_id="project-reader", @@ -1347,6 +1376,7 @@ def test_vnext_memories_router_partitions_preserve_global_route_sequence() -> No vnext_memories_router.source_review_router, [ ("GET", "/v0/vnext/sources/{source_id}"), + ("POST", "/v0/vnext/sources/{source_id}/regenerate"), ("POST", "/v0/vnext/sources/{source_id}/review"), ], ), @@ -1912,7 +1942,7 @@ def test_create_vnext_source_threads_project_scope_into_captured_memory(monkeypa candidates = store.list_memories(status="candidate") assert candidates, "capture must promote at least one candidate memory" - assert memory_project_scope(candidates[0]) == ("Project-Helios", "project-helios") + assert memory_project_scope(candidates[0]) == ("Project-Helios",) for memory in candidates: store.update_memory(memory_id=str(memory["id"]), patch={"status": "active"}, actor_type="system") diff --git a/tests/unit/test_vnext_store.py b/tests/unit/test_vnext_store.py index de7109f4d..ec0c424b9 100644 --- a/tests/unit/test_vnext_store.py +++ b/tests/unit/test_vnext_store.py @@ -27,6 +27,10 @@ def __init__( self.executed: list[tuple[str, tuple[object, ...] | None]] = [] self.fetchone_results = list(fetchone_results) self.fetchall_result = fetchall_result or [] + self.current_query = "" + self.graph_locked = False + self.labels_locked = False + self.labels_exclusive = False def __enter__(self) -> "RecordingCursor": return self @@ -38,13 +42,28 @@ def execute(self, query: str, params: tuple[object, ...] | None = None) -> None: if params is not None: assert query.count("%s") == len(params) self.executed.append((query, params)) + self.current_query = query + if "pg_advisory_xact_lock" in query: + if "vnext_supersession" in query: + self.graph_locked = True + elif "vnext_labels" in query: + self.labels_locked = True + self.labels_exclusive |= "pg_advisory_xact_lock_shared" not in query @property def statements(self) -> list[tuple[str, tuple[object, ...] | None]]: """Business SQL, with transaction guards retained separately in executed.""" - return [(query, params) for query, params in self.executed if "pg_advisory_xact_lock" not in query] + return [(query, params) for query, params in self.executed if not any(marker in query for marker in ( + "pg_advisory_xact_lock", "FROM pg_locks", "current_setting('lock_timeout')", + "SET LOCAL lock_timeout", "set_config('lock_timeout'", + ))] def fetchone(self) -> dict[str, Any] | None: + if "current_setting('lock_timeout')" in self.current_query: + return {"lock_timeout": "0"} + if "FROM pg_locks" in self.current_query: + return {"graph": self.graph_locked, "labels": self.labels_locked, + "exclusive": self.labels_exclusive} if not self.fetchone_results: return None return self.fetchone_results.pop(0) @@ -96,15 +115,12 @@ def test_source_crud_and_chunks_write_audit_events() -> None: {"id": source_id}, _event_row(source_id), {"id": source_id}, - { - "id": source_id, - "content_hash": "sha256:abc", - "dedupe_key": "capture-md5:legacy", - "domain": "project", - "sensitivity": "private", - "metadata_json": {"path": "docs/spec.md"}, - }, - {"id": source_id}, + # The unlocked label pre-read precedes the source row lock. + {"id": source_id, "content_hash": "sha256:abc", "dedupe_key": "capture-md5:legacy", + "domain": "project", "sensitivity": "private", "metadata_json": {"path": "docs/spec.md"}}, + {"id": source_id, "content_hash": "sha256:abc", "dedupe_key": "capture-md5:legacy", + "domain": "project", "sensitivity": "private", "metadata_json": {"path": "docs/spec.md"}}, + {"id": source_id, "domain": "project", "sensitivity": "private", "metadata_json": {"rev": 2}}, _event_row(source_id), {"id": source_id}, _event_row(source_id), @@ -293,7 +309,8 @@ def test_update_source_recomputes_postgres_dedupe_key_with_the_same_statement() } cursor = RecordingCursor( fetchone_results=[ - current, + current, # label pre-read + current, # locked capture identity None, {**current, "domain": "professional"}, _event_row(source_id), @@ -339,7 +356,7 @@ def test_update_source_postgres_collision_fails_before_mutation_event() -> None: "sensitivity": "private", "metadata_json": {"raw_text": raw_text, "project_scope": ["Alpha"]}, } - cursor = RecordingCursor(fetchone_results=[current, {"id": str(uuid4())}]) + cursor = RecordingCursor(fetchone_results=[current, current, {"id": str(uuid4())}]) store = PostgresVNextStore(RecordingConnection(cursor)) with pytest.raises(ContinuityStoreInvariantError, match="already belongs"): @@ -364,7 +381,8 @@ def test_update_source_postgres_releases_key_when_changed_identity_has_no_raw_te } cursor = RecordingCursor( fetchone_results=[ - current, + current, # label pre-read + current, # locked capture identity {**current, "dedupe_key": None, "metadata_json": {"project_scope": ["Beta"]}}, _event_row(source_id), ] @@ -721,7 +739,6 @@ def test_list_artifacts_applies_type_domain_sensitivity_and_limit_filters() -> N ["public", "private"], None, None, - None, 5, ) @@ -1409,6 +1426,7 @@ def test_project_people_belief_and_open_loop_methods_write_audit_events() -> Non {"id": project_id}, _event_row(project_id), {"id": project_id}, + {"id": project_id}, # label pre-read before the update {"id": project_id}, _event_row(project_id), {"id": person_id}, @@ -1426,6 +1444,7 @@ def test_project_people_belief_and_open_loop_methods_write_audit_events() -> Non {"id": loop_id}, {"id": loop_id}, _event_row(loop_id), + {"id": loop_id}, # label pre-read before the final update {"id": loop_id}, _event_row(loop_id), ] @@ -1464,14 +1483,14 @@ def test_project_people_belief_and_open_loop_methods_write_audit_events() -> Non None, 3, ) - assert "UPDATE projects" in cursor.statements[4][0] - assert "INSERT INTO people" in cursor.statements[6][0] - assert "UPDATE people" in cursor.statements[9][0] - assert "INSERT INTO beliefs" in cursor.statements[11][0] - assert "UPDATE beliefs" in cursor.statements[14][0] - assert "INSERT INTO open_loops" in cursor.statements[16][0] - assert "UPDATE open_loops" in cursor.statements[19][0] - assert "UPDATE open_loops" in cursor.statements[21][0] + assert "UPDATE projects" in cursor.statements[5][0] + assert "INSERT INTO people" in cursor.statements[7][0] + assert "UPDATE people" in cursor.statements[10][0] + assert "INSERT INTO beliefs" in cursor.statements[12][0] + assert "UPDATE beliefs" in cursor.statements[15][0] + assert "INSERT INTO open_loops" in cursor.statements[17][0] + assert "UPDATE open_loops" in cursor.statements[20][0] + assert "UPDATE open_loops" in cursor.statements[23][0] def test_get_artifact_for_update_locks_the_persisted_authorization_target() -> None: diff --git a/tests/unit/test_workspaces_router_split.py b/tests/unit/test_workspaces_router_split.py index 9f0aa630c..9718ba3fa 100644 --- a/tests/unit/test_workspaces_router_split.py +++ b/tests/unit/test_workspaces_router_split.py @@ -122,7 +122,9 @@ # lone_surrogates.py and main.py only registers it, so it adds no definition # here. Earlier re-pin (2026-09-26): _rewrite_user_id_json_body writes the # rewritten JSON into request._body before call_next. -EXPECTED_CARRIER_AST_SHA256 = "ab3fc6d61cb81a1b9c1a6573adc8e1e297cbbcf01e230effd4a0824dee2d8e2b" + +# Source regeneration adds one protected write route without new carrier definitions. +EXPECTED_CARRIER_AST_SHA256 = "fcd6d722e2d6c3be28b138449555e703f6930168f52b8e94b41dacbcb937b605" EXPECTED_ROUTE_NODE_SHA256 = { "get_vnext_workspace": "6c2151bf38b1b1311f016c00d14394afc7077a6ea219f7ce3dcfd9b701474ae7", "bootstrap_v1_workspace": "07b1fe2a4cd03a5ba69abe76e258a457e85e92b0bfba592520ee02d01d759c4b", @@ -587,7 +589,7 @@ def test_workspace_routes_preserve_mount_order_origins_and_operation_ids() -> No for method in sorted(getattr(route, "methods", None) or set()) if method in {"GET", "POST", "PUT", "PATCH", "DELETE"} ] - expected_indices = (84, 224, 225) if main_module.LEGACY_SURFACES_ENABLED else (38, 175, 176) + expected_indices = (84, 225, 226) if main_module.LEGACY_SURFACES_ENABLED else (38, 176, 177) assert all(effective_pairs.count((method, path)) == 1 for method, path, _name in EXPECTED_ROUTE_MANIFEST) observed_indices = tuple( effective_pairs.index((method, path)) From 429ea61816ca477ca87bfe8d11c66f0b87941359 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:55:16 +0200 Subject: [PATCH 165/270] Pipeline live capture locks before ordered inserts --- .../src/alicebot_api/vnext_label_writes.py | 5 ++- .../test_capture_label_batch_postgres.py | 43 ++++++++++++++++++- .../test_label_floor_ancestry_postgres.py | 21 ++++++--- 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 0efd18e34..68f905528 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -122,7 +122,10 @@ def capture_label_inputs(store: Any, source_id: str) -> Iterator[None]: batch = _CaptureLabelInputs(conn, transaction_id, int(getattr(conn, "_alice_label_rollback_counter", 0)), identifier(source_id)) token = _CAPTURE_LABEL_INPUTS.set(batch) try: - yield + # Every lock statement remains live and precedes its INSERT on the + # server. Fetching the INSERT result synchronizes the ordered queue. + with conn.pipeline(): + yield finally: _CAPTURE_LABEL_INPUTS.reset(token) diff --git a/tests/integration/test_capture_label_batch_postgres.py b/tests/integration/test_capture_label_batch_postgres.py index e7a6d8549..49f7410aa 100644 --- a/tests/integration/test_capture_label_batch_postgres.py +++ b/tests/integration/test_capture_label_batch_postgres.py @@ -114,7 +114,6 @@ def test_source_raise_invalidates_inputs_in_the_same_transaction(migrated_databa with writes.capture_label_inputs(store, str(source["id"])): assert _memory(store, "before", source)["sensitivity"] == "public" store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) - assert writes._current_capture_inputs(store).rows == {} assert _memory(store, "after", source)["sensitivity"] == "confidential" @@ -157,3 +156,45 @@ def test_one_store_starts_fresh_source_inputs_in_each_transaction(migrated_datab assert writes._current_capture_inputs(store) is None conn.commit() assert keys[0][0] != keys[1][0] + + +def test_pipeline_writer_waits_for_live_lock_and_reads_committed_source(migrated_database_urls): + from concurrent.futures import ThreadPoolExecutor + from threading import Event + from time import monotonic, sleep + + user = uuid4() + url = migrated_database_urls["app"] + with user_connection(url, user) as conn: + _user(conn, user) + source = _source(PostgresVNextStore(conn), user) + queued = Event() + def writer(): + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + real_lock = store.lock_label_writes + def lock(*, exclusive=False): + real_lock(exclusive=exclusive) + queued.set() + store.lock_label_writes = lock + with writes.capture_label_inputs(store, str(source["id"])): + return _memory(store, "waited", source) + with ThreadPoolExecutor(max_workers=1) as pool: + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) + future = pool.submit(writer) + assert queued.wait(3) + deadline = monotonic() + 3 + while monotonic() < deadline: + waiting = conn.execute("SELECT count(*) AS n FROM pg_locks WHERE locktype='advisory' AND NOT granted AND classid=(hashtext('vnext_labels')::bigint & 4294967295)::oid AND objid=(hashtext(app.current_user_id()::text)::bigint & 4294967295)::oid").fetchone()["n"] + if waiting: + break + sleep(0.01) + assert waiting == 1 + assert future.done() is False + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 0 + result = future.result(timeout=5) + assert result["sensitivity"] == "confidential" diff --git a/tests/integration/test_label_floor_ancestry_postgres.py b/tests/integration/test_label_floor_ancestry_postgres.py index 882999373..ce42fae48 100644 --- a/tests/integration/test_label_floor_ancestry_postgres.py +++ b/tests/integration/test_label_floor_ancestry_postgres.py @@ -117,16 +117,27 @@ def test_postgres_owner_edit_is_clamped_in_response_event_and_storage(migrated_d response = router.review_vnext_memory(UUID(str(memory["id"])), router.VNextMemoryReviewRequest(user_id=user_id, action="edit", domain="project", sensitivity="public"), authorization=None) assert response.status_code == 200 payload = json.loads(response.body) - assert payload["label_floor_applied"] is True - assert payload["memory"]["domain"] == "health" - assert payload["memory"]["sensitivity"] == "confidential" with user_connection(url, user_id) as conn: store = PostgresVNextStore(conn) stored = store.get_memory(str(memory["id"])) - assert stored["domain"] == "health" and stored["sensitivity"] == "confidential" assert stored["metadata_json"]["source_id"] == str(source["id"]) event = conn.execute("SELECT payload_json FROM event_log WHERE event_type='memory.labels_raised' AND target_id=%s", (str(memory["id"]),)).fetchone() - assert event["payload_json"]["cause"] == "floor_clamped" + observed = { + "response_flag": payload.get("label_floor_applied", False), + "response_domain": payload["memory"]["domain"], + "response_sensitivity": payload["memory"]["sensitivity"], + "stored_domain": stored["domain"], + "stored_sensitivity": stored["sensitivity"], + "event_cause": event["payload_json"].get("cause") if event else None, + } + assert observed == { + "response_flag": True, + "response_domain": "health", + "response_sensitivity": "confidential", + "stored_domain": "health", + "stored_sensitivity": "confidential", + "event_cause": "floor_clamped", + } encoded = json.dumps(event["payload_json"]) assert "Synthetic private input" not in encoded assert "Synthetic private observation" not in encoded From 9eaee88da4814be7916ac39ad51a855ae0873be5 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:53:05 +0200 Subject: [PATCH 166/270] Measure committed derived-label work in disposable PostgreSQL fixtures --- scripts/measure_derived_label_budgets.py | 211 +++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 scripts/measure_derived_label_budgets.py diff --git a/scripts/measure_derived_label_budgets.py b/scripts/measure_derived_label_budgets.py new file mode 100644 index 000000000..02ab97704 --- /dev/null +++ b/scripts/measure_derived_label_budgets.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Measure actual capture and relabel work in fresh disposable PostgreSQL databases.""" + +from __future__ import annotations + +import argparse +import cProfile +import json +import os +from pathlib import Path +import pstats +import statistics +import time +from uuid import uuid4 + +from alembic import command + +from alicebot_api.db import user_connection +from alicebot_api.migrations import make_alembic_config +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_capture import SourceCaptureInput, VNextCaptureService +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import _create_role_separated_database, _drop_database + + +def capture(url, repetitions): + samples = [] + for repeat in range(repetitions + 1): + user_id = uuid4() + with user_connection(url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"capture-{user_id}@example.invalid", "Capture") + store = PostgresVNextStore(conn) + text = "\n".join( + f"Decision: Synthetic item {index} is assigned to synthetic route {index}." for index in range(200) + ) + start = time.perf_counter() + profile = cProfile.Profile() if os.getenv("ALICE_BUDGET_PROFILE") else None + if profile: + profile.enable() + result = VNextCaptureService(store).capture_source( + SourceCaptureInput( + source_type="manual_text", + title="Synthetic capture", + raw_text=text, + domain="project", + sensitivity="public", + ) + ) + elapsed = time.perf_counter() - start + if profile: + profile.disable() + profile.dump_stats(os.environ["ALICE_BUDGET_PROFILE"]) + pstats.Stats(profile).sort_stats("cumulative").print_stats(45) + assert result.candidate_memory_count == 200, result.to_record() + with user_connection(url, user_id) as conn: + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 200 + if repeat: + samples.append(elapsed) + return { + "facts": 200, + "warmup_runs": 1, + "repetitions": repetitions, + "seconds": samples, + "median_seconds": statistics.median(samples), + "timing_includes_commit": True, + "fixture_validation_timed": False, + } + + +def relabel(url, repetitions, dependants, *, state=None, prepare=False): + from alicebot_api.vnext_derived_labels import with_derived_from + + user_id = uuid4() if state is None else state["user_id"] + if state is None: + with user_connection(url, user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"relabel-{user_id}@example.invalid", "Relabel") + store = PostgresVNextStore(conn) + source = store.create_source( + { + "source_type": "manual_text", + "title": "Synthetic input", + "content_hash": str(uuid4()), + "domain": "project", + "sensitivity": "public", + } + ) + source_id = str(source["id"]) + conn.execute( + """INSERT INTO memories(id,user_id,memory_key,value,title,canonical_text,status,domain,sensitivity,source_event_ids,metadata_json) + SELECT gen_random_uuid(),app.current_user_id(),'scale.'||n,'{}'::jsonb,'Synthetic row '||n,'Synthetic scale row '||n, + 'active','project','public','[]'::jsonb,CASE WHEN n<=%s THEN jsonb_build_object('source_id',%s::text) ELSE '{}'::jsonb END + FROM generate_series(1,50000) n""", + (dependants, source_id), + ) + record = with_derived_from({"workflow": "daily_brief"}, {"sources": [source]}) + conn.execute( + """INSERT INTO generated_artifacts(id,user_id,artifact_type,title,content_markdown,status,domain,sensitivity,generated_by,metadata_json) + SELECT gen_random_uuid(),app.current_user_id(),'daily_brief','Synthetic artifact '||n,'Synthetic scale artifact '||n, + 'needs_review','project','public','system',CASE WHEN n<=%s THEN %s::jsonb ELSE %s::jsonb END + FROM generate_series(1,2000) n""", + (dependants, json.dumps(record), json.dumps(with_derived_from({"workflow": "daily_brief"}, {}))), + ) + else: + source_id = state["source_id"] + if prepare: + return {"user_id": str(user_id), "source_id": source_id, "dependants": dependants} + samples = [] + # Commit is timed. Restore only this synthetic fixture outside each timed operation. + for repeat in range(repetitions + 1): + with user_connection(url, user_id) as conn: + store = PostgresVNextStore(conn) + before_events = conn.execute( + "SELECT count(*) AS n FROM event_log WHERE event_type LIKE '%%.labels_raised'" + ).fetchone()["n"] + start = time.perf_counter() + profile = cProfile.Profile() if os.getenv("ALICE_BUDGET_PROFILE") else None + if profile: + profile.enable() + store.update_source(source_id=source_id, patch={"sensitivity": "confidential"}) + elapsed = time.perf_counter() - start + if profile: + profile.disable() + profile.dump_stats(os.environ["ALICE_BUDGET_PROFILE"]) + pstats.Stats(profile).sort_stats("cumulative").print_stats(45) + with user_connection(url, user_id) as conn: + assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == 50000 + assert conn.execute("SELECT count(*) AS n FROM generated_artifacts").fetchone()["n"] == 2000 + assert ( + conn.execute("SELECT count(*) AS n FROM memories WHERE sensitivity='confidential'").fetchone()["n"] + == dependants + ) + assert ( + conn.execute( + "SELECT count(*) AS n FROM generated_artifacts WHERE sensitivity='confidential'" + ).fetchone()["n"] + == dependants + ) + assert ( + conn.execute("SELECT sensitivity FROM sources WHERE id=%s", (source_id,)).fetchone()["sensitivity"] + == "confidential" + ) + assert ( + conn.execute("SELECT count(*) AS n FROM event_log WHERE event_type LIKE '%%.labels_raised'").fetchone()[ + "n" + ] + - before_events + == 2 * dependants + ) + conn.execute("UPDATE sources SET sensitivity='public' WHERE id=%s", (source_id,)) + conn.execute("UPDATE memories SET sensitivity='public' WHERE sensitivity='confidential'") + conn.execute("UPDATE generated_artifacts SET sensitivity='public' WHERE sensitivity='confidential'") + if repeat: + samples.append(elapsed) + return { + "sources": 1, + "memories": 50000, + "artifacts": 2000, + "dependent_memories": dependants, + "dependent_artifacts": dependants, + "warmup_runs": 1, + "repetitions": repetitions, + "seconds": samples, + "median_seconds": statistics.median(samples), + "budget_seconds": 3, + "label_events_per_run": 2 * dependants, + "timing_includes_commit": True, + "fixture_validation_and_reset_timed": False, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=("capture", "relabel"), required=True) + parser.add_argument("--repetitions", type=int, default=5) + parser.add_argument("--dependants", type=int, default=100) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--prepare", type=Path) + parser.add_argument("--prepared", type=Path) + args = parser.parse_args() + prepared = json.loads(args.prepared.read_text()) if args.prepared else None + name = prepared["database_name"] if prepared else "alice_label_perf_" + uuid4().hex[:12] + urls = prepared["urls"] if prepared else _create_role_separated_database(name) + retained = False + try: + if prepared is None: + command.upgrade(make_alembic_config(urls["admin"]), "head") + if args.prepare: + state = relabel(urls["app"], 0, args.dependants, prepare=True) if args.mode == "relabel" else None + args.prepare.write_text(json.dumps({"database_name": name, "urls": urls, "state": state}, indent=2) + "\n") + retained = True + print("Synthetic scale fixture prepared") + return + record = ( + capture(urls["app"], args.repetitions) + if args.mode == "capture" + else relabel( + urls["app"], + args.repetitions, + prepared["state"]["dependants"] if prepared else args.dependants, + state=prepared["state"] if prepared else None, + ) + ) + args.output.write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps(record)) + finally: + if not retained: + _drop_database(name) + + +if __name__ == "__main__": + main() From 797c859e4b399238281eebc32f33590d3267b21f Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:00:06 +0200 Subject: [PATCH 167/270] Track producer-stage artifact project-floor query argument --- tests/unit/test_vnext_store.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_vnext_store.py b/tests/unit/test_vnext_store.py index ec0c424b9..3ff885712 100644 --- a/tests/unit/test_vnext_store.py +++ b/tests/unit/test_vnext_store.py @@ -739,6 +739,7 @@ def test_list_artifacts_applies_type_domain_sensitivity_and_limit_filters() -> N ["public", "private"], None, None, + None, 5, ) From f6d23e9273c036e804f6f70903620a16df6e269d Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:02:39 +0200 Subject: [PATCH 168/270] Apply effective loop reference admission at exact reader stage --- apps/api/src/alicebot_api/vnext_open_loop_references.py | 8 +++++++- tests/unit/test_main.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_open_loop_references.py b/apps/api/src/alicebot_api/vnext_open_loop_references.py index 92b51c8ff..958b595ed 100644 --- a/apps/api/src/alicebot_api/vnext_open_loop_references.py +++ b/apps/api/src/alicebot_api/vnext_open_loop_references.py @@ -135,7 +135,13 @@ def withhold_unreadable_references( source_rows = _rows_by_id(store, sorted(source_ids | metadata_ids), bulk="get_sources_by_ids", single="get_source") memory_rows = _rows_by_id(store, sorted(memory_ids | metadata_ids), bulk="get_memories_by_ids", single="get_memory") admitted_sources = frozenset(key for key, row in source_rows.items() if fence.admits(row)) - admitted_memories = frozenset(key for key, row in memory_rows.items() if fence.admits_memory(row)) + from alicebot_api.vnext_label_guard import LabelGuard + + guard = LabelGuard.for_fence(store, fence) + admitted_memories = frozenset( + key for key, row in memory_rows.items() + if isinstance(effective := guard.effective_row("memory", row), Mapping) and fence.admits_memory(effective) + ) refused = (set(source_rows) - admitted_sources) | (set(memory_rows) - admitted_memories) # Every id of a row the fence refuses, and every id at a reference position that names no admitted row (a refused, # a deleted, a removed or a missing one), is withheld at every position of every row of the response, in every diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 563513a30..6c8783108 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -645,7 +645,7 @@ def test_openapi_helper_backed_contracts_track_authoritative_response_types() -> def test_openapi_store_row_contracts_track_authoritative_column_sets() -> None: def column_fields(columns: str) -> set[str]: - return {column.strip() for column in columns.split(",") if column.strip()} + return {column.strip().rsplit(" AS ", 1)[-1] for column in columns.split(",") if column.strip()} row_contracts = { ("GET", "/v0/vnext/sources/{source_id}"): SOURCE_COLUMNS, From 248724a17af2d715e7cc3f918b4501df6d8d7fa7 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:56:27 +0200 Subject: [PATCH 169/270] Prove effective admission at both existing rollup readers --- ...est_derived_labels_group_scope_postgres.py | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_derived_labels_group_scope_postgres.py b/tests/integration/test_derived_labels_group_scope_postgres.py index 3b6f07192..0aec1cac4 100644 --- a/tests/integration/test_derived_labels_group_scope_postgres.py +++ b/tests/integration/test_derived_labels_group_scope_postgres.py @@ -15,7 +15,9 @@ @pytest.mark.parametrize("accept", (False, True)) -def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inserts_nothing(migrated_database_urls, accept): +def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inserts_nothing( + migrated_database_urls, accept +): user_id = uuid4() with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") @@ -28,15 +30,25 @@ def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inser assert candidate["metadata_json"]["project_scope"] == [] assert group_scope(candidate) == group_scope(members[0]) if accept: - assert VNextMemoryCommitService(store).accept_consolidation_candidate(first.candidate_ids[0], reason="Reviewed synthetic group")["status"] == "accepted" + assert ( + VNextMemoryCommitService(store).accept_consolidation_candidate( + first.candidate_ids[0], reason="Reviewed synthetic group" + )["status"] + == "accepted" + ) before = conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] second = service.propose_rollups(projects=(ALPHA,)) assert second.proposals == [] assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == before - assert any(group["state"] == ("already_covered_by_accepted" if accept else "existing_candidate") for group in second.groups), second + assert any( + group["state"] == ("already_covered_by_accepted" if accept else "existing_candidate") + for group in second.groups + ), second -def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossing_project_scopes(migrated_database_urls): +def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossing_project_scopes( + migrated_database_urls, +): user_id = uuid4() with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") @@ -48,4 +60,38 @@ def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossi metadata = dict(member["metadata_json"], project_scope=[], project_floor=[ALPHA]) conn.execute("UPDATE memories SET metadata_json=%s::jsonb WHERE id=%s", (json.dumps(metadata), member["id"])) with pytest.raises(VNextMemoryCommitValidationError, match="crosses project scopes"): - VNextMemoryCommitService(store).accept_consolidation_candidate(candidate_id, reason="Reviewed synthetic group") + VNextMemoryCommitService(store).accept_consolidation_candidate( + candidate_id, reason="Reviewed synthetic group" + ) + + +@pytest.mark.parametrize("accept", (False, True)) +def test_existing_rollup_state_admits_effective_labels_for_pending_and_accepted_cards(migrated_database_urls, accept): + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"existing-{user_id}@example.invalid", "Existing") + store = PostgresVNextStore(conn) + members = seed_members(store) + service = VNextRollupService(store) + proposal = service.propose_rollups(projects=(ALPHA,)) + candidate_id = proposal.candidate_ids[0] + candidate = store.get_memory(candidate_id) + metadata = candidate["metadata_json"] + if accept: + VNextMemoryCommitService(store).accept_consolidation_candidate( + candidate_id, reason="Reviewed synthetic group" + ) + arguments = { + "rollup_digests": (metadata["rollup_digest"],), + "rollup_keys": (metadata["rollup_key"],), + "domains": None, + "sensitivity_allowed": ["public", "internal"], + "projects": (ALPHA,), + } + pending, accepted = service._existing_rollup_state(**arguments) + assert bool(accepted if accept else pending) + # Simulate a stale stored card whose input now has a higher effective label. + conn.execute("UPDATE memories SET sensitivity='confidential' WHERE id=%s", (members[0]["id"],)) + pending, accepted = service._existing_rollup_state(**arguments) + assert not pending + assert not accepted From 95132456c2652253bed7cca71f2d7f8d625ac831 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:57:42 +0200 Subject: [PATCH 170/270] Exercise exact entrypoints and every retrieval stage against stale labels --- ...rived_labels_exact_entrypoints_postgres.py | 122 ++++++++++++++++ tests/unit/test_label_door_registry.py | 1 + tests/unit/test_label_reader_stage_matrix.py | 133 ++++++++++++++++++ 3 files changed, 256 insertions(+) create mode 100644 tests/integration/test_derived_labels_exact_entrypoints_postgres.py create mode 100644 tests/unit/test_label_reader_stage_matrix.py diff --git a/tests/integration/test_derived_labels_exact_entrypoints_postgres.py b/tests/integration/test_derived_labels_exact_entrypoints_postgres.py new file mode 100644 index 000000000..96437149c --- /dev/null +++ b/tests/integration/test_derived_labels_exact_entrypoints_postgres.py @@ -0,0 +1,122 @@ +"""Exact entrypoints use the shared effective project floor with real keys.""" +from __future__ import annotations + +import json +from uuid import UUID, uuid4 + +import pytest + +from alicebot_api.config import Settings +from alicebot_api.db import user_connection +from alicebot_api.mcp.registry import MCPToolNotFoundError, call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError +from alicebot_api.routers import vnext_memories, vnext_projects, vnext_retrieval, vnext_review +from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.test_derived_labels_read_acceptance_postgres import _user +from tests.unit.test_derived_labels_real_keys import ALPHA, real_reader_key + +DOORS = ("artifact_get", "artifact_trace", "artifact_export", "artifact_review", "artifact_feedback", "artifact_rating", + "legacy_artifact_get", "legacy_artifact_review", "memory_review_http", "memory_review_mcp", + "memory_correct_mcp", "memory_redact_mcp", "loop_review_http", "loop_review_mcp") + +def _invoke(door, *, app_url, user_id, target_id, key, tmp_path): + auth = f"Bearer {key}" if key else None + if door == "artifact_get": + return vnext_review.get_vnext_artifact(UUID(target_id), user_id, authorization=auth) + if door == "artifact_trace": + return vnext_retrieval.get_vnext_artifact_trace(UUID(target_id), user_id, authorization=auth) + if door == "artifact_export": + return vnext_review.export_vnext_artifact(UUID(target_id), vnext_review.VNextArtifactExportRequest(user_id=user_id, output_dir=str(tmp_path)), authorization=auth) + if door == "artifact_review": + return vnext_review.review_vnext_artifact(UUID(target_id), vnext_review.VNextArtifactReviewRequest(user_id=user_id, action="reject"), authorization=auth) + if door == "artifact_feedback": + return vnext_review.record_vnext_artifact_insight_feedback(UUID(target_id), vnext_review.VNextArtifactInsightFeedbackRequest(user_id=user_id, useful_insight="yes"), authorization=auth) + if door == "artifact_rating": + return vnext_review.rate_vnext_artifact_quality(UUID(target_id), vnext_review.VNextArtifactQualityRatingRequest(user_id=user_id), authorization=auth) + if door == "memory_review_http": + return vnext_memories.review_vnext_memory(UUID(target_id), vnext_memories.VNextMemoryReviewRequest(user_id=user_id, action="accept"), authorization=auth) + if door == "loop_review_http": + return vnext_projects.review_vnext_open_loop(target_id, vnext_projects.VNextOpenLoopReviewRequest(user_id=user_id, action="close"), authorization=auth) + name, args = { + "legacy_artifact_get": ("alice_vnext_artifact_get", {"artifact_id": target_id}), + "legacy_artifact_review": ("alice_vnext_artifact_review", {"artifact_id": target_id, "action": "reject"}), + "memory_review_mcp": ("alice_memory_review", {"review_item_id": target_id}), + "memory_correct_mcp": ("alice_memory_correct", {"review_item_id": target_id, "action": "approve"}), + "memory_redact_mcp": ("alice_memory_manage", {"memory_id": target_id, "action": "redact", "reason": "Synthetic test"}), + "loop_review_mcp": ("alice_open_loops", {"loop_id": target_id, "action": "close"}), + }[door] + return call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name=name, arguments=args) + +@pytest.mark.parametrize("door", DOORS) +@pytest.mark.parametrize("reader", ("owner", "bound_admin")) +def test_exact_entrypoint_checks_effective_floor(migrated_database_urls, monkeypatch, tmp_path, door, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + for module in (vnext_memories, vnext_projects, vnext_retrieval, vnext_review): + monkeypatch.setattr(module, "get_settings", lambda: Settings(database_url=app_url)) + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.setenv("ALICE_MCP_LEGACY_TOOLS", "1") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + key = real_reader_key(store, user_id, reader) + if key: + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + for hidden in (True, False): + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Parent", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": ["prj_" + "b" * 16 if hidden else ALPHA]}}) + metadata = {"source_id": str(source["id"]), "project_scope": [ALPHA]} + derived = {"v": 1, "sources": [str(source["id"])], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}} + with without_insert_floor(): + if "artifact" in door: + row = store.create_artifact({"artifact_type": "daily_brief", "title": "Door sentinel", "content_markdown": "Door sentinel", "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA], "derived_from": derived}}) + elif door.startswith("loop_"): + row = store.create_open_loop({"title": "Door sentinel", "source_id": str(source["id"]), "domain": "project", "sensitivity": "public", "metadata_json": {**metadata, "discovered_by": "vnext_daily_capture"}}) + else: + row = store.create_memory({"memory_key": str(uuid4()), "title": "Door sentinel", "canonical_text": "Door sentinel", "status": "candidate", "domain": "project", "sensitivity": "public", "metadata_json": metadata}) + blocked = hidden and reader == "bound_admin" + if door.startswith("legacy_") and key: + with pytest.raises(MCPToolNotFoundError, match="disabled whenever"): + _invoke(door, app_url=app_url, user_id=user_id, target_id=str(row["id"]), key=key, tmp_path=tmp_path) + continue + try: + result = _invoke(door, app_url=app_url, user_id=user_id, target_id=str(row["id"]), key=key, tmp_path=tmp_path) + except MCPToolError as exc: + assert blocked, (door, reader, hidden, type(exc).__name__, str(exc)) + assert "policy" in type(exc).__name__.lower() or "project" in str(exc).lower(), (type(exc).__name__, str(exc)) + else: + if hasattr(result, "status_code"): + assert (result.status_code == 403) is blocked, (door, reader, hidden, result.status_code, result.body) + if blocked: + assert str(row["id"]) not in result.body.decode() + assert "Door sentinel" not in result.body.decode() + else: + assert result.status_code in {200, 201}, (door, result.status_code, result.body) + else: + assert not blocked, (door, reader, hidden, json.dumps(result, default=str)) + assert str(row["id"]) in json.dumps(result, default=str) + + +@pytest.mark.parametrize("reader", ("owner", "bound_admin")) +def test_review_queue_list_uses_current_parent_scope(migrated_database_urls, monkeypatch, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + key = real_reader_key(store, user_id, reader) + rows = [] + for hidden in (False, True): + source = store.create_source({"source_type": "note", "title": "Parent", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": ["prj_" + "b" * 16 if hidden else ALPHA]}}) + with without_insert_floor(): + rows.append(store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Queue sentinel", "status": "candidate", "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA], "source_id": str(source["id"])}})) + if key: + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + result = call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name="alice_memory_review", arguments={"status": "all"}) + rendered = json.dumps(result, default=str) + assert str(rows[0]["id"]) in rendered + assert (str(rows[1]["id"]) in rendered) is (reader == "owner") diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py index 3e0350f58..6aebf88d8 100644 --- a/tests/unit/test_label_door_registry.py +++ b/tests/unit/test_label_door_registry.py @@ -57,6 +57,7 @@ DOORS = { "routers/_vnext_shared.py:_vnext_authorized_artifact": None, "vnext_source_fence.py:resolve_attachable_memory_id": None, + "vnext_open_loop_references.py:withhold_unreadable_references": None, "mcp/evidence_artifacts.py:_authorize_explain_resource": None, "mcp/evidence_artifacts.py:_authorize_entity_explain_target": "_authorize_explain_resource", "mcp/evidence_artifacts.py:_entity_backing_is_fully_authorized": "_authorize_explain_resource", diff --git a/tests/unit/test_label_reader_stage_matrix.py b/tests/unit/test_label_reader_stage_matrix.py new file mode 100644 index 000000000..59e9fba90 --- /dev/null +++ b/tests/unit/test_label_reader_stage_matrix.py @@ -0,0 +1,133 @@ +"""Each read stage holds back a stale public row and keeps a visible control.""" +from __future__ import annotations +from datetime import UTC, datetime, timedelta +from contextlib import contextmanager +from types import SimpleNamespace +from uuid import UUID, uuid4 + +import pytest + +from alicebot_api.project_view import ProjectView +from alicebot_api.session_briefing import compile_session_brief +from alicebot_api.vnext_agent_control import AgentIdentity +from alicebot_api.vnext_context_tree import ContextTreeRequest, VNextContextTreeService +from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService, VNextProjectValidationError +from alicebot_api.vnext_retrieval import VNextRetrievalService, _ResolvedRetrievalScope, expand_provenance_once +from alicebot_api.vnext_temporal_query import TemporalAnchor +from alicebot_api.vnext_open_loop_references import withhold_unreadable_references +from alicebot_api.vnext_source_fence import SourceReadFence +from alicebot_api.mcp import evidence_artifacts +from alicebot_api.routers import workspaces +from tests.unit.test_complete_readable_counts import PopulationStore, _quiet_services +from tests.unit.test_derived_labels_real_keys import ALPHA + +SOURCE = str(UUID(int=100)) +MEMORY = str(UUID(int=101)) +LOOP = str(UUID(int=102)) +ARTIFACT = str(UUID(int=103)) +PROJECT = str(UUID(int=104)) +ENTITY = str(UUID(int=105)) +SECRET = "Door sentinel" +CEILING = ["public", "internal", "private", "unknown"] + +class StageStore(PopulationStore): + def __init__(self): + super().__init__() + derived = {"v": 1, "sources": [SOURCE], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}} + self.rows["source"] = [{"id": SOURCE, "domain": "project", "sensitivity": "confidential", "metadata_json": {"project_scope": [ALPHA]}}] + for kind, identifier in (("memory", MEMORY), ("open_loop", LOOP), ("artifact", ARTIFACT), ("project", PROJECT)): + self.rows[kind] = [{"id": identifier, "domain": "project", "sensitivity": "public", "status": "open" if kind == "open_loop" else "active", + "title": SECRET, "name": SECRET, "canonical_text": SECRET, "content_markdown": SECRET, + "memory_type": "semantic", "artifact_type": "daily_brief", "source_id": SOURCE, + "metadata_json": {"project_scope": [ALPHA], "source_id": SOURCE, "discovered_by": "vnext_daily_capture", "derived_from": derived}}] + self.belief = {"id": str(UUID(int=106)), "memory_id": MEMORY, "claim": "The deployment pipeline is ready for production launch.", "status": "active"} + self.events = [{"id": str(UUID(int=107)), "event_type": "memory.labels_raised", "target_type": "memory", "target_id": MEMORY, "occurred_at": datetime.now(UTC).isoformat()}] + def search_memories_vector(self, **kwargs): return self.rows["memory"] + def search_memories_by_time(self, **kwargs): return self.rows["memory"] + def search_memories(self, **kwargs): return self.rows["memory"] + def search_sources(self, **kwargs): return self.rows["source"] + def search_sources_fts(self, **kwargs): return [] + def search_source_chunks_fts(self, **kwargs): return [] + def get_memories_by_ids(self, ids): return [row for row in self.rows["memory"] if row["id"] in ids] + def list_memories_referencing_source(self, **kwargs): return self.rows["memory"] + def list_beliefs(self, **kwargs): return [self.belief] + def list_memory_events(self, **kwargs): return self.events + def list_resume_memory_events(self, **kwargs): return [] + def list_open_loop_events(self, **kwargs): return [] + def find_entities_by_names(self, names): return [{"id": ENTITY, "name": "Alice", "entity_type": "person"}] + def list_memory_entity_edges(self, **kwargs): return [{"from_type": "memory", "from_id": MEMORY, "to_type": "entity", "to_id": ENTITY, "edge_type": "mentions", "observed_at": datetime.now(UTC).isoformat()}] + def append_event(self, event): return {"id": str(uuid4()), **event} + def upsert_agent_identity(self, *args, **kwargs): return None + def get_project(self, identifier): return {"id": identifier, "name": "Visible project", "domain": "project", "sensitivity": "public", "metadata_json": {}} + def get_entity_optional(self, identifier): return {"id": identifier, "source_memory_ids": [MEMORY]} + def list_entity_edges_for_entity(self, identifier): return [] + def list_edges(self, **kwargs): return [{"from_type": "memory", "from_id": MEMORY, "to_type": "entity", "to_id": ENTITY}] + +def _scope(scoped=False): + return _ResolvedRetrievalScope(projects=frozenset((ALPHA,)) if scoped else frozenset(), people=frozenset(), window_start=None, window_end=None, exclude_global_domains=frozenset()) + +STAGES = ("by_ids", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs") + +def _stage(stage, store): + service = VNextRetrievalService(store, embedding_provider=SimpleNamespace(provider="synthetic", model="synthetic", base_url="http://synthetic.invalid")) + kwargs = {"domains": ["project"], "sensitivity_allowed": CEILING, "limit": 10} + if stage == "by_ids": return list(service._memories_by_ids([MEMORY], domains=kwargs["domains"], sensitivity_allowed=CEILING).values()) + if stage == "vector": return service._memory_vector_rows(query="Alice", query_vector=[0.1], query_embedding_status="enabled", **kwargs)[0] + if stage == "graph": return service._memory_graph_rows(query="Alice", entity_read_fenced=True, **kwargs)[0] + if stage == "temporal": return service._memory_temporal_rows(anchor=TemporalAnchor(datetime.now(UTC)-timedelta(days=1), datetime.now(UTC), "synthetic"), **kwargs)[0] + if stage == "provenance": return expand_provenance_once(store, fts_memories=[], source_excerpts=[{"id": SOURCE}], already_selected_ids=set(), effective_domains=["project"], effective_sensitivity_allowed=CEILING, effective_project_scope=()) + if stage == "visibility": return [store.rows["memory"][0]] if service.memory_visibility(domains=["project"], sensitivity_allowed=CEILING, scope=None)(store.rows["memory"][0]) else [] + if "contradictions" in stage: + new = {"id": str(UUID(int=108)), "memory_type": "semantic", "canonical_text": "The deployment pipeline is not ready for production launch."} + return service._contradicting_evidence([new], requested=True, domains=["project"], sensitivity_allowed=CEILING, scope=_scope(stage.startswith("scoped_")), person_linked_memory_ids=frozenset())[0] + if "recent_changes" in stage: + return service._recent_changes(domains=["project"], sensitivity_allowed=CEILING, scope=_scope(stage.startswith("scoped_")), person_linked_memory_ids=frozenset()) + if stage == "session": return compile_session_brief(store, effective_domains=("project",), effective_sensitivity_allowed=tuple(CEILING), effective_project_scope=(), project_view=ProjectView.unscoped(), exclude_global_domains=frozenset(), query=None) + if stage.startswith("context_"): + tree = VNextContextTreeService(store).build_tree(ContextTreeRequest(domains=("project",), sensitivity_allowed=tuple(CEILING))) + return next(root["children"] for root in tree["roots"] if root["id"] == "root:" + stage.removeprefix("context_")) + if stage == "project_resolution": + try: return [VNextProjectService(store)._resolve_project(ProjectAutomationRequest(domains=("project",), sensitivity_allowed=tuple(CEILING)))] + except VNextProjectValidationError: return [] + if stage == "dashboard_lists": + result = VNextProjectService(store).project_dashboard(project_id=ALPHA, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent")) + return [*result["memories"], *result["open_loops"], *result["artifacts"]] + if stage.startswith("workspace_"): + field = stage.removeprefix("workspace_") + if field == "memories": + store.rows["memory"][0]["status"] = "candidate" + field = "review_memories" + return workspaces._vnext_workspace_payload(store)[field] + trusted = AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent") + if stage == "loop_memory_reference": + shown = withhold_unreadable_references(store, [{"id": LOOP, "memory_id": MEMORY, "metadata_json": {}}], fence=SourceReadFence.for_identity(trusted)) + return [shown[0]["memory_id"]] if shown[0]["memory_id"] else [] + if stage == "entity_backing": + return [store.rows["memory"][0]] if evidence_artifacts._entity_backing_is_fully_authorized(store, identity=trusted, entity_id=ENTITY) else [] + if stage == "entity_explain": + try: + evidence_artifacts._authorize_entity_explain_target(None, identity=trusted, entity_id=UUID(ENTITY)) + except evidence_artifacts._ExplainAuthorizationError: + return [] + return store.rows["memory"] + raise AssertionError(stage) + +@pytest.mark.parametrize("stage", STAGES) +def test_each_stage_uses_current_parent_label(stage, monkeypatch): + _quiet_services(monkeypatch) + store = StageStore() + @contextmanager + def context(_ignored): + yield store + monkeypatch.setattr(evidence_artifacts, "_store_context", context) + monkeypatch.setattr(evidence_artifacts, "_vnext_store_context", context) + hidden = _stage(stage, store) + assert SECRET not in str(hidden) + assert SOURCE not in str(hidden) and MEMORY not in str(hidden) and LOOP not in str(hidden) and ARTIFACT not in str(hidden) and PROJECT not in str(hidden) + store.rows["source"][0]["sensitivity"] = "public" + visible = _stage(stage, store) + assert visible, (stage, visible) + if stage != "session": + assert any(identifier in str(visible) for identifier in (SOURCE, MEMORY, LOOP, ARTIFACT, PROJECT, str(store.belief["id"]))), (stage, visible) + else: + assert SECRET in visible From 2c37dfcbe2967a1c959e56b46546a7af189d6456 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:09:56 +0200 Subject: [PATCH 171/270] Align read-filter fixtures with derived label admission --- tests/unit/test_derived_labels_real_keys.py | 23 +++++++++ tests/unit/test_vnext_main.py | 9 ++++ tests/unit/test_vnext_retrieval.py | 53 ++++++++++++++++----- tests/unit/test_workspaces_router_split.py | 27 +++++++---- 4 files changed, 92 insertions(+), 20 deletions(-) diff --git a/tests/unit/test_derived_labels_real_keys.py b/tests/unit/test_derived_labels_real_keys.py index 991454b09..ee873bcdf 100644 --- a/tests/unit/test_derived_labels_real_keys.py +++ b/tests/unit/test_derived_labels_real_keys.py @@ -84,3 +84,26 @@ def test_sqlite_real_key_core_doors(tmp_path, monkeypatch, reader): if not admitted: assert str(row["title"]) not in rendered + +@pytest.mark.parametrize("reader", READERS) +def test_original_loop_holds_back_stale_backing_memory_reference(tmp_path, monkeypatch, reader): + user_id = uuid4() + path = tmp_path / "loop-refs.sqlite3" + bootstrap_database(path, user_id=str(user_id), user_email="synthetic@example.invalid") + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.setenv("ALICE_PROJECT_SCOPING", "off") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + with sqlite_user_connection(path, user_id) as conn: + store = SQLiteVNextStore(conn, user_id) + memory = seed_read_rows(store)["verified_confidential"] + loop = store.create_open_loop({"title": "Visible original loop", "memory_id": str(memory["id"]), "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA]}}) + key = real_reader_key(store, user_id, reader) + if key: + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + result = call_mcp_tool(MCPRuntimeContext(database_url=sqlite_url_for_path(path), user_id=user_id), name="alice_open_loops", arguments={"sensitivity_allowed": list(ALL_SENSITIVITY)}) + item = next(row for row in result["items"] if str(row["id"]) == str(loop["id"])) + admitted = expected_read(reader, "verified_confidential") + assert item["memory_id"] == (str(memory["id"]) if admitted else None) + assert (str(memory["id"]) in json.dumps(result, default=str)) is admitted + diff --git a/tests/unit/test_vnext_main.py b/tests/unit/test_vnext_main.py index e9aade0df..c208bca2f 100644 --- a/tests/unit/test_vnext_main.py +++ b/tests/unit/test_vnext_main.py @@ -70,6 +70,12 @@ def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: "artifact": self.artifacts.values(), "belief": self.beliefs.values(), "project": self.projects.values()}.get(kind, []) return [dict(row) for row in collection if str(row.get("id")) in ids] + def iter_label_rows(self, kind: str): + collections = {"source": self.sources.values(), "memory": self.memories, + "open_loop": self.open_loops, "artifact": self.artifacts.values(), + "belief": self.beliefs.values(), "project": self.projects.values()} + yield [dict(row) for row in collections[kind]] + def _fetch_all(self, query: str, _params: tuple[object, ...]) -> list[dict[str, object]]: # The label dependant walker performs its exact canonical reference filter after this prefilter. for table, kind in (("memories", "memory"), ("open_loops", "open_loop"), ("generated_artifacts", "artifact"), ("projects", "project")): @@ -4476,6 +4482,9 @@ def test_dogfooding_dashboard_and_insight_feedback_api(monkeypatch) -> None: "sensitivity": "private", } ) + from alicebot_api.vnext_derived_labels import stamp_derived_from + + stamp_derived_from(artifact, {}) store.create_artifact_quality_rating( { "artifact_id": artifact["id"], diff --git a/tests/unit/test_vnext_retrieval.py b/tests/unit/test_vnext_retrieval.py index 70454a4a5..b2ccaebff 100644 --- a/tests/unit/test_vnext_retrieval.py +++ b/tests/unit/test_vnext_retrieval.py @@ -99,8 +99,10 @@ def __init__( entities: list[dict[str, object]] | None = None, edges: list[dict[str, object]] | None = None, source_chunks: list[dict[str, object]] | None = None, + stored_memories: list[dict[str, object]] | None = None, ) -> None: self.memories = memories + self.stored_memories = stored_memories or [] self.sources = sources self.open_loops = open_loops or [] self.provenance_links = provenance_links or [] @@ -121,6 +123,19 @@ def __init__( self.vector_limits: list[int] = [] self.memory_bulk_reads = 0 + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + # Complete persisted ancestry, including rows outside the FTS match list. + collections = { + "memory": [*self.memories, *self.stored_memories, *(self.vector_memories or [])], + "source": self.sources, "open_loop": self.open_loops, + "belief": self.beliefs or [], + } + from alicebot_api.vnext_derived_labels import identifier + + wanted = {identifier(item) for item in ids} + found = {str(row.get("id")): row for row in collections.get(kind, [])} + return [dict(row) for row in found.values() if identifier(row.get("id")) in wanted] + def append_event(self, event: dict[str, object]) -> dict[str, object]: self.events.append(event) return event @@ -649,7 +664,7 @@ def test_context_pack_degrades_to_fts_when_query_embedding_fails() -> None: assert pack["trace"]["stages"]["vector"]["candidate_count"] == 0 -def test_context_pack_filters_sensitive_memories_and_records_trace_exclusion() -> None: +def test_context_pack_filters_sensitive_memories_before_recording_trace() -> None: store = InMemoryVNextRetrievalStore( memories=[ _memory_row( @@ -680,8 +695,9 @@ def test_context_pack_filters_sensitive_memories_and_records_trace_exclusion() - ) assert [memory["id"] for memory in pack["relevant_memories"]] == ["memory-public"] - assert "sensitive_items_filtered" in pack["warnings"] - assert pack["trace"]["excluded_counts"] == {"sensitivity_filtered": 1} + assert pack["warnings"] == [] + assert pack["trace"]["excluded_counts"] == {} + assert "memory-secret" not in json.dumps(pack["trace"]) assert [record["target_id"] for record in pack["trace"]["selected"]] == ["memory-public"] @@ -1120,7 +1136,10 @@ def test_strict_count_candidate_statistic_without_selected_rollup_stays_trace_on ) for index in range(1, 4) ], - sources=[], + sources=[ + {"id": f"source-{index}", "source_type": "note", "domain": "project", + "sensitivity": "private"} for index in range(1, 4) + ], ) pack = VNextRetrievalService(store).compile_context_pack( @@ -1958,14 +1977,12 @@ def _capture_currency_source_lookup(rows, *, source_lookup): source_fence=SourceReadFence.unfenced() ) - assert source_resolver_results == [None] + assert source_resolver_results == [] + assert pack["relevant_memories"] == [] assert pack["sources"] == [] assert pack["supporting_evidence"] == [] assert "derived_values" not in pack assert all("event_time" not in memory for memory in pack["relevant_memories"]) - first = pack["relevant_memories"][0] - assert "source_created_at" not in first - assert first["metadata_json"] == {"project_id": "alicebot", "source_refs": []} assert "source-hermes" not in json.dumps(pack, sort_keys=True) @@ -2670,6 +2687,7 @@ def test_context_pack_populates_contradicting_evidence_from_active_beliefs() -> ) ], sources=[], + stored_memories=[_memory_row("memory-belief", "The deployment pipeline is ready for production launch.")], beliefs=[ { "id": "belief-1", @@ -3447,7 +3465,10 @@ def test_contradictions_first_lets_contradictions_survive_a_budget_that_drops_th } def compile_with(strategy: str, max_tokens: int | None) -> dict[str, object]: - store = InMemoryVNextRetrievalStore(memories=[dict(memory)], sources=[], beliefs=[dict(belief)]) + store = InMemoryVNextRetrievalStore( + memories=[dict(memory)], sources=[], beliefs=[dict(belief)], + stored_memories=[_memory_row("memory-belief", str(belief["claim"]))], + ) return VNextRetrievalService(store).compile_context_pack( VNextRetrievalRequest( query="deployment pipeline production launch", @@ -3687,6 +3708,7 @@ def build_store() -> InMemoryVNextRetrievalStore: return InMemoryVNextRetrievalStore( memories=[_memory_row("memory-1", "The deployment pipeline is not ready for production launch.")], sources=[], + stored_memories=[_memory_row("memory-belief", "The deployment pipeline is ready for production launch.")], beliefs=[ { "id": "belief-1", @@ -4494,7 +4516,12 @@ def build_store() -> InMemoryVNextRetrievalStore: for index in range(1, 25) ] swim = _memory_row("memory-swim", "Swimming laps review.", metadata_json={"source_id": "src-swim"}) - return InMemoryVNextRetrievalStore(memories=[*fillers, swim], sources=[]) + return InMemoryVNextRetrievalStore( + memories=[*fillers, swim], + sources=[{"id": source_id, "source_type": "note", "domain": "project", + "sensitivity": "private"} + for source_id in ["src-shared", *[f"src-{index:02d}" for index in range(3, 25)], "src-swim"]], + ) control_store = build_store() with monkeypatch.context() as patch: @@ -4729,7 +4756,11 @@ def test_naturally_selected_unrelated_rollup_does_not_turn_trace_count_into_answ ["actual-member-1", "actual-member-2", "actual-member-3"], ) pack = VNextRetrievalService( - InMemoryVNextRetrievalStore(memories=[*unrelated, card], sources=[]) + InMemoryVNextRetrievalStore( + memories=[*unrelated, card], sources=[], + stored_memories=[_memory_row(f"actual-member-{index}", f"A different activity {index}.") + for index in range(1, 4)], + ) ).compile_context_pack( VNextRetrievalRequest( query="How many times did I host board game night?", diff --git a/tests/unit/test_workspaces_router_split.py b/tests/unit/test_workspaces_router_split.py index a7aecd2e6..d38ef3b32 100644 --- a/tests/unit/test_workspaces_router_split.py +++ b/tests/unit/test_workspaces_router_split.py @@ -82,11 +82,14 @@ # The later typing repair adds Sequence and annotates _workspace_rows. # Two local helpers admit rows and check event targets; the payload uses them # before returning lists/counts. Route bodies, mounts and middleware are unchanged. -EXPECTED_ROUTE_AST_SHA256 = "fb8925ebcda058598b0c6d5e7eca83abe18606a0126bdb6cde0fd1c22444a795" -EXPECTED_SUPPORT_AST_SHA256 = "0e5708e69de1dd1358ca91709d4a60effeddcd262fc8beddb19ba490cab3b0f1" + +# Re-pin 2026-10-06: workspace reads authenticate the protected identity and +# admit rows through effective labels before totals or dashboard disclosure. +EXPECTED_ROUTE_AST_SHA256 = "b99f1435de67d9499819acb9ed7ed61b588a3fb0ff037e782eeca070b39af742" +EXPECTED_SUPPORT_AST_SHA256 = "8700f0ab3b8a87f9863a3e95132e834f87bf82725c6e8d33602345bc3253bce1" EXPECTED_ROUTE_NAME_MANIFEST_SHA256 = "225c57c08bd8314156c56352dd1c53ffed3f556ce285c666dd6fca125115d0b4" EXPECTED_OPERATION_MANIFEST_SHA256 = "c320979b62d7ee8de244fe38bde5bf3761a4f9d76f76bf3cd8576c30fce9857e" -EXPECTED_IMPORT_MANIFEST_SHA256 = "4a9df193d620b33578f1e42760b118a4267d3e1c8ff01ad92bee1f1bba1c1e9f" +EXPECTED_IMPORT_MANIFEST_SHA256 = "d8887934cacc6a4e5d52f080dae3c2c0d0cdf23d1518a4060a885a8c7f209c0c" EXPECTED_CARRIER_NAMES_SHA256 = "2c109fc234a05dd8f44e4c34bee49e797fbb5e49e92413391541a7e504da328b" # Re-pinned 2026-10-02 (DB-005, legacy /v0 routes). One definition changed, # found by a per-definition AST diff against the previous pin: @@ -130,13 +133,13 @@ # Source regeneration adds one protected write route without new carrier definitions. EXPECTED_CARRIER_AST_SHA256 = "fcd6d722e2d6c3be28b138449555e703f6930168f52b8e94b41dacbcb937b605" EXPECTED_ROUTE_NODE_SHA256 = { - "get_vnext_workspace": "6c2151bf38b1b1311f016c00d14394afc7077a6ea219f7ce3dcfd9b701474ae7", + "get_vnext_workspace": "52c12b20d7bb33759f8dafa2249b2d775b54666130402c0c75045e9ad57ed587", "bootstrap_v1_workspace": "07b1fe2a4cd03a5ba69abe76e258a457e85e92b0bfba592520ee02d01d759c4b", "get_v1_workspace_bootstrap_status": "2849d7126ee37b6e3ffd9ebe84b2a8e719eb0f811da750a29f7e0a0798305faa", } EXPECTED_SUPPORT_NODE_SHA256 = { "_vnext_status_counts": "0bf0ed228a14bd648a9d18fcd5f99ebf8c585bd29f4b5e81e1df17fe0201fd15", - "_vnext_workspace_payload": "4b6e4a3d59d16538b0e859c425ede21207e9c80f11ab31c3af2ce4d604e14edf", + "_vnext_workspace_payload": "83bc100509fc21e950702cba190ac8d94f33efe7774df673809d49daa6864136", "_workspace_rows": "070bdfbd1eae10608bd8208b08367e1c0ea10e2064f03ad5a84121a190ed4cf0", "_workspace_event_visible": "8343c060909326a5cb69fa6f671ac62f160630ecf989f04d78e792ea74c0ea90", } @@ -485,7 +488,7 @@ def test_workspace_import_direction_pruning_timing_and_runtime_identities_are_ex main_definitions = _top_level_definitions(main_tree) router_imports = _import_manifest(router_tree) - assert len(router_imports) == 32 + assert len(router_imports) == 38 assert hashlib.sha256(json.dumps(router_imports, separators=(",", ":")).encode()).hexdigest() == ( EXPECTED_IMPORT_MANIFEST_SHA256 ) @@ -514,6 +517,12 @@ def test_workspace_import_direction_pruning_timing_and_runtime_identities_are_ex assert MAIN_PRUNED_BINDINGS.isdisjoint(main_imports) assert MAIN_PRUNED_BINDINGS <= router_import_bindings assert main_imports & router_import_bindings == { + "AgentIdentity", + "AgentKeyAuthenticationError", + "Header", + "_vnext_agent_auth_error_response", + "agent_key_from_authorization", + "resolve_protected_agent_identity", "JSONResponse", "PostgresVNextStore", "Request", @@ -526,9 +535,9 @@ def test_workspace_import_direction_pruning_timing_and_runtime_identities_are_ex main_loads = { node.id for node in ast.walk(main_tree) if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) } - # /v1 agent-key authentication resolves the bound user in main, so no - # shared binding is a re-export-only import any more. - assert (main_imports & router_import_bindings) - main_loads == set() + # Workspace auth now consumes Header; the carrier retains its historical + # Header import while the other shared bindings remain runtime dependencies. + assert (main_imports & router_import_bindings) - main_loads == {"Header"} provider_module_imports = [ node From 36a8eaff2322022d1d78312e6cf5947e99701621 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:59:23 +0200 Subject: [PATCH 172/270] Pin context pack loops and fallback memory lookup controls --- tests/unit/test_label_reader_stage_matrix.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_label_reader_stage_matrix.py b/tests/unit/test_label_reader_stage_matrix.py index 59e9fba90..0d2193d61 100644 --- a/tests/unit/test_label_reader_stage_matrix.py +++ b/tests/unit/test_label_reader_stage_matrix.py @@ -12,7 +12,7 @@ from alicebot_api.vnext_agent_control import AgentIdentity from alicebot_api.vnext_context_tree import ContextTreeRequest, VNextContextTreeService from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService, VNextProjectValidationError -from alicebot_api.vnext_retrieval import VNextRetrievalService, _ResolvedRetrievalScope, expand_provenance_once +from alicebot_api.vnext_retrieval import VNextRetrievalRequest, VNextRetrievalService, _ResolvedRetrievalScope, expand_provenance_once from alicebot_api.vnext_temporal_query import TemporalAnchor from alicebot_api.vnext_open_loop_references import withhold_unreadable_references from alicebot_api.vnext_source_fence import SourceReadFence @@ -66,12 +66,15 @@ def list_edges(self, **kwargs): return [{"from_type": "memory", "from_id": MEMOR def _scope(scoped=False): return _ResolvedRetrievalScope(projects=frozenset((ALPHA,)) if scoped else frozenset(), people=frozenset(), window_start=None, window_end=None, exclude_global_domains=frozenset()) -STAGES = ("by_ids", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs") +STAGES = ("by_ids", "by_ids_fallback", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "pack_open_loops", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs") def _stage(stage, store): service = VNextRetrievalService(store, embedding_provider=SimpleNamespace(provider="synthetic", model="synthetic", base_url="http://synthetic.invalid")) kwargs = {"domains": ["project"], "sensitivity_allowed": CEILING, "limit": 10} - if stage == "by_ids": return list(service._memories_by_ids([MEMORY], domains=kwargs["domains"], sensitivity_allowed=CEILING).values()) + if stage.startswith("by_ids"): + if stage == "by_ids_fallback": + store.get_memories_by_ids = None + return list(service._memories_by_ids([MEMORY], domains=kwargs["domains"], sensitivity_allowed=CEILING).values()) if stage == "vector": return service._memory_vector_rows(query="Alice", query_vector=[0.1], query_embedding_status="enabled", **kwargs)[0] if stage == "graph": return service._memory_graph_rows(query="Alice", entity_read_fenced=True, **kwargs)[0] if stage == "temporal": return service._memory_temporal_rows(anchor=TemporalAnchor(datetime.now(UTC)-timedelta(days=1), datetime.now(UTC), "synthetic"), **kwargs)[0] @@ -83,6 +86,8 @@ def _stage(stage, store): if "recent_changes" in stage: return service._recent_changes(domains=["project"], sensitivity_allowed=CEILING, scope=_scope(stage.startswith("scoped_")), person_linked_memory_ids=frozenset()) if stage == "session": return compile_session_brief(store, effective_domains=("project",), effective_sensitivity_allowed=tuple(CEILING), effective_project_scope=(), project_view=ProjectView.unscoped(), exclude_global_domains=frozenset(), query=None) + if stage == "pack_open_loops": + return VNextRetrievalService(store).compile_context_pack(VNextRetrievalRequest(query="open loop", domains=("project",), sensitivity_allowed=tuple(CEILING), include_sources=False, include_contradictions=False), source_fence=SourceReadFence.unfenced())["open_loops"] if stage.startswith("context_"): tree = VNextContextTreeService(store).build_tree(ContextTreeRequest(domains=("project",), sensitivity_allowed=tuple(CEILING))) return next(root["children"] for root in tree["roots"] if root["id"] == "root:" + stage.removeprefix("context_")) From 04b6306507a2d3e9a18d63c59008fcda766b75e5 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:13:54 +0200 Subject: [PATCH 173/270] Withhold restricted backing references in workspace views --- .../src/alicebot_api/routers/_vnext_shared.py | 2 ++ .../src/alicebot_api/routers/workspaces.py | 3 ++ apps/api/src/alicebot_api/vnext_projects.py | 3 ++ ...derived_labels_read_acceptance_postgres.py | 33 ++++++++++++++++--- tests/unit/test_label_reader_stage_matrix.py | 9 ++++- .../test_open_loop_references_read_fence.py | 8 ++--- 6 files changed, 49 insertions(+), 9 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/_vnext_shared.py b/apps/api/src/alicebot_api/routers/_vnext_shared.py index 99fddf907..0ba0ac414 100644 --- a/apps/api/src/alicebot_api/routers/_vnext_shared.py +++ b/apps/api/src/alicebot_api/routers/_vnext_shared.py @@ -309,6 +309,7 @@ def _vnext_load_source_trace( from alicebot_api.vnext_agent_control import AgentIdentity from alicebot_api.vnext_label_guard import apply_sensitivity_ceiling + from alicebot_api.vnext_open_loop_references import withhold_unreadable_references caller = identity if isinstance(identity, AgentIdentity) else None if not apply_sensitivity_ceiling(store, kind="source", rows=[source], identity=caller): @@ -323,6 +324,7 @@ def _vnext_load_source_trace( open_loops, open_loops_complete = _vnext_readable_trace_rows( store, "open_loop", lambda limit: store.list_open_loops_referencing_source(source_id=source_id, limit=limit), caller ) + open_loops = withhold_unreadable_references(store, open_loops, fence=SourceReadFence.for_identity(caller)) kept_ids = {str(row.get("id")) for row in (*memories, *artifacts, *open_loops)} kept_ids.add(source_id) events, direct_events_complete = _vnext_readable_trace_rows( diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index 4948bc285..aff8f5f22 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -87,6 +87,8 @@ def _workspace_rows(store: PostgresVNextStore, kind: str, rows: Sequence[Mapping def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdentity | None = None) -> dict[str, object]: from alicebot_api.vnext_label_guard import LabelGuard, sensitivity_ceiling + from alicebot_api.vnext_open_loop_references import withhold_unreadable_references + from alicebot_api.vnext_source_fence import SourceReadFence sensitivity_allowed = ["public", "internal", "private", "unknown"] ceiling = sensitivity_ceiling(identity) @@ -127,6 +129,7 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti project_count = sum(guard.readable_status_counts("project").values()) fetched_loops = store.list_open_loops(status=None, sensitivity_allowed=sensitivity_allowed, limit=30) open_loops = guard.admit_rows("open_loop", fetched_loops) + open_loops = withhold_unreadable_references(store, open_loops, fence=SourceReadFence.for_identity(identity)) open_loop_status_counts = guard.readable_status_counts("open_loop") open_loop_count = open_loop_status_counts.get("open", 0) people = store.list_people(sensitivity_allowed=sensitivity_allowed, limit=12) diff --git a/apps/api/src/alicebot_api/vnext_projects.py b/apps/api/src/alicebot_api/vnext_projects.py index df7e58ef8..ff928c6f4 100644 --- a/apps/api/src/alicebot_api/vnext_projects.py +++ b/apps/api/src/alicebot_api/vnext_projects.py @@ -1372,6 +1372,8 @@ def project_dashboard( ) -> JsonObject: from alicebot_api.vnext_agent_control import AgentIdentity from alicebot_api.vnext_label_guard import admit_loaded, apply_sensitivity_ceiling + from alicebot_api.vnext_open_loop_references import withhold_unreadable_references + from alicebot_api.vnext_source_fence import SourceReadFence project = self.store.get_project(project_id) if project is None: @@ -1421,6 +1423,7 @@ def project_dashboard( sensitivity_allowed=sensitivity_allowed, projects=(project_id,), ) + open_loops = withhold_unreadable_references(self.store, open_loops, fence=SourceReadFence.for_identity(caller)) artifacts = admit_loaded( self.store, kind="artifact", diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py index beaeb4f87..8a9b0cb00 100644 --- a/tests/integration/test_derived_labels_read_acceptance_postgres.py +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -286,7 +286,32 @@ def test_direct_column_loop_references_reach_real_read_guard(migrated_database_u assert count_guard.readable_status_counts("open_loop").get("open", 0) == int(source_sensitivity == "public") if key: monkeypatch.setenv("ALICE_AGENT_API_KEY", key) - result = call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name="alice_open_loops", arguments={"sensitivity_allowed": list(ALL_SENSITIVITY)}) - rendered = json.dumps(result, default=str) - assert (str(loop["id"]) in rendered) is admitted - assert ("Direct loop sentinel" in rendered) is admitted + for tool in ("alice_open_loops", "alice_resume"): + result = call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name=tool, arguments={"sensitivity_allowed": list(ALL_SENSITIVITY)}) + rendered = json.dumps(result, default=str) + assert (str(loop["id"]) in rendered) is admitted, tool + assert ("Direct loop sentinel" in rendered) is admitted, tool + + +@pytest.mark.parametrize("reader", ("owner", "admin", "trusted")) +def test_workspace_and_dashboard_original_loop_references_use_actual_key(migrated_database_urls, monkeypatch, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + for module in (workspaces, vnext_projects, vnext_retrieval): + monkeypatch.setattr(module, "get_settings", lambda: Settings(database_url=app_url)) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + project = store.create_project({"name": "Visible project", "slug": "visible-project", "domain": "project", "sensitivity": "public"}) + project_id = str(project["id"]) + source = store.create_source({"source_type": "note", "title": "Restricted parent", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential", "metadata_json": {"project_scope": [project_id]}}) + visible_source = store.create_source({"source_type": "note", "title": "Visible source", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [project_id]}}) + with without_insert_floor(): + memory = store.create_memory({"memory_key": "hidden-backing", "canonical_text": "Restricted backing memory", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"]), "project_scope": [project_id]}}) + loop = store.create_open_loop({"title": "Visible original loop", "source_id": str(visible_source["id"]), "memory_id": str(memory["id"]), "project_id": project_id, "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [project_id]}}) + key = real_reader_key(store, user_id, reader) + auth = f"Bearer {key}" if key else None + responses = (workspaces.get_vnext_workspace(user_id, authorization=auth), vnext_projects.get_vnext_project_dashboard(project_id, user_id, authorization=auth), vnext_retrieval.get_vnext_source_trace(UUID(str(visible_source["id"])), user_id, authorization=auth)) + for response in responses: + assert response.status_code == 200, response.body + item = next(row for row in json.loads(response.body)["open_loops"] if str(row["id"]) == str(loop["id"])) + assert item["memory_id"] == (None if reader == "trusted" else str(memory["id"])) diff --git a/tests/unit/test_label_reader_stage_matrix.py b/tests/unit/test_label_reader_stage_matrix.py index 0d2193d61..cf882591c 100644 --- a/tests/unit/test_label_reader_stage_matrix.py +++ b/tests/unit/test_label_reader_stage_matrix.py @@ -66,7 +66,7 @@ def list_edges(self, **kwargs): return [{"from_type": "memory", "from_id": MEMOR def _scope(scoped=False): return _ResolvedRetrievalScope(projects=frozenset((ALPHA,)) if scoped else frozenset(), people=frozenset(), window_start=None, window_end=None, exclude_global_domains=frozenset()) -STAGES = ("by_ids", "by_ids_fallback", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "pack_open_loops", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs") +STAGES = ("by_ids", "by_ids_fallback", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "pack_open_loops", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs", "workspace_loop_refs", "dashboard_loop_refs") def _stage(stage, store): service = VNextRetrievalService(store, embedding_provider=SimpleNamespace(provider="synthetic", model="synthetic", base_url="http://synthetic.invalid")) @@ -97,6 +97,13 @@ def _stage(stage, store): if stage == "dashboard_lists": result = VNextProjectService(store).project_dashboard(project_id=ALPHA, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent")) return [*result["memories"], *result["open_loops"], *result["artifacts"]] + if stage.endswith("loop_refs"): + store.rows["open_loop"][0].update(title="Visible original loop", memory_id=MEMORY, source_id=None, metadata_json={"project_scope": [ALPHA]}) + if stage == "workspace_loop_refs": + body = workspaces._vnext_workspace_payload(store, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent")) + else: + body = VNextProjectService(store).project_dashboard(project_id=ALPHA, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent")) + return [body["open_loops"][0]["memory_id"]] if body["open_loops"][0]["memory_id"] else [] if stage.startswith("workspace_"): field = stage.removeprefix("workspace_") if field == "memories": diff --git a/tests/unit/test_open_loop_references_read_fence.py b/tests/unit/test_open_loop_references_read_fence.py index dcba0624a..7d5edb180 100644 --- a/tests/unit/test_open_loop_references_read_fence.py +++ b/tests/unit/test_open_loop_references_read_fence.py @@ -1285,17 +1285,17 @@ def test_the_reference_keys_include_every_key_the_reverse_lookup_of_a_source_rea ("list_open_loops", "mcp/projects.py", "_handle_alice_vnext_open_loops"): _FENCED, ("list_open_loops", "mcp/retrieval.py", "_vnext_resume"): _ALLOWLIST, ("list_open_loops", "memory.py", "list_open_loop_records"): _OWNER, - ("list_open_loops", "routers/workspaces.py", "_vnext_workspace_payload"): _OWNER, + ("list_open_loops", "routers/workspaces.py", "_vnext_workspace_payload"): _FENCED, ("list_open_loops", "session_briefing.py", "compile_session_brief"): _ALLOWLIST, ("list_open_loops", "vnext_context_tree.py", "build_tree"): _ALLOWLIST, ("list_open_loops", "vnext_dogfooding.py", "dashboard"): _OPERATOR, - ("list_open_loops", "vnext_projects.py", "project_dashboard"): _PRODUCER, + ("list_open_loops", "vnext_projects.py", "project_dashboard"): _FENCED, ("list_open_loops", "vnext_scheduler.py", "_generate_open_loop_review_artifact"): _FENCED, - ("list_open_loops_referencing_source", "routers/_vnext_shared.py", "_vnext_load_source_trace"): _SENSITIVITY, + ("list_open_loops_referencing_source", "routers/_vnext_shared.py", "_vnext_load_source_trace"): _FENCED, ("project_dashboard", "cli/automation.py", "_run_vnext_project_dashboard"): _OPERATOR, ("project_dashboard", "mcp/projects.py", "_handle_alice_project_dashboard"): _OWNER, ("project_dashboard", "routers/vnext_projects.py", "get_vnext_project_dashboard"): _SENSITIVITY, - ("project_dashboard", "routers/workspaces.py", "_vnext_workspace_payload"): _OWNER, + ("project_dashboard", "routers/workspaces.py", "_vnext_workspace_payload"): _FENCED, ("review_open_loop", "cli/automation.py", "_run_vnext_open_loop_review"): _OPERATOR, ("review_open_loop", "mcp/projects.py", "_handle_alice_open_loop_review"): _OWNER, ("review_open_loop", "mcp/retrieval.py", "_handle_alice_open_loops"): _FENCED, From 059fbad39d2f07bc784735b75291451df2a37478 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:14:02 +0200 Subject: [PATCH 174/270] Enable strict locks for original PostgreSQL behavior proofs --- .../test_derived_labels_main_behavior_postgres.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/integration/test_derived_labels_main_behavior_postgres.py b/tests/integration/test_derived_labels_main_behavior_postgres.py index c76fbe109..c7f5cdddd 100644 --- a/tests/integration/test_derived_labels_main_behavior_postgres.py +++ b/tests/integration/test_derived_labels_main_behavior_postgres.py @@ -1,12 +1,24 @@ """Execute the original stale-label behaviors even before the new kernel exists.""" +from importlib.util import find_spec from uuid import uuid4 +import pytest + from alicebot_api.db import user_connection from alicebot_api.store import ContinuityStore from alicebot_api.vnext_store import PostgresVNextStore +@pytest.fixture(autouse=True) +def strict_when_the_label_writer_exists(monkeypatch): + # Archived main has no writer module; its real stale-row behavior still runs. + if find_spec("alicebot_api.vnext_label_writes") is not None: + from alicebot_api import vnext_label_writes + + monkeypatch.setattr(vnext_label_writes, "STRICT_LOCK_ORDER", True) + + def test_source_relabel_reaches_an_existing_report_on_main(migrated_database_urls): user_id = uuid4() with user_connection(migrated_database_urls["app"], user_id) as conn: From 2369d8f504b71001f97e2180c5945a7a5007cd60 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:15:43 +0200 Subject: [PATCH 175/270] Provide recorded label inputs in the semantic rollup fixture --- tests/unit/test_vnext_rollups_semantic.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/test_vnext_rollups_semantic.py b/tests/unit/test_vnext_rollups_semantic.py index 85ad78280..4e2ba5d45 100644 --- a/tests/unit/test_vnext_rollups_semantic.py +++ b/tests/unit/test_vnext_rollups_semantic.py @@ -22,6 +22,7 @@ from alicebot_api.sqlite_schema import bootstrap_sqlite_schema from alicebot_api.sqlite_store import SQLiteVNextStore, ensure_sqlite_user +from alicebot_api.vnext_derived_labels import identifier from alicebot_api.vnext_embeddings import memory_embedding_text from alicebot_api.vnext_repositories import JsonObject from alicebot_api.vnext_rollups import ( @@ -61,6 +62,13 @@ def create_memory(self, memory: JsonObject, *, actor_type: str = "system") -> Js def list_memories(self, *, status: str | None = None) -> list[JsonObject]: return [dict(row) for row in self.memories if status is None or row.get("status") == status] + def read_label_rows(self, kind: str, ids) -> list[JsonObject]: + """Expose recorded inputs so an existing card's label can be verified.""" + if kind != "memory": + return [] + wanted = {identifier(item) for item in ids} + return [dict(row) for row in self.memories if identifier(row.get("id")) in wanted] + @staticmethod def _in_scope( row: JsonObject, From e31330e1fd2c89f53ec05e101a2c8486ab49b630 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:14:55 +0200 Subject: [PATCH 176/270] Scope historical migration fixtures to their declared identities --- .../integration/test_migration_0087_retry.py | 3 + tests/integration/test_migrations.py | 127 ++++++++++++++++-- 2 files changed, 116 insertions(+), 14 deletions(-) diff --git a/tests/integration/test_migration_0087_retry.py b/tests/integration/test_migration_0087_retry.py index 4a1c2800d..caeb0a8a5 100644 --- a/tests/integration/test_migration_0087_retry.py +++ b/tests/integration/test_migration_0087_retry.py @@ -5,6 +5,7 @@ import pytest from alicebot_api.migrations import make_alembic_config +from tests.integration.test_migrations import _set_fixture_identity _PARTIALLY_COMMITTED_SCHEMA = ( @@ -90,6 +91,7 @@ def test_0087_retries_after_committed_ddl_and_invalid_concurrent_unique_index( user_id = "00000000-0000-0000-0000-000000008701" with psycopg.connect(database_url) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: for statement in _PARTIALLY_COMMITTED_SCHEMA: cur.execute(statement) @@ -129,6 +131,7 @@ def test_0087_retries_after_committed_ddl_and_invalid_concurrent_unique_index( # catalog row. This models an operator correcting the build cause before # rerunning the still-unapplied Alembic revision. with psycopg.connect(database_url) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index 59320ba04..f276dec51 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -6,6 +6,7 @@ from psycopg.types.json import Jsonb import pytest from uuid import UUID +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from alicebot_api.db import user_connection from alicebot_api.migrations import make_alembic_config @@ -15,6 +16,38 @@ from alicebot_api.vnext_store import PostgresVNextStore +def _fixture_migration_config(database_url, *, user_id=None, user_account_id=None): + """Migrate one declared historical fixture identity under unchanged RLS. + + These data-bearing fixtures contain one user or one account. The empty + full-schema smoke and the multi-user 0096 acceptance keep their unbound + migrator connections. + """ + parsed = urlsplit(database_url) + query = dict(parse_qsl(parsed.query, keep_blank_values=True)) + options = [query.get("options", "")] + for setting, identity in ( + ("app.current_user_id", user_id), + ("app.current_user_account_id", user_account_id), + ): + if identity is not None: + options.append(f"-c {setting}={UUID(str(identity))}") + query["options"] = " ".join(option for option in options if option) + config = make_alembic_config(database_url) + config.attributes["explicit_database_url"] = urlunsplit(parsed._replace(query=urlencode(query))) + return config + + +def _set_fixture_identity(conn, *, user_id=None, user_account_id=None): + """Keep historical fixture reads and writes inside the existing RLS policy.""" + for setting, identity in ( + ("app.current_user_id", user_id), + ("app.current_user_account_id", user_account_id), + ): + if identity is not None: + conn.execute("SELECT set_config(%s, %s, %s)", (setting, str(identity), not conn.autocommit)) + + def test_vnext_kernel_upgrade_backfills_data_bearing_append_only_revisions(database_urls): """A real pre-vNext row must survive 0066 -> 0067. @@ -23,13 +56,14 @@ def test_vnext_kernel_upgrade_backfills_data_bearing_append_only_revisions(datab data-bearing upgrade path that an empty-schema migration smoke cannot exercise. """ - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000101" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) memory_id = "00000000-0000-0000-0000-000000000102" revision_id = "00000000-0000-0000-0000-000000000103" command.upgrade(config, "20260416_0066") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -62,6 +96,7 @@ def test_vnext_kernel_upgrade_backfills_data_bearing_append_only_revisions(datab command.upgrade(config, "20260510_0067") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -96,14 +131,15 @@ def test_vnext_kernel_upgrade_backfills_data_bearing_append_only_revisions(datab def test_lifecycle_invariant_upgrade_canonicalizes_retry_ids_and_installs_edge_trigger(database_urls): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000111" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) first_id = "00000000-0000-0000-0000-000000000112" second_id = "00000000-0000-0000-0000-000000000113" edge_id = "00000000-0000-0000-0000-000000000114" command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -144,6 +180,7 @@ def test_lifecycle_invariant_upgrade_canonicalizes_retry_ids_and_installs_edge_t command.upgrade(config, "20260711_0083") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -205,18 +242,20 @@ def test_lifecycle_invariant_upgrade_keeps_identifiers_on_live_row_over_tombston row; stranding it on the tombstone makes replay return nothing while the partial unique index blocks re-insertion of the same key. """ - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000131" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) tombstone_id = "00000000-0000-0000-0000-000000000132" live_id = "00000000-0000-0000-0000-000000000133" command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) _seed_tombstone_and_live_duplicate(conn, user_id=user_id, tombstone_id=tombstone_id, live_id=live_id) command.upgrade(config, "head") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -251,19 +290,21 @@ def test_lifecycle_identifier_repair_corrects_database_mis_upgraded_by_0083(data move it onto the oldest live row, and be safe to re-run on already-corrected data. """ - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000141" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) tombstone_id = "00000000-0000-0000-0000-000000000142" live_id = "00000000-0000-0000-0000-000000000143" command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) _seed_tombstone_and_live_duplicate(conn, user_id=user_id, tombstone_id=tombstone_id, live_id=live_id) # Apply only the shipped (buggy) 0083 and document the mis-assignment. command.upgrade(config, "20260711_0083") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -284,6 +325,7 @@ def test_lifecycle_identifier_repair_corrects_database_mis_upgraded_by_0083(data def _assert_corrected() -> None: with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -323,14 +365,15 @@ def test_released_0084_database_upgrades_through_current_head(database_urls): at the deleted former holder. Existing v0.9.4 databases will never rerun 0084, so only the new 0086 revision may repair that stale pointer. """ - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000151" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) tombstone_id = "00000000-0000-0000-0000-000000000152" canonical_live_id = "00000000-0000-0000-0000-000000000153" later_live_id = "00000000-0000-0000-0000-000000000154" command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) _seed_tombstone_and_live_duplicate( conn, user_id=user_id, @@ -362,6 +405,7 @@ def test_released_0084_database_upgrades_through_current_head(database_urls): command.upgrade(config, "20260712_0084") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute("SELECT version_num FROM alembic_version") assert cur.fetchone()["version_num"] == "20260712_0084" @@ -386,6 +430,7 @@ def test_released_0084_database_upgrades_through_current_head(database_urls): def _assert_all_pointers_truthful() -> None: with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -412,6 +457,7 @@ def _assert_all_pointers_truthful() -> None: _assert_all_pointers_truthful() with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute("SELECT version_num FROM alembic_version") assert cur.fetchone()["version_num"] == "20261005_0096" @@ -428,13 +474,14 @@ def _assert_all_pointers_truthful() -> None: def test_migration_0088_supports_rolling_provider_writes_and_rejects_token_rewind( database_urls, ): - config = make_alembic_config(database_urls["admin"]) user_account_id = "00000000-0000-0000-0000-000000000131" + config = _fixture_migration_config(database_urls["admin"], user_account_id=user_account_id) workspace_id = "00000000-0000-0000-0000-000000000132" provider_id = "00000000-0000-0000-0000-000000000133" command.upgrade(config, "20260713_0087") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_account_id=user_account_id) with conn.cursor() as cur: cur.execute( """ @@ -452,6 +499,13 @@ def test_migration_0088_supports_rolling_provider_writes_and_rejects_token_rewin """, (workspace_id, user_account_id), ) + cur.execute( + """ + INSERT INTO workspace_members (workspace_id, user_account_id, role) + VALUES (%s, %s, 'owner') + """, + (workspace_id, user_account_id), + ) cur.execute( """ INSERT INTO model_providers ( @@ -508,6 +562,7 @@ def test_migration_0088_supports_rolling_provider_writes_and_rejects_token_rewin autocommit=True, row_factory=dict_row, ) as conn: + _set_fixture_identity(conn, user_account_id=user_account_id) with conn.cursor() as cur: cur.execute( """ @@ -734,13 +789,14 @@ def test_migration_0088_supports_rolling_provider_writes_and_rejects_token_rewin def test_lifecycle_upgrade_promotes_and_reads_legacy_nested_multi_project_scope(database_urls): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000121" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) memory_id = "00000000-0000-0000-0000-000000000122" canonical_memory_id = "00000000-0000-0000-0000-000000000123" command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -779,6 +835,7 @@ def test_lifecycle_upgrade_promotes_and_reads_legacy_nested_multi_project_scope( ) with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) legacy_store = PostgresVNextStore(conn) legacy_row = legacy_store.get_memory(memory_id) assert legacy_row is not None @@ -794,6 +851,7 @@ def test_lifecycle_upgrade_promotes_and_reads_legacy_nested_multi_project_scope( command.upgrade(config, "20260711_0083") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "SELECT project_id, metadata_json FROM memories WHERE id = %s", @@ -836,8 +894,8 @@ def test_pre_lifecycle_upgrade_preserves_present_canonical_project_scope_to_head ): """0082 rows must not resurrect stale nested scope while upgrading to head.""" - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000124" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) row_ids = { "empty": "00000000-0000-0000-0000-000000000125", "null": "00000000-0000-0000-0000-000000000126", @@ -872,6 +930,7 @@ def test_pre_lifecycle_upgrade_preserves_present_canonical_project_scope_to_head command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -901,6 +960,7 @@ def test_pre_lifecycle_upgrade_preserves_present_canonical_project_scope_to_head command.upgrade(config, "head") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -956,14 +1016,15 @@ def test_pre_lifecycle_upgrade_preserves_unicode_project_whitespace_exactly_to_h ): """0083 must not reinterpret Unicode whitespace as the ASCII contract.""" - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000130" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) memory_id = "00000000-0000-0000-0000-000000000131" unicode_scope = "\u2003Alice\u2003" metadata = {"agentic_memory": {"project_scope": [unicode_scope]}} command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -986,6 +1047,7 @@ def test_pre_lifecycle_upgrade_preserves_unicode_project_whitespace_exactly_to_h command.upgrade(config, "head") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "SELECT project_id, metadata_json FROM memories WHERE id = %s", @@ -999,8 +1061,8 @@ def test_pre_lifecycle_upgrade_preserves_unicode_project_whitespace_exactly_to_h def test_tool_execution_task_step_linkage_migration_backfills_existing_rows(database_urls): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000001" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) thread_id = "00000000-0000-0000-0000-000000000002" trace_id = "00000000-0000-0000-0000-000000000003" tool_id = "00000000-0000-0000-0000-000000000004" @@ -1012,6 +1074,7 @@ def test_tool_execution_task_step_linkage_migration_backfills_existing_rows(data command.upgrade(config, "20260313_0020") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1216,6 +1279,7 @@ def test_tool_execution_task_step_linkage_migration_backfills_existing_rows(data command.upgrade(config, "head") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1241,13 +1305,14 @@ def test_tool_execution_task_step_linkage_migration_backfills_existing_rows(data def test_gmail_account_credentials_migration_round_trip_preserves_tokens(database_urls): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000101" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) gmail_account_id = "00000000-0000-0000-0000-000000000102" command.upgrade(config, "20260316_0026") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1284,6 +1349,7 @@ def test_gmail_account_credentials_migration_round_trip_preserves_tokens(databas command.upgrade(config, "20260316_0027") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1315,6 +1381,7 @@ def test_gmail_account_credentials_migration_round_trip_preserves_tokens(databas command.downgrade(config, "20260316_0026") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1342,13 +1409,14 @@ def test_gmail_account_credentials_migration_round_trip_preserves_tokens(databas def test_gmail_refresh_token_lifecycle_migration_round_trip_preserves_downgrade_compatibility( database_urls, ): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000201" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) gmail_account_id = "00000000-0000-0000-0000-000000000202" command.upgrade(config, "20260316_0027") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1403,6 +1471,7 @@ def test_gmail_refresh_token_lifecycle_migration_round_trip_preserves_downgrade_ command.upgrade(config, "20260316_0028") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1440,6 +1509,7 @@ def test_gmail_refresh_token_lifecycle_migration_round_trip_preserves_downgrade_ command.downgrade(config, "20260316_0027") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1462,13 +1532,14 @@ def test_gmail_refresh_token_lifecycle_migration_round_trip_preserves_downgrade_ def test_gmail_external_secret_manager_migration_round_trip_preserves_legacy_transition_rows( database_urls, ): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000301" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) gmail_account_id = "00000000-0000-0000-0000-000000000302" command.upgrade(config, "20260316_0028") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1527,6 +1598,7 @@ def test_gmail_external_secret_manager_migration_round_trip_preserves_legacy_tra command.upgrade(config, "20260316_0029") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1550,6 +1622,7 @@ def test_gmail_external_secret_manager_migration_round_trip_preserves_legacy_tra command.downgrade(config, "20260316_0028") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1570,14 +1643,15 @@ def test_gmail_external_secret_manager_migration_round_trip_preserves_legacy_tra def test_calendar_account_migration_round_trip_preserves_table_shape(database_urls): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000401" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) calendar_account_id = "00000000-0000-0000-0000-000000000402" command.upgrade(config, "20260316_0029") command.upgrade(config, "20260319_0030") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1633,6 +1707,7 @@ def test_calendar_account_migration_round_trip_preserves_table_shape(database_ur conn.commit() with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1658,6 +1733,7 @@ def test_calendar_account_migration_round_trip_preserves_table_shape(database_ur command.downgrade(config, "20260316_0029") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute("SELECT to_regclass('public.calendar_account_credentials')") assert cur.fetchone() == (None,) @@ -2323,6 +2399,7 @@ def test_project_scope_identity_upgrade_repairs_dedupe_without_widening_empty_sc config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260713_0089") user_id = "00000000-0000-0000-0000-000000000301" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) source_a = "00000000-0000-0000-0000-000000000302" source_b = "00000000-0000-0000-0000-000000000303" memory_id = "00000000-0000-0000-0000-000000000304" @@ -2348,6 +2425,7 @@ def test_project_scope_identity_upgrade_repairs_dedupe_without_widening_empty_sc ) with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -2450,6 +2528,7 @@ def test_project_scope_identity_upgrade_repairs_dedupe_without_widening_empty_sc sensitivity="private", ) with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -2545,6 +2624,7 @@ def test_project_scope_identity_upgrade_resolves_all_legacy_source_forms_and_blo config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260713_0089") user_id = "00000000-0000-0000-0000-000000000331" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) target_scope = "Legacy Project" cases = { "root_canonical": {"project_scope": [f" {target_scope} "]}, @@ -2563,6 +2643,7 @@ def test_project_scope_identity_upgrade_resolves_all_legacy_source_forms_and_blo raw_texts = {name: f"Fact: {name} keeps its migrated source scope." for name in cases} with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -2592,6 +2673,7 @@ def test_project_scope_identity_upgrade_resolves_all_legacy_source_forms_and_blo command.upgrade(config, "head") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -2638,6 +2720,7 @@ def test_project_scope_identity_upgrade_keeps_present_empty_nested_source_scope_ config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260713_0089") user_id = "00000000-0000-0000-0000-000000000351" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) stale_project = "Legacy Project" cases: dict[str, dict[str, object]] = { "nested_blank": { @@ -2666,6 +2749,7 @@ def test_project_scope_identity_upgrade_keeps_present_empty_nested_source_scope_ raw_texts = {name: f"Fact: migration {name} preserves nested scope presence." for name in cases} with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -2695,6 +2779,7 @@ def test_project_scope_identity_upgrade_keeps_present_empty_nested_source_scope_ command.upgrade(config, "20260714_0090") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -2733,6 +2818,7 @@ def test_project_scope_identity_upgrade_matches_python_strip_and_blocks_unicode_ config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260713_0089") user_id = "00000000-0000-0000-0000-000000000341" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) project_scope = ("Legacy Project",) raw_texts = { "nbsp": "\u00a0Fact: NBSP boundary\u00a0", @@ -2742,6 +2828,7 @@ def test_project_scope_identity_upgrade_matches_python_strip_and_blocks_unicode_ source_ids = {name: f"00000000-0000-0000-0005-{index:012d}" for index, name in enumerate(raw_texts, start=1)} with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -2785,6 +2872,7 @@ def test_project_scope_identity_upgrade_matches_python_strip_and_blocks_unicode_ for name, raw_text in raw_texts.items() } with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -2819,6 +2907,7 @@ def test_source_identity_0091_clears_only_live_whitespace_strings_and_installs_e config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260714_0090") user_id = "00000000-0000-0000-0007-000000000001" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) whitespace_cases = { "ascii": " \t\r\n", "unit_separator_control": "\u001c\u001f", @@ -2835,6 +2924,7 @@ def test_source_identity_0091_clears_only_live_whitespace_strings_and_installs_e deleted_id = "00000000-0000-0000-0007-000000000023" with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -2919,6 +3009,7 @@ def test_source_identity_0091_clears_only_live_whitespace_strings_and_installs_e def identity_snapshot() -> dict[str, str | None]: with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -2943,6 +3034,7 @@ def identity_snapshot() -> dict[str, str | None]: assert identity_snapshot() == expected with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -3018,6 +3110,7 @@ def identity_snapshot() -> dict[str, str | None]: # Re-crossing the forward boundary is data-idempotent. command.downgrade(config, "20260714_0090") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -3041,6 +3134,7 @@ def test_0092_backfills_prior_authorized_project_update_redaction(database_urls) config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260715_0091") user_id = "00000000-0000-0000-0092-000000000001" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) memory_id = "00000000-0000-0000-0092-000000000002" artifact_id = "00000000-0000-0000-0092-000000000003" revision_id = "00000000-0000-0000-0092-000000000004" @@ -3051,6 +3145,7 @@ def test_0092_backfills_prior_authorized_project_update_redaction(database_urls) sentinel = "0092-OLD-REDACTION-SECRET" with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -3212,6 +3307,7 @@ def test_0092_backfills_prior_authorized_project_update_redaction(database_urls) command.upgrade(config, "head") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -3311,6 +3407,7 @@ def test_0092_backfills_prior_authorized_project_update_redaction(database_urls) command.downgrade(config, "20260715_0091") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -3330,6 +3427,7 @@ def test_0092_backfills_prior_authorized_project_update_redaction(database_urls) assert cur.fetchone() == {"content_markdown": "[REDACTED]"} command.upgrade(config, "head") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -3371,6 +3469,7 @@ def test_postgres_source_constraints_reject_noncanonical_classifications(databas command.upgrade(config, "head") user_id = "00000000-0000-0000-0007-000000000030" with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", From af921d2d7938c045a3d5bfbfa5ce746d0479110d Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:16:47 +0200 Subject: [PATCH 177/270] Separate original saved quotes from derived-copy refusal controls --- ...test_saved_quotes_follow_the_source_fence.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_saved_quotes_follow_the_source_fence.py b/tests/unit/test_saved_quotes_follow_the_source_fence.py index c1d8a66b5..8b238b645 100644 --- a/tests/unit/test_saved_quotes_follow_the_source_fence.py +++ b/tests/unit/test_saved_quotes_follow_the_source_fence.py @@ -1161,8 +1161,8 @@ def test_a_source_with_no_project_is_outside_the_fence_of_a_key_bound_to_a_proje what ``alice_explain`` has always done for such a key. A key bound to no project reads it. On v0.20.0 the review returned all three to a key bound to a project. The release notes say this in one sentence. - The memory is planted with the two writes a review makes (the metadata copy and a link), as the lifecycle tests of the - review door do. + The memory is an original row with the two saved provenance copies that a review makes, as the lifecycle tests of + this projection do. The derived-copy control below separately verifies refusal of the whole captured copy. Mutation: pass ``require_explicit_project_scope=False`` in ``SourceReadFence._admits``: the keys bound to ``alpha`` read the link, the quote and the id again, and disagree with explain. @@ -1173,6 +1173,7 @@ def test_a_source_with_no_project_is_outside_the_fence_of_a_key_bound_to_a_proje assert json.loads(row["metadata_json"])["project_scope"] == [], "the owner's capture belongs to no project" candidate = vault._candidate("Projectlessnote: the alpha kiln shelf is cleaned on Fridays") _plant_saved_quote(vault, candidate, source_id) + vault._original_quote_fixture(candidate) for who in _KEY_SPECS: answer = vault.review(who, candidate) assert answer["is_error"] is False, (who, answer) @@ -1184,6 +1185,18 @@ def test_a_source_with_no_project_is_outside_the_fence_of_a_key_bound_to_a_proje +def test_a_derived_copy_of_a_projectless_source_is_denied_to_bound_keys(vault: _Vault) -> None: + source_id = vault.capture_source(as_owner=True) + candidate = vault._candidate("Projectlesscopy: the alpha kiln shelf is cleaned on Fridays") + _plant_saved_quote(vault, candidate, source_id) + for who in _KEY_SPECS: + answer = vault.review(who, candidate) + readable = who == "unbound" + assert answer["is_error"] is not readable, (who, answer) + assert _holds_quote(answer) is readable, who + assert _holds_source_id(answer, source_id) is readable, who + + def test_current_derived_copy_is_denied_as_a_whole_after_source_relabel(vault: _Vault) -> None: source_id = vault.capture_source() memory_id, _query = vault.edit_and_approve(source_id) From 3718799884717fa124fcc91c3197e8a09964f9fa Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:17:43 +0200 Subject: [PATCH 178/270] Retain original project identity in effective label lists --- .../api/src/alicebot_api/vnext_label_guard.py | 9 ++- ...t_label_guard_project_identity_postgres.py | 36 +++++++++++ .../unit/test_label_guard_project_identity.py | 61 +++++++++++++++++++ tests/unit/test_vnext_context_tree.py | 10 +++ tests/unit/test_vnext_date_precompute.py | 7 +++ 5 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_label_guard_project_identity_postgres.py create mode 100644 tests/unit/test_label_guard_project_identity.py diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index a9e4de7db..e229bec93 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -26,7 +26,7 @@ settle_labels, ) from alicebot_api.vnext_label_closure import collect_label_rows -from alicebot_api.vnext_project_scope import project_floor_shape, project_scopes_overlap, resolve_project_scope +from alicebot_api.vnext_project_scope import project_floor_shape, project_scope_identity, project_scopes_overlap, resolve_project_scope _GUARD_USER = "label-guard" @@ -226,7 +226,12 @@ def _admits_effective(self, row: Mapping[str, object], *, kind: str) -> bool: if self.projects: from alicebot_api.vnext_project_scope import source_project_scope - scope = source_project_scope(row) if canon_kind(kind) == "source" else resolve_project_scope(row).values + resolution = resolve_project_scope(row) + scope = source_project_scope(row) if canon_kind(kind) == "source" else resolution.values + if canon_kind(kind) == "project" and not resolution.present and not is_derived(kind, row): + # Original project rows are selected by ID, slug or name. A + # canonical or effective derived scope remains authoritative. + scope = project_scope_identity([*scope, str(row.get("id") or ""), row.get("slug"), row.get("name")]) _shape, floor = project_floor_shape(row) if not project_scopes_overlap(scope, self.projects, floor=floor): return False diff --git a/tests/integration/test_label_guard_project_identity_postgres.py b/tests/integration/test_label_guard_project_identity_postgres.py new file mode 100644 index 000000000..6cad4dbd2 --- /dev/null +++ b/tests/integration/test_label_guard_project_identity_postgres.py @@ -0,0 +1,36 @@ +"""Persisted original project aliases retain their scoped context-tree behavior.""" +from __future__ import annotations + +from uuid import uuid4 + +import pytest + +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_context_tree import ContextTreeRequest, VNextContextTreeService +from alicebot_api.vnext_label_guard import LabelGuard +from alicebot_api.vnext_store import PostgresVNextStore + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +@pytest.mark.parametrize("field", ("id", "slug", "name")) +def test_persisted_original_project_matches_its_id_slug_or_name(migrated_database_urls, field: str) -> None: + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"project-{user_id}@example.invalid", "Synthetic Project Reader") + store = PostgresVNextStore(conn) + project = store.create_project({"name": "Project Atlas", "slug": "launch-plan", "domain": "project", "sensitivity": "public"}) + assert "project_scope" not in project + assert "project_scope" not in project["metadata_json"] + scope = (str(project[field]).upper(),) + payload = VNextContextTreeService(store).build_tree(ContextTreeRequest(projects=scope, include_events=False)) + assert payload["summary"]["projects"] == 1 + assert payload["roots"][0]["children"][0]["ref"] == f"project:{project['id']}" + canonical = store.create_project({"name": "Canonical Project", "slug": ALPHA, "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": [BETA], "project_floor": [BETA]}}) + guard = LabelGuard.for_filters(store, None, ("public",), (ALPHA,)) + assert guard.admit_rows("project", [canonical]) == [] + guard = LabelGuard.for_filters(store, None, ("public",), (BETA,)) + assert guard.admit_rows("project", [canonical]) == [canonical] diff --git a/tests/unit/test_label_guard_project_identity.py b/tests/unit/test_label_guard_project_identity.py new file mode 100644 index 000000000..ab3053052 --- /dev/null +++ b/tests/unit/test_label_guard_project_identity.py @@ -0,0 +1,61 @@ +"""Original project identities survive filtering without widening derived scope.""" +from __future__ import annotations + +from uuid import UUID + +import pytest + +from alicebot_api.vnext_label_guard import LabelGuard +from alicebot_api.vnext_project_scope import GLOBAL_PROJECT_MARKER + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +def _guard(store: object, projects: tuple[str, ...], *, all_of: tuple[str, ...] | None = None) -> LabelGuard: + return LabelGuard.for_filters(store, None, ("public",), projects, all_of=all_of) + + +@pytest.mark.parametrize("field", ("id", "slug", "name")) +def test_original_project_is_admitted_by_its_own_identity(field: str) -> None: + row = {"id": "project-id", "slug": "other", "name": "Other", "sensitivity": "public", field: ALPHA.upper()} + assert _guard(object(), (ALPHA,)).admit_rows("project", [row]) == [row] + + +def test_original_postgres_uuid_project_id_is_compared_as_text() -> None: + project_id = UUID("12345678-1234-1234-1234-123456abcdef") + row = {"id": project_id, "slug": "other", "sensitivity": "public"} + assert _guard(object(), (str(project_id).upper(),)).admit_rows("project", [row]) == [row] + + +@pytest.mark.parametrize("scope", ([], [BETA])) +@pytest.mark.parametrize("at_root", (False, True)) +def test_canonical_original_project_scope_does_not_widen_to_its_slug(scope: list[str], at_root: bool) -> None: + row = {"id": "project-id", "slug": ALPHA, "sensitivity": "public"} + row["project_scope" if at_root else "metadata_json"] = scope if at_root else {"project_scope": scope} + assert _guard(object(), (ALPHA,)).admit_rows("project", [row]) == [] + + +def test_original_project_identity_does_not_bypass_its_floor_or_all_input_binding() -> None: + row = {"id": "project-id", "slug": "original-project", "sensitivity": "public", "metadata_json": {"project_floor": [BETA]}} + assert _guard(object(), (GLOBAL_PROJECT_MARKER,)).admit_rows("project", [row]) == [] + assert _guard(object(), (GLOBAL_PROJECT_MARKER, BETA)).admit_rows("project", [row]) == [row] + assert _guard(object(), (), all_of=(ALPHA, BETA)).admit_rows("project", [row]) == [] + + +def test_derived_project_slug_does_not_replace_its_effective_input_scope() -> None: + source = {"id": "source-id", "sensitivity": "public", "metadata_json": {"project_scope": [BETA]}} + + class Store: + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + return [source] if kind == "source" and source["id"] in ids else [] + + project = {"id": "project-id", "slug": ALPHA, "sensitivity": "public", "metadata_json": { + "derived_from": {"v": 1, "sources": [source["id"]], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], + "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}}, + }} + assert _guard(Store(), (ALPHA,)).admit_rows("project", [project]) == [] + assert _guard(Store(), (BETA,)).admit_rows("project", [project]) == [] + assert _guard(Store(), (), all_of=(ALPHA,)).admit_rows("project", [project]) == [] + assert _guard(Store(), (), all_of=(BETA,)).admit_rows("project", [project]) == [] + assert _guard(Store(), ()).admit_rows("project", [project]) == [project] diff --git a/tests/unit/test_vnext_context_tree.py b/tests/unit/test_vnext_context_tree.py index 253adf14c..fc4c216bd 100644 --- a/tests/unit/test_vnext_context_tree.py +++ b/tests/unit/test_vnext_context_tree.py @@ -150,6 +150,16 @@ def _limited(rows: list[dict[str, object]], kwargs: Mapping[str, object]) -> lis limit = kwargs.get("limit") return rows[:limit] if isinstance(limit, int) else rows + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + groups = { + "project": self.projects, + "memory": self.memories, + "source": self.sources, + "open_loop": self.open_loops, + "artifact": self.artifacts, + } + return [row for row in groups.get(kind, []) if str(row.get("id")) in ids] + def append_event(self, event: dict[str, object]) -> dict[str, object]: self.events.append(event) return event diff --git a/tests/unit/test_vnext_date_precompute.py b/tests/unit/test_vnext_date_precompute.py index 78c344edb..14fd24649 100644 --- a/tests/unit/test_vnext_date_precompute.py +++ b/tests/unit/test_vnext_date_precompute.py @@ -248,6 +248,13 @@ def list_provenance_links(self, *, target_type, target_id): del target_type, target_id return [] + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + if kind == "source": + return [self.source_rows_by_id[row_id] for row_id in ids if row_id in self.source_rows_by_id] + if kind == "memory": + return [row for row in self.memories if str(row.get("id")) in ids] + return [] + def get_source(self, source_id: str): self.get_source_calls += 1 return self.source_rows_by_id.get(str(source_id)) From dc23247d7e8e61f4f7e5a17e125daad4c4b62c10 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:17:43 +0200 Subject: [PATCH 179/270] Retain original project identity in effective label lists --- .../api/src/alicebot_api/vnext_label_guard.py | 9 ++- ...t_label_guard_project_identity_postgres.py | 36 +++++++++++ .../unit/test_label_guard_project_identity.py | 61 +++++++++++++++++++ tests/unit/test_vnext_context_tree.py | 10 +++ tests/unit/test_vnext_date_precompute.py | 7 +++ 5 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_label_guard_project_identity_postgres.py create mode 100644 tests/unit/test_label_guard_project_identity.py diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index a9e4de7db..e229bec93 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -26,7 +26,7 @@ settle_labels, ) from alicebot_api.vnext_label_closure import collect_label_rows -from alicebot_api.vnext_project_scope import project_floor_shape, project_scopes_overlap, resolve_project_scope +from alicebot_api.vnext_project_scope import project_floor_shape, project_scope_identity, project_scopes_overlap, resolve_project_scope _GUARD_USER = "label-guard" @@ -226,7 +226,12 @@ def _admits_effective(self, row: Mapping[str, object], *, kind: str) -> bool: if self.projects: from alicebot_api.vnext_project_scope import source_project_scope - scope = source_project_scope(row) if canon_kind(kind) == "source" else resolve_project_scope(row).values + resolution = resolve_project_scope(row) + scope = source_project_scope(row) if canon_kind(kind) == "source" else resolution.values + if canon_kind(kind) == "project" and not resolution.present and not is_derived(kind, row): + # Original project rows are selected by ID, slug or name. A + # canonical or effective derived scope remains authoritative. + scope = project_scope_identity([*scope, str(row.get("id") or ""), row.get("slug"), row.get("name")]) _shape, floor = project_floor_shape(row) if not project_scopes_overlap(scope, self.projects, floor=floor): return False diff --git a/tests/integration/test_label_guard_project_identity_postgres.py b/tests/integration/test_label_guard_project_identity_postgres.py new file mode 100644 index 000000000..6cad4dbd2 --- /dev/null +++ b/tests/integration/test_label_guard_project_identity_postgres.py @@ -0,0 +1,36 @@ +"""Persisted original project aliases retain their scoped context-tree behavior.""" +from __future__ import annotations + +from uuid import uuid4 + +import pytest + +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_context_tree import ContextTreeRequest, VNextContextTreeService +from alicebot_api.vnext_label_guard import LabelGuard +from alicebot_api.vnext_store import PostgresVNextStore + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +@pytest.mark.parametrize("field", ("id", "slug", "name")) +def test_persisted_original_project_matches_its_id_slug_or_name(migrated_database_urls, field: str) -> None: + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"project-{user_id}@example.invalid", "Synthetic Project Reader") + store = PostgresVNextStore(conn) + project = store.create_project({"name": "Project Atlas", "slug": "launch-plan", "domain": "project", "sensitivity": "public"}) + assert "project_scope" not in project + assert "project_scope" not in project["metadata_json"] + scope = (str(project[field]).upper(),) + payload = VNextContextTreeService(store).build_tree(ContextTreeRequest(projects=scope, include_events=False)) + assert payload["summary"]["projects"] == 1 + assert payload["roots"][0]["children"][0]["ref"] == f"project:{project['id']}" + canonical = store.create_project({"name": "Canonical Project", "slug": ALPHA, "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": [BETA], "project_floor": [BETA]}}) + guard = LabelGuard.for_filters(store, None, ("public",), (ALPHA,)) + assert guard.admit_rows("project", [canonical]) == [] + guard = LabelGuard.for_filters(store, None, ("public",), (BETA,)) + assert guard.admit_rows("project", [canonical]) == [canonical] diff --git a/tests/unit/test_label_guard_project_identity.py b/tests/unit/test_label_guard_project_identity.py new file mode 100644 index 000000000..ab3053052 --- /dev/null +++ b/tests/unit/test_label_guard_project_identity.py @@ -0,0 +1,61 @@ +"""Original project identities survive filtering without widening derived scope.""" +from __future__ import annotations + +from uuid import UUID + +import pytest + +from alicebot_api.vnext_label_guard import LabelGuard +from alicebot_api.vnext_project_scope import GLOBAL_PROJECT_MARKER + +ALPHA = "prj_" + "a" * 16 +BETA = "prj_" + "b" * 16 + + +def _guard(store: object, projects: tuple[str, ...], *, all_of: tuple[str, ...] | None = None) -> LabelGuard: + return LabelGuard.for_filters(store, None, ("public",), projects, all_of=all_of) + + +@pytest.mark.parametrize("field", ("id", "slug", "name")) +def test_original_project_is_admitted_by_its_own_identity(field: str) -> None: + row = {"id": "project-id", "slug": "other", "name": "Other", "sensitivity": "public", field: ALPHA.upper()} + assert _guard(object(), (ALPHA,)).admit_rows("project", [row]) == [row] + + +def test_original_postgres_uuid_project_id_is_compared_as_text() -> None: + project_id = UUID("12345678-1234-1234-1234-123456abcdef") + row = {"id": project_id, "slug": "other", "sensitivity": "public"} + assert _guard(object(), (str(project_id).upper(),)).admit_rows("project", [row]) == [row] + + +@pytest.mark.parametrize("scope", ([], [BETA])) +@pytest.mark.parametrize("at_root", (False, True)) +def test_canonical_original_project_scope_does_not_widen_to_its_slug(scope: list[str], at_root: bool) -> None: + row = {"id": "project-id", "slug": ALPHA, "sensitivity": "public"} + row["project_scope" if at_root else "metadata_json"] = scope if at_root else {"project_scope": scope} + assert _guard(object(), (ALPHA,)).admit_rows("project", [row]) == [] + + +def test_original_project_identity_does_not_bypass_its_floor_or_all_input_binding() -> None: + row = {"id": "project-id", "slug": "original-project", "sensitivity": "public", "metadata_json": {"project_floor": [BETA]}} + assert _guard(object(), (GLOBAL_PROJECT_MARKER,)).admit_rows("project", [row]) == [] + assert _guard(object(), (GLOBAL_PROJECT_MARKER, BETA)).admit_rows("project", [row]) == [row] + assert _guard(object(), (), all_of=(ALPHA, BETA)).admit_rows("project", [row]) == [] + + +def test_derived_project_slug_does_not_replace_its_effective_input_scope() -> None: + source = {"id": "source-id", "sensitivity": "public", "metadata_json": {"project_scope": [BETA]}} + + class Store: + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + return [source] if kind == "source" and source["id"] in ids else [] + + project = {"id": "project-id", "slug": ALPHA, "sensitivity": "public", "metadata_json": { + "derived_from": {"v": 1, "sources": [source["id"]], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], + "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}}, + }} + assert _guard(Store(), (ALPHA,)).admit_rows("project", [project]) == [] + assert _guard(Store(), (BETA,)).admit_rows("project", [project]) == [] + assert _guard(Store(), (), all_of=(ALPHA,)).admit_rows("project", [project]) == [] + assert _guard(Store(), (), all_of=(BETA,)).admit_rows("project", [project]) == [] + assert _guard(Store(), ()).admit_rows("project", [project]) == [project] diff --git a/tests/unit/test_vnext_context_tree.py b/tests/unit/test_vnext_context_tree.py index 253adf14c..fc4c216bd 100644 --- a/tests/unit/test_vnext_context_tree.py +++ b/tests/unit/test_vnext_context_tree.py @@ -150,6 +150,16 @@ def _limited(rows: list[dict[str, object]], kwargs: Mapping[str, object]) -> lis limit = kwargs.get("limit") return rows[:limit] if isinstance(limit, int) else rows + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + groups = { + "project": self.projects, + "memory": self.memories, + "source": self.sources, + "open_loop": self.open_loops, + "artifact": self.artifacts, + } + return [row for row in groups.get(kind, []) if str(row.get("id")) in ids] + def append_event(self, event: dict[str, object]) -> dict[str, object]: self.events.append(event) return event diff --git a/tests/unit/test_vnext_date_precompute.py b/tests/unit/test_vnext_date_precompute.py index 78c344edb..14fd24649 100644 --- a/tests/unit/test_vnext_date_precompute.py +++ b/tests/unit/test_vnext_date_precompute.py @@ -248,6 +248,13 @@ def list_provenance_links(self, *, target_type, target_id): del target_type, target_id return [] + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + if kind == "source": + return [self.source_rows_by_id[row_id] for row_id in ids if row_id in self.source_rows_by_id] + if kind == "memory": + return [row for row in self.memories if str(row.get("id")) in ids] + return [] + def get_source(self, source_id: str): self.get_source_calls += 1 return self.source_rows_by_id.get(str(source_id)) From 13b21b14b6aef92100885e1edaba0bfa2eb742a2 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:17:50 +0200 Subject: [PATCH 180/270] Pin every dogfooding sample to effective reader admission --- tests/unit/test_label_reader_stage_matrix.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/test_label_reader_stage_matrix.py b/tests/unit/test_label_reader_stage_matrix.py index cf882591c..d595362ed 100644 --- a/tests/unit/test_label_reader_stage_matrix.py +++ b/tests/unit/test_label_reader_stage_matrix.py @@ -11,6 +11,7 @@ from alicebot_api.session_briefing import compile_session_brief from alicebot_api.vnext_agent_control import AgentIdentity from alicebot_api.vnext_context_tree import ContextTreeRequest, VNextContextTreeService +from alicebot_api.vnext_dogfooding import VNextDogfoodingService from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService, VNextProjectValidationError from alicebot_api.vnext_retrieval import VNextRetrievalRequest, VNextRetrievalService, _ResolvedRetrievalScope, expand_provenance_once from alicebot_api.vnext_temporal_query import TemporalAnchor @@ -143,3 +144,14 @@ def context(_ignored): assert any(identifier in str(visible) for identifier in (SOURCE, MEMORY, LOOP, ARTIFACT, PROJECT, str(store.belief["id"]))), (stage, visible) else: assert SECRET in visible + + +@pytest.mark.parametrize("kind", ("sources", "memories", "artifacts", "open_loops", "events")) +def test_each_dogfooding_sample_counts_only_currently_readable_rows(kind, monkeypatch): + _quiet_services(monkeypatch) + store = StageStore() + hidden = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(CEILING)) + assert hidden["sample_scope"][kind]["returned_count"] == 0 + store.rows["source"][0]["sensitivity"] = "public" + visible = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(CEILING)) + assert visible["sample_scope"][kind]["returned_count"] == 1 From aa191488823f5c1c7247f7aa75a92311904d7cd6 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:14:02 +0200 Subject: [PATCH 181/270] Enable strict locks for original PostgreSQL behavior proofs --- ...t_derived_labels_main_behavior_postgres.py | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 tests/integration/test_derived_labels_main_behavior_postgres.py diff --git a/tests/integration/test_derived_labels_main_behavior_postgres.py b/tests/integration/test_derived_labels_main_behavior_postgres.py new file mode 100644 index 000000000..c7f5cdddd --- /dev/null +++ b/tests/integration/test_derived_labels_main_behavior_postgres.py @@ -0,0 +1,71 @@ +"""Execute the original stale-label behaviors even before the new kernel exists.""" + +from importlib.util import find_spec +from uuid import uuid4 + +import pytest + +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_store import PostgresVNextStore + + +@pytest.fixture(autouse=True) +def strict_when_the_label_writer_exists(monkeypatch): + # Archived main has no writer module; its real stale-row behavior still runs. + if find_spec("alicebot_api.vnext_label_writes") is not None: + from alicebot_api import vnext_label_writes + + monkeypatch.setattr(vnext_label_writes, "STRICT_LOCK_ORDER", True) + + +def test_source_relabel_reaches_an_existing_report_on_main(migrated_database_urls): + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user( + user_id, f"main-behavior-{user_id}@example.invalid", "Synthetic main behavior" + ) + store = PostgresVNextStore(conn) + store.lock_graph_mutation() + source = store.create_source( + {"source_type": "note", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public"} + ) + report = store.create_artifact( + { + "artifact_type": "daily_brief", + "title": "Synthetic report", + "content_markdown": "Synthetic report", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"workflow": "daily_brief", "source_refs": [str(source["id"])]}, + } + ) + lock = getattr(store, "lock_label_writes", None) + if callable(lock): + lock(exclusive=True) + store.update_source(source_id=str(source["id"]), patch={"domain": "health", "sensitivity": "confidential"}) + updated = store.get_artifact(str(report["id"])) + assert updated["domain"] == "health" + assert updated["sensitivity"] == "confidential" + + +def test_insert_floor_reads_current_source_labels_on_main(migrated_database_urls): + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"main-floor-{user_id}@example.invalid", "Synthetic main floor") + store = PostgresVNextStore(conn) + source = store.create_source( + {"source_type": "note", "content_hash": str(uuid4()), "domain": "health", "sensitivity": "confidential"} + ) + report = store.create_artifact( + { + "artifact_type": "daily_brief", + "title": "Synthetic report", + "content_markdown": "Synthetic report", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"workflow": "daily_brief", "source_refs": [str(source["id"])]}, + } + ) + assert report["domain"] == "health" + assert report["sensitivity"] == "confidential" From b6e2ca35fe78ad9b0f459f41662672a732b38202 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:56:27 +0200 Subject: [PATCH 182/270] Prove effective admission at both existing rollup readers --- ...est_derived_labels_group_scope_postgres.py | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_derived_labels_group_scope_postgres.py b/tests/integration/test_derived_labels_group_scope_postgres.py index 3b6f07192..0aec1cac4 100644 --- a/tests/integration/test_derived_labels_group_scope_postgres.py +++ b/tests/integration/test_derived_labels_group_scope_postgres.py @@ -15,7 +15,9 @@ @pytest.mark.parametrize("accept", (False, True)) -def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inserts_nothing(migrated_database_urls, accept): +def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inserts_nothing( + migrated_database_urls, accept +): user_id = uuid4() with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") @@ -28,15 +30,25 @@ def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inser assert candidate["metadata_json"]["project_scope"] == [] assert group_scope(candidate) == group_scope(members[0]) if accept: - assert VNextMemoryCommitService(store).accept_consolidation_candidate(first.candidate_ids[0], reason="Reviewed synthetic group")["status"] == "accepted" + assert ( + VNextMemoryCommitService(store).accept_consolidation_candidate( + first.candidate_ids[0], reason="Reviewed synthetic group" + )["status"] + == "accepted" + ) before = conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] second = service.propose_rollups(projects=(ALPHA,)) assert second.proposals == [] assert conn.execute("SELECT count(*) AS n FROM memories").fetchone()["n"] == before - assert any(group["state"] == ("already_covered_by_accepted" if accept else "existing_candidate") for group in second.groups), second + assert any( + group["state"] == ("already_covered_by_accepted" if accept else "existing_candidate") + for group in second.groups + ), second -def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossing_project_scopes(migrated_database_urls): +def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossing_project_scopes( + migrated_database_urls, +): user_id = uuid4() with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") @@ -48,4 +60,38 @@ def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossi metadata = dict(member["metadata_json"], project_scope=[], project_floor=[ALPHA]) conn.execute("UPDATE memories SET metadata_json=%s::jsonb WHERE id=%s", (json.dumps(metadata), member["id"])) with pytest.raises(VNextMemoryCommitValidationError, match="crosses project scopes"): - VNextMemoryCommitService(store).accept_consolidation_candidate(candidate_id, reason="Reviewed synthetic group") + VNextMemoryCommitService(store).accept_consolidation_candidate( + candidate_id, reason="Reviewed synthetic group" + ) + + +@pytest.mark.parametrize("accept", (False, True)) +def test_existing_rollup_state_admits_effective_labels_for_pending_and_accepted_cards(migrated_database_urls, accept): + user_id = uuid4() + with user_connection(migrated_database_urls["app"], user_id) as conn: + ContinuityStore(conn).create_user(user_id, f"existing-{user_id}@example.invalid", "Existing") + store = PostgresVNextStore(conn) + members = seed_members(store) + service = VNextRollupService(store) + proposal = service.propose_rollups(projects=(ALPHA,)) + candidate_id = proposal.candidate_ids[0] + candidate = store.get_memory(candidate_id) + metadata = candidate["metadata_json"] + if accept: + VNextMemoryCommitService(store).accept_consolidation_candidate( + candidate_id, reason="Reviewed synthetic group" + ) + arguments = { + "rollup_digests": (metadata["rollup_digest"],), + "rollup_keys": (metadata["rollup_key"],), + "domains": None, + "sensitivity_allowed": ["public", "internal"], + "projects": (ALPHA,), + } + pending, accepted = service._existing_rollup_state(**arguments) + assert bool(accepted if accept else pending) + # Simulate a stale stored card whose input now has a higher effective label. + conn.execute("UPDATE memories SET sensitivity='confidential' WHERE id=%s", (members[0]["id"],)) + pending, accepted = service._existing_rollup_state(**arguments) + assert not pending + assert not accepted From b3b54062d19575d01bc2a69141999296f450ec7b Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:57:42 +0200 Subject: [PATCH 183/270] Exercise exact entrypoints and every retrieval stage against stale labels --- ...rived_labels_exact_entrypoints_postgres.py | 122 ++++++++++++++++ tests/unit/test_label_door_registry.py | 1 + tests/unit/test_label_reader_stage_matrix.py | 133 ++++++++++++++++++ 3 files changed, 256 insertions(+) create mode 100644 tests/integration/test_derived_labels_exact_entrypoints_postgres.py create mode 100644 tests/unit/test_label_reader_stage_matrix.py diff --git a/tests/integration/test_derived_labels_exact_entrypoints_postgres.py b/tests/integration/test_derived_labels_exact_entrypoints_postgres.py new file mode 100644 index 000000000..96437149c --- /dev/null +++ b/tests/integration/test_derived_labels_exact_entrypoints_postgres.py @@ -0,0 +1,122 @@ +"""Exact entrypoints use the shared effective project floor with real keys.""" +from __future__ import annotations + +import json +from uuid import UUID, uuid4 + +import pytest + +from alicebot_api.config import Settings +from alicebot_api.db import user_connection +from alicebot_api.mcp.registry import MCPToolNotFoundError, call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError +from alicebot_api.routers import vnext_memories, vnext_projects, vnext_retrieval, vnext_review +from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.test_derived_labels_read_acceptance_postgres import _user +from tests.unit.test_derived_labels_real_keys import ALPHA, real_reader_key + +DOORS = ("artifact_get", "artifact_trace", "artifact_export", "artifact_review", "artifact_feedback", "artifact_rating", + "legacy_artifact_get", "legacy_artifact_review", "memory_review_http", "memory_review_mcp", + "memory_correct_mcp", "memory_redact_mcp", "loop_review_http", "loop_review_mcp") + +def _invoke(door, *, app_url, user_id, target_id, key, tmp_path): + auth = f"Bearer {key}" if key else None + if door == "artifact_get": + return vnext_review.get_vnext_artifact(UUID(target_id), user_id, authorization=auth) + if door == "artifact_trace": + return vnext_retrieval.get_vnext_artifact_trace(UUID(target_id), user_id, authorization=auth) + if door == "artifact_export": + return vnext_review.export_vnext_artifact(UUID(target_id), vnext_review.VNextArtifactExportRequest(user_id=user_id, output_dir=str(tmp_path)), authorization=auth) + if door == "artifact_review": + return vnext_review.review_vnext_artifact(UUID(target_id), vnext_review.VNextArtifactReviewRequest(user_id=user_id, action="reject"), authorization=auth) + if door == "artifact_feedback": + return vnext_review.record_vnext_artifact_insight_feedback(UUID(target_id), vnext_review.VNextArtifactInsightFeedbackRequest(user_id=user_id, useful_insight="yes"), authorization=auth) + if door == "artifact_rating": + return vnext_review.rate_vnext_artifact_quality(UUID(target_id), vnext_review.VNextArtifactQualityRatingRequest(user_id=user_id), authorization=auth) + if door == "memory_review_http": + return vnext_memories.review_vnext_memory(UUID(target_id), vnext_memories.VNextMemoryReviewRequest(user_id=user_id, action="accept"), authorization=auth) + if door == "loop_review_http": + return vnext_projects.review_vnext_open_loop(target_id, vnext_projects.VNextOpenLoopReviewRequest(user_id=user_id, action="close"), authorization=auth) + name, args = { + "legacy_artifact_get": ("alice_vnext_artifact_get", {"artifact_id": target_id}), + "legacy_artifact_review": ("alice_vnext_artifact_review", {"artifact_id": target_id, "action": "reject"}), + "memory_review_mcp": ("alice_memory_review", {"review_item_id": target_id}), + "memory_correct_mcp": ("alice_memory_correct", {"review_item_id": target_id, "action": "approve"}), + "memory_redact_mcp": ("alice_memory_manage", {"memory_id": target_id, "action": "redact", "reason": "Synthetic test"}), + "loop_review_mcp": ("alice_open_loops", {"loop_id": target_id, "action": "close"}), + }[door] + return call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name=name, arguments=args) + +@pytest.mark.parametrize("door", DOORS) +@pytest.mark.parametrize("reader", ("owner", "bound_admin")) +def test_exact_entrypoint_checks_effective_floor(migrated_database_urls, monkeypatch, tmp_path, door, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + for module in (vnext_memories, vnext_projects, vnext_retrieval, vnext_review): + monkeypatch.setattr(module, "get_settings", lambda: Settings(database_url=app_url)) + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.setenv("ALICE_MCP_LEGACY_TOOLS", "1") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + key = real_reader_key(store, user_id, reader) + if key: + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + for hidden in (True, False): + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + source = store.create_source({"source_type": "note", "title": "Parent", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": ["prj_" + "b" * 16 if hidden else ALPHA]}}) + metadata = {"source_id": str(source["id"]), "project_scope": [ALPHA]} + derived = {"v": 1, "sources": [str(source["id"])], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}} + with without_insert_floor(): + if "artifact" in door: + row = store.create_artifact({"artifact_type": "daily_brief", "title": "Door sentinel", "content_markdown": "Door sentinel", "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA], "derived_from": derived}}) + elif door.startswith("loop_"): + row = store.create_open_loop({"title": "Door sentinel", "source_id": str(source["id"]), "domain": "project", "sensitivity": "public", "metadata_json": {**metadata, "discovered_by": "vnext_daily_capture"}}) + else: + row = store.create_memory({"memory_key": str(uuid4()), "title": "Door sentinel", "canonical_text": "Door sentinel", "status": "candidate", "domain": "project", "sensitivity": "public", "metadata_json": metadata}) + blocked = hidden and reader == "bound_admin" + if door.startswith("legacy_") and key: + with pytest.raises(MCPToolNotFoundError, match="disabled whenever"): + _invoke(door, app_url=app_url, user_id=user_id, target_id=str(row["id"]), key=key, tmp_path=tmp_path) + continue + try: + result = _invoke(door, app_url=app_url, user_id=user_id, target_id=str(row["id"]), key=key, tmp_path=tmp_path) + except MCPToolError as exc: + assert blocked, (door, reader, hidden, type(exc).__name__, str(exc)) + assert "policy" in type(exc).__name__.lower() or "project" in str(exc).lower(), (type(exc).__name__, str(exc)) + else: + if hasattr(result, "status_code"): + assert (result.status_code == 403) is blocked, (door, reader, hidden, result.status_code, result.body) + if blocked: + assert str(row["id"]) not in result.body.decode() + assert "Door sentinel" not in result.body.decode() + else: + assert result.status_code in {200, 201}, (door, result.status_code, result.body) + else: + assert not blocked, (door, reader, hidden, json.dumps(result, default=str)) + assert str(row["id"]) in json.dumps(result, default=str) + + +@pytest.mark.parametrize("reader", ("owner", "bound_admin")) +def test_review_queue_list_uses_current_parent_scope(migrated_database_urls, monkeypatch, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + key = real_reader_key(store, user_id, reader) + rows = [] + for hidden in (False, True): + source = store.create_source({"source_type": "note", "title": "Parent", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": ["prj_" + "b" * 16 if hidden else ALPHA]}}) + with without_insert_floor(): + rows.append(store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Queue sentinel", "status": "candidate", "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [ALPHA], "source_id": str(source["id"])}})) + if key: + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + result = call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name="alice_memory_review", arguments={"status": "all"}) + rendered = json.dumps(result, default=str) + assert str(rows[0]["id"]) in rendered + assert (str(rows[1]["id"]) in rendered) is (reader == "owner") diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py index 3e0350f58..6aebf88d8 100644 --- a/tests/unit/test_label_door_registry.py +++ b/tests/unit/test_label_door_registry.py @@ -57,6 +57,7 @@ DOORS = { "routers/_vnext_shared.py:_vnext_authorized_artifact": None, "vnext_source_fence.py:resolve_attachable_memory_id": None, + "vnext_open_loop_references.py:withhold_unreadable_references": None, "mcp/evidence_artifacts.py:_authorize_explain_resource": None, "mcp/evidence_artifacts.py:_authorize_entity_explain_target": "_authorize_explain_resource", "mcp/evidence_artifacts.py:_entity_backing_is_fully_authorized": "_authorize_explain_resource", diff --git a/tests/unit/test_label_reader_stage_matrix.py b/tests/unit/test_label_reader_stage_matrix.py new file mode 100644 index 000000000..59e9fba90 --- /dev/null +++ b/tests/unit/test_label_reader_stage_matrix.py @@ -0,0 +1,133 @@ +"""Each read stage holds back a stale public row and keeps a visible control.""" +from __future__ import annotations +from datetime import UTC, datetime, timedelta +from contextlib import contextmanager +from types import SimpleNamespace +from uuid import UUID, uuid4 + +import pytest + +from alicebot_api.project_view import ProjectView +from alicebot_api.session_briefing import compile_session_brief +from alicebot_api.vnext_agent_control import AgentIdentity +from alicebot_api.vnext_context_tree import ContextTreeRequest, VNextContextTreeService +from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService, VNextProjectValidationError +from alicebot_api.vnext_retrieval import VNextRetrievalService, _ResolvedRetrievalScope, expand_provenance_once +from alicebot_api.vnext_temporal_query import TemporalAnchor +from alicebot_api.vnext_open_loop_references import withhold_unreadable_references +from alicebot_api.vnext_source_fence import SourceReadFence +from alicebot_api.mcp import evidence_artifacts +from alicebot_api.routers import workspaces +from tests.unit.test_complete_readable_counts import PopulationStore, _quiet_services +from tests.unit.test_derived_labels_real_keys import ALPHA + +SOURCE = str(UUID(int=100)) +MEMORY = str(UUID(int=101)) +LOOP = str(UUID(int=102)) +ARTIFACT = str(UUID(int=103)) +PROJECT = str(UUID(int=104)) +ENTITY = str(UUID(int=105)) +SECRET = "Door sentinel" +CEILING = ["public", "internal", "private", "unknown"] + +class StageStore(PopulationStore): + def __init__(self): + super().__init__() + derived = {"v": 1, "sources": [SOURCE], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}} + self.rows["source"] = [{"id": SOURCE, "domain": "project", "sensitivity": "confidential", "metadata_json": {"project_scope": [ALPHA]}}] + for kind, identifier in (("memory", MEMORY), ("open_loop", LOOP), ("artifact", ARTIFACT), ("project", PROJECT)): + self.rows[kind] = [{"id": identifier, "domain": "project", "sensitivity": "public", "status": "open" if kind == "open_loop" else "active", + "title": SECRET, "name": SECRET, "canonical_text": SECRET, "content_markdown": SECRET, + "memory_type": "semantic", "artifact_type": "daily_brief", "source_id": SOURCE, + "metadata_json": {"project_scope": [ALPHA], "source_id": SOURCE, "discovered_by": "vnext_daily_capture", "derived_from": derived}}] + self.belief = {"id": str(UUID(int=106)), "memory_id": MEMORY, "claim": "The deployment pipeline is ready for production launch.", "status": "active"} + self.events = [{"id": str(UUID(int=107)), "event_type": "memory.labels_raised", "target_type": "memory", "target_id": MEMORY, "occurred_at": datetime.now(UTC).isoformat()}] + def search_memories_vector(self, **kwargs): return self.rows["memory"] + def search_memories_by_time(self, **kwargs): return self.rows["memory"] + def search_memories(self, **kwargs): return self.rows["memory"] + def search_sources(self, **kwargs): return self.rows["source"] + def search_sources_fts(self, **kwargs): return [] + def search_source_chunks_fts(self, **kwargs): return [] + def get_memories_by_ids(self, ids): return [row for row in self.rows["memory"] if row["id"] in ids] + def list_memories_referencing_source(self, **kwargs): return self.rows["memory"] + def list_beliefs(self, **kwargs): return [self.belief] + def list_memory_events(self, **kwargs): return self.events + def list_resume_memory_events(self, **kwargs): return [] + def list_open_loop_events(self, **kwargs): return [] + def find_entities_by_names(self, names): return [{"id": ENTITY, "name": "Alice", "entity_type": "person"}] + def list_memory_entity_edges(self, **kwargs): return [{"from_type": "memory", "from_id": MEMORY, "to_type": "entity", "to_id": ENTITY, "edge_type": "mentions", "observed_at": datetime.now(UTC).isoformat()}] + def append_event(self, event): return {"id": str(uuid4()), **event} + def upsert_agent_identity(self, *args, **kwargs): return None + def get_project(self, identifier): return {"id": identifier, "name": "Visible project", "domain": "project", "sensitivity": "public", "metadata_json": {}} + def get_entity_optional(self, identifier): return {"id": identifier, "source_memory_ids": [MEMORY]} + def list_entity_edges_for_entity(self, identifier): return [] + def list_edges(self, **kwargs): return [{"from_type": "memory", "from_id": MEMORY, "to_type": "entity", "to_id": ENTITY}] + +def _scope(scoped=False): + return _ResolvedRetrievalScope(projects=frozenset((ALPHA,)) if scoped else frozenset(), people=frozenset(), window_start=None, window_end=None, exclude_global_domains=frozenset()) + +STAGES = ("by_ids", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs") + +def _stage(stage, store): + service = VNextRetrievalService(store, embedding_provider=SimpleNamespace(provider="synthetic", model="synthetic", base_url="http://synthetic.invalid")) + kwargs = {"domains": ["project"], "sensitivity_allowed": CEILING, "limit": 10} + if stage == "by_ids": return list(service._memories_by_ids([MEMORY], domains=kwargs["domains"], sensitivity_allowed=CEILING).values()) + if stage == "vector": return service._memory_vector_rows(query="Alice", query_vector=[0.1], query_embedding_status="enabled", **kwargs)[0] + if stage == "graph": return service._memory_graph_rows(query="Alice", entity_read_fenced=True, **kwargs)[0] + if stage == "temporal": return service._memory_temporal_rows(anchor=TemporalAnchor(datetime.now(UTC)-timedelta(days=1), datetime.now(UTC), "synthetic"), **kwargs)[0] + if stage == "provenance": return expand_provenance_once(store, fts_memories=[], source_excerpts=[{"id": SOURCE}], already_selected_ids=set(), effective_domains=["project"], effective_sensitivity_allowed=CEILING, effective_project_scope=()) + if stage == "visibility": return [store.rows["memory"][0]] if service.memory_visibility(domains=["project"], sensitivity_allowed=CEILING, scope=None)(store.rows["memory"][0]) else [] + if "contradictions" in stage: + new = {"id": str(UUID(int=108)), "memory_type": "semantic", "canonical_text": "The deployment pipeline is not ready for production launch."} + return service._contradicting_evidence([new], requested=True, domains=["project"], sensitivity_allowed=CEILING, scope=_scope(stage.startswith("scoped_")), person_linked_memory_ids=frozenset())[0] + if "recent_changes" in stage: + return service._recent_changes(domains=["project"], sensitivity_allowed=CEILING, scope=_scope(stage.startswith("scoped_")), person_linked_memory_ids=frozenset()) + if stage == "session": return compile_session_brief(store, effective_domains=("project",), effective_sensitivity_allowed=tuple(CEILING), effective_project_scope=(), project_view=ProjectView.unscoped(), exclude_global_domains=frozenset(), query=None) + if stage.startswith("context_"): + tree = VNextContextTreeService(store).build_tree(ContextTreeRequest(domains=("project",), sensitivity_allowed=tuple(CEILING))) + return next(root["children"] for root in tree["roots"] if root["id"] == "root:" + stage.removeprefix("context_")) + if stage == "project_resolution": + try: return [VNextProjectService(store)._resolve_project(ProjectAutomationRequest(domains=("project",), sensitivity_allowed=tuple(CEILING)))] + except VNextProjectValidationError: return [] + if stage == "dashboard_lists": + result = VNextProjectService(store).project_dashboard(project_id=ALPHA, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent")) + return [*result["memories"], *result["open_loops"], *result["artifacts"]] + if stage.startswith("workspace_"): + field = stage.removeprefix("workspace_") + if field == "memories": + store.rows["memory"][0]["status"] = "candidate" + field = "review_memories" + return workspaces._vnext_workspace_payload(store)[field] + trusted = AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent") + if stage == "loop_memory_reference": + shown = withhold_unreadable_references(store, [{"id": LOOP, "memory_id": MEMORY, "metadata_json": {}}], fence=SourceReadFence.for_identity(trusted)) + return [shown[0]["memory_id"]] if shown[0]["memory_id"] else [] + if stage == "entity_backing": + return [store.rows["memory"][0]] if evidence_artifacts._entity_backing_is_fully_authorized(store, identity=trusted, entity_id=ENTITY) else [] + if stage == "entity_explain": + try: + evidence_artifacts._authorize_entity_explain_target(None, identity=trusted, entity_id=UUID(ENTITY)) + except evidence_artifacts._ExplainAuthorizationError: + return [] + return store.rows["memory"] + raise AssertionError(stage) + +@pytest.mark.parametrize("stage", STAGES) +def test_each_stage_uses_current_parent_label(stage, monkeypatch): + _quiet_services(monkeypatch) + store = StageStore() + @contextmanager + def context(_ignored): + yield store + monkeypatch.setattr(evidence_artifacts, "_store_context", context) + monkeypatch.setattr(evidence_artifacts, "_vnext_store_context", context) + hidden = _stage(stage, store) + assert SECRET not in str(hidden) + assert SOURCE not in str(hidden) and MEMORY not in str(hidden) and LOOP not in str(hidden) and ARTIFACT not in str(hidden) and PROJECT not in str(hidden) + store.rows["source"][0]["sensitivity"] = "public" + visible = _stage(stage, store) + assert visible, (stage, visible) + if stage != "session": + assert any(identifier in str(visible) for identifier in (SOURCE, MEMORY, LOOP, ARTIFACT, PROJECT, str(store.belief["id"]))), (stage, visible) + else: + assert SECRET in visible From 799cf203927507c0e664c175a660f2c71111c70d Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:59:23 +0200 Subject: [PATCH 184/270] Pin context pack loops and fallback memory lookup controls --- tests/unit/test_label_reader_stage_matrix.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_label_reader_stage_matrix.py b/tests/unit/test_label_reader_stage_matrix.py index 59e9fba90..0d2193d61 100644 --- a/tests/unit/test_label_reader_stage_matrix.py +++ b/tests/unit/test_label_reader_stage_matrix.py @@ -12,7 +12,7 @@ from alicebot_api.vnext_agent_control import AgentIdentity from alicebot_api.vnext_context_tree import ContextTreeRequest, VNextContextTreeService from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService, VNextProjectValidationError -from alicebot_api.vnext_retrieval import VNextRetrievalService, _ResolvedRetrievalScope, expand_provenance_once +from alicebot_api.vnext_retrieval import VNextRetrievalRequest, VNextRetrievalService, _ResolvedRetrievalScope, expand_provenance_once from alicebot_api.vnext_temporal_query import TemporalAnchor from alicebot_api.vnext_open_loop_references import withhold_unreadable_references from alicebot_api.vnext_source_fence import SourceReadFence @@ -66,12 +66,15 @@ def list_edges(self, **kwargs): return [{"from_type": "memory", "from_id": MEMOR def _scope(scoped=False): return _ResolvedRetrievalScope(projects=frozenset((ALPHA,)) if scoped else frozenset(), people=frozenset(), window_start=None, window_end=None, exclude_global_domains=frozenset()) -STAGES = ("by_ids", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs") +STAGES = ("by_ids", "by_ids_fallback", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "pack_open_loops", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs") def _stage(stage, store): service = VNextRetrievalService(store, embedding_provider=SimpleNamespace(provider="synthetic", model="synthetic", base_url="http://synthetic.invalid")) kwargs = {"domains": ["project"], "sensitivity_allowed": CEILING, "limit": 10} - if stage == "by_ids": return list(service._memories_by_ids([MEMORY], domains=kwargs["domains"], sensitivity_allowed=CEILING).values()) + if stage.startswith("by_ids"): + if stage == "by_ids_fallback": + store.get_memories_by_ids = None + return list(service._memories_by_ids([MEMORY], domains=kwargs["domains"], sensitivity_allowed=CEILING).values()) if stage == "vector": return service._memory_vector_rows(query="Alice", query_vector=[0.1], query_embedding_status="enabled", **kwargs)[0] if stage == "graph": return service._memory_graph_rows(query="Alice", entity_read_fenced=True, **kwargs)[0] if stage == "temporal": return service._memory_temporal_rows(anchor=TemporalAnchor(datetime.now(UTC)-timedelta(days=1), datetime.now(UTC), "synthetic"), **kwargs)[0] @@ -83,6 +86,8 @@ def _stage(stage, store): if "recent_changes" in stage: return service._recent_changes(domains=["project"], sensitivity_allowed=CEILING, scope=_scope(stage.startswith("scoped_")), person_linked_memory_ids=frozenset()) if stage == "session": return compile_session_brief(store, effective_domains=("project",), effective_sensitivity_allowed=tuple(CEILING), effective_project_scope=(), project_view=ProjectView.unscoped(), exclude_global_domains=frozenset(), query=None) + if stage == "pack_open_loops": + return VNextRetrievalService(store).compile_context_pack(VNextRetrievalRequest(query="open loop", domains=("project",), sensitivity_allowed=tuple(CEILING), include_sources=False, include_contradictions=False), source_fence=SourceReadFence.unfenced())["open_loops"] if stage.startswith("context_"): tree = VNextContextTreeService(store).build_tree(ContextTreeRequest(domains=("project",), sensitivity_allowed=tuple(CEILING))) return next(root["children"] for root in tree["roots"] if root["id"] == "root:" + stage.removeprefix("context_")) From 1fb6582a8f470325507d1c6846f27d7dadb66eb6 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:13:54 +0200 Subject: [PATCH 185/270] Withhold restricted backing references in workspace views --- .../src/alicebot_api/routers/_vnext_shared.py | 2 ++ .../src/alicebot_api/routers/workspaces.py | 3 ++ apps/api/src/alicebot_api/vnext_projects.py | 3 ++ ...derived_labels_read_acceptance_postgres.py | 33 ++++++++++++++++--- tests/unit/test_label_reader_stage_matrix.py | 9 ++++- .../test_open_loop_references_read_fence.py | 8 ++--- 6 files changed, 49 insertions(+), 9 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/_vnext_shared.py b/apps/api/src/alicebot_api/routers/_vnext_shared.py index 99fddf907..0ba0ac414 100644 --- a/apps/api/src/alicebot_api/routers/_vnext_shared.py +++ b/apps/api/src/alicebot_api/routers/_vnext_shared.py @@ -309,6 +309,7 @@ def _vnext_load_source_trace( from alicebot_api.vnext_agent_control import AgentIdentity from alicebot_api.vnext_label_guard import apply_sensitivity_ceiling + from alicebot_api.vnext_open_loop_references import withhold_unreadable_references caller = identity if isinstance(identity, AgentIdentity) else None if not apply_sensitivity_ceiling(store, kind="source", rows=[source], identity=caller): @@ -323,6 +324,7 @@ def _vnext_load_source_trace( open_loops, open_loops_complete = _vnext_readable_trace_rows( store, "open_loop", lambda limit: store.list_open_loops_referencing_source(source_id=source_id, limit=limit), caller ) + open_loops = withhold_unreadable_references(store, open_loops, fence=SourceReadFence.for_identity(caller)) kept_ids = {str(row.get("id")) for row in (*memories, *artifacts, *open_loops)} kept_ids.add(source_id) events, direct_events_complete = _vnext_readable_trace_rows( diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index 4948bc285..aff8f5f22 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -87,6 +87,8 @@ def _workspace_rows(store: PostgresVNextStore, kind: str, rows: Sequence[Mapping def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdentity | None = None) -> dict[str, object]: from alicebot_api.vnext_label_guard import LabelGuard, sensitivity_ceiling + from alicebot_api.vnext_open_loop_references import withhold_unreadable_references + from alicebot_api.vnext_source_fence import SourceReadFence sensitivity_allowed = ["public", "internal", "private", "unknown"] ceiling = sensitivity_ceiling(identity) @@ -127,6 +129,7 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti project_count = sum(guard.readable_status_counts("project").values()) fetched_loops = store.list_open_loops(status=None, sensitivity_allowed=sensitivity_allowed, limit=30) open_loops = guard.admit_rows("open_loop", fetched_loops) + open_loops = withhold_unreadable_references(store, open_loops, fence=SourceReadFence.for_identity(identity)) open_loop_status_counts = guard.readable_status_counts("open_loop") open_loop_count = open_loop_status_counts.get("open", 0) people = store.list_people(sensitivity_allowed=sensitivity_allowed, limit=12) diff --git a/apps/api/src/alicebot_api/vnext_projects.py b/apps/api/src/alicebot_api/vnext_projects.py index df7e58ef8..ff928c6f4 100644 --- a/apps/api/src/alicebot_api/vnext_projects.py +++ b/apps/api/src/alicebot_api/vnext_projects.py @@ -1372,6 +1372,8 @@ def project_dashboard( ) -> JsonObject: from alicebot_api.vnext_agent_control import AgentIdentity from alicebot_api.vnext_label_guard import admit_loaded, apply_sensitivity_ceiling + from alicebot_api.vnext_open_loop_references import withhold_unreadable_references + from alicebot_api.vnext_source_fence import SourceReadFence project = self.store.get_project(project_id) if project is None: @@ -1421,6 +1423,7 @@ def project_dashboard( sensitivity_allowed=sensitivity_allowed, projects=(project_id,), ) + open_loops = withhold_unreadable_references(self.store, open_loops, fence=SourceReadFence.for_identity(caller)) artifacts = admit_loaded( self.store, kind="artifact", diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py index beaeb4f87..8a9b0cb00 100644 --- a/tests/integration/test_derived_labels_read_acceptance_postgres.py +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -286,7 +286,32 @@ def test_direct_column_loop_references_reach_real_read_guard(migrated_database_u assert count_guard.readable_status_counts("open_loop").get("open", 0) == int(source_sensitivity == "public") if key: monkeypatch.setenv("ALICE_AGENT_API_KEY", key) - result = call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name="alice_open_loops", arguments={"sensitivity_allowed": list(ALL_SENSITIVITY)}) - rendered = json.dumps(result, default=str) - assert (str(loop["id"]) in rendered) is admitted - assert ("Direct loop sentinel" in rendered) is admitted + for tool in ("alice_open_loops", "alice_resume"): + result = call_mcp_tool(MCPRuntimeContext(database_url=app_url, user_id=user_id), name=tool, arguments={"sensitivity_allowed": list(ALL_SENSITIVITY)}) + rendered = json.dumps(result, default=str) + assert (str(loop["id"]) in rendered) is admitted, tool + assert ("Direct loop sentinel" in rendered) is admitted, tool + + +@pytest.mark.parametrize("reader", ("owner", "admin", "trusted")) +def test_workspace_and_dashboard_original_loop_references_use_actual_key(migrated_database_urls, monkeypatch, reader): + app_url = migrated_database_urls["app"] + user_id = _user(app_url) + for module in (workspaces, vnext_projects, vnext_retrieval): + monkeypatch.setattr(module, "get_settings", lambda: Settings(database_url=app_url)) + with user_connection(app_url, user_id) as conn: + store = PostgresVNextStore(conn) + project = store.create_project({"name": "Visible project", "slug": "visible-project", "domain": "project", "sensitivity": "public"}) + project_id = str(project["id"]) + source = store.create_source({"source_type": "note", "title": "Restricted parent", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential", "metadata_json": {"project_scope": [project_id]}}) + visible_source = store.create_source({"source_type": "note", "title": "Visible source", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [project_id]}}) + with without_insert_floor(): + memory = store.create_memory({"memory_key": "hidden-backing", "canonical_text": "Restricted backing memory", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(source["id"]), "project_scope": [project_id]}}) + loop = store.create_open_loop({"title": "Visible original loop", "source_id": str(visible_source["id"]), "memory_id": str(memory["id"]), "project_id": project_id, "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [project_id]}}) + key = real_reader_key(store, user_id, reader) + auth = f"Bearer {key}" if key else None + responses = (workspaces.get_vnext_workspace(user_id, authorization=auth), vnext_projects.get_vnext_project_dashboard(project_id, user_id, authorization=auth), vnext_retrieval.get_vnext_source_trace(UUID(str(visible_source["id"])), user_id, authorization=auth)) + for response in responses: + assert response.status_code == 200, response.body + item = next(row for row in json.loads(response.body)["open_loops"] if str(row["id"]) == str(loop["id"])) + assert item["memory_id"] == (None if reader == "trusted" else str(memory["id"])) diff --git a/tests/unit/test_label_reader_stage_matrix.py b/tests/unit/test_label_reader_stage_matrix.py index 0d2193d61..cf882591c 100644 --- a/tests/unit/test_label_reader_stage_matrix.py +++ b/tests/unit/test_label_reader_stage_matrix.py @@ -66,7 +66,7 @@ def list_edges(self, **kwargs): return [{"from_type": "memory", "from_id": MEMOR def _scope(scoped=False): return _ResolvedRetrievalScope(projects=frozenset((ALPHA,)) if scoped else frozenset(), people=frozenset(), window_start=None, window_end=None, exclude_global_domains=frozenset()) -STAGES = ("by_ids", "by_ids_fallback", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "pack_open_loops", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs") +STAGES = ("by_ids", "by_ids_fallback", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "pack_open_loops", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs", "workspace_loop_refs", "dashboard_loop_refs") def _stage(stage, store): service = VNextRetrievalService(store, embedding_provider=SimpleNamespace(provider="synthetic", model="synthetic", base_url="http://synthetic.invalid")) @@ -97,6 +97,13 @@ def _stage(stage, store): if stage == "dashboard_lists": result = VNextProjectService(store).project_dashboard(project_id=ALPHA, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent")) return [*result["memories"], *result["open_loops"], *result["artifacts"]] + if stage.endswith("loop_refs"): + store.rows["open_loop"][0].update(title="Visible original loop", memory_id=MEMORY, source_id=None, metadata_json={"project_scope": [ALPHA]}) + if stage == "workspace_loop_refs": + body = workspaces._vnext_workspace_payload(store, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent")) + else: + body = VNextProjectService(store).project_dashboard(project_id=ALPHA, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent")) + return [body["open_loops"][0]["memory_id"]] if body["open_loops"][0]["memory_id"] else [] if stage.startswith("workspace_"): field = stage.removeprefix("workspace_") if field == "memories": diff --git a/tests/unit/test_open_loop_references_read_fence.py b/tests/unit/test_open_loop_references_read_fence.py index d2061a741..c8fbbf5ae 100644 --- a/tests/unit/test_open_loop_references_read_fence.py +++ b/tests/unit/test_open_loop_references_read_fence.py @@ -1181,17 +1181,17 @@ def test_the_reference_keys_include_every_key_the_reverse_lookup_of_a_source_rea ("list_open_loops", "mcp/projects.py", "_handle_alice_vnext_open_loops"): _FENCED, ("list_open_loops", "mcp/retrieval.py", "_vnext_resume"): _ALLOWLIST, ("list_open_loops", "memory.py", "list_open_loop_records"): _OWNER, - ("list_open_loops", "routers/workspaces.py", "_vnext_workspace_payload"): _OWNER, + ("list_open_loops", "routers/workspaces.py", "_vnext_workspace_payload"): _FENCED, ("list_open_loops", "session_briefing.py", "compile_session_brief"): _ALLOWLIST, ("list_open_loops", "vnext_context_tree.py", "build_tree"): _ALLOWLIST, ("list_open_loops", "vnext_dogfooding.py", "dashboard"): _OPERATOR, - ("list_open_loops", "vnext_projects.py", "project_dashboard"): _PRODUCER, + ("list_open_loops", "vnext_projects.py", "project_dashboard"): _FENCED, ("list_open_loops", "vnext_scheduler.py", "_generate_open_loop_review_artifact"): _FENCED, - ("list_open_loops_referencing_source", "routers/_vnext_shared.py", "_vnext_load_source_trace"): _SENSITIVITY, + ("list_open_loops_referencing_source", "routers/_vnext_shared.py", "_vnext_load_source_trace"): _FENCED, ("project_dashboard", "cli/automation.py", "_run_vnext_project_dashboard"): _OPERATOR, ("project_dashboard", "mcp/projects.py", "_handle_alice_project_dashboard"): _OWNER, ("project_dashboard", "routers/vnext_projects.py", "get_vnext_project_dashboard"): _SENSITIVITY, - ("project_dashboard", "routers/workspaces.py", "_vnext_workspace_payload"): _OWNER, + ("project_dashboard", "routers/workspaces.py", "_vnext_workspace_payload"): _FENCED, ("review_open_loop", "cli/automation.py", "_run_vnext_open_loop_review"): _OPERATOR, ("review_open_loop", "mcp/projects.py", "_handle_alice_open_loop_review"): _OWNER, ("review_open_loop", "mcp/retrieval.py", "_handle_alice_open_loops"): _FENCED, From 6cfbaec955707ce0017c4a6fe1560dfdef8bdf4c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:16:47 +0200 Subject: [PATCH 186/270] Separate original saved quotes from derived-copy refusal controls --- ...test_saved_quotes_follow_the_source_fence.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_saved_quotes_follow_the_source_fence.py b/tests/unit/test_saved_quotes_follow_the_source_fence.py index c1d8a66b5..8b238b645 100644 --- a/tests/unit/test_saved_quotes_follow_the_source_fence.py +++ b/tests/unit/test_saved_quotes_follow_the_source_fence.py @@ -1161,8 +1161,8 @@ def test_a_source_with_no_project_is_outside_the_fence_of_a_key_bound_to_a_proje what ``alice_explain`` has always done for such a key. A key bound to no project reads it. On v0.20.0 the review returned all three to a key bound to a project. The release notes say this in one sentence. - The memory is planted with the two writes a review makes (the metadata copy and a link), as the lifecycle tests of the - review door do. + The memory is an original row with the two saved provenance copies that a review makes, as the lifecycle tests of + this projection do. The derived-copy control below separately verifies refusal of the whole captured copy. Mutation: pass ``require_explicit_project_scope=False`` in ``SourceReadFence._admits``: the keys bound to ``alpha`` read the link, the quote and the id again, and disagree with explain. @@ -1173,6 +1173,7 @@ def test_a_source_with_no_project_is_outside_the_fence_of_a_key_bound_to_a_proje assert json.loads(row["metadata_json"])["project_scope"] == [], "the owner's capture belongs to no project" candidate = vault._candidate("Projectlessnote: the alpha kiln shelf is cleaned on Fridays") _plant_saved_quote(vault, candidate, source_id) + vault._original_quote_fixture(candidate) for who in _KEY_SPECS: answer = vault.review(who, candidate) assert answer["is_error"] is False, (who, answer) @@ -1184,6 +1185,18 @@ def test_a_source_with_no_project_is_outside_the_fence_of_a_key_bound_to_a_proje +def test_a_derived_copy_of_a_projectless_source_is_denied_to_bound_keys(vault: _Vault) -> None: + source_id = vault.capture_source(as_owner=True) + candidate = vault._candidate("Projectlesscopy: the alpha kiln shelf is cleaned on Fridays") + _plant_saved_quote(vault, candidate, source_id) + for who in _KEY_SPECS: + answer = vault.review(who, candidate) + readable = who == "unbound" + assert answer["is_error"] is not readable, (who, answer) + assert _holds_quote(answer) is readable, who + assert _holds_source_id(answer, source_id) is readable, who + + def test_current_derived_copy_is_denied_as_a_whole_after_source_relabel(vault: _Vault) -> None: source_id = vault.capture_source() memory_id, _query = vault.edit_and_approve(source_id) From 96f7addc046c4811b213cc07a8aba471536bda3c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:19:05 +0200 Subject: [PATCH 187/270] Provide current ancestry labels in dashboard test stores --- tests/unit/test_cli.py | 38 +++++++++++++++++++++++++++++++ tests/unit/test_vnext_projects.py | 37 ++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 776760fc1..b7f0a4b1b 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -422,6 +422,25 @@ def append_event(self, event: dict[str, object]) -> dict[str, object]: self.events.append(event) return event + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + """Mirror narrow dependency reads for this fixture's stored rows.""" + + from alicebot_api.vnext_derived_labels import identifier + + rows = { + "source": self.sources, + "memory": self.memories, + "open_loop": self.open_loops, + "artifact": list(self.artifacts.values()), + "project": list(self.projects.values()), + }.get(kind, []) + wanted = {identifier(value) for value in ids} + fields = ("id", "user_id", "domain", "sensitivity", "metadata_json", "value", "project_id", "source_id", "memory_id", "status", "memory_type", "artifact_type") + return [ + {field: row[field] for field in fields if field in row} + for row in rows if identifier(row.get("id")) in wanted + ] + def upsert_agent_identity(self, identity: dict[str, object], **_kwargs) -> dict[str, object]: row = { **identity, @@ -1685,6 +1704,16 @@ def test_vnext_contradiction_and_belief_cli(monkeypatch) -> None: "memory_type": "belief", } + store.memories.append({ + "id": "memory-belief-1", + "canonical_text": "Alice should auto-promote generated artifacts into memory.", + "memory_type": "belief", + "status": "active", + "domain": "project", + "sensitivity": "private", + "metadata_json": {}, + }) + @contextmanager def fake_vnext_store_context(_ctx): yield store @@ -1794,6 +1823,15 @@ def fake_vnext_store_context(_ctx): assert review_loop_payload["due_at"] == "2026-05-12T09:00:00Z" assert dashboard_payload["counts"]["open_loops"] == 1 + # The CLI must keep the dashboard's current-input admission checks. + store.sources[0]["domain"] = "health" + store.sources[0]["sensitivity"] = "regulated" + restricted = json.loads(dashboard_args.handler(ctx, dashboard_args)) + assert restricted["counts"]["open_loops"] == 0 + store.sources.clear() + missing = json.loads(dashboard_args.handler(ctx, dashboard_args)) + assert missing["counts"]["open_loops"] == 0 + def test_vnext_queue_cli_add_process_review_and_export(monkeypatch, tmp_path: Path) -> None: store = FakeVNextCliStore() diff --git a/tests/unit/test_vnext_projects.py b/tests/unit/test_vnext_projects.py index 01f164afb..eec4242b1 100644 --- a/tests/unit/test_vnext_projects.py +++ b/tests/unit/test_vnext_projects.py @@ -77,6 +77,25 @@ def list_project_update_events( rows.append(event) return rows + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + """Mirror narrow dependency reads for this fixture's stored rows.""" + + from alicebot_api.vnext_derived_labels import identifier + + rows = { + "source": self.sources, + "memory": list(self.memories.values()), + "open_loop": list(self.open_loops.values()), + "artifact": list(self.artifacts.values()), + "project": list(self.projects.values()), + }.get(kind, []) + wanted = {identifier(value) for value in ids} + fields = ("id", "user_id", "domain", "sensitivity", "metadata_json", "value", "project_id", "source_id", "memory_id", "status", "memory_type", "artifact_type") + return [ + {field: row[field] for field in fields if field in row} + for row in rows if identifier(row.get("id")) in wanted + ] + def create_artifact(self, artifact: dict[str, object], **_kwargs) -> dict[str, object]: row = {**artifact, "id": f"artifact-{len(self.artifacts) + 1}"} self.artifacts[str(row["id"])] = row @@ -1562,6 +1581,24 @@ def test_open_loop_extraction_and_review_support_source_owner_and_filters() -> N assert dashboard["counts"]["open_loops"] == 1 + +@pytest.mark.parametrize("parent_change", ["missing", "restricted"]) +def test_dashboard_withholds_loops_after_their_source_becomes_unreadable(parent_change) -> None: + store = _seed_store() + service = VNextProjectService(store) + loops = service.extract_open_loops(ProjectAutomationRequest(project_id="project-1", domains=("project",))) + assert service.project_dashboard(project_id="project-1")["counts"]["open_loops"] == 2 + if parent_change == "missing": + store.sources.clear() + else: + store.sources[0]["domain"] = "health" + store.sources[0]["sensitivity"] = "regulated" + dashboard = service.project_dashboard(project_id="project-1") + assert dashboard["counts"]["open_loops"] == 0 + assert dashboard["open_loops"] == [] + assert all(loop["source_id"] == "source-1" for loop in loops) + + def test_project_service_validation_errors() -> None: service = VNextProjectService(InMemoryVNextProjectStore()) From ad719ad2f67e60c47bd889f35138f89074458bb5 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:19:05 +0200 Subject: [PATCH 188/270] Provide current ancestry labels in dashboard test stores --- tests/unit/test_cli.py | 38 +++++++++++++++++++++++++++++++ tests/unit/test_vnext_projects.py | 37 ++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 776760fc1..b7f0a4b1b 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -422,6 +422,25 @@ def append_event(self, event: dict[str, object]) -> dict[str, object]: self.events.append(event) return event + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + """Mirror narrow dependency reads for this fixture's stored rows.""" + + from alicebot_api.vnext_derived_labels import identifier + + rows = { + "source": self.sources, + "memory": self.memories, + "open_loop": self.open_loops, + "artifact": list(self.artifacts.values()), + "project": list(self.projects.values()), + }.get(kind, []) + wanted = {identifier(value) for value in ids} + fields = ("id", "user_id", "domain", "sensitivity", "metadata_json", "value", "project_id", "source_id", "memory_id", "status", "memory_type", "artifact_type") + return [ + {field: row[field] for field in fields if field in row} + for row in rows if identifier(row.get("id")) in wanted + ] + def upsert_agent_identity(self, identity: dict[str, object], **_kwargs) -> dict[str, object]: row = { **identity, @@ -1685,6 +1704,16 @@ def test_vnext_contradiction_and_belief_cli(monkeypatch) -> None: "memory_type": "belief", } + store.memories.append({ + "id": "memory-belief-1", + "canonical_text": "Alice should auto-promote generated artifacts into memory.", + "memory_type": "belief", + "status": "active", + "domain": "project", + "sensitivity": "private", + "metadata_json": {}, + }) + @contextmanager def fake_vnext_store_context(_ctx): yield store @@ -1794,6 +1823,15 @@ def fake_vnext_store_context(_ctx): assert review_loop_payload["due_at"] == "2026-05-12T09:00:00Z" assert dashboard_payload["counts"]["open_loops"] == 1 + # The CLI must keep the dashboard's current-input admission checks. + store.sources[0]["domain"] = "health" + store.sources[0]["sensitivity"] = "regulated" + restricted = json.loads(dashboard_args.handler(ctx, dashboard_args)) + assert restricted["counts"]["open_loops"] == 0 + store.sources.clear() + missing = json.loads(dashboard_args.handler(ctx, dashboard_args)) + assert missing["counts"]["open_loops"] == 0 + def test_vnext_queue_cli_add_process_review_and_export(monkeypatch, tmp_path: Path) -> None: store = FakeVNextCliStore() diff --git a/tests/unit/test_vnext_projects.py b/tests/unit/test_vnext_projects.py index 01f164afb..eec4242b1 100644 --- a/tests/unit/test_vnext_projects.py +++ b/tests/unit/test_vnext_projects.py @@ -77,6 +77,25 @@ def list_project_update_events( rows.append(event) return rows + def read_label_rows(self, kind: str, ids: list[str]) -> list[dict[str, object]]: + """Mirror narrow dependency reads for this fixture's stored rows.""" + + from alicebot_api.vnext_derived_labels import identifier + + rows = { + "source": self.sources, + "memory": list(self.memories.values()), + "open_loop": list(self.open_loops.values()), + "artifact": list(self.artifacts.values()), + "project": list(self.projects.values()), + }.get(kind, []) + wanted = {identifier(value) for value in ids} + fields = ("id", "user_id", "domain", "sensitivity", "metadata_json", "value", "project_id", "source_id", "memory_id", "status", "memory_type", "artifact_type") + return [ + {field: row[field] for field in fields if field in row} + for row in rows if identifier(row.get("id")) in wanted + ] + def create_artifact(self, artifact: dict[str, object], **_kwargs) -> dict[str, object]: row = {**artifact, "id": f"artifact-{len(self.artifacts) + 1}"} self.artifacts[str(row["id"])] = row @@ -1562,6 +1581,24 @@ def test_open_loop_extraction_and_review_support_source_owner_and_filters() -> N assert dashboard["counts"]["open_loops"] == 1 + +@pytest.mark.parametrize("parent_change", ["missing", "restricted"]) +def test_dashboard_withholds_loops_after_their_source_becomes_unreadable(parent_change) -> None: + store = _seed_store() + service = VNextProjectService(store) + loops = service.extract_open_loops(ProjectAutomationRequest(project_id="project-1", domains=("project",))) + assert service.project_dashboard(project_id="project-1")["counts"]["open_loops"] == 2 + if parent_change == "missing": + store.sources.clear() + else: + store.sources[0]["domain"] = "health" + store.sources[0]["sensitivity"] = "regulated" + dashboard = service.project_dashboard(project_id="project-1") + assert dashboard["counts"]["open_loops"] == 0 + assert dashboard["open_loops"] == [] + assert all(loop["source_id"] == "source-1" for loop in loops) + + def test_project_service_validation_errors() -> None: service = VNextProjectService(InMemoryVNextProjectStore()) From c2369835fce09020bde8c99547f7754e46fe41de Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:20:51 +0200 Subject: [PATCH 189/270] Stage dashboard reader controls with the read filters --- tests/unit/test_cli.py | 9 --------- tests/unit/test_vnext_projects.py | 15 --------------- 2 files changed, 24 deletions(-) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index b7f0a4b1b..0a9fdf937 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -1823,15 +1823,6 @@ def fake_vnext_store_context(_ctx): assert review_loop_payload["due_at"] == "2026-05-12T09:00:00Z" assert dashboard_payload["counts"]["open_loops"] == 1 - # The CLI must keep the dashboard's current-input admission checks. - store.sources[0]["domain"] = "health" - store.sources[0]["sensitivity"] = "regulated" - restricted = json.loads(dashboard_args.handler(ctx, dashboard_args)) - assert restricted["counts"]["open_loops"] == 0 - store.sources.clear() - missing = json.loads(dashboard_args.handler(ctx, dashboard_args)) - assert missing["counts"]["open_loops"] == 0 - def test_vnext_queue_cli_add_process_review_and_export(monkeypatch, tmp_path: Path) -> None: store = FakeVNextCliStore() diff --git a/tests/unit/test_vnext_projects.py b/tests/unit/test_vnext_projects.py index eec4242b1..a4cbef901 100644 --- a/tests/unit/test_vnext_projects.py +++ b/tests/unit/test_vnext_projects.py @@ -1582,21 +1582,6 @@ def test_open_loop_extraction_and_review_support_source_owner_and_filters() -> N -@pytest.mark.parametrize("parent_change", ["missing", "restricted"]) -def test_dashboard_withholds_loops_after_their_source_becomes_unreadable(parent_change) -> None: - store = _seed_store() - service = VNextProjectService(store) - loops = service.extract_open_loops(ProjectAutomationRequest(project_id="project-1", domains=("project",))) - assert service.project_dashboard(project_id="project-1")["counts"]["open_loops"] == 2 - if parent_change == "missing": - store.sources.clear() - else: - store.sources[0]["domain"] = "health" - store.sources[0]["sensitivity"] = "regulated" - dashboard = service.project_dashboard(project_id="project-1") - assert dashboard["counts"]["open_loops"] == 0 - assert dashboard["open_loops"] == [] - assert all(loop["source_id"] == "source-1" for loop in loops) def test_project_service_validation_errors() -> None: From e466ce7e45df30b3f1d8ea471bb35d4037b90f7c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:15:43 +0200 Subject: [PATCH 190/270] Provide recorded label inputs in the semantic rollup fixture --- tests/unit/test_vnext_rollups_semantic.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/test_vnext_rollups_semantic.py b/tests/unit/test_vnext_rollups_semantic.py index 85ad78280..4e2ba5d45 100644 --- a/tests/unit/test_vnext_rollups_semantic.py +++ b/tests/unit/test_vnext_rollups_semantic.py @@ -22,6 +22,7 @@ from alicebot_api.sqlite_schema import bootstrap_sqlite_schema from alicebot_api.sqlite_store import SQLiteVNextStore, ensure_sqlite_user +from alicebot_api.vnext_derived_labels import identifier from alicebot_api.vnext_embeddings import memory_embedding_text from alicebot_api.vnext_repositories import JsonObject from alicebot_api.vnext_rollups import ( @@ -61,6 +62,13 @@ def create_memory(self, memory: JsonObject, *, actor_type: str = "system") -> Js def list_memories(self, *, status: str | None = None) -> list[JsonObject]: return [dict(row) for row in self.memories if status is None or row.get("status") == status] + def read_label_rows(self, kind: str, ids) -> list[JsonObject]: + """Expose recorded inputs so an existing card's label can be verified.""" + if kind != "memory": + return [] + wanted = {identifier(item) for item in ids} + return [dict(row) for row in self.memories if identifier(row.get("id")) in wanted] + @staticmethod def _in_scope( row: JsonObject, From 05bf1ea0c5daf1a4cf171bc6ca696a4861358a36 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:21:42 +0200 Subject: [PATCH 191/270] Refresh workspace receipts for withheld loop references --- tests/unit/test_workspaces_router_split.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_workspaces_router_split.py b/tests/unit/test_workspaces_router_split.py index d38ef3b32..82ce7ae11 100644 --- a/tests/unit/test_workspaces_router_split.py +++ b/tests/unit/test_workspaces_router_split.py @@ -86,7 +86,7 @@ # Re-pin 2026-10-06: workspace reads authenticate the protected identity and # admit rows through effective labels before totals or dashboard disclosure. EXPECTED_ROUTE_AST_SHA256 = "b99f1435de67d9499819acb9ed7ed61b588a3fb0ff037e782eeca070b39af742" -EXPECTED_SUPPORT_AST_SHA256 = "8700f0ab3b8a87f9863a3e95132e834f87bf82725c6e8d33602345bc3253bce1" +EXPECTED_SUPPORT_AST_SHA256 = "052f564a10f2f32c090f857142eda105f62d5e44b698d3e7475506c422a5ca47" EXPECTED_ROUTE_NAME_MANIFEST_SHA256 = "225c57c08bd8314156c56352dd1c53ffed3f556ce285c666dd6fca125115d0b4" EXPECTED_OPERATION_MANIFEST_SHA256 = "c320979b62d7ee8de244fe38bde5bf3761a4f9d76f76bf3cd8576c30fce9857e" EXPECTED_IMPORT_MANIFEST_SHA256 = "d8887934cacc6a4e5d52f080dae3c2c0d0cdf23d1518a4060a885a8c7f209c0c" @@ -139,7 +139,7 @@ } EXPECTED_SUPPORT_NODE_SHA256 = { "_vnext_status_counts": "0bf0ed228a14bd648a9d18fcd5f99ebf8c585bd29f4b5e81e1df17fe0201fd15", - "_vnext_workspace_payload": "83bc100509fc21e950702cba190ac8d94f33efe7774df673809d49daa6864136", + "_vnext_workspace_payload": "c2d41b35c27496c469fd70ebdf6bba01d526dac24ed00bddf983cf6dee57022a", "_workspace_rows": "070bdfbd1eae10608bd8208b08367e1c0ea10e2064f03ad5a84121a190ed4cf0", "_workspace_event_visible": "8343c060909326a5cb69fa6f671ac62f160630ecf989f04d78e792ea74c0ea90", } From 339670d1c32367b2b7c0df02b27b370309fb2e0d Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:21:25 +0200 Subject: [PATCH 192/270] Take graph and label locks before project review fence --- apps/api/src/alicebot_api/routers/vnext_review.py | 4 ++++ tests/unit/test_vnext_main.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/apps/api/src/alicebot_api/routers/vnext_review.py b/apps/api/src/alicebot_api/routers/vnext_review.py index 9a94db0bc..6050c133b 100644 --- a/apps/api/src/alicebot_api/routers/vnext_review.py +++ b/apps/api/src/alicebot_api/routers/vnext_review.py @@ -1034,6 +1034,10 @@ def review_vnext_project_update_candidate( identity = _vnext_authenticated_agent_identity( store, request, user_id=request.user_id, authorization=authorization ) + store.lock_graph_mutation() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) _artifact, decision = _vnext_authorized_artifact( store=store, identity=identity, diff --git a/tests/unit/test_vnext_main.py b/tests/unit/test_vnext_main.py index 7676ebad7..7274a8041 100644 --- a/tests/unit/test_vnext_main.py +++ b/tests/unit/test_vnext_main.py @@ -2692,6 +2692,12 @@ def test_vnext_project_and_open_loop_endpoints(monkeypatch) -> None: update_response = vnext_review_router.generate_vnext_project_update_candidate(request) update_payload = json.loads(update_response.body) extract_response = vnext_projects_router.extract_vnext_open_loops(request) + original_row_locker = store.get_artifact_for_update + def checked_row_locker(artifact_id): + assert store.graph_locked is True + assert store.labels_exclusive is True + return original_row_locker(artifact_id) + monkeypatch.setattr(store, "get_artifact_for_update", checked_row_locker) review_update_response = vnext_review_router.review_vnext_project_update_candidate( update_payload["id"], vnext_review_router.VNextProjectUpdateReviewRequest( From 857df228862b6477b21954e086611e59c81eb930 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:21:42 +0200 Subject: [PATCH 193/270] Refresh workspace receipts for withheld loop references --- tests/unit/test_workspaces_router_split.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_workspaces_router_split.py b/tests/unit/test_workspaces_router_split.py index 9a5b4230b..74d5cf205 100644 --- a/tests/unit/test_workspaces_router_split.py +++ b/tests/unit/test_workspaces_router_split.py @@ -86,7 +86,7 @@ # Re-pin 2026-10-06: workspace reads authenticate the protected identity and # admit rows through effective labels before totals or dashboard disclosure. EXPECTED_ROUTE_AST_SHA256 = "b99f1435de67d9499819acb9ed7ed61b588a3fb0ff037e782eeca070b39af742" -EXPECTED_SUPPORT_AST_SHA256 = "8700f0ab3b8a87f9863a3e95132e834f87bf82725c6e8d33602345bc3253bce1" +EXPECTED_SUPPORT_AST_SHA256 = "052f564a10f2f32c090f857142eda105f62d5e44b698d3e7475506c422a5ca47" EXPECTED_ROUTE_NAME_MANIFEST_SHA256 = "225c57c08bd8314156c56352dd1c53ffed3f556ce285c666dd6fca125115d0b4" EXPECTED_OPERATION_MANIFEST_SHA256 = "c320979b62d7ee8de244fe38bde5bf3761a4f9d76f76bf3cd8576c30fce9857e" EXPECTED_IMPORT_MANIFEST_SHA256 = "d8887934cacc6a4e5d52f080dae3c2c0d0cdf23d1518a4060a885a8c7f209c0c" @@ -140,7 +140,7 @@ } EXPECTED_SUPPORT_NODE_SHA256 = { "_vnext_status_counts": "0bf0ed228a14bd648a9d18fcd5f99ebf8c585bd29f4b5e81e1df17fe0201fd15", - "_vnext_workspace_payload": "83bc100509fc21e950702cba190ac8d94f33efe7774df673809d49daa6864136", + "_vnext_workspace_payload": "c2d41b35c27496c469fd70ebdf6bba01d526dac24ed00bddf983cf6dee57022a", "_workspace_rows": "070bdfbd1eae10608bd8208b08367e1c0ea10e2064f03ad5a84121a190ed4cf0", "_workspace_event_visible": "8343c060909326a5cb69fa6f671ac62f160630ecf989f04d78e792ea74c0ea90", } From 7c8b7fd11701c409c8cc93879559cf2f7b358d83 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:21:25 +0200 Subject: [PATCH 194/270] Take graph and label locks before project review fence --- apps/api/src/alicebot_api/routers/vnext_review.py | 4 ++++ tests/unit/test_vnext_main.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/apps/api/src/alicebot_api/routers/vnext_review.py b/apps/api/src/alicebot_api/routers/vnext_review.py index ea4004c89..7c7cc14fb 100644 --- a/apps/api/src/alicebot_api/routers/vnext_review.py +++ b/apps/api/src/alicebot_api/routers/vnext_review.py @@ -983,6 +983,10 @@ def review_vnext_project_update_candidate( identity = _vnext_authenticated_agent_identity( store, request, user_id=request.user_id, authorization=authorization ) + store.lock_graph_mutation() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) _artifact, decision = _vnext_authorized_artifact( store=store, identity=identity, diff --git a/tests/unit/test_vnext_main.py b/tests/unit/test_vnext_main.py index 59cbcdb3a..ebc4564c2 100644 --- a/tests/unit/test_vnext_main.py +++ b/tests/unit/test_vnext_main.py @@ -2677,6 +2677,12 @@ def test_vnext_project_and_open_loop_endpoints(monkeypatch) -> None: update_response = vnext_review_router.generate_vnext_project_update_candidate(request) update_payload = json.loads(update_response.body) extract_response = vnext_projects_router.extract_vnext_open_loops(request) + original_row_locker = store.get_artifact_for_update + def checked_row_locker(artifact_id): + assert store.graph_locked is True + assert store.labels_exclusive is True + return original_row_locker(artifact_id) + monkeypatch.setattr(store, "get_artifact_for_update", checked_row_locker) review_update_response = vnext_review_router.review_vnext_project_update_candidate( update_payload["id"], vnext_review_router.VNextProjectUpdateReviewRequest( From da282df66ee1a352132df092c9a6494b5a6e9cf7 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:17:50 +0200 Subject: [PATCH 195/270] Pin every dogfooding sample to effective reader admission --- tests/unit/test_label_reader_stage_matrix.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/test_label_reader_stage_matrix.py b/tests/unit/test_label_reader_stage_matrix.py index cf882591c..d595362ed 100644 --- a/tests/unit/test_label_reader_stage_matrix.py +++ b/tests/unit/test_label_reader_stage_matrix.py @@ -11,6 +11,7 @@ from alicebot_api.session_briefing import compile_session_brief from alicebot_api.vnext_agent_control import AgentIdentity from alicebot_api.vnext_context_tree import ContextTreeRequest, VNextContextTreeService +from alicebot_api.vnext_dogfooding import VNextDogfoodingService from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService, VNextProjectValidationError from alicebot_api.vnext_retrieval import VNextRetrievalRequest, VNextRetrievalService, _ResolvedRetrievalScope, expand_provenance_once from alicebot_api.vnext_temporal_query import TemporalAnchor @@ -143,3 +144,14 @@ def context(_ignored): assert any(identifier in str(visible) for identifier in (SOURCE, MEMORY, LOOP, ARTIFACT, PROJECT, str(store.belief["id"]))), (stage, visible) else: assert SECRET in visible + + +@pytest.mark.parametrize("kind", ("sources", "memories", "artifacts", "open_loops", "events")) +def test_each_dogfooding_sample_counts_only_currently_readable_rows(kind, monkeypatch): + _quiet_services(monkeypatch) + store = StageStore() + hidden = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(CEILING)) + assert hidden["sample_scope"][kind]["returned_count"] == 0 + store.rows["source"][0]["sensitivity"] = "public" + visible = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(CEILING)) + assert visible["sample_scope"][kind]["returned_count"] == 1 From 7a40021213abc9b25c92c57e6b981294ab75cba5 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:22:50 +0200 Subject: [PATCH 196/270] Align PostgreSQL fixtures with scoped owner access --- tests/integration/conftest.py | 21 ++++++++ .../integration/test_calendar_accounts_api.py | 3 +- .../test_capture_dedupe_postgres.py | 3 ++ tests/integration/test_context_compile.py | 13 +++-- tests/integration/test_continuity_api.py | 15 ++++-- .../integration/test_continuity_recall_api.py | 21 ++++++-- .../test_continuity_resumption_api.py | 21 ++++++-- tests/integration/test_continuity_store.py | 24 +++++---- .../test_credential_floor_every_door_api.py | 2 + .../test_default_surface_integration.py | 2 +- ...est_derived_labels_group_scope_postgres.py | 3 ++ .../test_derived_labels_migration_postgres.py | 53 ++++++++++++++++++- tests/integration/test_gmail_accounts_api.py | 8 ++- .../test_mcp_refusal_before_state_postgres.py | 2 + tests/integration/test_memory_admission.py | 3 +- .../test_memory_review_labels_api.py | 29 +++++----- ...t_pending_project_update_guard_postgres.py | 2 + .../integration/test_provider_runtime_api.py | 32 ++++++++++- tests/integration/test_proxy_execution_api.py | 8 ++- .../integration/test_saved_quotes_postgres.py | 4 ++ tests/integration/test_task_artifacts_api.py | 5 +- tests/integration/test_temporal_state_api.py | 7 ++- .../test_temporal_state_mcp_cli.py | 7 ++- .../test_vnext_consolidation_postgres.py | 2 + .../test_vnext_live_workspace_api.py | 4 ++ 25 files changed, 245 insertions(+), 49 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 404554c6e..5d8d3667f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -171,3 +171,24 @@ def migrated_database_urls(migrated_template_database: str) -> Iterator[dict[str yield urls finally: _drop_database(database_name) + + +def lock_label_fixture(store) -> None: + """Seed and mutate one transaction with the production lock order.""" + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + store.lock_graph_mutation() + acquire_exclusive_label_lock(store) + + +def assert_append_only_mutation_refused(conn, *, snapshot_sql, mutation_sql, params) -> None: + """Prove refusal under both forced RLS and the append-only trigger.""" + before = conn.execute(snapshot_sql, params).fetchall() + assert before, "the synthetic target must be visible before attempting its mutation" + try: + with conn.transaction(): + changed = conn.execute(mutation_sql, params).rowcount + assert changed == 0, "an append-only row was changed" + except psycopg.Error as error: + assert "append-only" in str(error) + assert conn.execute(snapshot_sql, params).fetchall() == before diff --git a/tests/integration/test_calendar_accounts_api.py b/tests/integration/test_calendar_accounts_api.py index 708abcc89..f221aac06 100644 --- a/tests/integration/test_calendar_accounts_api.py +++ b/tests/integration/test_calendar_accounts_api.py @@ -13,7 +13,7 @@ from alicebot_api.routers import legacy_gated as legacy_gated_router from alicebot_api.config import Settings import alicebot_api.calendar as calendar_module -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -251,6 +251,7 @@ def test_calendar_account_endpoints_connect_list_detail_and_isolate( assert '"access_token":' not in json.dumps(detail_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_capture_dedupe_postgres.py b/tests/integration/test_capture_dedupe_postgres.py index e54f73ec6..475419ce3 100644 --- a/tests/integration/test_capture_dedupe_postgres.py +++ b/tests/integration/test_capture_dedupe_postgres.py @@ -15,6 +15,7 @@ content_hash_for_text, ) from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import lock_label_fixture def test_two_connections_claim_one_source_dedupe_identity( @@ -144,6 +145,7 @@ def test_source_scope_mutation_rotates_postgres_identity_and_releases_old_captur "Capture mutation", ) store = PostgresVNextStore(conn) + lock_label_fixture(store) service = VNextCaptureService(store) text = "Fact: Reviewed source scope changes must rotate identity atomically." first = service.capture_text( @@ -209,6 +211,7 @@ def test_source_scope_mutation_collision_rolls_back_postgres_row( "Capture collision", ) store = PostgresVNextStore(conn) + lock_label_fixture(store) service = VNextCaptureService(store) text = "Fact: Collision rollback leaves both source identities intact." alpha = service.capture_text( diff --git a/tests/integration/test_context_compile.py b/tests/integration/test_context_compile.py index 905b26da4..012996646 100644 --- a/tests/integration/test_context_compile.py +++ b/tests/integration/test_context_compile.py @@ -12,8 +12,9 @@ import alicebot_api.main as main_module from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore +from tests.integration.conftest import assert_append_only_mutation_refused def invoke_compile_context(payload: dict[str, Any]) -> tuple[int, dict[str, Any]]: @@ -662,9 +663,13 @@ def test_compile_context_endpoint_persists_trace_and_trace_events(migrated_datab assert trace_events[-1]["payload"]["excluded_entity_edge_limit_count"] == 1 with psycopg.connect(migrated_database_urls["admin"]) as conn: - with conn.cursor() as cur: - with pytest.raises(psycopg.Error, match="append-only"): - cur.execute("UPDATE trace_events SET kind = 'mutated' WHERE trace_id = %s", (trace_id,)) + set_current_user(conn, user_id) + assert_append_only_mutation_refused( + conn, + snapshot_sql="SELECT * FROM trace_events WHERE trace_id = %s ORDER BY id", + mutation_sql="UPDATE trace_events SET kind = 'mutated' WHERE trace_id = %s", + params=(trace_id,), + ) def test_compile_context_prefers_updated_active_memory_within_same_transaction( diff --git a/tests/integration/test_continuity_api.py b/tests/integration/test_continuity_api.py index 1502722ba..e3d45d5a7 100644 --- a/tests/integration/test_continuity_api.py +++ b/tests/integration/test_continuity_api.py @@ -12,7 +12,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -103,12 +103,14 @@ def seed_user_with_continuity(database_url: str, *, email: str) -> dict[str, obj def set_thread_timestamps( admin_database_url: str, + user_id: UUID, *, thread_id: UUID, created_at: datetime, updated_at: datetime, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( "UPDATE threads SET created_at = %s, updated_at = %s WHERE id = %s", @@ -118,13 +120,15 @@ def set_thread_timestamps( def set_session_timestamps( admin_database_url: str, + user_id: UUID, *, session_id: UUID, started_at: datetime, ended_at: datetime | None, created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( "UPDATE sessions SET started_at = %s, ended_at = %s, created_at = %s WHERE id = %s", @@ -219,24 +223,28 @@ def test_thread_continuity_endpoints_create_list_detail_sessions_and_events( set_thread_timestamps( migrated_database_urls["admin"], + user_id=seeded["user_id"], thread_id=seeded["first_thread"]["id"], created_at=shared_created_at, updated_at=shared_created_at, ) set_thread_timestamps( migrated_database_urls["admin"], + user_id=seeded["user_id"], thread_id=seeded["second_thread"]["id"], created_at=shared_created_at, updated_at=shared_created_at, ) set_thread_timestamps( migrated_database_urls["admin"], + user_id=seeded["user_id"], thread_id=api_thread_id, created_at=newer_created_at, updated_at=newer_created_at, ) set_session_timestamps( migrated_database_urls["admin"], + user_id=seeded["user_id"], session_id=seeded["first_session"]["id"], started_at=first_session_start, ended_at=first_session_end, @@ -244,6 +252,7 @@ def test_thread_continuity_endpoints_create_list_detail_sessions_and_events( ) set_session_timestamps( migrated_database_urls["admin"], + user_id=seeded["user_id"], session_id=seeded["second_session"]["id"], started_at=second_session_start, ended_at=None, diff --git a/tests/integration/test_continuity_recall_api.py b/tests/integration/test_continuity_recall_api.py index d30bc10e6..7d71d206c 100644 --- a/tests/integration/test_continuity_recall_api.py +++ b/tests/integration/test_continuity_recall_api.py @@ -12,7 +12,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import continuity as continuity_router from alicebot_api.config import Settings -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -70,11 +70,13 @@ def seed_user(database_url: str, *, email: str) -> UUID: def set_continuity_timestamps( admin_database_url: str, + user_id: UUID, *, continuity_object_id: UUID, created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( "UPDATE continuity_objects SET created_at = %s, updated_at = %s WHERE id = %s", @@ -84,6 +86,7 @@ def set_continuity_timestamps( def set_continuity_lifecycle_flags( admin_database_url: str, + user_id: UUID, *, continuity_object_id: UUID, is_searchable: bool | None = None, @@ -101,7 +104,8 @@ def set_continuity_lifecycle_flags( return values.append(continuity_object_id) - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( f"UPDATE continuity_objects SET {', '.join(assignments)} WHERE id = %s", @@ -170,11 +174,13 @@ def test_continuity_recall_api_returns_provenance_backed_scoped_results( set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=primary_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=other_object["id"], created_at=datetime(2026, 3, 29, 9, 0, tzinfo=UTC), ) @@ -316,6 +322,7 @@ def test_continuity_recall_debug_api_persists_and_exposes_trace( set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=continuity_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) @@ -393,6 +400,7 @@ def test_continuity_resumption_debug_api_includes_underlying_retrieval_trace( set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=decision["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) @@ -486,16 +494,19 @@ def test_continuity_recall_api_prefers_confirmed_fresh_active_truth_over_superse set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=current_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=stale_object["id"], created_at=datetime(2026, 3, 20, 9, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=superseded_object["id"], created_at=datetime(2026, 3, 10, 8, 0, tzinfo=UTC), ) @@ -573,16 +584,19 @@ def test_continuity_recall_api_excludes_preserved_but_non_searchable_objects( set_continuity_lifecycle_flags( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=hidden_object["id"], is_searchable=False, ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=hidden_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=visible_object["id"], created_at=datetime(2026, 3, 29, 10, 5, tzinfo=UTC), ) @@ -637,6 +651,7 @@ def test_continuity_lifecycle_debug_endpoints_expose_flags( set_continuity_lifecycle_flags( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=continuity_object["id"], is_promotable=False, ) diff --git a/tests/integration/test_continuity_resumption_api.py b/tests/integration/test_continuity_resumption_api.py index 9397924ff..665d96d70 100644 --- a/tests/integration/test_continuity_resumption_api.py +++ b/tests/integration/test_continuity_resumption_api.py @@ -12,7 +12,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import continuity as continuity_router from alicebot_api.config import Settings -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -74,11 +74,13 @@ def seed_user(database_url: str, *, email: str) -> UUID: def set_continuity_timestamps( admin_database_url: str, + user_id: UUID, *, continuity_object_id: UUID, created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( "UPDATE continuity_objects SET created_at = %s, updated_at = %s WHERE id = %s", @@ -88,11 +90,13 @@ def set_continuity_timestamps( def set_continuity_lifecycle_flags( admin_database_url: str, + user_id: UUID, *, continuity_object_id: UUID, is_promotable: bool, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( "UPDATE continuity_objects SET is_promotable = %s WHERE id = %s", @@ -176,21 +180,25 @@ def test_continuity_resumption_api_returns_required_sections( set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=decision_object["id"], created_at=datetime(2026, 3, 29, 9, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=waiting_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=next_object["id"], created_at=datetime(2026, 3, 29, 10, 5, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=latest_decision_object["id"], created_at=datetime(2026, 3, 29, 10, 10, tzinfo=UTC), ) @@ -263,6 +271,7 @@ def test_continuity_resumption_api_returns_explicit_empty_states( set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=continuity_object["id"], created_at=datetime(2026, 3, 29, 9, 0, tzinfo=UTC), ) @@ -376,17 +385,20 @@ def test_continuity_resumption_api_selects_latest_sections_beyond_recall_limit( for index, continuity_object_id in enumerate(historical_object_ids): set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=continuity_object_id, created_at=base_time + timedelta(minutes=index), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=latest_decision_object["id"], created_at=base_time + timedelta(minutes=200), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=latest_next_action_object["id"], created_at=base_time + timedelta(minutes=201), ) @@ -465,16 +477,19 @@ def test_continuity_resumption_api_uses_promotable_facts_by_default_with_overrid set_continuity_lifecycle_flags( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=fact_object["id"], is_promotable=False, ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=fact_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=decision_object["id"], created_at=datetime(2026, 3, 29, 10, 5, tzinfo=UTC), ) diff --git a/tests/integration/test_continuity_store.py b/tests/integration/test_continuity_store.py index 956156388..fd91100fc 100644 --- a/tests/integration/test_continuity_store.py +++ b/tests/integration/test_continuity_store.py @@ -9,6 +9,7 @@ from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore +from tests.integration.conftest import assert_append_only_mutation_refused def test_thread_session_and_event_persistence(migrated_database_urls): @@ -40,12 +41,13 @@ def test_thread_session_and_event_persistence(migrated_database_urls): assert events[0]["payload"]["text"] == "hello" with psycopg.connect(migrated_database_urls["admin"]) as conn: - with pytest.raises(psycopg.Error, match="append-only"): - with conn.cursor() as cur: - cur.execute( - "UPDATE events SET kind = 'message.mutated' WHERE id = %s", - (first_event["id"],), - ) + set_current_user(conn, user_id) + assert_append_only_mutation_refused( + conn, + snapshot_sql="SELECT * FROM events WHERE id = %s", + mutation_sql="UPDATE events SET kind = 'message.mutated' WHERE id = %s", + params=(first_event['id'],), + ) def test_event_deletes_are_rejected_at_database_level(migrated_database_urls): @@ -59,9 +61,13 @@ def test_event_deletes_are_rejected_at_database_level(migrated_database_urls): event = store.append_event(thread["id"], session["id"], "message.user", {"text": "keep"}) with psycopg.connect(migrated_database_urls["admin"]) as conn: - with pytest.raises(psycopg.Error, match="append-only"): - with conn.cursor() as cur: - cur.execute("DELETE FROM events WHERE id = %s", (event["id"],)) + set_current_user(conn, user_id) + assert_append_only_mutation_refused( + conn, + snapshot_sql="SELECT * FROM events WHERE id = %s", + mutation_sql='DELETE FROM events WHERE id = %s', + params=(event['id'],), + ) def test_continuity_rls_blocks_cross_user_access(migrated_database_urls): diff --git a/tests/integration/test_credential_floor_every_door_api.py b/tests/integration/test_credential_floor_every_door_api.py index 04b39f349..fe3d782ee 100644 --- a/tests/integration/test_credential_floor_every_door_api.py +++ b/tests/integration/test_credential_floor_every_door_api.py @@ -40,6 +40,7 @@ from alicebot_api.vnext_store import PostgresVNextStore from tests.integration.test_memory_mutations_api import identity_header, invoke_request, seed_user +from tests.integration.conftest import lock_label_fixture # Built rather than written out so the source carries no scanner-shaped token. @@ -638,6 +639,7 @@ def test_round2_c2_project_update_accept_refuses_a_credential_state(migrated_dat user_id = seed_user(app_url, email="floor-c2-project@example.com") with user_connection(app_url, user_id) as conn: store = PostgresVNextStore(conn) + lock_label_fixture(store) project = store.create_project( { "name": "Deploy pipeline", diff --git a/tests/integration/test_default_surface_integration.py b/tests/integration/test_default_surface_integration.py index 5b4549320..d3c48329a 100644 --- a/tests/integration/test_default_surface_integration.py +++ b/tests/integration/test_default_surface_integration.py @@ -20,7 +20,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] -DEFAULT_HTTP_OPERATION_COUNT = 183 +DEFAULT_HTTP_OPERATION_COUNT = 184 DEFAULT_MCP_TOOL_NAMES = [ "alice_memory_commit", "alice_recall", diff --git a/tests/integration/test_derived_labels_group_scope_postgres.py b/tests/integration/test_derived_labels_group_scope_postgres.py index 0aec1cac4..e9014da48 100644 --- a/tests/integration/test_derived_labels_group_scope_postgres.py +++ b/tests/integration/test_derived_labels_group_scope_postgres.py @@ -12,6 +12,7 @@ from alicebot_api.vnext_rollups import VNextRollupService from alicebot_api.vnext_store import PostgresVNextStore from tests.unit.test_group_scope_sqlite import ALPHA, seed_members, seed_promoted_members +from tests.integration.conftest import lock_label_fixture @pytest.mark.parametrize("accept", (False, True)) @@ -22,6 +23,7 @@ def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inser with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") store = PostgresVNextStore(conn) + lock_label_fixture(store) members = seed_members(store) service = VNextRollupService(store) first = service.propose_rollups(projects=(ALPHA,)) @@ -53,6 +55,7 @@ def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossi with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") store = PostgresVNextStore(conn) + lock_label_fixture(store) members = seed_promoted_members(store) first = VNextRollupService(store).propose_rollups() candidate_id = first.candidate_ids[0] diff --git a/tests/integration/test_derived_labels_migration_postgres.py b/tests/integration/test_derived_labels_migration_postgres.py index 3f24a8a19..39b177099 100644 --- a/tests/integration/test_derived_labels_migration_postgres.py +++ b/tests/integration/test_derived_labels_migration_postgres.py @@ -7,9 +7,11 @@ import threading from types import SimpleNamespace from uuid import uuid4 +from urllib.parse import quote, urlsplit, urlunsplit from alembic import command, op import psycopg +from psycopg import sql import pytest import alicebot_api.main as main_module @@ -29,13 +31,60 @@ from alicebot_api.vnext_label_writes import without_insert_floor from alicebot_api.vnext_store import PostgresVNextStore from tests.integration.test_vnext_omitted_domains_api import invoke_request -from tests.integration.conftest import _create_role_separated_database, _drop_database -from urllib.parse import urlsplit +from tests.integration.conftest import _create_role_separated_database, _drop_database, _role_urls TABLES = ("sources", "memories", "open_loops", "generated_artifacts", "beliefs", "event_log", "projects") SENTINEL = "Violet private migration sentinel" +@pytest.fixture +def database_urls(monkeypatch): + """Run migration acceptance with an owner that cannot bypass forced RLS. + + CI uses a superuser administrator for historical migrations. These tests + deliberately use a separate restricted owner for the current acceptance. + """ + admin_root, _app, lifecycle_root, *_ = _role_urls("unused") + role_name = None + with psycopg.connect(admin_root) as conn: + posture = conn.execute( + "SELECT rolsuper, rolbypassrls FROM pg_roles WHERE rolname=current_user" + ).fetchone() + if posture != (False, False): + role_name = "alicebot_repair_owner_" + uuid4().hex[:12] + password = uuid4().hex + with psycopg.connect(lifecycle_root, autocommit=True) as conn: + conn.execute( + sql.SQL("CREATE ROLE {} LOGIN PASSWORD {} NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS").format( + sql.Identifier(role_name), sql.Literal(password) + ) + ) + parsed = urlsplit(admin_root) + server = parsed.netloc.rsplit("@", 1)[-1] + strict_url = urlunsplit( + (parsed.scheme, f"{quote(role_name)}:{quote(password)}@{server}", parsed.path, parsed.query, parsed.fragment) + ) + # Preserve the original bootstrap actor when no explicit lifecycle URL is set. + monkeypatch.setenv("DATABASE_LIFECYCLE_URL", lifecycle_root) + monkeypatch.setenv("DATABASE_ADMIN_URL", strict_url) + name = "alicebot_repair_" + uuid4().hex[:12] + try: + urls = _create_role_separated_database(name) + yield urls + finally: + close_connection_pools() + _drop_database(name) + if role_name is not None: + with psycopg.connect(lifecycle_root, autocommit=True) as conn: + conn.execute(sql.SQL("DROP ROLE {}").format(sql.Identifier(role_name))) + + +@pytest.fixture +def migrated_database_urls(database_urls): + command.upgrade(make_alembic_config(database_urls["admin"]), "head") + return database_urls + + @contextmanager def owner_bracket(url): with psycopg.connect(url) as conn: diff --git a/tests/integration/test_gmail_accounts_api.py b/tests/integration/test_gmail_accounts_api.py index 0869c5f61..73d407c70 100644 --- a/tests/integration/test_gmail_accounts_api.py +++ b/tests/integration/test_gmail_accounts_api.py @@ -14,7 +14,7 @@ from alicebot_api.routers import legacy_gated as legacy_gated_router from alicebot_api.config import Settings import alicebot_api.gmail as gmail_module -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -273,6 +273,7 @@ def test_gmail_account_endpoints_connect_list_detail_and_isolate( assert '"client_secret":' not in json.dumps(create_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -495,6 +496,7 @@ def fake_fetch_gmail_message_raw_bytes(*, access_token: str, **_kwargs) -> bytes assert '"client_secret":' not in json.dumps(ingest_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -636,6 +638,7 @@ def fake_fetch_gmail_message_raw_bytes(*, access_token: str, **_kwargs) -> bytes assert '"client_secret":' not in json.dumps(ingest_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -757,6 +760,7 @@ def fail_fetch(**_kwargs): assert store.list_task_artifacts_for_task(owner["task_id"]) == [] with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -891,6 +895,7 @@ def fail_fetch(**_kwargs): ) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( "DELETE FROM gmail_account_credentials WHERE gmail_account_id = %s", @@ -964,6 +969,7 @@ def test_gmail_message_ingestion_endpoint_rejects_missing_external_secret_withou ) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_mcp_refusal_before_state_postgres.py b/tests/integration/test_mcp_refusal_before_state_postgres.py index 8a8a0b455..8d8f43c1f 100644 --- a/tests/integration/test_mcp_refusal_before_state_postgres.py +++ b/tests/integration/test_mcp_refusal_before_state_postgres.py @@ -30,6 +30,7 @@ from alicebot_api.vnext_store import PostgresVNextStore from tests.integration.test_vnext_live_workspace_api import invoke_request, seed_user +from tests.integration.conftest import lock_label_fixture _FIXED_MESSAGE = "The tool request could not be processed" _OWN_PROJECT = "alicebot" @@ -63,6 +64,7 @@ def _states(app_url: str, user_id: UUID) -> dict[str, str]: with user_connection(app_url, user_id) as conn: store = PostgresVNextStore(conn) + lock_label_fixture(store) ordinary = _memory(store) pending = _memory( store, diff --git a/tests/integration/test_memory_admission.py b/tests/integration/test_memory_admission.py index 31a04e735..202f597a1 100644 --- a/tests/integration/test_memory_admission.py +++ b/tests/integration/test_memory_admission.py @@ -11,7 +11,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -240,6 +240,7 @@ def test_admit_memory_endpoint_persists_add_update_and_delete_revisions( assert revisions[2]["new_value"] is None with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: with pytest.raises(psycopg.Error, match="append-only"): cur.execute( diff --git a/tests/integration/test_memory_review_labels_api.py b/tests/integration/test_memory_review_labels_api.py index 288fc03b6..ab80e4ad8 100644 --- a/tests/integration/test_memory_review_labels_api.py +++ b/tests/integration/test_memory_review_labels_api.py @@ -13,9 +13,10 @@ from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings from alicebot_api.contracts import MemoryCandidateInput -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.memory import admit_memory_candidate from alicebot_api.store import ContinuityStore +from tests.integration.conftest import assert_append_only_mutation_refused def invoke_request( @@ -296,20 +297,22 @@ def test_memory_review_labels_reject_update_and_delete_at_database_level(migrate ) with psycopg.connect(migrated_database_urls["admin"]) as conn: - with pytest.raises(psycopg.Error, match="append-only"): - with conn.cursor() as cur: - cur.execute( - "UPDATE memory_review_labels SET label = 'incorrect' WHERE id = %s", - (label["id"],), - ) + set_current_user(conn, UUID(seeded["user_id"])) + assert_append_only_mutation_refused( + conn, + snapshot_sql="SELECT * FROM memory_review_labels WHERE id = %s", + mutation_sql="UPDATE memory_review_labels SET label = 'incorrect' WHERE id = %s", + params=(label['id'],), + ) with psycopg.connect(migrated_database_urls["admin"]) as conn: - with pytest.raises(psycopg.Error, match="append-only"): - with conn.cursor() as cur: - cur.execute( - "DELETE FROM memory_review_labels WHERE id = %s", - (label["id"],), - ) + set_current_user(conn, UUID(seeded["user_id"])) + assert_append_only_mutation_refused( + conn, + snapshot_sql="SELECT * FROM memory_review_labels WHERE id = %s", + mutation_sql='DELETE FROM memory_review_labels WHERE id = %s', + params=(label['id'],), + ) def test_memory_review_label_endpoints_enforce_per_user_isolation_and_not_found_behavior( diff --git a/tests/integration/test_pending_project_update_guard_postgres.py b/tests/integration/test_pending_project_update_guard_postgres.py index 70849733a..c9e4dc6b7 100644 --- a/tests/integration/test_pending_project_update_guard_postgres.py +++ b/tests/integration/test_pending_project_update_guard_postgres.py @@ -12,6 +12,7 @@ from alicebot_api.vnext_memory_commit import VNextMemoryCommitService, VNextMemoryCommitValidationError from alicebot_api.vnext_project_update_guard import PENDING_PROJECT_UPDATE_MEMORY_MUTATION_MESSAGE from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import lock_label_fixture @pytest.mark.parametrize("marker", ["workflow", "memory_key"]) @@ -29,6 +30,7 @@ def test_postgres_pending_project_update_candidate_blocks_generic_memory_mutatio "Pending project guard", ) store = PostgresVNextStore(conn) + lock_label_fixture(store) metadata: dict[str, object] = {"candidate": True} memory_key = f"ordinary.pending.{uuid4().hex}" if marker == "workflow": diff --git a/tests/integration/test_provider_runtime_api.py b/tests/integration/test_provider_runtime_api.py index 27b5dc408..63adf3ce3 100644 --- a/tests/integration/test_provider_runtime_api.py +++ b/tests/integration/test_provider_runtime_api.py @@ -15,7 +15,7 @@ import alicebot_api.main as main_module from alicebot_api.config import Settings, WorkspaceProviderConfig -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, set_current_user_account, user_connection from alicebot_api.provider_configuration import provider_config_fingerprint from alicebot_api.public_errors import UPSTREAM_FAILURE from alicebot_api.provider_secrets import decode_provider_secret_ref, resolve_provider_api_key @@ -148,6 +148,8 @@ def _bootstrap_local_workspace(email: str) -> tuple[str, str, str]: def _seed_thread_for_user(*, admin_db_url: str, user_id: str, email: str) -> str: thread_id = str(uuid4()) with psycopg.connect(admin_db_url) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -842,6 +844,8 @@ def test_openai_compatible_registration_still_works(migrated_database_urls, monk assert any(record["url"] == "https://provider.example/v1/models" for record in captured_requests) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -929,6 +933,8 @@ def test_openai_compatible_no_auth_update_omits_auth_for_test_and_runtime( assert len(captured_requests) == 4 assert all("authorization" not in {str(key).lower() for key in record["headers"]} for record in captured_requests) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( "SELECT auth_mode, api_key FROM model_providers WHERE id = %s AND workspace_id = %s", @@ -965,6 +971,8 @@ def test_provider_update_uses_atomic_cas_and_hides_stale_capability( migrated_database_urls["admin"], row_factory=psycopg.rows.dict_row, ) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) store = ContinuityStore(conn) original = store.get_model_provider_for_workspace_optional( provider_id=provider_id, @@ -1017,6 +1025,8 @@ def test_provider_update_uses_atomic_cas_and_hides_stale_capability( migrated_database_urls["admin"], row_factory=psycopg.rows.dict_row, ) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) lost_update = ContinuityStore(conn).update_model_provider( provider_id=provider_id, workspace_id=UUID(workspace_id), @@ -1460,6 +1470,8 @@ def test_provider_invocation_telemetry_persists_for_test_and_runtime( assert provider_secret not in caplog.text with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -1680,6 +1692,8 @@ def fake_urlopen(request, timeout, enforce_public_peer): assert test_payload["result"]["usage"]["total_tokens"] == 14 with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -1738,6 +1752,8 @@ def fake_urlopen(request, timeout, enforce_public_peer): provider_id = register_payload["provider"]["id"] with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -1777,6 +1793,8 @@ def fake_urlopen(request, timeout, enforce_public_peer): assert updated_payload["capabilities"]["snapshot"]["azure_auth_mode"] == ("azure_ad_token") with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -1997,6 +2015,8 @@ def private_dns_getaddrinfo(hostname: str, port, type=0, proto=0): } with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2019,7 +2039,7 @@ def test_provider_test_and_runtime_reject_disallowed_target_without_outbound( user_id, workspace_id, user_account_id = _bootstrap_local_workspace("provider-security-blocked-runtime@example.com") urlopen_call_count = 0 - def fake_urlopen(_request, _timeout): + def fake_urlopen(_request, timeout, enforce_public_peer): nonlocal urlopen_call_count urlopen_call_count += 1 raise AssertionError("outbound request should not be attempted for blocked targets") @@ -2042,6 +2062,8 @@ def fake_urlopen(_request, _timeout): provider_id = register_payload["provider"]["id"] with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2117,6 +2139,8 @@ def test_provider_rejects_userinfo_and_redacts_legacy_rows( legacy_provider_id: str with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2235,6 +2259,8 @@ def fake_urlopen(request, timeout, enforce_public_peer): assert sensitive_detail not in json.dumps(test_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2272,6 +2298,8 @@ def fake_urlopen(request, timeout, enforce_public_peer): assert sensitive_detail not in json.dumps(runtime_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_proxy_execution_api.py b/tests/integration/test_proxy_execution_api.py index 24ca106e8..68909d9d9 100644 --- a/tests/integration/test_proxy_execution_api.py +++ b/tests/integration/test_proxy_execution_api.py @@ -11,7 +11,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import legacy_gated as legacy_gated_router from alicebot_api.config import Settings -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -167,11 +167,13 @@ def create_execution_budget( def set_execution_executed_at( admin_database_url: str, + user_id: UUID, *, execution_id: UUID, executed_at_sql: str, ) -> None: with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) conn.execute( f"UPDATE tool_executions SET executed_at = {executed_at_sql} WHERE id = %s", (execution_id,), @@ -181,11 +183,13 @@ def set_execution_executed_at( def set_approval_request_thread_id( admin_database_url: str, + user_id: UUID, *, approval_id: UUID, request_thread_id: str, ) -> None: with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) conn.execute( """ UPDATE approvals @@ -851,6 +855,7 @@ def test_execute_approved_proxy_endpoint_fail_closes_when_runtime_context_is_inv set_approval_request_thread_id( migrated_database_urls["admin"], + user_id=owner["user_id"], approval_id=UUID(create_payload["approval"]["id"]), request_thread_id="not-a-uuid", ) @@ -1317,6 +1322,7 @@ def test_execute_approved_proxy_endpoint_excludes_old_window_history_and_keeps_c set_execution_executed_at( migrated_database_urls["admin"], + user_id=owner["user_id"], execution_id=owner_first_execution_id, executed_at_sql="clock_timestamp() - interval '2 hours'", ) diff --git a/tests/integration/test_saved_quotes_postgres.py b/tests/integration/test_saved_quotes_postgres.py index b08fe6d74..8b7e65569 100644 --- a/tests/integration/test_saved_quotes_postgres.py +++ b/tests/integration/test_saved_quotes_postgres.py @@ -38,6 +38,7 @@ from alicebot_api.vnext_retrieval import VNextRetrievalRequest, VNextRetrievalService from alicebot_api.vnext_source_fence import SavedProvenanceReader, SourceReadFence from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import lock_label_fixture _QUOTE = "zinnwald-quote-8841 cone ten firing kiln log marlin-oxide-5520" @@ -84,6 +85,7 @@ def _read(store: PostgresVNextStore, memory_id: str, fence: SourceReadFence) -> def run_lifecycle(store: PostgresVNextStore) -> None: """The lifecycle over one store: save a quote, make the source confidential, archive it, read it three ways.""" + lock_label_fixture(store) source = store.create_source( { "source_type": "document", @@ -198,6 +200,7 @@ def _pack_for(store: PostgresVNextStore, fence: SourceReadFence) -> dict[str, ob def run_linkless_and_sibling_lifecycle(store: PostgresVNextStore) -> None: """A memory with no link and a memory with two links of the same quote, read before and after a source changes.""" + lock_label_fixture(store) refused = _make_source(store, "Alpha held log") kept = _make_source(store, "Alpha second log") linkless = store.create_memory( @@ -309,6 +312,7 @@ def test_a_memory_with_no_link_and_a_sibling_quote_are_withheld_on_postgres(migr def run_nested_reference_lifecycle(store: PostgresVNextStore) -> None: """Two memories that cite a readable source and one that is reclassified, in one ref, read before and after.""" + lock_label_fixture(store) readable = _make_source(store, "Alpha second log") refused = _make_source(store, "Alpha nested log") shapes = { diff --git a/tests/integration/test_task_artifacts_api.py b/tests/integration/test_task_artifacts_api.py index e5092e9e0..fdf78c113 100644 --- a/tests/integration/test_task_artifacts_api.py +++ b/tests/integration/test_task_artifacts_api.py @@ -18,7 +18,7 @@ from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings from alicebot_api.artifacts import TASK_ARTIFACT_CHUNK_RETRIEVAL_MATCHING_RULE -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -1742,6 +1742,7 @@ def test_task_artifact_ingestion_enforces_rooted_workspace_paths( assert register_status == 201 with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -1821,6 +1822,7 @@ def test_task_artifact_docx_ingestion_enforces_rooted_workspace_paths( assert register_status == 201 with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -1900,6 +1902,7 @@ def test_task_artifact_rfc822_ingestion_enforces_rooted_workspace_paths( assert register_status == 201 with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_temporal_state_api.py b/tests/integration/test_temporal_state_api.py index 0a7bc76f0..fd81b2c3a 100644 --- a/tests/integration/test_temporal_state_api.py +++ b/tests/integration/test_temporal_state_api.py @@ -14,7 +14,7 @@ from alicebot_api.routers import continuity as continuity_router from alicebot_api.config import Settings from alicebot_api.contracts import MemoryCandidateInput -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.memory import admit_memory_candidate from alicebot_api.store import ContinuityStore @@ -69,13 +69,15 @@ async def send(message: dict[str, object]) -> None: def _set_temporal_timestamps( admin_database_url: str, + user_id: UUID, *, entity_id: UUID, edge_id: UUID, entity_created_at: datetime, edge_created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( "UPDATE entities SET created_at = %s WHERE id = %s", @@ -150,6 +152,7 @@ def seed_temporal_entity_graph(database_url: str, admin_database_url: str) -> di midpoint = add_at + (update_at - add_at) / 2 _set_temporal_timestamps( admin_database_url, + user_id=user_id, entity_id=entity["id"], edge_id=edge["id"], entity_created_at=add_at - timedelta(seconds=1), diff --git a/tests/integration/test_temporal_state_mcp_cli.py b/tests/integration/test_temporal_state_mcp_cli.py index 5550497f4..1e434a508 100644 --- a/tests/integration/test_temporal_state_mcp_cli.py +++ b/tests/integration/test_temporal_state_mcp_cli.py @@ -12,7 +12,7 @@ import psycopg from alicebot_api.contracts import MemoryCandidateInput -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.memory import admit_memory_candidate from alicebot_api.store import ContinuityStore @@ -137,13 +137,15 @@ def _call_tool(client: MCPClient, *, name: str, arguments: dict[str, object]) -> def _set_temporal_timestamps( admin_database_url: str, + user_id: UUID, *, entity_id: UUID, edge_id: UUID, entity_created_at: datetime, edge_created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute("UPDATE entities SET created_at = %s WHERE id = %s", (entity_created_at, entity_id)) cur.execute( @@ -214,6 +216,7 @@ def seed_temporal_entity_graph(database_url: str, admin_database_url: str) -> di midpoint = add_at + (update_at - add_at) / 2 _set_temporal_timestamps( admin_database_url, + user_id=user_id, entity_id=entity["id"], edge_id=edge["id"], entity_created_at=add_at - timedelta(seconds=1), diff --git a/tests/integration/test_vnext_consolidation_postgres.py b/tests/integration/test_vnext_consolidation_postgres.py index 63c08eec5..1b125b2aa 100644 --- a/tests/integration/test_vnext_consolidation_postgres.py +++ b/tests/integration/test_vnext_consolidation_postgres.py @@ -22,6 +22,7 @@ from alicebot_api.vnext_embeddings import pad_embedding_vector from alicebot_api.vnext_memory_commit import VNextMemoryCommitService from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import lock_label_fixture def test_generate_memory_consolidation_persists_its_artifact(migrated_database_urls) -> None: @@ -67,6 +68,7 @@ def test_rollup_candidate_round_trips_and_acceptance_promotes_it( with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, "rollups@example.invalid", "Rollups") store = PostgresVNextStore(conn) + lock_label_fixture(store) members = [] for index, (text, day) in enumerate( ( diff --git a/tests/integration/test_vnext_live_workspace_api.py b/tests/integration/test_vnext_live_workspace_api.py index 504648ec4..73653f62d 100644 --- a/tests/integration/test_vnext_live_workspace_api.py +++ b/tests/integration/test_vnext_live_workspace_api.py @@ -30,6 +30,7 @@ VNextProjectTerminalConsistencyError, ) from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import lock_label_fixture def invoke_request( @@ -282,6 +283,7 @@ def test_project_update_true_redaction_scrubs_the_role_separated_coupled_graph( with user_connection(migrated_database_urls["app"], user_id) as conn: store = PostgresVNextStore(conn) + lock_label_fixture(store) project = store.create_project( { "name": f"Option A {action} {sentinel}", @@ -785,6 +787,7 @@ def test_project_update_terminal_replay_survives_authorized_true_redaction( ) with user_connection(migrated_database_urls["app"], user_id) as conn: store = PostgresVNextStore(conn) + lock_label_fixture(store) project = store.create_project( { "name": f"{terminal_status.title()} redaction replay", @@ -1006,6 +1009,7 @@ def test_project_update_terminal_replay_rejects_competing_postgres_decision_with ) with user_connection(migrated_database_urls["app"], user_id) as conn: store = PostgresVNextStore(conn) + lock_label_fixture(store) project = store.create_project( { "name": f"{action.title()} competing decision", From be1aaca89458ab341cc8e97ff711f63fea854e6f Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:25:47 +0200 Subject: [PATCH 197/270] Exercise current ancestry admission at dashboard read stage --- tests/unit/test_cli.py | 9 +++++++++ tests/unit/test_vnext_projects.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 0a9fdf937..b7f0a4b1b 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -1823,6 +1823,15 @@ def fake_vnext_store_context(_ctx): assert review_loop_payload["due_at"] == "2026-05-12T09:00:00Z" assert dashboard_payload["counts"]["open_loops"] == 1 + # The CLI must keep the dashboard's current-input admission checks. + store.sources[0]["domain"] = "health" + store.sources[0]["sensitivity"] = "regulated" + restricted = json.loads(dashboard_args.handler(ctx, dashboard_args)) + assert restricted["counts"]["open_loops"] == 0 + store.sources.clear() + missing = json.loads(dashboard_args.handler(ctx, dashboard_args)) + assert missing["counts"]["open_loops"] == 0 + def test_vnext_queue_cli_add_process_review_and_export(monkeypatch, tmp_path: Path) -> None: store = FakeVNextCliStore() diff --git a/tests/unit/test_vnext_projects.py b/tests/unit/test_vnext_projects.py index a4cbef901..996e49c5c 100644 --- a/tests/unit/test_vnext_projects.py +++ b/tests/unit/test_vnext_projects.py @@ -1584,6 +1584,24 @@ def test_open_loop_extraction_and_review_support_source_owner_and_filters() -> N +@pytest.mark.parametrize("parent_change", ["missing", "restricted"]) +def test_dashboard_withholds_loops_after_their_source_becomes_unreadable(parent_change) -> None: + store = _seed_store() + service = VNextProjectService(store) + loops = service.extract_open_loops(ProjectAutomationRequest(project_id="project-1", domains=("project",))) + assert service.project_dashboard(project_id="project-1")["counts"]["open_loops"] == 2 + if parent_change == "missing": + store.sources.clear() + else: + store.sources[0]["domain"] = "health" + store.sources[0]["sensitivity"] = "regulated" + dashboard = service.project_dashboard(project_id="project-1") + assert dashboard["counts"]["open_loops"] == 0 + assert dashboard["open_loops"] == [] + assert all(loop["source_id"] == "source-1" for loop in loops) + + + def test_project_service_validation_errors() -> None: service = VNextProjectService(InMemoryVNextProjectStore()) From 4c179e759f3624647acb241620413d33ce2ab59f Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:27:49 +0200 Subject: [PATCH 198/270] Count source regeneration in the default surface smoke --- tests/integration/test_default_surface_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_default_surface_integration.py b/tests/integration/test_default_surface_integration.py index 5b4549320..d3c48329a 100644 --- a/tests/integration/test_default_surface_integration.py +++ b/tests/integration/test_default_surface_integration.py @@ -20,7 +20,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] -DEFAULT_HTTP_OPERATION_COUNT = 183 +DEFAULT_HTTP_OPERATION_COUNT = 184 DEFAULT_MCP_TOOL_NAMES = [ "alice_memory_commit", "alice_recall", From 0685f813e64fee461ef00326b97ccb70bb929945 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:33:00 +0200 Subject: [PATCH 199/270] Provide ordered lock protocol in deferred review fixture --- tests/unit/test_main.py | 48 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 6c8783108..476f198c7 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5416,6 +5416,7 @@ def test_vnext_project_review_defers_embedding_and_preserves_human_attribution(m user_id = uuid4() transaction_depth = 0 calls: list[str] = [] + lock_calls: list[str] = [] review_kwargs: dict[str, object] = {} deferred_input = object() decision = main_module.PolicyDecision( @@ -5434,6 +5435,48 @@ def fake_user_connection(_database_url: str, _user_id): finally: transaction_depth -= 1 + class FakeLockCursor: + def execute(self, query: str, params=None) -> None: + assert transaction_depth == 1 + assert query in { + "SELECT current_setting('lock_timeout') AS lock_timeout", + "SET LOCAL lock_timeout = '3s'", + "SELECT set_config('lock_timeout', %s, true)", + } + if query.startswith("SELECT set_config"): + assert params == ("0",) + + def fetchone(self): + return {"lock_timeout": "0"} + + class FakeLockConnection: + @contextmanager + def cursor(self): + yield FakeLockCursor() + + class FakeStore: + conn = FakeLockConnection() + + def lock_graph_mutation(self) -> None: + assert transaction_depth == 1 + assert lock_calls == [] + lock_calls.append("graph") + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + assert transaction_depth == 1 + assert exclusive is True + assert lock_calls == ["graph"] + lock_calls.append("exclusive_labels") + + store = FakeStore() + + def fake_authorized_artifact(**_kwargs): + assert transaction_depth == 1 + assert lock_calls == ["graph", "exclusive_labels"] + assert _kwargs["store"] is store + lock_calls.append("authorize") + return {"id": "artifact-1", "status": "needs_review"}, decision + class FakeProjectService: def __init__(self, _store, *, defer_embeddings: bool = False) -> None: assert transaction_depth == 1 @@ -5456,12 +5499,12 @@ def fake_persist(**kwargs) -> None: monkeypatch.setattr(vnext_review_router, "get_settings", lambda: Settings(database_url="postgresql://db")) monkeypatch.setattr(vnext_review_router, "user_connection", fake_user_connection) - monkeypatch.setattr(vnext_review_router, "PostgresVNextStore", lambda _conn: object()) + monkeypatch.setattr(vnext_review_router, "PostgresVNextStore", lambda _conn: store) monkeypatch.setattr(vnext_review_router, "_vnext_authenticated_agent_identity", lambda *_args, **_kwargs: None) monkeypatch.setattr( vnext_review_router, "_vnext_authorized_artifact", - lambda **_kwargs: ({"id": "artifact-1", "status": "needs_review"}, decision), + fake_authorized_artifact, ) monkeypatch.setattr(vnext_review_router, "VNextProjectService", FakeProjectService) monkeypatch.setattr(vnext_review_router, "_persist_vnext_deferred_embeddings", fake_persist) @@ -5477,6 +5520,7 @@ def fake_persist(**kwargs) -> None: assert response.status_code == 200 assert calls == ["review", "embedding"] + assert lock_calls == ["graph", "exclusive_labels", "authorize"] assert review_kwargs["actor_type"] == "user" assert review_kwargs["actor_id"] == str(user_id) assert review_kwargs["trace_id"] == "request-trace-1" From 8b89b7723553b4242dbe094ba9f06177b5a1f1bc Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:33:00 +0200 Subject: [PATCH 200/270] Provide ordered lock protocol in deferred review fixture --- tests/unit/test_main.py | 48 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 6c8783108..476f198c7 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5416,6 +5416,7 @@ def test_vnext_project_review_defers_embedding_and_preserves_human_attribution(m user_id = uuid4() transaction_depth = 0 calls: list[str] = [] + lock_calls: list[str] = [] review_kwargs: dict[str, object] = {} deferred_input = object() decision = main_module.PolicyDecision( @@ -5434,6 +5435,48 @@ def fake_user_connection(_database_url: str, _user_id): finally: transaction_depth -= 1 + class FakeLockCursor: + def execute(self, query: str, params=None) -> None: + assert transaction_depth == 1 + assert query in { + "SELECT current_setting('lock_timeout') AS lock_timeout", + "SET LOCAL lock_timeout = '3s'", + "SELECT set_config('lock_timeout', %s, true)", + } + if query.startswith("SELECT set_config"): + assert params == ("0",) + + def fetchone(self): + return {"lock_timeout": "0"} + + class FakeLockConnection: + @contextmanager + def cursor(self): + yield FakeLockCursor() + + class FakeStore: + conn = FakeLockConnection() + + def lock_graph_mutation(self) -> None: + assert transaction_depth == 1 + assert lock_calls == [] + lock_calls.append("graph") + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + assert transaction_depth == 1 + assert exclusive is True + assert lock_calls == ["graph"] + lock_calls.append("exclusive_labels") + + store = FakeStore() + + def fake_authorized_artifact(**_kwargs): + assert transaction_depth == 1 + assert lock_calls == ["graph", "exclusive_labels"] + assert _kwargs["store"] is store + lock_calls.append("authorize") + return {"id": "artifact-1", "status": "needs_review"}, decision + class FakeProjectService: def __init__(self, _store, *, defer_embeddings: bool = False) -> None: assert transaction_depth == 1 @@ -5456,12 +5499,12 @@ def fake_persist(**kwargs) -> None: monkeypatch.setattr(vnext_review_router, "get_settings", lambda: Settings(database_url="postgresql://db")) monkeypatch.setattr(vnext_review_router, "user_connection", fake_user_connection) - monkeypatch.setattr(vnext_review_router, "PostgresVNextStore", lambda _conn: object()) + monkeypatch.setattr(vnext_review_router, "PostgresVNextStore", lambda _conn: store) monkeypatch.setattr(vnext_review_router, "_vnext_authenticated_agent_identity", lambda *_args, **_kwargs: None) monkeypatch.setattr( vnext_review_router, "_vnext_authorized_artifact", - lambda **_kwargs: ({"id": "artifact-1", "status": "needs_review"}, decision), + fake_authorized_artifact, ) monkeypatch.setattr(vnext_review_router, "VNextProjectService", FakeProjectService) monkeypatch.setattr(vnext_review_router, "_persist_vnext_deferred_embeddings", fake_persist) @@ -5477,6 +5520,7 @@ def fake_persist(**kwargs) -> None: assert response.status_code == 200 assert calls == ["review", "embedding"] + assert lock_calls == ["graph", "exclusive_labels", "authorize"] assert review_kwargs["actor_type"] == "user" assert review_kwargs["actor_id"] == str(user_id) assert review_kwargs["trace_id"] == "request-trace-1" From b46473fb31a64a891d9393d8af56ce665316d631 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:30:05 +0200 Subject: [PATCH 201/270] Pin remaining registered event and rating reader controls --- tests/unit/test_label_reader_stage_matrix.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_label_reader_stage_matrix.py b/tests/unit/test_label_reader_stage_matrix.py index d595362ed..d695cdea6 100644 --- a/tests/unit/test_label_reader_stage_matrix.py +++ b/tests/unit/test_label_reader_stage_matrix.py @@ -67,7 +67,7 @@ def list_edges(self, **kwargs): return [{"from_type": "memory", "from_id": MEMOR def _scope(scoped=False): return _ResolvedRetrievalScope(projects=frozenset((ALPHA,)) if scoped else frozenset(), people=frozenset(), window_start=None, window_end=None, exclude_global_domains=frozenset()) -STAGES = ("by_ids", "by_ids_fallback", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "pack_open_loops", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs", "workspace_loop_refs", "dashboard_loop_refs") +STAGES = ("by_ids", "by_ids_fallback", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "pack_open_loops", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs", "workspace_event_helper", "workspace_loop_refs", "dashboard_loop_refs") def _stage(stage, store): service = VNextRetrievalService(store, embedding_provider=SimpleNamespace(provider="synthetic", model="synthetic", base_url="http://synthetic.invalid")) @@ -105,6 +105,8 @@ def _stage(stage, store): else: body = VNextProjectService(store).project_dashboard(project_id=ALPHA, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent")) return [body["open_loops"][0]["memory_id"]] if body["open_loops"][0]["memory_id"] else [] + if stage == "workspace_event_helper": + return store.events if workspaces._workspace_event_visible(store, store.events[0], CEILING) else [] if stage.startswith("workspace_"): field = stage.removeprefix("workspace_") if field == "memories": @@ -155,3 +157,16 @@ def test_each_dogfooding_sample_counts_only_currently_readable_rows(kind, monkey store.rows["source"][0]["sensitivity"] = "public" visible = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(CEILING)) assert visible["sample_scope"][kind]["returned_count"] == 1 + + +def test_dogfooding_rating_count_uses_the_current_artifact_label(monkeypatch): + _quiet_services(monkeypatch) + store = StageStore() + store.list_artifact_quality_ratings = lambda **kwargs: [{"id": str(UUID(int=109)), "artifact_id": ARTIFACT, "usefulness": 5}] + hidden = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(CEILING)) + assert hidden["artifact_quality_rating_count"] == 0 + assert hidden["artifact_quality_average"] is None + store.rows["source"][0]["sensitivity"] = "public" + visible = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(CEILING)) + assert visible["artifact_quality_rating_count"] == 1 + assert visible["artifact_quality_average"] == 5 From 7e9cc413bbecb90e6295353438222b377b290ea7 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:33:00 +0200 Subject: [PATCH 202/270] Provide ordered lock protocol in deferred review fixture --- tests/unit/test_main.py | 48 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 6c8783108..476f198c7 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5416,6 +5416,7 @@ def test_vnext_project_review_defers_embedding_and_preserves_human_attribution(m user_id = uuid4() transaction_depth = 0 calls: list[str] = [] + lock_calls: list[str] = [] review_kwargs: dict[str, object] = {} deferred_input = object() decision = main_module.PolicyDecision( @@ -5434,6 +5435,48 @@ def fake_user_connection(_database_url: str, _user_id): finally: transaction_depth -= 1 + class FakeLockCursor: + def execute(self, query: str, params=None) -> None: + assert transaction_depth == 1 + assert query in { + "SELECT current_setting('lock_timeout') AS lock_timeout", + "SET LOCAL lock_timeout = '3s'", + "SELECT set_config('lock_timeout', %s, true)", + } + if query.startswith("SELECT set_config"): + assert params == ("0",) + + def fetchone(self): + return {"lock_timeout": "0"} + + class FakeLockConnection: + @contextmanager + def cursor(self): + yield FakeLockCursor() + + class FakeStore: + conn = FakeLockConnection() + + def lock_graph_mutation(self) -> None: + assert transaction_depth == 1 + assert lock_calls == [] + lock_calls.append("graph") + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + assert transaction_depth == 1 + assert exclusive is True + assert lock_calls == ["graph"] + lock_calls.append("exclusive_labels") + + store = FakeStore() + + def fake_authorized_artifact(**_kwargs): + assert transaction_depth == 1 + assert lock_calls == ["graph", "exclusive_labels"] + assert _kwargs["store"] is store + lock_calls.append("authorize") + return {"id": "artifact-1", "status": "needs_review"}, decision + class FakeProjectService: def __init__(self, _store, *, defer_embeddings: bool = False) -> None: assert transaction_depth == 1 @@ -5456,12 +5499,12 @@ def fake_persist(**kwargs) -> None: monkeypatch.setattr(vnext_review_router, "get_settings", lambda: Settings(database_url="postgresql://db")) monkeypatch.setattr(vnext_review_router, "user_connection", fake_user_connection) - monkeypatch.setattr(vnext_review_router, "PostgresVNextStore", lambda _conn: object()) + monkeypatch.setattr(vnext_review_router, "PostgresVNextStore", lambda _conn: store) monkeypatch.setattr(vnext_review_router, "_vnext_authenticated_agent_identity", lambda *_args, **_kwargs: None) monkeypatch.setattr( vnext_review_router, "_vnext_authorized_artifact", - lambda **_kwargs: ({"id": "artifact-1", "status": "needs_review"}, decision), + fake_authorized_artifact, ) monkeypatch.setattr(vnext_review_router, "VNextProjectService", FakeProjectService) monkeypatch.setattr(vnext_review_router, "_persist_vnext_deferred_embeddings", fake_persist) @@ -5477,6 +5520,7 @@ def fake_persist(**kwargs) -> None: assert response.status_code == 200 assert calls == ["review", "embedding"] + assert lock_calls == ["graph", "exclusive_labels", "authorize"] assert review_kwargs["actor_type"] == "user" assert review_kwargs["actor_id"] == str(user_id) assert review_kwargs["trace_id"] == "request-trace-1" From 6cf579d45045f3fb7b75a79d186bdc21154a9432 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:34:00 +0200 Subject: [PATCH 203/270] Take exclusive labels before memory review row locks --- apps/api/src/alicebot_api/mcp/memories.py | 3 +++ apps/api/src/alicebot_api/mcp/review.py | 3 +++ apps/api/src/alicebot_api/routers/vnext_memories.py | 9 +++++---- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/apps/api/src/alicebot_api/mcp/memories.py b/apps/api/src/alicebot_api/mcp/memories.py index 0beb74c31..5d470cae7 100644 --- a/apps/api/src/alicebot_api/mcp/memories.py +++ b/apps/api/src/alicebot_api/mcp/memories.py @@ -412,6 +412,9 @@ def redact_memory_flow( raise VNextMemoryCommitValidationError("reason is required to redact a memory") memory_service = VNextMemoryCommitService(store) memory_service.lock_supersession_graph() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) memory = store.get_memory_for_redaction(memory_id) if memory is None: raise MemoryNotFoundError("memory was not found") diff --git a/apps/api/src/alicebot_api/mcp/review.py b/apps/api/src/alicebot_api/mcp/review.py index 3f5384013..ba82ed3c7 100644 --- a/apps/api/src/alicebot_api/mcp/review.py +++ b/apps/api/src/alicebot_api/mcp/review.py @@ -571,6 +571,9 @@ def _vnext_memory_correct(context: MCPRuntimeContext, arguments: Mapping[str, ob # approval activates a memory too, so it must not be a row-first # exception to the lifecycle mutation boundary. memory_service.lock_supersession_graph() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) get_memory_for_update = getattr(store, "get_memory_for_update", None) memory = get_memory_for_update(memory_id) if callable(get_memory_for_update) else store.get_memory(memory_id) if memory is None: diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index b8e09b65e..2ace1800f 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -1068,16 +1068,17 @@ def review_vnext_memory( # graph boundary before the route takes any candidate/member row lock; # delegated service calls may safely reacquire the transaction lock. memory_service.lock_supersession_graph() + # A status-only review can also raise stale derived labels at the + # owner floor, so acquire the label lock before reading for update. + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) label_change = ( request.domain is not None or request.sensitivity is not None or request.project_id is not None or action in {"private", "assign_project"} ) - if label_change: - from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock - - acquire_exclusive_label_lock(store) preview = store.get_memory(str(memory_id)) if preview is None: return _vnext_public_error_response(status_code=404, detail="vNext memory was not found") From cffdb0a0abb9bfc5b2658509e405ecda32f95205 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:34:00 +0200 Subject: [PATCH 204/270] Take exclusive labels before memory review row locks --- apps/api/src/alicebot_api/mcp/memories.py | 3 +++ apps/api/src/alicebot_api/mcp/review.py | 3 +++ apps/api/src/alicebot_api/routers/vnext_memories.py | 9 +++++---- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/apps/api/src/alicebot_api/mcp/memories.py b/apps/api/src/alicebot_api/mcp/memories.py index 6165fe47c..1e8757c13 100644 --- a/apps/api/src/alicebot_api/mcp/memories.py +++ b/apps/api/src/alicebot_api/mcp/memories.py @@ -412,6 +412,9 @@ def redact_memory_flow( raise VNextMemoryCommitValidationError("reason is required to redact a memory") memory_service = VNextMemoryCommitService(store) memory_service.lock_supersession_graph() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) memory = store.get_memory_for_redaction(memory_id) if memory is None: raise MemoryNotFoundError("memory was not found") diff --git a/apps/api/src/alicebot_api/mcp/review.py b/apps/api/src/alicebot_api/mcp/review.py index 297df7896..515a746de 100644 --- a/apps/api/src/alicebot_api/mcp/review.py +++ b/apps/api/src/alicebot_api/mcp/review.py @@ -542,6 +542,9 @@ def _vnext_memory_correct(context: MCPRuntimeContext, arguments: Mapping[str, ob # approval activates a memory too, so it must not be a row-first # exception to the lifecycle mutation boundary. memory_service.lock_supersession_graph() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) get_memory_for_update = getattr(store, "get_memory_for_update", None) memory = get_memory_for_update(memory_id) if callable(get_memory_for_update) else store.get_memory(memory_id) if memory is None: diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index f8d44a3ca..f312817b1 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -1040,16 +1040,17 @@ def review_vnext_memory( # graph boundary before the route takes any candidate/member row lock; # delegated service calls may safely reacquire the transaction lock. memory_service.lock_supersession_graph() + # A status-only review can also raise stale derived labels at the + # owner floor, so acquire the label lock before reading for update. + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) label_change = ( request.domain is not None or request.sensitivity is not None or request.project_id is not None or action in {"private", "assign_project"} ) - if label_change: - from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock - - acquire_exclusive_label_lock(store) preview = store.get_memory(str(memory_id)) if preview is None: return _vnext_public_error_response(status_code=404, detail="vNext memory was not found") From 406bea83a7962bf34c459864cad803826a9e9252 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:35:11 +0200 Subject: [PATCH 205/270] Verify memory review lock state and rollup fixture order --- ...rived_labels_exact_entrypoints_postgres.py | 27 ++++++++++++++++++- ...est_derived_labels_group_scope_postgres.py | 1 + 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_derived_labels_exact_entrypoints_postgres.py b/tests/integration/test_derived_labels_exact_entrypoints_postgres.py index 96437149c..476991fea 100644 --- a/tests/integration/test_derived_labels_exact_entrypoints_postgres.py +++ b/tests/integration/test_derived_labels_exact_entrypoints_postgres.py @@ -11,7 +11,7 @@ from alicebot_api.mcp.registry import MCPToolNotFoundError, call_mcp_tool from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError from alicebot_api.routers import vnext_memories, vnext_projects, vnext_retrieval, vnext_review -from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_label_writes import held_label_locks, without_insert_floor from alicebot_api.vnext_store import PostgresVNextStore from tests.integration.test_derived_labels_read_acceptance_postgres import _user from tests.unit.test_derived_labels_real_keys import ALPHA, real_reader_key @@ -59,6 +59,18 @@ def test_exact_entrypoint_checks_effective_floor(migrated_database_urls, monkeyp monkeypatch.setenv("ALICE_MCP_LEGACY_TOOLS", "1") monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + mutation_locks = [] + if door in {"memory_review_http", "memory_correct_mcp", "memory_redact_mcp"}: + for method_name in ("get_memory_for_update", "get_memory_for_redaction"): + original = getattr(PostgresVNextStore, method_name) + + def checked(store, *args, _original=original, **kwargs): + locks = held_label_locks(store) + assert locks == (True, True, True), (door, locks) + mutation_locks.append(locks) + return _original(store, *args, **kwargs) + + monkeypatch.setattr(PostgresVNextStore, method_name, checked) with user_connection(app_url, user_id) as conn: store = PostgresVNextStore(conn) key = real_reader_key(store, user_id, reader) @@ -78,6 +90,7 @@ def test_exact_entrypoint_checks_effective_floor(migrated_database_urls, monkeyp else: row = store.create_memory({"memory_key": str(uuid4()), "title": "Door sentinel", "canonical_text": "Door sentinel", "status": "candidate", "domain": "project", "sensitivity": "public", "metadata_json": metadata}) blocked = hidden and reader == "bound_admin" + mutation_locks.clear() if door.startswith("legacy_") and key: with pytest.raises(MCPToolNotFoundError, match="disabled whenever"): _invoke(door, app_url=app_url, user_id=user_id, target_id=str(row["id"]), key=key, tmp_path=tmp_path) @@ -98,6 +111,18 @@ def test_exact_entrypoint_checks_effective_floor(migrated_database_urls, monkeyp else: assert not blocked, (door, reader, hidden, json.dumps(result, default=str)) assert str(row["id"]) in json.dumps(result, default=str) + if door in {"memory_review_http", "memory_correct_mcp", "memory_redact_mcp"}: + if not blocked: + assert mutation_locks, (door, reader, hidden) + with user_connection(app_url, user_id) as conn: + stored = conn.execute("SELECT status, deleted_at FROM memories WHERE id=%s", (row["id"],)).fetchone() + if blocked: + assert stored["status"] == "candidate" + assert stored["deleted_at"] is None + elif door == "memory_redact_mcp": + assert stored["deleted_at"] is not None + else: + assert stored["status"] == "active" @pytest.mark.parametrize("reader", ("owner", "bound_admin")) diff --git a/tests/integration/test_derived_labels_group_scope_postgres.py b/tests/integration/test_derived_labels_group_scope_postgres.py index e9014da48..7fc47c861 100644 --- a/tests/integration/test_derived_labels_group_scope_postgres.py +++ b/tests/integration/test_derived_labels_group_scope_postgres.py @@ -74,6 +74,7 @@ def test_existing_rollup_state_admits_effective_labels_for_pending_and_accepted_ with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, f"existing-{user_id}@example.invalid", "Existing") store = PostgresVNextStore(conn) + lock_label_fixture(store) members = seed_members(store) service = VNextRollupService(store) proposal = service.propose_rollups(projects=(ALPHA,)) From 6aaba446aebe5eeb5b1b4aeeff862507120075cc Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:36:07 +0200 Subject: [PATCH 206/270] Bootstrap vector for restricted owner acceptance fixtures --- tests/integration/test_derived_labels_migration_postgres.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_derived_labels_migration_postgres.py b/tests/integration/test_derived_labels_migration_postgres.py index 39b177099..2a586b226 100644 --- a/tests/integration/test_derived_labels_migration_postgres.py +++ b/tests/integration/test_derived_labels_migration_postgres.py @@ -70,6 +70,9 @@ def database_urls(monkeypatch): name = "alicebot_repair_" + uuid4().hex[:12] try: urls = _create_role_separated_database(name) + # CI preloads vector in its root database rather than template1. + with psycopg.connect(_role_urls(name)[-1], autocommit=True) as conn: + conn.execute("CREATE EXTENSION IF NOT EXISTS vector") yield urls finally: close_connection_pools() From 2aaa931048f8d0cb4257fe9116817cff40d45b1c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:37:52 +0200 Subject: [PATCH 207/270] Seed rollup reader fixtures in graph and label lock order --- tests/integration/conftest.py | 8 ++++++++ .../test_derived_labels_group_scope_postgres.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 404554c6e..7e0a3be98 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -171,3 +171,11 @@ def migrated_database_urls(migrated_template_database: str) -> Iterator[dict[str yield urls finally: _drop_database(database_name) + + +def lock_label_fixture(store) -> None: + """Seed and mutate one transaction with the production lock order.""" + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + store.lock_graph_mutation() + acquire_exclusive_label_lock(store) diff --git a/tests/integration/test_derived_labels_group_scope_postgres.py b/tests/integration/test_derived_labels_group_scope_postgres.py index 0aec1cac4..e9014da48 100644 --- a/tests/integration/test_derived_labels_group_scope_postgres.py +++ b/tests/integration/test_derived_labels_group_scope_postgres.py @@ -12,6 +12,7 @@ from alicebot_api.vnext_rollups import VNextRollupService from alicebot_api.vnext_store import PostgresVNextStore from tests.unit.test_group_scope_sqlite import ALPHA, seed_members, seed_promoted_members +from tests.integration.conftest import lock_label_fixture @pytest.mark.parametrize("accept", (False, True)) @@ -22,6 +23,7 @@ def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inser with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") store = PostgresVNextStore(conn) + lock_label_fixture(store) members = seed_members(store) service = VNextRollupService(store) first = service.propose_rollups(projects=(ALPHA,)) @@ -53,6 +55,7 @@ def test_a_cluster_of_promoted_copies_with_different_floors_is_refused_as_crossi with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") store = PostgresVNextStore(conn) + lock_label_fixture(store) members = seed_promoted_members(store) first = VNextRollupService(store).propose_rollups() candidate_id = first.candidate_ids[0] From 902d3b143b2179bf1b3fbfc96682215fcae9cb07 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:35:11 +0200 Subject: [PATCH 208/270] Verify memory review lock state and rollup fixture order --- ...rived_labels_exact_entrypoints_postgres.py | 27 ++++++++++++++++++- ...est_derived_labels_group_scope_postgres.py | 1 + 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_derived_labels_exact_entrypoints_postgres.py b/tests/integration/test_derived_labels_exact_entrypoints_postgres.py index 96437149c..476991fea 100644 --- a/tests/integration/test_derived_labels_exact_entrypoints_postgres.py +++ b/tests/integration/test_derived_labels_exact_entrypoints_postgres.py @@ -11,7 +11,7 @@ from alicebot_api.mcp.registry import MCPToolNotFoundError, call_mcp_tool from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError from alicebot_api.routers import vnext_memories, vnext_projects, vnext_retrieval, vnext_review -from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_label_writes import held_label_locks, without_insert_floor from alicebot_api.vnext_store import PostgresVNextStore from tests.integration.test_derived_labels_read_acceptance_postgres import _user from tests.unit.test_derived_labels_real_keys import ALPHA, real_reader_key @@ -59,6 +59,18 @@ def test_exact_entrypoint_checks_effective_floor(migrated_database_urls, monkeyp monkeypatch.setenv("ALICE_MCP_LEGACY_TOOLS", "1") monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + mutation_locks = [] + if door in {"memory_review_http", "memory_correct_mcp", "memory_redact_mcp"}: + for method_name in ("get_memory_for_update", "get_memory_for_redaction"): + original = getattr(PostgresVNextStore, method_name) + + def checked(store, *args, _original=original, **kwargs): + locks = held_label_locks(store) + assert locks == (True, True, True), (door, locks) + mutation_locks.append(locks) + return _original(store, *args, **kwargs) + + monkeypatch.setattr(PostgresVNextStore, method_name, checked) with user_connection(app_url, user_id) as conn: store = PostgresVNextStore(conn) key = real_reader_key(store, user_id, reader) @@ -78,6 +90,7 @@ def test_exact_entrypoint_checks_effective_floor(migrated_database_urls, monkeyp else: row = store.create_memory({"memory_key": str(uuid4()), "title": "Door sentinel", "canonical_text": "Door sentinel", "status": "candidate", "domain": "project", "sensitivity": "public", "metadata_json": metadata}) blocked = hidden and reader == "bound_admin" + mutation_locks.clear() if door.startswith("legacy_") and key: with pytest.raises(MCPToolNotFoundError, match="disabled whenever"): _invoke(door, app_url=app_url, user_id=user_id, target_id=str(row["id"]), key=key, tmp_path=tmp_path) @@ -98,6 +111,18 @@ def test_exact_entrypoint_checks_effective_floor(migrated_database_urls, monkeyp else: assert not blocked, (door, reader, hidden, json.dumps(result, default=str)) assert str(row["id"]) in json.dumps(result, default=str) + if door in {"memory_review_http", "memory_correct_mcp", "memory_redact_mcp"}: + if not blocked: + assert mutation_locks, (door, reader, hidden) + with user_connection(app_url, user_id) as conn: + stored = conn.execute("SELECT status, deleted_at FROM memories WHERE id=%s", (row["id"],)).fetchone() + if blocked: + assert stored["status"] == "candidate" + assert stored["deleted_at"] is None + elif door == "memory_redact_mcp": + assert stored["deleted_at"] is not None + else: + assert stored["status"] == "active" @pytest.mark.parametrize("reader", ("owner", "bound_admin")) diff --git a/tests/integration/test_derived_labels_group_scope_postgres.py b/tests/integration/test_derived_labels_group_scope_postgres.py index e9014da48..7fc47c861 100644 --- a/tests/integration/test_derived_labels_group_scope_postgres.py +++ b/tests/integration/test_derived_labels_group_scope_postgres.py @@ -74,6 +74,7 @@ def test_existing_rollup_state_admits_effective_labels_for_pending_and_accepted_ with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, f"existing-{user_id}@example.invalid", "Existing") store = PostgresVNextStore(conn) + lock_label_fixture(store) members = seed_members(store) service = VNextRollupService(store) proposal = service.propose_rollups(projects=(ALPHA,)) From 7d40e96ba8f720aa0cfe6ecbbb5bd833582396ca Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:30:05 +0200 Subject: [PATCH 209/270] Pin remaining registered event and rating reader controls --- tests/unit/test_label_reader_stage_matrix.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_label_reader_stage_matrix.py b/tests/unit/test_label_reader_stage_matrix.py index d595362ed..d695cdea6 100644 --- a/tests/unit/test_label_reader_stage_matrix.py +++ b/tests/unit/test_label_reader_stage_matrix.py @@ -67,7 +67,7 @@ def list_edges(self, **kwargs): return [{"from_type": "memory", "from_id": MEMOR def _scope(scoped=False): return _ResolvedRetrievalScope(projects=frozenset((ALPHA,)) if scoped else frozenset(), people=frozenset(), window_start=None, window_end=None, exclude_global_domains=frozenset()) -STAGES = ("by_ids", "by_ids_fallback", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "pack_open_loops", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs", "workspace_loop_refs", "dashboard_loop_refs") +STAGES = ("by_ids", "by_ids_fallback", "vector", "graph", "temporal", "provenance", "visibility", "contradictions", "scoped_contradictions", "recent_changes", "scoped_recent_changes", "session", "pack_open_loops", "context_projects", "context_memories", "context_open_loops", "context_artifacts", "context_sources", "project_resolution", "dashboard_lists", "loop_memory_reference", "entity_explain", "entity_backing", "workspace_projects", "workspace_memories", "workspace_open_loops", "workspace_artifacts", "workspace_beliefs", "workspace_event_helper", "workspace_loop_refs", "dashboard_loop_refs") def _stage(stage, store): service = VNextRetrievalService(store, embedding_provider=SimpleNamespace(provider="synthetic", model="synthetic", base_url="http://synthetic.invalid")) @@ -105,6 +105,8 @@ def _stage(stage, store): else: body = VNextProjectService(store).project_dashboard(project_id=ALPHA, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent")) return [body["open_loops"][0]["memory_id"]] if body["open_loops"][0]["memory_id"] else [] + if stage == "workspace_event_helper": + return store.events if workspaces._workspace_event_visible(store, store.events[0], CEILING) else [] if stage.startswith("workspace_"): field = stage.removeprefix("workspace_") if field == "memories": @@ -155,3 +157,16 @@ def test_each_dogfooding_sample_counts_only_currently_readable_rows(kind, monkey store.rows["source"][0]["sensitivity"] = "public" visible = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(CEILING)) assert visible["sample_scope"][kind]["returned_count"] == 1 + + +def test_dogfooding_rating_count_uses_the_current_artifact_label(monkeypatch): + _quiet_services(monkeypatch) + store = StageStore() + store.list_artifact_quality_ratings = lambda **kwargs: [{"id": str(UUID(int=109)), "artifact_id": ARTIFACT, "usefulness": 5}] + hidden = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(CEILING)) + assert hidden["artifact_quality_rating_count"] == 0 + assert hidden["artifact_quality_average"] is None + store.rows["source"][0]["sensitivity"] = "public" + visible = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(CEILING)) + assert visible["artifact_quality_rating_count"] == 1 + assert visible["artifact_quality_average"] == 5 From 5d7e5dd31586b3e18ab546c87a3d95428b237bea Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:41:32 +0200 Subject: [PATCH 210/270] Model exclusive labels before deferred memory review row locks --- tests/unit/test_main.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 476f198c7..7899762b3 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5279,6 +5279,7 @@ def test_vnext_memory_review_defers_embedding_until_primary_transaction_closes(m memory_id = uuid4() transaction_depth = 0 calls: list[str] = [] + lock_calls: list[str] = [] deferred_input = object() memory = { "id": str(memory_id), @@ -5300,11 +5301,41 @@ def fake_user_connection(_database_url: str, _user_id): finally: transaction_depth -= 1 + class FakeLockCursor: + def execute(self, query: str, params=None) -> None: + assert transaction_depth == 1 + assert query in { + "SELECT current_setting('lock_timeout') AS lock_timeout", + "SET LOCAL lock_timeout = '3s'", + "SELECT set_config('lock_timeout', %s, true)", + } + if query.startswith("SELECT set_config"): + assert params == ("0",) + + def fetchone(self): + return {"lock_timeout": "0"} + + class FakeLockConnection: + @contextmanager + def cursor(self): + yield FakeLockCursor() + class FakeStore: + conn = FakeLockConnection() + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + assert transaction_depth == 1 + assert exclusive is True + assert lock_calls == ["graph"] + lock_calls.append("exclusive_labels") + def get_memory(self, _memory_id: str): return memory def get_memory_for_update(self, _memory_id: str): + assert transaction_depth == 1 + assert lock_calls == ["graph", "exclusive_labels"] + lock_calls.append("row") return memory def update_memory(self, *, memory_id: str, patch: dict[str, object], **_kwargs): @@ -5329,7 +5360,9 @@ def __init__(self, _store, *, defer_embeddings: bool = False) -> None: self.deferred_embedding_inputs = (deferred_input,) def lock_supersession_graph(self) -> None: - pass + assert transaction_depth == 1 + assert lock_calls == [] + lock_calls.append("graph") def refresh_memory_derived_state(self, _memory, **_kwargs) -> None: assert transaction_depth == 1 @@ -5356,6 +5389,7 @@ def fake_persist(**kwargs) -> None: assert response.status_code == 200 assert calls == ["refresh", "embedding"] + assert lock_calls == ["graph", "exclusive_labels", "row"] def test_vnext_consolidation_defers_embedding_until_primary_transaction_closes(monkeypatch) -> None: From 49547fc29f40651f7234e38c8b0932e90472fc52 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:41:32 +0200 Subject: [PATCH 211/270] Model exclusive labels before deferred memory review row locks --- tests/unit/test_main.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 476f198c7..7899762b3 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5279,6 +5279,7 @@ def test_vnext_memory_review_defers_embedding_until_primary_transaction_closes(m memory_id = uuid4() transaction_depth = 0 calls: list[str] = [] + lock_calls: list[str] = [] deferred_input = object() memory = { "id": str(memory_id), @@ -5300,11 +5301,41 @@ def fake_user_connection(_database_url: str, _user_id): finally: transaction_depth -= 1 + class FakeLockCursor: + def execute(self, query: str, params=None) -> None: + assert transaction_depth == 1 + assert query in { + "SELECT current_setting('lock_timeout') AS lock_timeout", + "SET LOCAL lock_timeout = '3s'", + "SELECT set_config('lock_timeout', %s, true)", + } + if query.startswith("SELECT set_config"): + assert params == ("0",) + + def fetchone(self): + return {"lock_timeout": "0"} + + class FakeLockConnection: + @contextmanager + def cursor(self): + yield FakeLockCursor() + class FakeStore: + conn = FakeLockConnection() + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + assert transaction_depth == 1 + assert exclusive is True + assert lock_calls == ["graph"] + lock_calls.append("exclusive_labels") + def get_memory(self, _memory_id: str): return memory def get_memory_for_update(self, _memory_id: str): + assert transaction_depth == 1 + assert lock_calls == ["graph", "exclusive_labels"] + lock_calls.append("row") return memory def update_memory(self, *, memory_id: str, patch: dict[str, object], **_kwargs): @@ -5329,7 +5360,9 @@ def __init__(self, _store, *, defer_embeddings: bool = False) -> None: self.deferred_embedding_inputs = (deferred_input,) def lock_supersession_graph(self) -> None: - pass + assert transaction_depth == 1 + assert lock_calls == [] + lock_calls.append("graph") def refresh_memory_derived_state(self, _memory, **_kwargs) -> None: assert transaction_depth == 1 @@ -5356,6 +5389,7 @@ def fake_persist(**kwargs) -> None: assert response.status_code == 200 assert calls == ["refresh", "embedding"] + assert lock_calls == ["graph", "exclusive_labels", "row"] def test_vnext_consolidation_defers_embedding_until_primary_transaction_closes(monkeypatch) -> None: From 34b7477d7a06e4b6d796a6bc9c80cd14b765bbb1 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:45:25 +0200 Subject: [PATCH 212/270] Align existing PostgreSQL fixtures with ordered label writes --- tests/integration/conftest.py | 21 ++++++++++++ .../integration/test_calendar_accounts_api.py | 3 +- .../test_capture_dedupe_postgres.py | 3 ++ tests/integration/test_context_compile.py | 13 +++++--- tests/integration/test_continuity_api.py | 15 +++++++-- .../integration/test_continuity_recall_api.py | 21 ++++++++++-- .../test_continuity_resumption_api.py | 21 ++++++++++-- tests/integration/test_continuity_store.py | 24 ++++++++------ .../test_credential_floor_every_door_api.py | 2 ++ .../test_derived_domain_postgres.py | 2 ++ tests/integration/test_gmail_accounts_api.py | 8 ++++- .../test_mcp_refusal_before_state_postgres.py | 2 ++ tests/integration/test_memory_admission.py | 3 +- .../test_memory_review_labels_api.py | 29 +++++++++-------- ...t_pending_project_update_guard_postgres.py | 2 ++ .../integration/test_provider_runtime_api.py | 32 +++++++++++++++++-- tests/integration/test_proxy_execution_api.py | 8 ++++- .../integration/test_saved_quotes_postgres.py | 4 +++ tests/integration/test_task_artifacts_api.py | 5 ++- tests/integration/test_temporal_state_api.py | 7 ++-- .../test_temporal_state_mcp_cli.py | 7 ++-- .../test_vnext_consolidation_postgres.py | 2 ++ .../test_vnext_live_workspace_api.py | 4 +++ 23 files changed, 192 insertions(+), 46 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 404554c6e..5d8d3667f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -171,3 +171,24 @@ def migrated_database_urls(migrated_template_database: str) -> Iterator[dict[str yield urls finally: _drop_database(database_name) + + +def lock_label_fixture(store) -> None: + """Seed and mutate one transaction with the production lock order.""" + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + store.lock_graph_mutation() + acquire_exclusive_label_lock(store) + + +def assert_append_only_mutation_refused(conn, *, snapshot_sql, mutation_sql, params) -> None: + """Prove refusal under both forced RLS and the append-only trigger.""" + before = conn.execute(snapshot_sql, params).fetchall() + assert before, "the synthetic target must be visible before attempting its mutation" + try: + with conn.transaction(): + changed = conn.execute(mutation_sql, params).rowcount + assert changed == 0, "an append-only row was changed" + except psycopg.Error as error: + assert "append-only" in str(error) + assert conn.execute(snapshot_sql, params).fetchall() == before diff --git a/tests/integration/test_calendar_accounts_api.py b/tests/integration/test_calendar_accounts_api.py index 708abcc89..f221aac06 100644 --- a/tests/integration/test_calendar_accounts_api.py +++ b/tests/integration/test_calendar_accounts_api.py @@ -13,7 +13,7 @@ from alicebot_api.routers import legacy_gated as legacy_gated_router from alicebot_api.config import Settings import alicebot_api.calendar as calendar_module -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -251,6 +251,7 @@ def test_calendar_account_endpoints_connect_list_detail_and_isolate( assert '"access_token":' not in json.dumps(detail_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_capture_dedupe_postgres.py b/tests/integration/test_capture_dedupe_postgres.py index e54f73ec6..475419ce3 100644 --- a/tests/integration/test_capture_dedupe_postgres.py +++ b/tests/integration/test_capture_dedupe_postgres.py @@ -15,6 +15,7 @@ content_hash_for_text, ) from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import lock_label_fixture def test_two_connections_claim_one_source_dedupe_identity( @@ -144,6 +145,7 @@ def test_source_scope_mutation_rotates_postgres_identity_and_releases_old_captur "Capture mutation", ) store = PostgresVNextStore(conn) + lock_label_fixture(store) service = VNextCaptureService(store) text = "Fact: Reviewed source scope changes must rotate identity atomically." first = service.capture_text( @@ -209,6 +211,7 @@ def test_source_scope_mutation_collision_rolls_back_postgres_row( "Capture collision", ) store = PostgresVNextStore(conn) + lock_label_fixture(store) service = VNextCaptureService(store) text = "Fact: Collision rollback leaves both source identities intact." alpha = service.capture_text( diff --git a/tests/integration/test_context_compile.py b/tests/integration/test_context_compile.py index 905b26da4..012996646 100644 --- a/tests/integration/test_context_compile.py +++ b/tests/integration/test_context_compile.py @@ -12,8 +12,9 @@ import alicebot_api.main as main_module from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore +from tests.integration.conftest import assert_append_only_mutation_refused def invoke_compile_context(payload: dict[str, Any]) -> tuple[int, dict[str, Any]]: @@ -662,9 +663,13 @@ def test_compile_context_endpoint_persists_trace_and_trace_events(migrated_datab assert trace_events[-1]["payload"]["excluded_entity_edge_limit_count"] == 1 with psycopg.connect(migrated_database_urls["admin"]) as conn: - with conn.cursor() as cur: - with pytest.raises(psycopg.Error, match="append-only"): - cur.execute("UPDATE trace_events SET kind = 'mutated' WHERE trace_id = %s", (trace_id,)) + set_current_user(conn, user_id) + assert_append_only_mutation_refused( + conn, + snapshot_sql="SELECT * FROM trace_events WHERE trace_id = %s ORDER BY id", + mutation_sql="UPDATE trace_events SET kind = 'mutated' WHERE trace_id = %s", + params=(trace_id,), + ) def test_compile_context_prefers_updated_active_memory_within_same_transaction( diff --git a/tests/integration/test_continuity_api.py b/tests/integration/test_continuity_api.py index 1502722ba..e3d45d5a7 100644 --- a/tests/integration/test_continuity_api.py +++ b/tests/integration/test_continuity_api.py @@ -12,7 +12,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -103,12 +103,14 @@ def seed_user_with_continuity(database_url: str, *, email: str) -> dict[str, obj def set_thread_timestamps( admin_database_url: str, + user_id: UUID, *, thread_id: UUID, created_at: datetime, updated_at: datetime, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( "UPDATE threads SET created_at = %s, updated_at = %s WHERE id = %s", @@ -118,13 +120,15 @@ def set_thread_timestamps( def set_session_timestamps( admin_database_url: str, + user_id: UUID, *, session_id: UUID, started_at: datetime, ended_at: datetime | None, created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( "UPDATE sessions SET started_at = %s, ended_at = %s, created_at = %s WHERE id = %s", @@ -219,24 +223,28 @@ def test_thread_continuity_endpoints_create_list_detail_sessions_and_events( set_thread_timestamps( migrated_database_urls["admin"], + user_id=seeded["user_id"], thread_id=seeded["first_thread"]["id"], created_at=shared_created_at, updated_at=shared_created_at, ) set_thread_timestamps( migrated_database_urls["admin"], + user_id=seeded["user_id"], thread_id=seeded["second_thread"]["id"], created_at=shared_created_at, updated_at=shared_created_at, ) set_thread_timestamps( migrated_database_urls["admin"], + user_id=seeded["user_id"], thread_id=api_thread_id, created_at=newer_created_at, updated_at=newer_created_at, ) set_session_timestamps( migrated_database_urls["admin"], + user_id=seeded["user_id"], session_id=seeded["first_session"]["id"], started_at=first_session_start, ended_at=first_session_end, @@ -244,6 +252,7 @@ def test_thread_continuity_endpoints_create_list_detail_sessions_and_events( ) set_session_timestamps( migrated_database_urls["admin"], + user_id=seeded["user_id"], session_id=seeded["second_session"]["id"], started_at=second_session_start, ended_at=None, diff --git a/tests/integration/test_continuity_recall_api.py b/tests/integration/test_continuity_recall_api.py index d30bc10e6..7d71d206c 100644 --- a/tests/integration/test_continuity_recall_api.py +++ b/tests/integration/test_continuity_recall_api.py @@ -12,7 +12,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import continuity as continuity_router from alicebot_api.config import Settings -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -70,11 +70,13 @@ def seed_user(database_url: str, *, email: str) -> UUID: def set_continuity_timestamps( admin_database_url: str, + user_id: UUID, *, continuity_object_id: UUID, created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( "UPDATE continuity_objects SET created_at = %s, updated_at = %s WHERE id = %s", @@ -84,6 +86,7 @@ def set_continuity_timestamps( def set_continuity_lifecycle_flags( admin_database_url: str, + user_id: UUID, *, continuity_object_id: UUID, is_searchable: bool | None = None, @@ -101,7 +104,8 @@ def set_continuity_lifecycle_flags( return values.append(continuity_object_id) - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( f"UPDATE continuity_objects SET {', '.join(assignments)} WHERE id = %s", @@ -170,11 +174,13 @@ def test_continuity_recall_api_returns_provenance_backed_scoped_results( set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=primary_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=other_object["id"], created_at=datetime(2026, 3, 29, 9, 0, tzinfo=UTC), ) @@ -316,6 +322,7 @@ def test_continuity_recall_debug_api_persists_and_exposes_trace( set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=continuity_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) @@ -393,6 +400,7 @@ def test_continuity_resumption_debug_api_includes_underlying_retrieval_trace( set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=decision["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) @@ -486,16 +494,19 @@ def test_continuity_recall_api_prefers_confirmed_fresh_active_truth_over_superse set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=current_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=stale_object["id"], created_at=datetime(2026, 3, 20, 9, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=superseded_object["id"], created_at=datetime(2026, 3, 10, 8, 0, tzinfo=UTC), ) @@ -573,16 +584,19 @@ def test_continuity_recall_api_excludes_preserved_but_non_searchable_objects( set_continuity_lifecycle_flags( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=hidden_object["id"], is_searchable=False, ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=hidden_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=visible_object["id"], created_at=datetime(2026, 3, 29, 10, 5, tzinfo=UTC), ) @@ -637,6 +651,7 @@ def test_continuity_lifecycle_debug_endpoints_expose_flags( set_continuity_lifecycle_flags( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=continuity_object["id"], is_promotable=False, ) diff --git a/tests/integration/test_continuity_resumption_api.py b/tests/integration/test_continuity_resumption_api.py index 9397924ff..665d96d70 100644 --- a/tests/integration/test_continuity_resumption_api.py +++ b/tests/integration/test_continuity_resumption_api.py @@ -12,7 +12,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import continuity as continuity_router from alicebot_api.config import Settings -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -74,11 +74,13 @@ def seed_user(database_url: str, *, email: str) -> UUID: def set_continuity_timestamps( admin_database_url: str, + user_id: UUID, *, continuity_object_id: UUID, created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( "UPDATE continuity_objects SET created_at = %s, updated_at = %s WHERE id = %s", @@ -88,11 +90,13 @@ def set_continuity_timestamps( def set_continuity_lifecycle_flags( admin_database_url: str, + user_id: UUID, *, continuity_object_id: UUID, is_promotable: bool, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( "UPDATE continuity_objects SET is_promotable = %s WHERE id = %s", @@ -176,21 +180,25 @@ def test_continuity_resumption_api_returns_required_sections( set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=decision_object["id"], created_at=datetime(2026, 3, 29, 9, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=waiting_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=next_object["id"], created_at=datetime(2026, 3, 29, 10, 5, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=latest_decision_object["id"], created_at=datetime(2026, 3, 29, 10, 10, tzinfo=UTC), ) @@ -263,6 +271,7 @@ def test_continuity_resumption_api_returns_explicit_empty_states( set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=continuity_object["id"], created_at=datetime(2026, 3, 29, 9, 0, tzinfo=UTC), ) @@ -376,17 +385,20 @@ def test_continuity_resumption_api_selects_latest_sections_beyond_recall_limit( for index, continuity_object_id in enumerate(historical_object_ids): set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=continuity_object_id, created_at=base_time + timedelta(minutes=index), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=latest_decision_object["id"], created_at=base_time + timedelta(minutes=200), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=latest_next_action_object["id"], created_at=base_time + timedelta(minutes=201), ) @@ -465,16 +477,19 @@ def test_continuity_resumption_api_uses_promotable_facts_by_default_with_overrid set_continuity_lifecycle_flags( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=fact_object["id"], is_promotable=False, ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=fact_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], + user_id=user_id, continuity_object_id=decision_object["id"], created_at=datetime(2026, 3, 29, 10, 5, tzinfo=UTC), ) diff --git a/tests/integration/test_continuity_store.py b/tests/integration/test_continuity_store.py index 956156388..fd91100fc 100644 --- a/tests/integration/test_continuity_store.py +++ b/tests/integration/test_continuity_store.py @@ -9,6 +9,7 @@ from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore +from tests.integration.conftest import assert_append_only_mutation_refused def test_thread_session_and_event_persistence(migrated_database_urls): @@ -40,12 +41,13 @@ def test_thread_session_and_event_persistence(migrated_database_urls): assert events[0]["payload"]["text"] == "hello" with psycopg.connect(migrated_database_urls["admin"]) as conn: - with pytest.raises(psycopg.Error, match="append-only"): - with conn.cursor() as cur: - cur.execute( - "UPDATE events SET kind = 'message.mutated' WHERE id = %s", - (first_event["id"],), - ) + set_current_user(conn, user_id) + assert_append_only_mutation_refused( + conn, + snapshot_sql="SELECT * FROM events WHERE id = %s", + mutation_sql="UPDATE events SET kind = 'message.mutated' WHERE id = %s", + params=(first_event['id'],), + ) def test_event_deletes_are_rejected_at_database_level(migrated_database_urls): @@ -59,9 +61,13 @@ def test_event_deletes_are_rejected_at_database_level(migrated_database_urls): event = store.append_event(thread["id"], session["id"], "message.user", {"text": "keep"}) with psycopg.connect(migrated_database_urls["admin"]) as conn: - with pytest.raises(psycopg.Error, match="append-only"): - with conn.cursor() as cur: - cur.execute("DELETE FROM events WHERE id = %s", (event["id"],)) + set_current_user(conn, user_id) + assert_append_only_mutation_refused( + conn, + snapshot_sql="SELECT * FROM events WHERE id = %s", + mutation_sql='DELETE FROM events WHERE id = %s', + params=(event['id'],), + ) def test_continuity_rls_blocks_cross_user_access(migrated_database_urls): diff --git a/tests/integration/test_credential_floor_every_door_api.py b/tests/integration/test_credential_floor_every_door_api.py index 04b39f349..fe3d782ee 100644 --- a/tests/integration/test_credential_floor_every_door_api.py +++ b/tests/integration/test_credential_floor_every_door_api.py @@ -40,6 +40,7 @@ from alicebot_api.vnext_store import PostgresVNextStore from tests.integration.test_memory_mutations_api import identity_header, invoke_request, seed_user +from tests.integration.conftest import lock_label_fixture # Built rather than written out so the source carries no scanner-shaped token. @@ -638,6 +639,7 @@ def test_round2_c2_project_update_accept_refuses_a_credential_state(migrated_dat user_id = seed_user(app_url, email="floor-c2-project@example.com") with user_connection(app_url, user_id) as conn: store = PostgresVNextStore(conn) + lock_label_fixture(store) project = store.create_project( { "name": "Deploy pipeline", diff --git a/tests/integration/test_derived_domain_postgres.py b/tests/integration/test_derived_domain_postgres.py index 3c68bc2dc..accabda86 100644 --- a/tests/integration/test_derived_domain_postgres.py +++ b/tests/integration/test_derived_domain_postgres.py @@ -297,6 +297,8 @@ def test_promoted_artifact_uuid_alias_repaired(database_urls): with without_insert_floor(), user_connection(database_urls["app"], user) as conn: ContinuityStore(conn).create_user(user, "alias@example.invalid", "Alias fixture") store = PostgresVNextStore(conn) + # Promotion takes the graph lock before any label-table writes. + store.lock_graph_mutation() memory = store.create_memory({"memory_key": "health", "canonical_text": "Private observation", "domain": "health", "sensitivity": "public", "status": "active"}) artifact = store.create_artifact({"artifact_type": "daily_brief", "title": "Fixture brief", diff --git a/tests/integration/test_gmail_accounts_api.py b/tests/integration/test_gmail_accounts_api.py index 0869c5f61..73d407c70 100644 --- a/tests/integration/test_gmail_accounts_api.py +++ b/tests/integration/test_gmail_accounts_api.py @@ -14,7 +14,7 @@ from alicebot_api.routers import legacy_gated as legacy_gated_router from alicebot_api.config import Settings import alicebot_api.gmail as gmail_module -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -273,6 +273,7 @@ def test_gmail_account_endpoints_connect_list_detail_and_isolate( assert '"client_secret":' not in json.dumps(create_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -495,6 +496,7 @@ def fake_fetch_gmail_message_raw_bytes(*, access_token: str, **_kwargs) -> bytes assert '"client_secret":' not in json.dumps(ingest_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -636,6 +638,7 @@ def fake_fetch_gmail_message_raw_bytes(*, access_token: str, **_kwargs) -> bytes assert '"client_secret":' not in json.dumps(ingest_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -757,6 +760,7 @@ def fail_fetch(**_kwargs): assert store.list_task_artifacts_for_task(owner["task_id"]) == [] with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -891,6 +895,7 @@ def fail_fetch(**_kwargs): ) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( "DELETE FROM gmail_account_credentials WHERE gmail_account_id = %s", @@ -964,6 +969,7 @@ def test_gmail_message_ingestion_endpoint_rejects_missing_external_secret_withou ) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_mcp_refusal_before_state_postgres.py b/tests/integration/test_mcp_refusal_before_state_postgres.py index 8a8a0b455..8d8f43c1f 100644 --- a/tests/integration/test_mcp_refusal_before_state_postgres.py +++ b/tests/integration/test_mcp_refusal_before_state_postgres.py @@ -30,6 +30,7 @@ from alicebot_api.vnext_store import PostgresVNextStore from tests.integration.test_vnext_live_workspace_api import invoke_request, seed_user +from tests.integration.conftest import lock_label_fixture _FIXED_MESSAGE = "The tool request could not be processed" _OWN_PROJECT = "alicebot" @@ -63,6 +64,7 @@ def _states(app_url: str, user_id: UUID) -> dict[str, str]: with user_connection(app_url, user_id) as conn: store = PostgresVNextStore(conn) + lock_label_fixture(store) ordinary = _memory(store) pending = _memory( store, diff --git a/tests/integration/test_memory_admission.py b/tests/integration/test_memory_admission.py index 31a04e735..202f597a1 100644 --- a/tests/integration/test_memory_admission.py +++ b/tests/integration/test_memory_admission.py @@ -11,7 +11,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -240,6 +240,7 @@ def test_admit_memory_endpoint_persists_add_update_and_delete_revisions( assert revisions[2]["new_value"] is None with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: with pytest.raises(psycopg.Error, match="append-only"): cur.execute( diff --git a/tests/integration/test_memory_review_labels_api.py b/tests/integration/test_memory_review_labels_api.py index 288fc03b6..ab80e4ad8 100644 --- a/tests/integration/test_memory_review_labels_api.py +++ b/tests/integration/test_memory_review_labels_api.py @@ -13,9 +13,10 @@ from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings from alicebot_api.contracts import MemoryCandidateInput -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.memory import admit_memory_candidate from alicebot_api.store import ContinuityStore +from tests.integration.conftest import assert_append_only_mutation_refused def invoke_request( @@ -296,20 +297,22 @@ def test_memory_review_labels_reject_update_and_delete_at_database_level(migrate ) with psycopg.connect(migrated_database_urls["admin"]) as conn: - with pytest.raises(psycopg.Error, match="append-only"): - with conn.cursor() as cur: - cur.execute( - "UPDATE memory_review_labels SET label = 'incorrect' WHERE id = %s", - (label["id"],), - ) + set_current_user(conn, UUID(seeded["user_id"])) + assert_append_only_mutation_refused( + conn, + snapshot_sql="SELECT * FROM memory_review_labels WHERE id = %s", + mutation_sql="UPDATE memory_review_labels SET label = 'incorrect' WHERE id = %s", + params=(label['id'],), + ) with psycopg.connect(migrated_database_urls["admin"]) as conn: - with pytest.raises(psycopg.Error, match="append-only"): - with conn.cursor() as cur: - cur.execute( - "DELETE FROM memory_review_labels WHERE id = %s", - (label["id"],), - ) + set_current_user(conn, UUID(seeded["user_id"])) + assert_append_only_mutation_refused( + conn, + snapshot_sql="SELECT * FROM memory_review_labels WHERE id = %s", + mutation_sql='DELETE FROM memory_review_labels WHERE id = %s', + params=(label['id'],), + ) def test_memory_review_label_endpoints_enforce_per_user_isolation_and_not_found_behavior( diff --git a/tests/integration/test_pending_project_update_guard_postgres.py b/tests/integration/test_pending_project_update_guard_postgres.py index 70849733a..c9e4dc6b7 100644 --- a/tests/integration/test_pending_project_update_guard_postgres.py +++ b/tests/integration/test_pending_project_update_guard_postgres.py @@ -12,6 +12,7 @@ from alicebot_api.vnext_memory_commit import VNextMemoryCommitService, VNextMemoryCommitValidationError from alicebot_api.vnext_project_update_guard import PENDING_PROJECT_UPDATE_MEMORY_MUTATION_MESSAGE from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import lock_label_fixture @pytest.mark.parametrize("marker", ["workflow", "memory_key"]) @@ -29,6 +30,7 @@ def test_postgres_pending_project_update_candidate_blocks_generic_memory_mutatio "Pending project guard", ) store = PostgresVNextStore(conn) + lock_label_fixture(store) metadata: dict[str, object] = {"candidate": True} memory_key = f"ordinary.pending.{uuid4().hex}" if marker == "workflow": diff --git a/tests/integration/test_provider_runtime_api.py b/tests/integration/test_provider_runtime_api.py index 27b5dc408..63adf3ce3 100644 --- a/tests/integration/test_provider_runtime_api.py +++ b/tests/integration/test_provider_runtime_api.py @@ -15,7 +15,7 @@ import alicebot_api.main as main_module from alicebot_api.config import Settings, WorkspaceProviderConfig -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, set_current_user_account, user_connection from alicebot_api.provider_configuration import provider_config_fingerprint from alicebot_api.public_errors import UPSTREAM_FAILURE from alicebot_api.provider_secrets import decode_provider_secret_ref, resolve_provider_api_key @@ -148,6 +148,8 @@ def _bootstrap_local_workspace(email: str) -> tuple[str, str, str]: def _seed_thread_for_user(*, admin_db_url: str, user_id: str, email: str) -> str: thread_id = str(uuid4()) with psycopg.connect(admin_db_url) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -842,6 +844,8 @@ def test_openai_compatible_registration_still_works(migrated_database_urls, monk assert any(record["url"] == "https://provider.example/v1/models" for record in captured_requests) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -929,6 +933,8 @@ def test_openai_compatible_no_auth_update_omits_auth_for_test_and_runtime( assert len(captured_requests) == 4 assert all("authorization" not in {str(key).lower() for key in record["headers"]} for record in captured_requests) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( "SELECT auth_mode, api_key FROM model_providers WHERE id = %s AND workspace_id = %s", @@ -965,6 +971,8 @@ def test_provider_update_uses_atomic_cas_and_hides_stale_capability( migrated_database_urls["admin"], row_factory=psycopg.rows.dict_row, ) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) store = ContinuityStore(conn) original = store.get_model_provider_for_workspace_optional( provider_id=provider_id, @@ -1017,6 +1025,8 @@ def test_provider_update_uses_atomic_cas_and_hides_stale_capability( migrated_database_urls["admin"], row_factory=psycopg.rows.dict_row, ) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) lost_update = ContinuityStore(conn).update_model_provider( provider_id=provider_id, workspace_id=UUID(workspace_id), @@ -1460,6 +1470,8 @@ def test_provider_invocation_telemetry_persists_for_test_and_runtime( assert provider_secret not in caplog.text with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -1680,6 +1692,8 @@ def fake_urlopen(request, timeout, enforce_public_peer): assert test_payload["result"]["usage"]["total_tokens"] == 14 with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -1738,6 +1752,8 @@ def fake_urlopen(request, timeout, enforce_public_peer): provider_id = register_payload["provider"]["id"] with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -1777,6 +1793,8 @@ def fake_urlopen(request, timeout, enforce_public_peer): assert updated_payload["capabilities"]["snapshot"]["azure_auth_mode"] == ("azure_ad_token") with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -1997,6 +2015,8 @@ def private_dns_getaddrinfo(hostname: str, port, type=0, proto=0): } with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2019,7 +2039,7 @@ def test_provider_test_and_runtime_reject_disallowed_target_without_outbound( user_id, workspace_id, user_account_id = _bootstrap_local_workspace("provider-security-blocked-runtime@example.com") urlopen_call_count = 0 - def fake_urlopen(_request, _timeout): + def fake_urlopen(_request, timeout, enforce_public_peer): nonlocal urlopen_call_count urlopen_call_count += 1 raise AssertionError("outbound request should not be attempted for blocked targets") @@ -2042,6 +2062,8 @@ def fake_urlopen(_request, _timeout): provider_id = register_payload["provider"]["id"] with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2117,6 +2139,8 @@ def test_provider_rejects_userinfo_and_redacts_legacy_rows( legacy_provider_id: str with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2235,6 +2259,8 @@ def fake_urlopen(request, timeout, enforce_public_peer): assert sensitive_detail not in json.dumps(test_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2272,6 +2298,8 @@ def fake_urlopen(request, timeout, enforce_public_peer): assert sensitive_detail not in json.dumps(runtime_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, UUID(user_id)) + set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_proxy_execution_api.py b/tests/integration/test_proxy_execution_api.py index 24ca106e8..68909d9d9 100644 --- a/tests/integration/test_proxy_execution_api.py +++ b/tests/integration/test_proxy_execution_api.py @@ -11,7 +11,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import legacy_gated as legacy_gated_router from alicebot_api.config import Settings -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -167,11 +167,13 @@ def create_execution_budget( def set_execution_executed_at( admin_database_url: str, + user_id: UUID, *, execution_id: UUID, executed_at_sql: str, ) -> None: with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) conn.execute( f"UPDATE tool_executions SET executed_at = {executed_at_sql} WHERE id = %s", (execution_id,), @@ -181,11 +183,13 @@ def set_execution_executed_at( def set_approval_request_thread_id( admin_database_url: str, + user_id: UUID, *, approval_id: UUID, request_thread_id: str, ) -> None: with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) conn.execute( """ UPDATE approvals @@ -851,6 +855,7 @@ def test_execute_approved_proxy_endpoint_fail_closes_when_runtime_context_is_inv set_approval_request_thread_id( migrated_database_urls["admin"], + user_id=owner["user_id"], approval_id=UUID(create_payload["approval"]["id"]), request_thread_id="not-a-uuid", ) @@ -1317,6 +1322,7 @@ def test_execute_approved_proxy_endpoint_excludes_old_window_history_and_keeps_c set_execution_executed_at( migrated_database_urls["admin"], + user_id=owner["user_id"], execution_id=owner_first_execution_id, executed_at_sql="clock_timestamp() - interval '2 hours'", ) diff --git a/tests/integration/test_saved_quotes_postgres.py b/tests/integration/test_saved_quotes_postgres.py index b08fe6d74..8b7e65569 100644 --- a/tests/integration/test_saved_quotes_postgres.py +++ b/tests/integration/test_saved_quotes_postgres.py @@ -38,6 +38,7 @@ from alicebot_api.vnext_retrieval import VNextRetrievalRequest, VNextRetrievalService from alicebot_api.vnext_source_fence import SavedProvenanceReader, SourceReadFence from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import lock_label_fixture _QUOTE = "zinnwald-quote-8841 cone ten firing kiln log marlin-oxide-5520" @@ -84,6 +85,7 @@ def _read(store: PostgresVNextStore, memory_id: str, fence: SourceReadFence) -> def run_lifecycle(store: PostgresVNextStore) -> None: """The lifecycle over one store: save a quote, make the source confidential, archive it, read it three ways.""" + lock_label_fixture(store) source = store.create_source( { "source_type": "document", @@ -198,6 +200,7 @@ def _pack_for(store: PostgresVNextStore, fence: SourceReadFence) -> dict[str, ob def run_linkless_and_sibling_lifecycle(store: PostgresVNextStore) -> None: """A memory with no link and a memory with two links of the same quote, read before and after a source changes.""" + lock_label_fixture(store) refused = _make_source(store, "Alpha held log") kept = _make_source(store, "Alpha second log") linkless = store.create_memory( @@ -309,6 +312,7 @@ def test_a_memory_with_no_link_and_a_sibling_quote_are_withheld_on_postgres(migr def run_nested_reference_lifecycle(store: PostgresVNextStore) -> None: """Two memories that cite a readable source and one that is reclassified, in one ref, read before and after.""" + lock_label_fixture(store) readable = _make_source(store, "Alpha second log") refused = _make_source(store, "Alpha nested log") shapes = { diff --git a/tests/integration/test_task_artifacts_api.py b/tests/integration/test_task_artifacts_api.py index e5092e9e0..fdf78c113 100644 --- a/tests/integration/test_task_artifacts_api.py +++ b/tests/integration/test_task_artifacts_api.py @@ -18,7 +18,7 @@ from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings from alicebot_api.artifacts import TASK_ARTIFACT_CHUNK_RETRIEVAL_MATCHING_RULE -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore @@ -1742,6 +1742,7 @@ def test_task_artifact_ingestion_enforces_rooted_workspace_paths( assert register_status == 201 with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -1821,6 +1822,7 @@ def test_task_artifact_docx_ingestion_enforces_rooted_workspace_paths( assert register_status == 201 with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -1900,6 +1902,7 @@ def test_task_artifact_rfc822_ingestion_enforces_rooted_workspace_paths( assert register_status == 201 with psycopg.connect(migrated_database_urls["admin"]) as conn: + set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_temporal_state_api.py b/tests/integration/test_temporal_state_api.py index 0a7bc76f0..fd81b2c3a 100644 --- a/tests/integration/test_temporal_state_api.py +++ b/tests/integration/test_temporal_state_api.py @@ -14,7 +14,7 @@ from alicebot_api.routers import continuity as continuity_router from alicebot_api.config import Settings from alicebot_api.contracts import MemoryCandidateInput -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.memory import admit_memory_candidate from alicebot_api.store import ContinuityStore @@ -69,13 +69,15 @@ async def send(message: dict[str, object]) -> None: def _set_temporal_timestamps( admin_database_url: str, + user_id: UUID, *, entity_id: UUID, edge_id: UUID, entity_created_at: datetime, edge_created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute( "UPDATE entities SET created_at = %s WHERE id = %s", @@ -150,6 +152,7 @@ def seed_temporal_entity_graph(database_url: str, admin_database_url: str) -> di midpoint = add_at + (update_at - add_at) / 2 _set_temporal_timestamps( admin_database_url, + user_id=user_id, entity_id=entity["id"], edge_id=edge["id"], entity_created_at=add_at - timedelta(seconds=1), diff --git a/tests/integration/test_temporal_state_mcp_cli.py b/tests/integration/test_temporal_state_mcp_cli.py index 5550497f4..1e434a508 100644 --- a/tests/integration/test_temporal_state_mcp_cli.py +++ b/tests/integration/test_temporal_state_mcp_cli.py @@ -12,7 +12,7 @@ import psycopg from alicebot_api.contracts import MemoryCandidateInput -from alicebot_api.db import user_connection +from alicebot_api.db import set_current_user, user_connection from alicebot_api.memory import admit_memory_candidate from alicebot_api.store import ContinuityStore @@ -137,13 +137,15 @@ def _call_tool(client: MCPClient, *, name: str, arguments: dict[str, object]) -> def _set_temporal_timestamps( admin_database_url: str, + user_id: UUID, *, entity_id: UUID, edge_id: UUID, entity_created_at: datetime, edge_created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url, autocommit=True) as conn: + with psycopg.connect(admin_database_url) as conn: + set_current_user(conn, user_id) with conn.cursor() as cur: cur.execute("UPDATE entities SET created_at = %s WHERE id = %s", (entity_created_at, entity_id)) cur.execute( @@ -214,6 +216,7 @@ def seed_temporal_entity_graph(database_url: str, admin_database_url: str) -> di midpoint = add_at + (update_at - add_at) / 2 _set_temporal_timestamps( admin_database_url, + user_id=user_id, entity_id=entity["id"], edge_id=edge["id"], entity_created_at=add_at - timedelta(seconds=1), diff --git a/tests/integration/test_vnext_consolidation_postgres.py b/tests/integration/test_vnext_consolidation_postgres.py index 63c08eec5..1b125b2aa 100644 --- a/tests/integration/test_vnext_consolidation_postgres.py +++ b/tests/integration/test_vnext_consolidation_postgres.py @@ -22,6 +22,7 @@ from alicebot_api.vnext_embeddings import pad_embedding_vector from alicebot_api.vnext_memory_commit import VNextMemoryCommitService from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import lock_label_fixture def test_generate_memory_consolidation_persists_its_artifact(migrated_database_urls) -> None: @@ -67,6 +68,7 @@ def test_rollup_candidate_round_trips_and_acceptance_promotes_it( with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, "rollups@example.invalid", "Rollups") store = PostgresVNextStore(conn) + lock_label_fixture(store) members = [] for index, (text, day) in enumerate( ( diff --git a/tests/integration/test_vnext_live_workspace_api.py b/tests/integration/test_vnext_live_workspace_api.py index 504648ec4..73653f62d 100644 --- a/tests/integration/test_vnext_live_workspace_api.py +++ b/tests/integration/test_vnext_live_workspace_api.py @@ -30,6 +30,7 @@ VNextProjectTerminalConsistencyError, ) from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import lock_label_fixture def invoke_request( @@ -282,6 +283,7 @@ def test_project_update_true_redaction_scrubs_the_role_separated_coupled_graph( with user_connection(migrated_database_urls["app"], user_id) as conn: store = PostgresVNextStore(conn) + lock_label_fixture(store) project = store.create_project( { "name": f"Option A {action} {sentinel}", @@ -785,6 +787,7 @@ def test_project_update_terminal_replay_survives_authorized_true_redaction( ) with user_connection(migrated_database_urls["app"], user_id) as conn: store = PostgresVNextStore(conn) + lock_label_fixture(store) project = store.create_project( { "name": f"{terminal_status.title()} redaction replay", @@ -1006,6 +1009,7 @@ def test_project_update_terminal_replay_rejects_competing_postgres_decision_with ) with user_connection(migrated_database_urls["app"], user_id) as conn: store = PostgresVNextStore(conn) + lock_label_fixture(store) project = store.create_project( { "name": f"{action.title()} competing decision", From dcea6ecbb3148c646eb0c938a55c658ccc937946 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:21:25 +0200 Subject: [PATCH 213/270] Take graph and label locks before project review fence --- apps/api/src/alicebot_api/routers/vnext_review.py | 4 ++++ tests/unit/test_vnext_main.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/apps/api/src/alicebot_api/routers/vnext_review.py b/apps/api/src/alicebot_api/routers/vnext_review.py index 6ebc447e0..0369d0663 100644 --- a/apps/api/src/alicebot_api/routers/vnext_review.py +++ b/apps/api/src/alicebot_api/routers/vnext_review.py @@ -973,6 +973,10 @@ def review_vnext_project_update_candidate( identity = _vnext_authenticated_agent_identity( store, request, user_id=request.user_id, authorization=authorization ) + store.lock_graph_mutation() + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + acquire_exclusive_label_lock(store) _artifact, decision = _vnext_authorized_artifact( store=store, identity=identity, diff --git a/tests/unit/test_vnext_main.py b/tests/unit/test_vnext_main.py index 90335d3dd..de77572fe 100644 --- a/tests/unit/test_vnext_main.py +++ b/tests/unit/test_vnext_main.py @@ -2675,6 +2675,12 @@ def test_vnext_project_and_open_loop_endpoints(monkeypatch) -> None: update_response = vnext_review_router.generate_vnext_project_update_candidate(request) update_payload = json.loads(update_response.body) extract_response = vnext_projects_router.extract_vnext_open_loops(request) + original_row_locker = store.get_artifact_for_update + def checked_row_locker(artifact_id): + assert store.graph_locked is True + assert store.labels_exclusive is True + return original_row_locker(artifact_id) + monkeypatch.setattr(store, "get_artifact_for_update", checked_row_locker) review_update_response = vnext_review_router.review_vnext_project_update_candidate( update_payload["id"], vnext_review_router.VNextProjectUpdateReviewRequest( From 0a3d0c03319205de1ee7560766ca9923dd51591a Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:33:00 +0200 Subject: [PATCH 214/270] Provide ordered lock protocol in deferred review fixture --- tests/unit/test_main.py | 48 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 563513a30..1e1c559c3 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5416,6 +5416,7 @@ def test_vnext_project_review_defers_embedding_and_preserves_human_attribution(m user_id = uuid4() transaction_depth = 0 calls: list[str] = [] + lock_calls: list[str] = [] review_kwargs: dict[str, object] = {} deferred_input = object() decision = main_module.PolicyDecision( @@ -5434,6 +5435,48 @@ def fake_user_connection(_database_url: str, _user_id): finally: transaction_depth -= 1 + class FakeLockCursor: + def execute(self, query: str, params=None) -> None: + assert transaction_depth == 1 + assert query in { + "SELECT current_setting('lock_timeout') AS lock_timeout", + "SET LOCAL lock_timeout = '3s'", + "SELECT set_config('lock_timeout', %s, true)", + } + if query.startswith("SELECT set_config"): + assert params == ("0",) + + def fetchone(self): + return {"lock_timeout": "0"} + + class FakeLockConnection: + @contextmanager + def cursor(self): + yield FakeLockCursor() + + class FakeStore: + conn = FakeLockConnection() + + def lock_graph_mutation(self) -> None: + assert transaction_depth == 1 + assert lock_calls == [] + lock_calls.append("graph") + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + assert transaction_depth == 1 + assert exclusive is True + assert lock_calls == ["graph"] + lock_calls.append("exclusive_labels") + + store = FakeStore() + + def fake_authorized_artifact(**_kwargs): + assert transaction_depth == 1 + assert lock_calls == ["graph", "exclusive_labels"] + assert _kwargs["store"] is store + lock_calls.append("authorize") + return {"id": "artifact-1", "status": "needs_review"}, decision + class FakeProjectService: def __init__(self, _store, *, defer_embeddings: bool = False) -> None: assert transaction_depth == 1 @@ -5456,12 +5499,12 @@ def fake_persist(**kwargs) -> None: monkeypatch.setattr(vnext_review_router, "get_settings", lambda: Settings(database_url="postgresql://db")) monkeypatch.setattr(vnext_review_router, "user_connection", fake_user_connection) - monkeypatch.setattr(vnext_review_router, "PostgresVNextStore", lambda _conn: object()) + monkeypatch.setattr(vnext_review_router, "PostgresVNextStore", lambda _conn: store) monkeypatch.setattr(vnext_review_router, "_vnext_authenticated_agent_identity", lambda *_args, **_kwargs: None) monkeypatch.setattr( vnext_review_router, "_vnext_authorized_artifact", - lambda **_kwargs: ({"id": "artifact-1", "status": "needs_review"}, decision), + fake_authorized_artifact, ) monkeypatch.setattr(vnext_review_router, "VNextProjectService", FakeProjectService) monkeypatch.setattr(vnext_review_router, "_persist_vnext_deferred_embeddings", fake_persist) @@ -5477,6 +5520,7 @@ def fake_persist(**kwargs) -> None: assert response.status_code == 200 assert calls == ["review", "embedding"] + assert lock_calls == ["graph", "exclusive_labels", "authorize"] assert review_kwargs["actor_type"] == "user" assert review_kwargs["actor_id"] == str(user_id) assert review_kwargs["trace_id"] == "request-trace-1" From e100914530e83097a86e6f22b45f6f545b5d8ef4 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:41:32 +0200 Subject: [PATCH 215/270] Model exclusive labels before deferred memory review row locks --- tests/unit/test_main.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 1e1c559c3..8d431d216 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5279,6 +5279,7 @@ def test_vnext_memory_review_defers_embedding_until_primary_transaction_closes(m memory_id = uuid4() transaction_depth = 0 calls: list[str] = [] + lock_calls: list[str] = [] deferred_input = object() memory = { "id": str(memory_id), @@ -5300,11 +5301,41 @@ def fake_user_connection(_database_url: str, _user_id): finally: transaction_depth -= 1 + class FakeLockCursor: + def execute(self, query: str, params=None) -> None: + assert transaction_depth == 1 + assert query in { + "SELECT current_setting('lock_timeout') AS lock_timeout", + "SET LOCAL lock_timeout = '3s'", + "SELECT set_config('lock_timeout', %s, true)", + } + if query.startswith("SELECT set_config"): + assert params == ("0",) + + def fetchone(self): + return {"lock_timeout": "0"} + + class FakeLockConnection: + @contextmanager + def cursor(self): + yield FakeLockCursor() + class FakeStore: + conn = FakeLockConnection() + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + assert transaction_depth == 1 + assert exclusive is True + assert lock_calls == ["graph"] + lock_calls.append("exclusive_labels") + def get_memory(self, _memory_id: str): return memory def get_memory_for_update(self, _memory_id: str): + assert transaction_depth == 1 + assert lock_calls == ["graph", "exclusive_labels"] + lock_calls.append("row") return memory def update_memory(self, *, memory_id: str, patch: dict[str, object], **_kwargs): @@ -5329,7 +5360,9 @@ def __init__(self, _store, *, defer_embeddings: bool = False) -> None: self.deferred_embedding_inputs = (deferred_input,) def lock_supersession_graph(self) -> None: - pass + assert transaction_depth == 1 + assert lock_calls == [] + lock_calls.append("graph") def refresh_memory_derived_state(self, _memory, **_kwargs) -> None: assert transaction_depth == 1 @@ -5356,6 +5389,7 @@ def fake_persist(**kwargs) -> None: assert response.status_code == 200 assert calls == ["refresh", "embedding"] + assert lock_calls == ["graph", "exclusive_labels", "row"] def test_vnext_consolidation_defers_embedding_until_primary_transaction_closes(monkeypatch) -> None: From fd66925f546d6c7451afc2c3e20f8f1d46a00090 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:45:52 +0200 Subject: [PATCH 216/270] Import protected identity helpers for the staged memory audit route --- apps/api/src/alicebot_api/routers/vnext_memories.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index abb13be2c..6af8f8f74 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -61,6 +61,8 @@ ) from alicebot_api.vnext_agent_keys import ( AgentKeyAuthenticationError, + agent_key_from_authorization, + resolve_protected_agent_identity, ) from alicebot_api.vnext_capture import ( VNextCaptureService, From f9d26b0177742c83bbda4d04d28c9164ddf108ca Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:50:20 +0200 Subject: [PATCH 217/270] Model PostgreSQL lock order in CLI redaction fixtures --- tests/unit/test_cli.py | 54 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 0a9fdf937..dd8c72013 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -398,8 +398,40 @@ def test_parser_preserves_explicit_vnext_sensitivity_filter() -> None: assert cli_module._vnext_sensitivity_allowed(omitted) == ("public", "internal", "private", "unknown") +class FakeVNextCliLockCursor: + def __init__(self, conn) -> None: + self.conn = conn + + def execute(self, query: str, params=None) -> None: + if query == "SELECT current_setting('lock_timeout') AS lock_timeout": + assert params is None + elif query == "SET LOCAL lock_timeout = '3s'": + assert params is None + self.conn.lock_timeout = "3s" + else: + assert query == "SELECT set_config('lock_timeout', %s, true)" + assert params == ("0",) + self.conn.lock_timeout = params[0] + + def fetchone(self): + return {"lock_timeout": self.conn.lock_timeout} + + +class FakeVNextCliLockConnection: + def __init__(self) -> None: + self.lock_timeout = "0" + + @contextmanager + def cursor(self): + yield FakeVNextCliLockCursor(self) + + class FakeVNextCliStore: def __init__(self) -> None: + self.conn = FakeVNextCliLockConnection() + self.graph_locked = False + self.labels_exclusive = False + self.lock_calls: list[str] = [] self.sources: list[dict[str, object]] = [] self.chunks: list[dict[str, object]] = [] self.memories: list[dict[str, object]] = [] @@ -418,6 +450,17 @@ def __init__(self) -> None: self.scheduler_workflows: dict[str, dict[str, object]] = {} self.scheduler_runs: list[dict[str, object]] = [] + def lock_graph_mutation(self) -> None: + self.graph_locked = True + self.lock_calls.append("graph") + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + assert self.graph_locked + if exclusive: + assert self.conn.lock_timeout == "3s" + self.labels_exclusive |= exclusive + self.lock_calls.append("exclusive_labels" if exclusive else "shared_labels") + def append_event(self, event: dict[str, object]) -> dict[str, object]: self.events.append(event) return event @@ -512,6 +555,9 @@ def get_memory_for_update(self, memory_id: str) -> dict[str, object] | None: return self.get_memory(memory_id) def get_memory_for_redaction(self, memory_id: str) -> dict[str, object] | None: + assert self.graph_locked + assert self.labels_exclusive + self.lock_calls.append("redaction_row") return self.get_memory(memory_id) def lock_project_update_artifacts_for_redaction(self, memory_id: str) -> list[dict[str, object]]: @@ -2497,12 +2543,17 @@ def fake_vnext_store_context(_ctx): ) first = json.loads(cli_module._run_vnext_memory_redact(context, args)) + assert store.lock_calls[:3] == ["graph", "exclusive_labels", "redaction_row"] + assert store.conn.lock_timeout == "0" assert first["status"] == "redacted" assert first["forgotten_first"] is True assert first["idempotent_replay"] is False frozen = deepcopy((store.memories, store.artifacts, store.revisions, store.events)) + previous_lock_count = len(store.lock_calls) second = json.loads(cli_module._run_vnext_memory_redact(context, args)) + assert store.lock_calls[previous_lock_count:][:3] == ["graph", "exclusive_labels", "redaction_row"] + assert store.conn.lock_timeout == "0" assert second["status"] == "redacted" assert second["forgotten_first"] is False assert second["idempotent_replay"] is True @@ -2716,6 +2767,9 @@ def test_cli_generic_memory_mutations_cannot_strand_pending_project_update_candi ) assert (store.projects, store.memories, store.artifacts, store.revisions) == state_before assert [event.get("event_type") for event in store.events] == event_types_before + if operation == "redact": + assert store.lock_calls[-3:] == ["graph", "exclusive_labels", "redaction_row"] + assert store.conn.lock_timeout == "0" def _apply_supported_cli_memory_lifecycle( From 385ccd8aa923eef01ea00829dc70d90c7bbd7aeb Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:50:20 +0200 Subject: [PATCH 218/270] Model PostgreSQL lock order in CLI redaction fixtures --- tests/unit/test_cli.py | 54 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index b7f0a4b1b..db8f918a7 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -398,8 +398,40 @@ def test_parser_preserves_explicit_vnext_sensitivity_filter() -> None: assert cli_module._vnext_sensitivity_allowed(omitted) == ("public", "internal", "private", "unknown") +class FakeVNextCliLockCursor: + def __init__(self, conn) -> None: + self.conn = conn + + def execute(self, query: str, params=None) -> None: + if query == "SELECT current_setting('lock_timeout') AS lock_timeout": + assert params is None + elif query == "SET LOCAL lock_timeout = '3s'": + assert params is None + self.conn.lock_timeout = "3s" + else: + assert query == "SELECT set_config('lock_timeout', %s, true)" + assert params == ("0",) + self.conn.lock_timeout = params[0] + + def fetchone(self): + return {"lock_timeout": self.conn.lock_timeout} + + +class FakeVNextCliLockConnection: + def __init__(self) -> None: + self.lock_timeout = "0" + + @contextmanager + def cursor(self): + yield FakeVNextCliLockCursor(self) + + class FakeVNextCliStore: def __init__(self) -> None: + self.conn = FakeVNextCliLockConnection() + self.graph_locked = False + self.labels_exclusive = False + self.lock_calls: list[str] = [] self.sources: list[dict[str, object]] = [] self.chunks: list[dict[str, object]] = [] self.memories: list[dict[str, object]] = [] @@ -418,6 +450,17 @@ def __init__(self) -> None: self.scheduler_workflows: dict[str, dict[str, object]] = {} self.scheduler_runs: list[dict[str, object]] = [] + def lock_graph_mutation(self) -> None: + self.graph_locked = True + self.lock_calls.append("graph") + + def lock_label_writes(self, *, exclusive: bool = False) -> None: + assert self.graph_locked + if exclusive: + assert self.conn.lock_timeout == "3s" + self.labels_exclusive |= exclusive + self.lock_calls.append("exclusive_labels" if exclusive else "shared_labels") + def append_event(self, event: dict[str, object]) -> dict[str, object]: self.events.append(event) return event @@ -512,6 +555,9 @@ def get_memory_for_update(self, memory_id: str) -> dict[str, object] | None: return self.get_memory(memory_id) def get_memory_for_redaction(self, memory_id: str) -> dict[str, object] | None: + assert self.graph_locked + assert self.labels_exclusive + self.lock_calls.append("redaction_row") return self.get_memory(memory_id) def lock_project_update_artifacts_for_redaction(self, memory_id: str) -> list[dict[str, object]]: @@ -2506,12 +2552,17 @@ def fake_vnext_store_context(_ctx): ) first = json.loads(cli_module._run_vnext_memory_redact(context, args)) + assert store.lock_calls[:3] == ["graph", "exclusive_labels", "redaction_row"] + assert store.conn.lock_timeout == "0" assert first["status"] == "redacted" assert first["forgotten_first"] is True assert first["idempotent_replay"] is False frozen = deepcopy((store.memories, store.artifacts, store.revisions, store.events)) + previous_lock_count = len(store.lock_calls) second = json.loads(cli_module._run_vnext_memory_redact(context, args)) + assert store.lock_calls[previous_lock_count:][:3] == ["graph", "exclusive_labels", "redaction_row"] + assert store.conn.lock_timeout == "0" assert second["status"] == "redacted" assert second["forgotten_first"] is False assert second["idempotent_replay"] is True @@ -2725,6 +2776,9 @@ def test_cli_generic_memory_mutations_cannot_strand_pending_project_update_candi ) assert (store.projects, store.memories, store.artifacts, store.revisions) == state_before assert [event.get("event_type") for event in store.events] == event_types_before + if operation == "redact": + assert store.lock_calls[-3:] == ["graph", "exclusive_labels", "redaction_row"] + assert store.conn.lock_timeout == "0" def _apply_supported_cli_memory_lifecycle( From 98949374745b3b808328f2974b4c810603d8351d Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:57:14 +0200 Subject: [PATCH 219/270] test: acquire label locks before producer fixture writes --- tests/integration/test_derived_labels_group_scope_postgres.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_derived_labels_group_scope_postgres.py b/tests/integration/test_derived_labels_group_scope_postgres.py index 39c093d19..8397a4ed2 100644 --- a/tests/integration/test_derived_labels_group_scope_postgres.py +++ b/tests/integration/test_derived_labels_group_scope_postgres.py @@ -10,6 +10,7 @@ from alicebot_api.vnext_memory_commit import VNextMemoryCommitService from alicebot_api.vnext_rollups import VNextRollupService from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.conftest import lock_label_fixture from tests.unit.test_group_scope_sqlite import ALPHA, seed_members @@ -19,6 +20,7 @@ def test_a_second_scoped_rollup_run_over_the_same_group_finds_its_card_and_inser with user_connection(migrated_database_urls["app"], user_id) as conn: ContinuityStore(conn).create_user(user_id, f"group-{user_id}@example.invalid", "Group") store = PostgresVNextStore(conn) + lock_label_fixture(store) members = seed_members(store) service = VNextRollupService(store) first = service.propose_rollups(projects=(ALPHA,)) From 1063f922b5b74b38ab65c791e1bf1e1b5959313c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 01:14:55 +0200 Subject: [PATCH 220/270] Scope historical migration fixtures to their declared identities --- .../integration/test_migration_0087_retry.py | 3 + tests/integration/test_migrations.py | 127 ++++++++++++++++-- 2 files changed, 116 insertions(+), 14 deletions(-) diff --git a/tests/integration/test_migration_0087_retry.py b/tests/integration/test_migration_0087_retry.py index 4a1c2800d..caeb0a8a5 100644 --- a/tests/integration/test_migration_0087_retry.py +++ b/tests/integration/test_migration_0087_retry.py @@ -5,6 +5,7 @@ import pytest from alicebot_api.migrations import make_alembic_config +from tests.integration.test_migrations import _set_fixture_identity _PARTIALLY_COMMITTED_SCHEMA = ( @@ -90,6 +91,7 @@ def test_0087_retries_after_committed_ddl_and_invalid_concurrent_unique_index( user_id = "00000000-0000-0000-0000-000000008701" with psycopg.connect(database_url) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: for statement in _PARTIALLY_COMMITTED_SCHEMA: cur.execute(statement) @@ -129,6 +131,7 @@ def test_0087_retries_after_committed_ddl_and_invalid_concurrent_unique_index( # catalog row. This models an operator correcting the build cause before # rerunning the still-unapplied Alembic revision. with psycopg.connect(database_url) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index 59320ba04..f276dec51 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -6,6 +6,7 @@ from psycopg.types.json import Jsonb import pytest from uuid import UUID +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from alicebot_api.db import user_connection from alicebot_api.migrations import make_alembic_config @@ -15,6 +16,38 @@ from alicebot_api.vnext_store import PostgresVNextStore +def _fixture_migration_config(database_url, *, user_id=None, user_account_id=None): + """Migrate one declared historical fixture identity under unchanged RLS. + + These data-bearing fixtures contain one user or one account. The empty + full-schema smoke and the multi-user 0096 acceptance keep their unbound + migrator connections. + """ + parsed = urlsplit(database_url) + query = dict(parse_qsl(parsed.query, keep_blank_values=True)) + options = [query.get("options", "")] + for setting, identity in ( + ("app.current_user_id", user_id), + ("app.current_user_account_id", user_account_id), + ): + if identity is not None: + options.append(f"-c {setting}={UUID(str(identity))}") + query["options"] = " ".join(option for option in options if option) + config = make_alembic_config(database_url) + config.attributes["explicit_database_url"] = urlunsplit(parsed._replace(query=urlencode(query))) + return config + + +def _set_fixture_identity(conn, *, user_id=None, user_account_id=None): + """Keep historical fixture reads and writes inside the existing RLS policy.""" + for setting, identity in ( + ("app.current_user_id", user_id), + ("app.current_user_account_id", user_account_id), + ): + if identity is not None: + conn.execute("SELECT set_config(%s, %s, %s)", (setting, str(identity), not conn.autocommit)) + + def test_vnext_kernel_upgrade_backfills_data_bearing_append_only_revisions(database_urls): """A real pre-vNext row must survive 0066 -> 0067. @@ -23,13 +56,14 @@ def test_vnext_kernel_upgrade_backfills_data_bearing_append_only_revisions(datab data-bearing upgrade path that an empty-schema migration smoke cannot exercise. """ - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000101" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) memory_id = "00000000-0000-0000-0000-000000000102" revision_id = "00000000-0000-0000-0000-000000000103" command.upgrade(config, "20260416_0066") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -62,6 +96,7 @@ def test_vnext_kernel_upgrade_backfills_data_bearing_append_only_revisions(datab command.upgrade(config, "20260510_0067") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -96,14 +131,15 @@ def test_vnext_kernel_upgrade_backfills_data_bearing_append_only_revisions(datab def test_lifecycle_invariant_upgrade_canonicalizes_retry_ids_and_installs_edge_trigger(database_urls): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000111" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) first_id = "00000000-0000-0000-0000-000000000112" second_id = "00000000-0000-0000-0000-000000000113" edge_id = "00000000-0000-0000-0000-000000000114" command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -144,6 +180,7 @@ def test_lifecycle_invariant_upgrade_canonicalizes_retry_ids_and_installs_edge_t command.upgrade(config, "20260711_0083") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -205,18 +242,20 @@ def test_lifecycle_invariant_upgrade_keeps_identifiers_on_live_row_over_tombston row; stranding it on the tombstone makes replay return nothing while the partial unique index blocks re-insertion of the same key. """ - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000131" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) tombstone_id = "00000000-0000-0000-0000-000000000132" live_id = "00000000-0000-0000-0000-000000000133" command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) _seed_tombstone_and_live_duplicate(conn, user_id=user_id, tombstone_id=tombstone_id, live_id=live_id) command.upgrade(config, "head") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -251,19 +290,21 @@ def test_lifecycle_identifier_repair_corrects_database_mis_upgraded_by_0083(data move it onto the oldest live row, and be safe to re-run on already-corrected data. """ - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000141" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) tombstone_id = "00000000-0000-0000-0000-000000000142" live_id = "00000000-0000-0000-0000-000000000143" command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) _seed_tombstone_and_live_duplicate(conn, user_id=user_id, tombstone_id=tombstone_id, live_id=live_id) # Apply only the shipped (buggy) 0083 and document the mis-assignment. command.upgrade(config, "20260711_0083") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -284,6 +325,7 @@ def test_lifecycle_identifier_repair_corrects_database_mis_upgraded_by_0083(data def _assert_corrected() -> None: with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -323,14 +365,15 @@ def test_released_0084_database_upgrades_through_current_head(database_urls): at the deleted former holder. Existing v0.9.4 databases will never rerun 0084, so only the new 0086 revision may repair that stale pointer. """ - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000151" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) tombstone_id = "00000000-0000-0000-0000-000000000152" canonical_live_id = "00000000-0000-0000-0000-000000000153" later_live_id = "00000000-0000-0000-0000-000000000154" command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) _seed_tombstone_and_live_duplicate( conn, user_id=user_id, @@ -362,6 +405,7 @@ def test_released_0084_database_upgrades_through_current_head(database_urls): command.upgrade(config, "20260712_0084") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute("SELECT version_num FROM alembic_version") assert cur.fetchone()["version_num"] == "20260712_0084" @@ -386,6 +430,7 @@ def test_released_0084_database_upgrades_through_current_head(database_urls): def _assert_all_pointers_truthful() -> None: with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -412,6 +457,7 @@ def _assert_all_pointers_truthful() -> None: _assert_all_pointers_truthful() with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute("SELECT version_num FROM alembic_version") assert cur.fetchone()["version_num"] == "20261005_0096" @@ -428,13 +474,14 @@ def _assert_all_pointers_truthful() -> None: def test_migration_0088_supports_rolling_provider_writes_and_rejects_token_rewind( database_urls, ): - config = make_alembic_config(database_urls["admin"]) user_account_id = "00000000-0000-0000-0000-000000000131" + config = _fixture_migration_config(database_urls["admin"], user_account_id=user_account_id) workspace_id = "00000000-0000-0000-0000-000000000132" provider_id = "00000000-0000-0000-0000-000000000133" command.upgrade(config, "20260713_0087") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_account_id=user_account_id) with conn.cursor() as cur: cur.execute( """ @@ -452,6 +499,13 @@ def test_migration_0088_supports_rolling_provider_writes_and_rejects_token_rewin """, (workspace_id, user_account_id), ) + cur.execute( + """ + INSERT INTO workspace_members (workspace_id, user_account_id, role) + VALUES (%s, %s, 'owner') + """, + (workspace_id, user_account_id), + ) cur.execute( """ INSERT INTO model_providers ( @@ -508,6 +562,7 @@ def test_migration_0088_supports_rolling_provider_writes_and_rejects_token_rewin autocommit=True, row_factory=dict_row, ) as conn: + _set_fixture_identity(conn, user_account_id=user_account_id) with conn.cursor() as cur: cur.execute( """ @@ -734,13 +789,14 @@ def test_migration_0088_supports_rolling_provider_writes_and_rejects_token_rewin def test_lifecycle_upgrade_promotes_and_reads_legacy_nested_multi_project_scope(database_urls): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000121" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) memory_id = "00000000-0000-0000-0000-000000000122" canonical_memory_id = "00000000-0000-0000-0000-000000000123" command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -779,6 +835,7 @@ def test_lifecycle_upgrade_promotes_and_reads_legacy_nested_multi_project_scope( ) with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) legacy_store = PostgresVNextStore(conn) legacy_row = legacy_store.get_memory(memory_id) assert legacy_row is not None @@ -794,6 +851,7 @@ def test_lifecycle_upgrade_promotes_and_reads_legacy_nested_multi_project_scope( command.upgrade(config, "20260711_0083") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "SELECT project_id, metadata_json FROM memories WHERE id = %s", @@ -836,8 +894,8 @@ def test_pre_lifecycle_upgrade_preserves_present_canonical_project_scope_to_head ): """0082 rows must not resurrect stale nested scope while upgrading to head.""" - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000124" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) row_ids = { "empty": "00000000-0000-0000-0000-000000000125", "null": "00000000-0000-0000-0000-000000000126", @@ -872,6 +930,7 @@ def test_pre_lifecycle_upgrade_preserves_present_canonical_project_scope_to_head command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -901,6 +960,7 @@ def test_pre_lifecycle_upgrade_preserves_present_canonical_project_scope_to_head command.upgrade(config, "head") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -956,14 +1016,15 @@ def test_pre_lifecycle_upgrade_preserves_unicode_project_whitespace_exactly_to_h ): """0083 must not reinterpret Unicode whitespace as the ASCII contract.""" - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000130" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) memory_id = "00000000-0000-0000-0000-000000000131" unicode_scope = "\u2003Alice\u2003" metadata = {"agentic_memory": {"project_scope": [unicode_scope]}} command.upgrade(config, "20260707_0082") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -986,6 +1047,7 @@ def test_pre_lifecycle_upgrade_preserves_unicode_project_whitespace_exactly_to_h command.upgrade(config, "head") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "SELECT project_id, metadata_json FROM memories WHERE id = %s", @@ -999,8 +1061,8 @@ def test_pre_lifecycle_upgrade_preserves_unicode_project_whitespace_exactly_to_h def test_tool_execution_task_step_linkage_migration_backfills_existing_rows(database_urls): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000001" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) thread_id = "00000000-0000-0000-0000-000000000002" trace_id = "00000000-0000-0000-0000-000000000003" tool_id = "00000000-0000-0000-0000-000000000004" @@ -1012,6 +1074,7 @@ def test_tool_execution_task_step_linkage_migration_backfills_existing_rows(data command.upgrade(config, "20260313_0020") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1216,6 +1279,7 @@ def test_tool_execution_task_step_linkage_migration_backfills_existing_rows(data command.upgrade(config, "head") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1241,13 +1305,14 @@ def test_tool_execution_task_step_linkage_migration_backfills_existing_rows(data def test_gmail_account_credentials_migration_round_trip_preserves_tokens(database_urls): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000101" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) gmail_account_id = "00000000-0000-0000-0000-000000000102" command.upgrade(config, "20260316_0026") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1284,6 +1349,7 @@ def test_gmail_account_credentials_migration_round_trip_preserves_tokens(databas command.upgrade(config, "20260316_0027") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1315,6 +1381,7 @@ def test_gmail_account_credentials_migration_round_trip_preserves_tokens(databas command.downgrade(config, "20260316_0026") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1342,13 +1409,14 @@ def test_gmail_account_credentials_migration_round_trip_preserves_tokens(databas def test_gmail_refresh_token_lifecycle_migration_round_trip_preserves_downgrade_compatibility( database_urls, ): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000201" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) gmail_account_id = "00000000-0000-0000-0000-000000000202" command.upgrade(config, "20260316_0027") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1403,6 +1471,7 @@ def test_gmail_refresh_token_lifecycle_migration_round_trip_preserves_downgrade_ command.upgrade(config, "20260316_0028") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1440,6 +1509,7 @@ def test_gmail_refresh_token_lifecycle_migration_round_trip_preserves_downgrade_ command.downgrade(config, "20260316_0027") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1462,13 +1532,14 @@ def test_gmail_refresh_token_lifecycle_migration_round_trip_preserves_downgrade_ def test_gmail_external_secret_manager_migration_round_trip_preserves_legacy_transition_rows( database_urls, ): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000301" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) gmail_account_id = "00000000-0000-0000-0000-000000000302" command.upgrade(config, "20260316_0028") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1527,6 +1598,7 @@ def test_gmail_external_secret_manager_migration_round_trip_preserves_legacy_tra command.upgrade(config, "20260316_0029") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1550,6 +1622,7 @@ def test_gmail_external_secret_manager_migration_round_trip_preserves_legacy_tra command.downgrade(config, "20260316_0028") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1570,14 +1643,15 @@ def test_gmail_external_secret_manager_migration_round_trip_preserves_legacy_tra def test_calendar_account_migration_round_trip_preserves_table_shape(database_urls): - config = make_alembic_config(database_urls["admin"]) user_id = "00000000-0000-0000-0000-000000000401" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) calendar_account_id = "00000000-0000-0000-0000-000000000402" command.upgrade(config, "20260316_0029") command.upgrade(config, "20260319_0030") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1633,6 +1707,7 @@ def test_calendar_account_migration_round_trip_preserves_table_shape(database_ur conn.commit() with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -1658,6 +1733,7 @@ def test_calendar_account_migration_round_trip_preserves_table_shape(database_ur command.downgrade(config, "20260316_0029") with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute("SELECT to_regclass('public.calendar_account_credentials')") assert cur.fetchone() == (None,) @@ -2323,6 +2399,7 @@ def test_project_scope_identity_upgrade_repairs_dedupe_without_widening_empty_sc config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260713_0089") user_id = "00000000-0000-0000-0000-000000000301" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) source_a = "00000000-0000-0000-0000-000000000302" source_b = "00000000-0000-0000-0000-000000000303" memory_id = "00000000-0000-0000-0000-000000000304" @@ -2348,6 +2425,7 @@ def test_project_scope_identity_upgrade_repairs_dedupe_without_widening_empty_sc ) with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -2450,6 +2528,7 @@ def test_project_scope_identity_upgrade_repairs_dedupe_without_widening_empty_sc sensitivity="private", ) with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -2545,6 +2624,7 @@ def test_project_scope_identity_upgrade_resolves_all_legacy_source_forms_and_blo config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260713_0089") user_id = "00000000-0000-0000-0000-000000000331" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) target_scope = "Legacy Project" cases = { "root_canonical": {"project_scope": [f" {target_scope} "]}, @@ -2563,6 +2643,7 @@ def test_project_scope_identity_upgrade_resolves_all_legacy_source_forms_and_blo raw_texts = {name: f"Fact: {name} keeps its migrated source scope." for name in cases} with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -2592,6 +2673,7 @@ def test_project_scope_identity_upgrade_resolves_all_legacy_source_forms_and_blo command.upgrade(config, "head") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -2638,6 +2720,7 @@ def test_project_scope_identity_upgrade_keeps_present_empty_nested_source_scope_ config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260713_0089") user_id = "00000000-0000-0000-0000-000000000351" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) stale_project = "Legacy Project" cases: dict[str, dict[str, object]] = { "nested_blank": { @@ -2666,6 +2749,7 @@ def test_project_scope_identity_upgrade_keeps_present_empty_nested_source_scope_ raw_texts = {name: f"Fact: migration {name} preserves nested scope presence." for name in cases} with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -2695,6 +2779,7 @@ def test_project_scope_identity_upgrade_keeps_present_empty_nested_source_scope_ command.upgrade(config, "20260714_0090") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -2733,6 +2818,7 @@ def test_project_scope_identity_upgrade_matches_python_strip_and_blocks_unicode_ config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260713_0089") user_id = "00000000-0000-0000-0000-000000000341" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) project_scope = ("Legacy Project",) raw_texts = { "nbsp": "\u00a0Fact: NBSP boundary\u00a0", @@ -2742,6 +2828,7 @@ def test_project_scope_identity_upgrade_matches_python_strip_and_blocks_unicode_ source_ids = {name: f"00000000-0000-0000-0005-{index:012d}" for index, name in enumerate(raw_texts, start=1)} with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -2785,6 +2872,7 @@ def test_project_scope_identity_upgrade_matches_python_strip_and_blocks_unicode_ for name, raw_text in raw_texts.items() } with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -2819,6 +2907,7 @@ def test_source_identity_0091_clears_only_live_whitespace_strings_and_installs_e config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260714_0090") user_id = "00000000-0000-0000-0007-000000000001" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) whitespace_cases = { "ascii": " \t\r\n", "unit_separator_control": "\u001c\u001f", @@ -2835,6 +2924,7 @@ def test_source_identity_0091_clears_only_live_whitespace_strings_and_installs_e deleted_id = "00000000-0000-0000-0007-000000000023" with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -2919,6 +3009,7 @@ def test_source_identity_0091_clears_only_live_whitespace_strings_and_installs_e def identity_snapshot() -> dict[str, str | None]: with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -2943,6 +3034,7 @@ def identity_snapshot() -> dict[str, str | None]: assert identity_snapshot() == expected with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -3018,6 +3110,7 @@ def identity_snapshot() -> dict[str, str | None]: # Re-crossing the forward boundary is data-idempotent. command.downgrade(config, "20260714_0090") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -3041,6 +3134,7 @@ def test_0092_backfills_prior_authorized_project_update_redaction(database_urls) config = make_alembic_config(database_urls["admin"]) command.upgrade(config, "20260715_0091") user_id = "00000000-0000-0000-0092-000000000001" + config = _fixture_migration_config(database_urls["admin"], user_id=user_id) memory_id = "00000000-0000-0000-0092-000000000002" artifact_id = "00000000-0000-0000-0092-000000000003" revision_id = "00000000-0000-0000-0092-000000000004" @@ -3051,6 +3145,7 @@ def test_0092_backfills_prior_authorized_project_update_redaction(database_urls) sentinel = "0092-OLD-REDACTION-SECRET" with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", @@ -3212,6 +3307,7 @@ def test_0092_backfills_prior_authorized_project_update_redaction(database_urls) command.upgrade(config, "head") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -3311,6 +3407,7 @@ def test_0092_backfills_prior_authorized_project_update_redaction(database_urls) command.downgrade(config, "20260715_0091") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -3330,6 +3427,7 @@ def test_0092_backfills_prior_authorized_project_update_redaction(database_urls) assert cur.fetchone() == {"content_markdown": "[REDACTED]"} command.upgrade(config, "head") with psycopg.connect(database_urls["admin"], row_factory=dict_row) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ @@ -3371,6 +3469,7 @@ def test_postgres_source_constraints_reject_noncanonical_classifications(databas command.upgrade(config, "head") user_id = "00000000-0000-0000-0007-000000000030" with psycopg.connect(database_urls["admin"]) as conn: + _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( "INSERT INTO users (id, email, display_name) VALUES (%s, %s, %s)", From 5e4bcf41f0b3c0d6d1c92ae67e0a617796b3ea30 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 02:06:22 +0200 Subject: [PATCH 221/270] Keep migration acceptance under a restricted owner in CI --- .../test_derived_labels_migration_postgres.py | 56 ++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_derived_labels_migration_postgres.py b/tests/integration/test_derived_labels_migration_postgres.py index 3f24a8a19..2a586b226 100644 --- a/tests/integration/test_derived_labels_migration_postgres.py +++ b/tests/integration/test_derived_labels_migration_postgres.py @@ -7,9 +7,11 @@ import threading from types import SimpleNamespace from uuid import uuid4 +from urllib.parse import quote, urlsplit, urlunsplit from alembic import command, op import psycopg +from psycopg import sql import pytest import alicebot_api.main as main_module @@ -29,13 +31,63 @@ from alicebot_api.vnext_label_writes import without_insert_floor from alicebot_api.vnext_store import PostgresVNextStore from tests.integration.test_vnext_omitted_domains_api import invoke_request -from tests.integration.conftest import _create_role_separated_database, _drop_database -from urllib.parse import urlsplit +from tests.integration.conftest import _create_role_separated_database, _drop_database, _role_urls TABLES = ("sources", "memories", "open_loops", "generated_artifacts", "beliefs", "event_log", "projects") SENTINEL = "Violet private migration sentinel" +@pytest.fixture +def database_urls(monkeypatch): + """Run migration acceptance with an owner that cannot bypass forced RLS. + + CI uses a superuser administrator for historical migrations. These tests + deliberately use a separate restricted owner for the current acceptance. + """ + admin_root, _app, lifecycle_root, *_ = _role_urls("unused") + role_name = None + with psycopg.connect(admin_root) as conn: + posture = conn.execute( + "SELECT rolsuper, rolbypassrls FROM pg_roles WHERE rolname=current_user" + ).fetchone() + if posture != (False, False): + role_name = "alicebot_repair_owner_" + uuid4().hex[:12] + password = uuid4().hex + with psycopg.connect(lifecycle_root, autocommit=True) as conn: + conn.execute( + sql.SQL("CREATE ROLE {} LOGIN PASSWORD {} NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS").format( + sql.Identifier(role_name), sql.Literal(password) + ) + ) + parsed = urlsplit(admin_root) + server = parsed.netloc.rsplit("@", 1)[-1] + strict_url = urlunsplit( + (parsed.scheme, f"{quote(role_name)}:{quote(password)}@{server}", parsed.path, parsed.query, parsed.fragment) + ) + # Preserve the original bootstrap actor when no explicit lifecycle URL is set. + monkeypatch.setenv("DATABASE_LIFECYCLE_URL", lifecycle_root) + monkeypatch.setenv("DATABASE_ADMIN_URL", strict_url) + name = "alicebot_repair_" + uuid4().hex[:12] + try: + urls = _create_role_separated_database(name) + # CI preloads vector in its root database rather than template1. + with psycopg.connect(_role_urls(name)[-1], autocommit=True) as conn: + conn.execute("CREATE EXTENSION IF NOT EXISTS vector") + yield urls + finally: + close_connection_pools() + _drop_database(name) + if role_name is not None: + with psycopg.connect(lifecycle_root, autocommit=True) as conn: + conn.execute(sql.SQL("DROP ROLE {}").format(sql.Identifier(role_name))) + + +@pytest.fixture +def migrated_database_urls(database_urls): + command.upgrade(make_alembic_config(database_urls["admin"]), "head") + return database_urls + + @contextmanager def owner_bracket(url): with psycopg.connect(url) as conn: From 3aaff2d245cd1270d0af1c0db5d7d0c630663a78 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 02:07:12 +0200 Subject: [PATCH 222/270] Retain strict concurrency and propagation acceptance coverage --- .../src/alicebot_api/vnext_derived_labels.py | 6 - .../derived_labels_postgres_support.py | 216 +++++++++ ...est_derived_labels_concurrency_postgres.py | 427 ++++++++++++++++++ ...est_derived_labels_propagation_postgres.py | 210 +++++++++ 4 files changed, 853 insertions(+), 6 deletions(-) create mode 100644 tests/integration/derived_labels_postgres_support.py create mode 100644 tests/integration/test_derived_labels_concurrency_postgres.py create mode 100644 tests/integration/test_derived_labels_propagation_postgres.py diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 24bdafa23..e9dae28b8 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -1302,12 +1302,6 @@ def scope_is_global(scope: object) -> bool: return is_global_scope(scope) - - - - - - __all__ = [ "DERIVED_ARTIFACT_TYPES", "DERIVED_WORKFLOWS", diff --git a/tests/integration/derived_labels_postgres_support.py b/tests/integration/derived_labels_postgres_support.py new file mode 100644 index 000000000..c29f47aad --- /dev/null +++ b/tests/integration/derived_labels_postgres_support.py @@ -0,0 +1,216 @@ +"""Synthetic, role-separated fixtures for the label atomicity acceptance tests.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime +import json +import time +from urllib.parse import urlencode +from uuid import UUID, uuid4 + +import anyio +import psycopg +from psycopg.rows import dict_row +import pytest + +import alicebot_api.main as main_module +from alicebot_api.config import Settings +from alicebot_api.db import user_connection +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_store import PostgresVNextStore + + +def invoke(method, path, *, user_id, payload=None, key=None): + """Invoke the mounted HTTP application, preserving response headers.""" + messages = [] + body = json.dumps(payload).encode() if payload is not None else b"" + received = False + + async def receive(): + nonlocal received + if received: + return {"type": "http.disconnect"} + received = True + return {"type": "http.request", "body": body, "more_body": False} + + async def send(message): + messages.append(message) + + headers = [(b"host", b"127.0.0.1:8000"), (b"content-type", b"application/json")] + if key: + headers.append((b"authorization", f"Bearer {key}".encode())) + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": method, + "scheme": "http", + "path": path, + "raw_path": path.encode(), + "query_string": urlencode({"user_id": str(user_id)}).encode(), + "headers": headers, + "client": ("127.0.0.1", 50000), + "server": ("testserver", 80), + "root_path": "", + } + anyio.run(main_module.app, scope, receive, send) + start = next(item for item in messages if item["type"] == "http.response.start") + raw = b"".join(item.get("body", b"") for item in messages if item["type"] == "http.response.body") + return int(start["status"]), json.loads(raw), dict(start.get("headers", [])) + + +@dataclass +class LabelHarness: + urls: dict[str, str] + user_id: UUID + + @contextmanager + def store(self) -> Iterator[PostgresVNextStore]: + with user_connection(self.urls["app"], self.user_id) as conn: + yield PostgresVNextStore(conn) + + def request(self, method, path, *, payload=None, key=None): + if payload is not None: + payload = {"user_id": str(self.user_id), **payload} + return invoke(method, path, user_id=self.user_id, payload=payload, key=key) + + def relabel(self, kind, row_id, **labels): + # Call the real owner handler independently of the middleware's key-provisioning gate. + from alicebot_api.routers import vnext_memories as router + + if kind == "source": + result = router.review_vnext_source( + UUID(str(row_id)), router.VNextSourceReviewRequest(user_id=self.user_id, action="update", **labels) + ) + else: + result = router.review_vnext_memory( + UUID(str(row_id)), + router.VNextMemoryReviewRequest(user_id=self.user_id, action="edit", **labels), + authorization=None, + ) + return ( + result.status_code, + json.loads(result.body), + {key.encode(): value.encode() for key, value in result.headers.items()}, + ) + + def key(self, profile, *, project=None): + with self.store() as store: + _record, raw = create_agent_key( + store, user_id=self.user_id, agent_id=str(uuid4()), permission_profile=profile, project_scope=project + ) + return raw + + def source(self, *, scope=(), text="TODO: Synthetic acceptance task", sensitivity="public"): + with self.store() as store: + return store.create_source( + { + "source_type": "note", + "title": "Synthetic acceptance", + "content_hash": str(uuid4()), + "domain": "project", + "sensitivity": sensitivity, + "metadata_json": {"project_scope": list(scope), "raw_text": text}, + } + ) + + def memory(self, *, source=None, parents=(), scope=()): + metadata = {"project_scope": list(scope)} + if source is not None: + metadata["source_id"] = str(source["id"]) + if parents: + metadata["consolidation"] = {"cluster_member_ids": [str(row["id"]) for row in parents]} + with self.store() as store: + return store.create_memory( + { + "memory_key": str(uuid4()), + "canonical_text": "Synthetic acceptance memory", + "status": "active", + "domain": "project", + "sensitivity": "public", + "metadata_json": metadata, + } + ) + + def snapshot(self): + with self.store() as store: + result = {} + with store.conn.cursor() as cur: + for table in ("sources", "memories", "open_loops", "generated_artifacts", "projects", "event_log"): + cur.execute(f"SELECT row_to_json(t) FROM {table} t ORDER BY id") # closed table names + result[table] = [row["row_to_json"] for row in cur.fetchall()] + return result + + def label_locks(self): + # Monitoring through a fresh app connection avoids disturbing the transaction under test. + with psycopg.connect(self.urls["app"], autocommit=True, row_factory=dict_row) as conn: + return conn.execute( + "SELECT pid, mode, granted FROM pg_locks WHERE locktype='advisory' AND classid=(hashtext('vnext_labels')::bigint & 4294967295) AND objid=(hashtext(%s)::bigint & 4294967295)", + (str(self.user_id),), + ).fetchall() + + def wait_exclusive(self, *, timeout=2.0): + end = time.monotonic() + timeout + while time.monotonic() < end: + rows = self.label_locks() + if any(row["mode"] == "ExclusiveLock" and not row["granted"] for row in rows): + return rows + time.sleep(0.01) + raise AssertionError(f"no waiting exclusive label lock in pg_locks: {self.label_locks()}") + + def wait_relabel(self, *, timeout=2.0): + """A review may serialise the relabel on S or L, in that order.""" + end = time.monotonic() + timeout + with psycopg.connect(self.urls["app"], autocommit=True, row_factory=dict_row) as conn: + while time.monotonic() < end: + rows = conn.execute( + "SELECT pid, mode, granted FROM pg_locks WHERE locktype='advisory' " + "AND classid IN ((hashtext('vnext_labels')::bigint & 4294967295), " + "(hashtext('vnext_supersession')::bigint & 4294967295)) " + "AND objid=(hashtext(%s)::bigint & 4294967295)", + (str(self.user_id),), + ).fetchall() + if any(row["mode"] == "ExclusiveLock" and not row["granted"] for row in rows): + return rows + time.sleep(0.01) + raise AssertionError(f"no waiting graph or label relabel lock: {rows}") + + +@pytest.fixture +def label_harness(migrated_database_urls, monkeypatch): + from alicebot_api import vnext_label_writes + from alicebot_api.routers import vnext_memories, vnext_projects, vnext_retrieval, vnext_review, workspaces + + settings = Settings(database_url=migrated_database_urls["app"]) + for module in (main_module, vnext_memories, vnext_projects, vnext_retrieval, vnext_review, workspaces): + monkeypatch.setattr(module, "get_settings", lambda: settings) + monkeypatch.setattr(vnext_label_writes, "STRICT_LOCK_ORDER", True) + monkeypatch.setenv("ALICE_LEGACY_SURFACES", "1") + monkeypatch.setenv("ALICE_MCP_LEGACY_TOOLS", "1") + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + harness = LabelHarness(migrated_database_urls, uuid4()) + with harness.store() as store: + ContinuityStore(store.conn).create_user( + harness.user_id, f"labels-{harness.user_id}@example.invalid", "Synthetic acceptance" + ) + return harness + + +def today(): + return datetime.now(UTC).date().isoformat() + + +def assert_raised(row, *, domain="health", sensitivity="confidential"): + assert row["domain"] == domain + assert row["sensitivity"] == sensitivity + + +def join_thread(thread, failures, *, timeout=6.0): + thread.join(timeout) + assert not thread.is_alive(), "worker did not finish within the acceptance deadline" + assert not failures, repr(failures) diff --git a/tests/integration/test_derived_labels_concurrency_postgres.py b/tests/integration/test_derived_labels_concurrency_postgres.py new file mode 100644 index 000000000..793e87f57 --- /dev/null +++ b/tests/integration/test_derived_labels_concurrency_postgres.py @@ -0,0 +1,427 @@ +"""Real PostgreSQL acceptance races for derived labels, always in strict mode.""" + +from __future__ import annotations + +import argparse +from copy import deepcopy +from datetime import UTC, datetime, timedelta +from threading import Barrier, Event, Thread +import time +from types import SimpleNamespace + +import pytest + +from alicebot_api.cli.automation import _run_vnext_artifact_review +from alicebot_api.cli.models import CLIContext +from alicebot_api.config import Settings +from alicebot_api.mcp_tools import MCPRuntimeContext, call_mcp_tool +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService +from alicebot_api.vnext_queue import VNextQueueService +from alicebot_api.vnext_scheduler import _StagedSchedulerStore, VNextSchedulerService +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.derived_labels_postgres_support import ( + assert_raised, + join_thread, + label_harness, + today, +) + + +def _thread(fn): + failures = [] + + def run(): + try: + fn() + except BaseException as exc: + failures.append(exc) + + thread = Thread(target=run, daemon=True) + thread.start() + return thread, failures + + +def test_a_report_built_from_stale_inputs_is_floored_at_insert(label_harness, monkeypatch): + h = label_harness + source = h.source() + reader = h.key("trusted_local_agent") + with h.store() as store: + start = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0) + stale = deepcopy( + VNextBrainService(store)._load_inputs( + BrainArtifactRequest(generated_for=today()), window_start=start, window_end=start + timedelta(days=1) + ) + ) + assert str(source["id"]) in [str(row["id"]) for row in stale[0]] + assert h.relabel("source", source["id"], domain="health", sensitivity="confidential")[0] == 200 + monkeypatch.setattr(VNextBrainService, "_load_inputs", lambda *_args, **_kwargs: deepcopy(stale)) + status, body, _ = h.request( + "POST", + "/v0/vnext/artifacts/generate/daily-brief", + payload={"options": {"generated_for": today(), "discover_open_loops": False}}, + key=reader, + ) + assert status == 201, body + assert "Synthetic acceptance" in body["content_markdown"] + # The 201 is the read made at selection; the durable row has the current floor. + with h.store() as store: + assert_raised(store.get_artifact(str(body["id"]))) + assert h.request("GET", f"/v0/vnext/artifacts/{body['id']}", key=reader)[0] == 403 + + +def test_a_relabel_waits_for_an_open_generation_and_then_labels_its_report(label_harness): + h = label_harness + source = h.source() + response = [] + with h.store() as store: + report = VNextBrainService(store).generate_daily_brief( + BrainArtifactRequest(generated_for=today(), discover_open_loops=False) + ) + thread, failures = _thread( + lambda: response.append(h.relabel("source", source["id"], domain="health", sensitivity="confidential")) + ) + try: + time.sleep(0.5) + assert thread.is_alive(), response + rows = h.wait_exclusive() + assert any(row["mode"] == "ShareLock" and row["granted"] for row in rows) + finally: + # Leaving the context commits the uncommitted report before joining A. + pass + join_thread(thread, failures) + assert response[0][0] == 200, response + with h.store() as store: + assert_raised(store.get_artifact(str(report["id"]))) + + +def test_a_relabel_behind_a_slow_provider_call_answers_retryable_and_changes_nothing(label_harness, monkeypatch): + h = label_harness + source = h.source() + h.memory(source=source) + before = h.snapshot() + provider_entered, release = Event(), Event() + generated = [] + + def provider(self, **kwargs): + provider_entered.set() + assert release.wait(8), "provider release deadline exceeded" + return SimpleNamespace( + content_markdown=kwargs["deterministic_markdown"], prompt_hash="synthetic", model_info={}, metadata={} + ) + + monkeypatch.setattr(VNextBrainService, "_model_backed_artifact", provider) + + def generate(): + with h.store() as store: + generated.append( + VNextBrainService(store).generate_daily_brief( + BrainArtifactRequest(generated_for=today(), generation_mode="model_backed") + ) + ) + + thread, failures = _thread(generate) + try: + assert provider_entered.wait(2) + assert any(row["mode"] == "ShareLock" and row["granted"] for row in h.label_locks()) + result = [] + started = time.monotonic() + relabel_thread, relabel_failures = _thread( + lambda: result.append(h.relabel("source", source["id"], domain="health", sensitivity="confidential")) + ) + relabel_thread.join(4.0) + assert not relabel_thread.is_alive(), "relabel exceeded 3 s bound plus 1 s" + assert not relabel_failures, relabel_failures + assert time.monotonic() - started < 4.0 + status, body, headers = result[0] + assert status == 503, body + assert headers[b"retry-after"] == b"2" + assert "nothing was changed" in body["detail"] + assert h.snapshot() == before + finally: + release.set() + join_thread(thread, failures) + assert h.relabel("source", source["id"], domain="health", sensitivity="confidential")[0] == 200 + with h.store() as store: + assert_raised(store.get_artifact(str(generated[0]["id"]))) + loops = store.list_open_loops( + status=None, sensitivity_allowed=["public", "private", "confidential", "regulated", "unknown"] + ) + assert loops, "model-backed generation must write its candidate before the provider" + for loop in loops: + assert_raised(loop) + + +def test_a_scheduler_plan_staged_before_a_relabel_is_floored_at_publish(label_harness): + h = label_harness + source = h.source() + with h.store() as store: + staged = _StagedSchedulerStore(store) + report = VNextBrainService(staged).generate_daily_brief( + BrainArtifactRequest(generated_for=today(), discover_open_loops=False) + ) + plan = staged.plan(report) + assert h.relabel("source", source["id"], domain="health", sensitivity="confidential")[0] == 200 + with h.store() as store: + published = plan.publish(store) + assert_raised(published) + assert any(row["mode"] == "ShareLock" and row["granted"] for row in h.label_locks()) + with h.store() as store: + assert_raised(store.get_artifact(str(published["id"]))) + + +def test_scheduler_direct_create_takes_the_shared_publish_lock(label_harness): + h = label_harness + source = h.source() + with h.store() as store: + staged = _StagedSchedulerStore(store) + artifact = staged.create_artifact( + { + "artifact_type": "daily_brief", + "title": "Synthetic staged report", + "content_markdown": "Synthetic staged report", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"workflow": "daily_brief", "source_refs": [str(source["id"])]}, + } + ) + plan = staged.plan(artifact) + assert h.relabel("source", source["id"], domain="health", sensitivity="confidential")[0] == 200 + with h.store() as store: + assert_raised(plan.publish(store)) + assert any(row["mode"] == "ShareLock" and row["granted"] for row in h.label_locks()) + + +def test_a_staged_staleness_mark_cannot_undo_a_relabel(label_harness): + h = label_harness + alpha, beta = "prj_" + "a" * 16, "prj_" + "b" * 16 + memory = h.memory(scope=(alpha,)) + with h.store() as store: + staged = _StagedSchedulerStore(store) + marked = VNextSchedulerService(staged)._mark_memory_stale( + memory, reason="synthetic", note="Synthetic mark", metadata={} + ) + plan = staged.plan(marked) + assert h.relabel("memory", memory["id"], domain="health", sensitivity="confidential")[0] == 200 + assert ( + h.request( + "POST", + f"/v0/vnext/memories/{memory['id']}/review", + payload={"action": "assign_project", "project_id": beta}, + )[0] + == 200 + ) + with h.store() as store: + before = store.get_memory(str(memory["id"])) + plan.publish(store) + after = store.get_memory(str(memory["id"])) + assert_raised(after) + assert after["metadata_json"]["project_scope"] == [beta] + assert after["metadata_json"].get("project_floor") == before["metadata_json"].get("project_floor") + assert "staleness" in after["metadata_json"] + + +def test_an_update_that_omits_a_marker_keeps_it(label_harness): + h = label_harness + original = h.memory() + derived = h.memory(parents=(original,)) + with h.store() as store: + before = store.get_memory(str(derived["id"])) + after = store.update_memory( + memory_id=str(derived["id"]), patch={"metadata_json": {"staleness": {"note": "synthetic"}}} + ) + for marker in ("consolidation", "derived_from"): + if marker in before["metadata_json"]: + assert after["metadata_json"][marker] == before["metadata_json"][marker] + assert "consolidation" in after["metadata_json"] + + +@pytest.mark.parametrize( + "entrypoint", ["http", "mcp", "cli", "project_accept", "project_edit", "project_reject", "direct_promote"] +) +def test_every_entry_point_that_locks_a_row_and_a_relabel_never_deadlock(label_harness, monkeypatch, entrypoint): + h = label_harness + original_fetch = PostgresVNextStore._fetch_optional_one + for round_no in range(25): + source = h.source(text=f"TODO: Synthetic acceptance task {entrypoint} {round_no}") + with h.store() as store: + if entrypoint.startswith("project_"): + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + project = store.create_project( + { + "name": f"Synthetic acceptance {round_no}", + "slug": f"synthetic-{round_no}", + "domain": "project", + "sensitivity": "public", + } + ) + # A real producer selects a source in the requested project. + store.update_source( + source_id=str(source["id"]), patch={"metadata_json": {"project_scope": [str(project["id"])]}} + ) + artifact = VNextProjectService(store).generate_project_update_candidate( + ProjectAutomationRequest(project_id=str(project["id"])) + ) + else: + artifact = VNextBrainService(store).generate_daily_brief( + BrainArtifactRequest(generated_for=today(), discover_open_loops=False) + ) + row_locked, release = Event(), Event() + + def paused_fetch(self, *args, **kwargs): + row = original_fetch(self, *args, **kwargs) + if ( + row + and str(row.get("id")) == str(artifact["id"]) + and "FOR UPDATE" in str(args).upper() + and not row_locked.is_set() + ): + row_locked.set() + assert release.wait(4), "review lock release deadline exceeded" + return row + + monkeypatch.setattr(PostgresVNextStore, "_fetch_optional_one", paused_fetch) + + def review(): + artifact_id = str(artifact["id"]) + if entrypoint == "http": + result = h.request("POST", f"/v0/vnext/artifacts/{artifact_id}/review", payload={"action": "promote"}) + assert result[0] == 200, result + elif entrypoint == "mcp": + call_mcp_tool( + MCPRuntimeContext(database_url=h.urls["app"], user_id=h.user_id), + name="alice_vnext_artifact_review", + arguments={"artifact_id": artifact_id, "action": "promote"}, + ) + elif entrypoint == "cli": + _run_vnext_artifact_review( + CLIContext(Settings(database_url=h.urls["app"]), h.urls["app"], h.user_id), + argparse.Namespace(artifact_id=artifact_id, action="promote"), + ) + else: + with h.store() as store: + if entrypoint.startswith("project_"): + VNextProjectService(store, defer_embeddings=True).review_project_update( + artifact_id=artifact_id, + action=entrypoint.split("_", 1)[1], + edited_current_state="Synthetic edited state" if entrypoint == "project_edit" else None, + ) + else: + VNextQueueService(store, defer_embeddings=True)._promote_artifact( + artifact_id=artifact_id, + actor_type="user", + actor_id=str(h.user_id), + trace_id=None, + run_id=None, + ) + + reviewer, review_failures = _thread(review) + result = [] + relabeller = None + try: + assert row_locked.wait(2), (entrypoint, round_no, review_failures) + relabeller, relabel_failures = _thread( + lambda: result.append(h.relabel("source", source["id"], domain="health", sensitivity="confidential")) + ) + h.wait_relabel() + finally: + release.set() + join_thread(reviewer, review_failures) + if relabeller is not None: + join_thread(relabeller, relabel_failures) + monkeypatch.setattr(PostgresVNextStore, "_fetch_optional_one", original_fetch) + assert result[0][0] == 200, (entrypoint, round_no, result) + with h.store() as store: + assert_raised(store.get_artifact(str(artifact["id"]))) + from alicebot_api.vnext_label_writes import walk_dependants + + for row in walk_dependants(store, [str(source["id"])]): + assert_raised(row) + + +def test_two_relabels_with_a_shared_dependant_do_not_deadlock(label_harness): + h = label_harness + first, second = h.memory(), h.memory() + shared = h.memory(parents=(first, second)) + barrier = Barrier(2) + results = [] + + def change(memory): + barrier.wait(2) + results.append(h.relabel("memory", memory["id"], domain="health", sensitivity="confidential")) + + a, af = _thread(lambda: change(first)) + b, bf = _thread(lambda: change(second)) + join_thread(a, af) + join_thread(b, bf) + assert [row[0] for row in results] == [200, 200], results + with h.store() as store: + assert_raised(store.get_memory(str(shared["id"]))) + + +@pytest.mark.parametrize("method", ["get_artifact_for_update", "get_memory_for_update", "get_project_for_update"]) +def test_each_row_locker_holds_the_shared_label_lock(label_harness, method): + h = label_harness + memory = h.memory() + with h.store() as store: + project = store.create_project({"name": "Synthetic locker", "slug": "synthetic-locker"}) + artifact = VNextBrainService(store).generate_daily_brief( + BrainArtifactRequest(generated_for=today(), discover_open_loops=False) + ) + row_id = { + "get_artifact_for_update": artifact["id"], + "get_memory_for_update": memory["id"], + "get_project_for_update": project["id"], + }[method] + with h.store() as store: + getattr(store, method)(str(row_id)) + assert any(row["mode"] == "ShareLock" and row["granted"] for row in h.label_locks()), method + + +@pytest.mark.parametrize( + "method", ["create_memory", "create_open_loop", "create_artifact", "upsert_artifact_by_workflow_digest"] +) +def test_each_postgres_creator_floors_a_stale_source_copy(label_harness, method): + from uuid import uuid4 + + h = label_harness + source = h.source() + assert h.relabel("source", source["id"], domain="health", sensitivity="confidential")[0] == 200 + with h.store() as store: + metadata = {"source_id": str(source["id"])} + if method == "create_memory": + row = store.create_memory( + { + "memory_key": str(uuid4()), + "canonical_text": "Synthetic copy", + "domain": "project", + "sensitivity": "public", + "metadata_json": metadata, + } + ) + elif method == "create_open_loop": + row = store.create_open_loop( + { + "title": "Synthetic copy", + "source_id": str(source["id"]), + "domain": "project", + "sensitivity": "public", + "metadata_json": {**metadata, "discovered_by": "vnext_daily_brief"}, + } + ) + else: + payload = { + "artifact_type": "daily_brief", + "title": "Synthetic copy", + "content_markdown": "Synthetic copy", + "domain": "project", + "sensitivity": "public", + "metadata_json": {"workflow": "daily_brief", "source_refs": [str(source["id"])]}, + } + row = ( + store.create_artifact(payload) + if method == "create_artifact" + else store.upsert_artifact_by_workflow_digest(payload, workflow="daily_brief", digest=str(uuid4())) + ) + assert_raised(row) diff --git a/tests/integration/test_derived_labels_propagation_postgres.py b/tests/integration/test_derived_labels_propagation_postgres.py new file mode 100644 index 000000000..237a50fe4 --- /dev/null +++ b/tests/integration/test_derived_labels_propagation_postgres.py @@ -0,0 +1,210 @@ +"""Propagation must finish with its original, over actual generated chains.""" + +from __future__ import annotations + +import json + +import psycopg +import pytest + +from alicebot_api import vnext_label_writes +from alicebot_api.vnext_derived_domain_backfill import require_changed +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from alicebot_api.vnext_capture import VNextCaptureService +from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService +from alicebot_api.vnext_queue import VNextQueueService +from tests.integration.derived_labels_postgres_support import assert_raised, label_harness, today + + +@pytest.mark.parametrize("failure_kind", ["database", "writer"]) +def test_a_failed_propagation_rolls_the_original_back_with_it(label_harness, monkeypatch, failure_kind): + h = label_harness + source = h.source() + copies = [h.memory(source=source) for _ in range(4)] + h.memory(parents=tuple(copies)) + before = h.snapshot() + original = vnext_label_writes.write_settled_label + attempts = [] + + def fail_third(store, **kwargs): + attempts.append(kwargs["row_id"]) + if len(attempts) == 3: + if failure_kind == "writer": + require_changed(0, "memories", kwargs["row_id"]) + # A genuine database exception must abort the transaction after two writes/events. + store.conn.execute("SELECT 1 / 0") + return original(store, **kwargs) + + monkeypatch.setattr(vnext_label_writes, "write_settled_label", fail_third) + try: + status, body, _ = h.relabel("source", source["id"], domain="health", sensitivity="confidential") + except psycopg.errors.DivisionByZero: + # Baseline exposes the error; atomic rollback is still asserted, separately from mapping. + status = None + assert len(attempts) == 3 + assert h.snapshot() == before + assert status == 409, "all database propagation failures must return the fixed whole-refusal answer" + assert "nothing was changed" in body["detail"] + + +def _build_chain(h, *, source=None, project=None, label=("project", "public")): + with h.store() as store: + # Taking S first also verifies capture's real strict-order path when it reacquires S. + store.lock_graph_mutation() + if project is None: + project = store.create_project( + {"name": "SyntheticChain", "slug": "synthetic-chain", "domain": "project", "sensitivity": "public"} + ) + if source is None: + capture = VNextCaptureService(store, defer_embeddings=True).capture_text( + "Fact: SyntheticChain acceptance is prepared.\nTODO: SyntheticChain followup", + title="SyntheticChain acceptance", + domain=label[0], + sensitivity=label[1], + project_scope=(str(project["id"]),), + ) + assert capture.status == "imported", capture + assert capture.candidate_memory_count == 2 + source = store.get_source(str(capture.source_id)) + copies = store.list_memories_referencing_source(source_id=str(source["id"])) + assert len(copies) >= 2 + copies = [ + store.update_memory(memory_id=str(row["id"]), patch={"status": "accepted"}, actor_type="user") + if row["status"] == "candidate" + else row + for row in copies + ] + brain = VNextBrainService(store) + request = BrainArtifactRequest( + generated_for=today(), + projects=(str(project["id"]),), + sensitivity_allowed=("public", "internal", "private", "confidential", "regulated", "unknown"), + ) + daily = brain.generate_daily_brief(request) + assert {str(row["id"]) for row in copies}.issubset(set(daily["metadata_json"]["derived_from"]["memories"])) + weekly = brain.generate_weekly_synthesis(request) + assert str(daily["id"]) in weekly["metadata_json"]["derived_from"]["artifacts"] + promoted = VNextQueueService(store, defer_embeddings=True)._promote_artifact( + artifact_id=str(weekly["id"]), actor_type="user", actor_id=str(h.user_id), trace_id=None, run_id=None + ) + update = VNextProjectService(store, defer_embeddings=True).generate_project_update_candidate( + ProjectAutomationRequest(project_id=str(project["id"]), sensitivity_allowed=request.sensitivity_allowed) + ) + VNextProjectService(store, defer_embeddings=True).review_project_update( + artifact_id=str(update["id"]), action="accept", actor_type="user" + ) + rows = [ + *(("memory", str(row["id"])) for row in copies), + ("artifact", str(daily["id"])), + ("artifact", str(weekly["id"])), + ("memory", str(promoted["promoted_memory_id"])), + ("artifact", str(update["id"])), + ("memory", str(update["metadata_json"]["candidate_memory_id"])), + ("project", str(project["id"])), + ] + rows.extend(("memory", str(row_id)) for row_id in weekly["metadata_json"]["candidate_memory_ids"]) + loops = store.list_open_loops(status=None, sensitivity_allowed=list(request.sensitivity_allowed)) + loops = [row for row in loops if str(row.get("source_id")) == str(source["id"])] + assert loops, "daily producer must create a candidate loop" + rows.extend(("open_loop", str(row["id"])) for row in loops) + assert store.get_project(str(project["id"]))["metadata_json"]["derived_from"] + return source, project, rows + + +def _read_row(store, kind, row_id): + return { + "memory": store.get_memory, + "artifact": store.get_artifact, + "open_loop": store.get_open_loop, + "project": store.get_project, + }[kind](row_id) + + +def _assert_chain(h, rows, *, domain, sensitivity): + with h.store() as store: + for kind, row_id in rows: + row = _read_row(store, kind, row_id) + assert_raised(row, domain=domain, sensitivity=sensitivity) + + +def test_a_source_relabel_reaches_the_extracted_memories_the_loop_every_report_and_the_project_state(label_harness): + h = label_harness + source, project, rows = _build_chain(h) + assert h.relabel("source", source["id"], domain="health")[0] == 200 + _assert_chain(h, rows, domain="health", sensitivity="public") + assert h.relabel("source", source["id"], sensitivity="confidential")[0] == 200 + _assert_chain(h, rows, domain="health", sensitivity="confidential") + # All original roles in the chain must be recreated at the raised floor. + with h.store() as store: + updated_source = store.get_source(str(source["id"])) + _control_source, _control_project, control = _build_chain(h, source=updated_source, project=project) + _assert_chain(h, control, domain="health", sensitivity="confidential") + + withheld = {row_id for _kind, row_id in rows} + for profile in ("read_only_agent", "trusted_local_agent"): + key = h.key(profile) + for kind, row_id in rows: + if kind == "artifact": + exact = f"/v0/vnext/artifacts/{row_id}" + elif kind == "memory": + exact = f"/v0/vnext/memories/{row_id}/audit" + elif kind == "open_loop": + # The coupled review door performs the exact target admission before mutation. + status, body, _ = h.request( + "POST", f"/v0/vnext/open-loops/{row_id}/review", payload={"action": "close"}, key=key + ) + assert status == 403, (profile, kind, body) + continue + else: + exact = f"/v0/vnext/projects/{row_id}/dashboard" + status, body, _ = h.request("GET", exact, key=key) + if kind == "project": + # O12 operator screens redact the response rather than requiring an exact policy refusal. + assert row_id not in json.dumps(body), (profile, kind, status, body) + else: + assert status == 403, (profile, kind, status, body) + for path in ("/v0/vnext/artifacts", "/v0/vnext/projects", "/v0/vnext/workspace", "/v0/vnext/context-tree"): + status, body, _ = h.request("GET", path, key=key) + # Read-only keys cannot use operator routes. Their refusal must carry no row data. + assert status == (403 if profile == "read_only_agent" else 200), (path, body) + encoded = json.dumps(body) + assert not withheld.intersection({row_id for row_id in withheld if row_id in encoded}), ( + profile, + path, + body, + ) + status, body, _ = h.request( + "POST", + "/v0/vnext/context-packs", + payload={"query": "SyntheticChain", "scope": {}, "options": {"include_sources": True}}, + key=key, + ) + assert status == 201, body + assert all(row_id not in json.dumps(body) for row_id in withheld) + + # A project-bound key loses the entire moved chain; provenance and a union floor survive. + bound = h.key("admin_agent", project=str(project["id"])) + beta = "prj_" + "b" * 16 + from alicebot_api.routers import vnext_memories as router + from uuid import UUID + + result = router.review_vnext_source( + UUID(str(source["id"])), + router.VNextSourceReviewRequest( + user_id=h.user_id, action="assign_project", project_id=beta, confirm_label_hide=True + ), + ) + move_status, move = result.status_code, json.loads(result.body) + assert move_status == 200, move + with h.store() as store: + for kind, row_id in rows: + row = _read_row(store, kind, row_id) + assert_raised(row) + if kind == "project": + # A project state has domain and sensitivity but carries no project scope. + assert not row["metadata_json"].get("project_scope") + assert not row["metadata_json"].get("project_floor") + else: + assert {str(project["id"]), beta}.issubset(row["metadata_json"]["project_floor"]), (kind, row) + if kind == "artifact": + assert h.request("GET", f"/v0/vnext/artifacts/{row_id}", key=bound)[0] == 403 From 1319c69f807a1f17e769194e86fba83a3aa6ec65 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 00:13:34 +0200 Subject: [PATCH 223/270] Verify replacement memories preserve dependency edges --- .../test_sqlite_derived_labels_write_path.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/unit/test_sqlite_derived_labels_write_path.py b/tests/unit/test_sqlite_derived_labels_write_path.py index ad231c68a..ee6a00024 100644 --- a/tests/unit/test_sqlite_derived_labels_write_path.py +++ b/tests/unit/test_sqlite_derived_labels_write_path.py @@ -232,6 +232,45 @@ def test_merge_protected_metadata_keeps_label_keys_unless_the_write_is_a_relabel assert relabel["consolidation"] == {"cluster_member_ids": ["m"]} +def test_a_replacement_memory_keeps_dependencies_and_floor(tmp_path: Path, monkeypatch) -> None: + from uuid import UUID + + from alicebot_api.mcp.registry import call_mcp_tool + from alicebot_api.mcp.types import MCPRuntimeContext + from alicebot_api.onramp import sqlite_url_for_path + from alicebot_api.vnext_derived_labels import with_derived_from + + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.delenv("ALICE_EMBEDDINGS_BASE_URL", raising=False) + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + path = tmp_path / "replacement.sqlite3" + with _vault(path) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source({"source_type": "note", "title": "Synthetic observation", "content_hash": "replacement", "domain": "health", "sensitivity": "confidential", "metadata_json": {"project_scope": [ALPHA]}}) + metadata = with_derived_from({"source_id": str(source["id"]), "project_scope": [ALPHA]}, {"sources": [source]}) + original = store.create_memory({"memory_key": "replacement-original", "canonical_text": "Synthetic original observation", "status": "active", "domain": "health", "sensitivity": "confidential", "metadata_json": metadata}) + link = store.create_provenance_link({"target_type": "memory", "target_id": str(original["id"]), "source_id": str(source["id"]), "evidence_role": "supports", "confidence": 1.0}) + result = call_mcp_tool( + MCPRuntimeContext(database_url=sqlite_url_for_path(path), user_id=UUID(USER)), + name="alice_memory_correct", + arguments={"review_item_id": str(original["id"]), "action": "supersede-existing", "replacement_title": "Synthetic corrected observation", "replacement_provenance": {"source_id": str(source["id"]), "evidence_role": "supports", "confidence": 1.0}}, + ) + replacement_id = str(result["replacement_object"]["id"]) + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + replacement = store.get_memory(replacement_id) + assert replacement["metadata_json"]["derived_from"] == metadata["derived_from"] + assert replacement["metadata_json"]["source_id"] == source["id"] + assert replacement["metadata_json"]["project_scope"] == [ALPHA] + assert replacement["metadata_json"]["project_floor"] == [ALPHA] + assert replacement["sensitivity"] == "confidential" + assert store.get_memory(str(original["id"]))["canonical_text"] == original["canonical_text"] + assert str(store.list_provenance_links(target_type="memory", target_id=str(original["id"]))[0]["id"]) == str(link["id"]) + assert str(store.list_provenance_links(target_type="memory", target_id=replacement_id)[0]["source_id"]) == str(source["id"]) + store.update_source(source_id=str(source["id"]), patch={"sensitivity": "regulated"}) + assert store.get_memory(replacement_id)["sensitivity"] == "regulated" + + def test_insert_floor_survives_two_hops(tmp_path: Path): db = tmp_path / 'labels.sqlite3' bootstrap_database(db, user_id=USER, user_email='synthetic@example.test') From 6297861fa98d47c918deb7a6b99b6838014054ba Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 02:16:21 +0200 Subject: [PATCH 224/270] Align saved quote and counting fixtures with exact source guards --- ...st_saved_quotes_follow_the_source_fence.py | 17 ++++++++++-- tests/unit/test_vnext_retrieval.py | 26 ++++++++++++++++--- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_saved_quotes_follow_the_source_fence.py b/tests/unit/test_saved_quotes_follow_the_source_fence.py index c1d8a66b5..8b238b645 100644 --- a/tests/unit/test_saved_quotes_follow_the_source_fence.py +++ b/tests/unit/test_saved_quotes_follow_the_source_fence.py @@ -1161,8 +1161,8 @@ def test_a_source_with_no_project_is_outside_the_fence_of_a_key_bound_to_a_proje what ``alice_explain`` has always done for such a key. A key bound to no project reads it. On v0.20.0 the review returned all three to a key bound to a project. The release notes say this in one sentence. - The memory is planted with the two writes a review makes (the metadata copy and a link), as the lifecycle tests of the - review door do. + The memory is an original row with the two saved provenance copies that a review makes, as the lifecycle tests of + this projection do. The derived-copy control below separately verifies refusal of the whole captured copy. Mutation: pass ``require_explicit_project_scope=False`` in ``SourceReadFence._admits``: the keys bound to ``alpha`` read the link, the quote and the id again, and disagree with explain. @@ -1173,6 +1173,7 @@ def test_a_source_with_no_project_is_outside_the_fence_of_a_key_bound_to_a_proje assert json.loads(row["metadata_json"])["project_scope"] == [], "the owner's capture belongs to no project" candidate = vault._candidate("Projectlessnote: the alpha kiln shelf is cleaned on Fridays") _plant_saved_quote(vault, candidate, source_id) + vault._original_quote_fixture(candidate) for who in _KEY_SPECS: answer = vault.review(who, candidate) assert answer["is_error"] is False, (who, answer) @@ -1184,6 +1185,18 @@ def test_a_source_with_no_project_is_outside_the_fence_of_a_key_bound_to_a_proje +def test_a_derived_copy_of_a_projectless_source_is_denied_to_bound_keys(vault: _Vault) -> None: + source_id = vault.capture_source(as_owner=True) + candidate = vault._candidate("Projectlesscopy: the alpha kiln shelf is cleaned on Fridays") + _plant_saved_quote(vault, candidate, source_id) + for who in _KEY_SPECS: + answer = vault.review(who, candidate) + readable = who == "unbound" + assert answer["is_error"] is not readable, (who, answer) + assert _holds_quote(answer) is readable, who + assert _holds_source_id(answer, source_id) is readable, who + + def test_current_derived_copy_is_denied_as_a_whole_after_source_relabel(vault: _Vault) -> None: source_id = vault.capture_source() memory_id, _query = vault.edit_and_approve(source_id) diff --git a/tests/unit/test_vnext_retrieval.py b/tests/unit/test_vnext_retrieval.py index 773b819cb..70454a4a5 100644 --- a/tests/unit/test_vnext_retrieval.py +++ b/tests/unit/test_vnext_retrieval.py @@ -1040,10 +1040,30 @@ def test_keyword_query_that_and_matches_does_not_use_the_fallback_on_sqlite() -> def test_count_candidate_statistic_uses_real_sqlite_fts_mode_and_provenance_dedup() -> None: store = _sqlite_retrieval_store() + source_a = store.create_source( + { + "source_type": "note", + "title": "Bike service source A", + "content_hash": "sha256:bike-a", + "domain": "personal", + "sensitivity": "private", + "captured_at": "2026-01-05T00:00:00Z", + } + ) + source_b = store.create_source( + { + "source_type": "note", + "title": "Bike service source B", + "content_hash": "sha256:bike-b", + "domain": "personal", + "sensitivity": "private", + "captured_at": "2026-01-06T00:00:00Z", + } + ) provenance = ( - ("record-1", "source-a", "chunk-a"), - ("record-2", "source-a", "chunk-a"), # restatement of the same captured turn - ("record-3", "source-b", "chunk-b"), + ("record-1", source_a["id"], "chunk-a"), + ("record-2", source_a["id"], "chunk-a"), # restatement of the same captured turn + ("record-3", source_b["id"], "chunk-b"), ) for memory_key, source_id, chunk_id in provenance: store.create_memory( From 87cd9d0b2ba9d1ccd9617211035eeab2b4d6fb07 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 02:18:46 +0200 Subject: [PATCH 225/270] Seed real source parents for SQLite count-candidate fixture --- tests/unit/test_vnext_retrieval.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_vnext_retrieval.py b/tests/unit/test_vnext_retrieval.py index 773b819cb..70454a4a5 100644 --- a/tests/unit/test_vnext_retrieval.py +++ b/tests/unit/test_vnext_retrieval.py @@ -1040,10 +1040,30 @@ def test_keyword_query_that_and_matches_does_not_use_the_fallback_on_sqlite() -> def test_count_candidate_statistic_uses_real_sqlite_fts_mode_and_provenance_dedup() -> None: store = _sqlite_retrieval_store() + source_a = store.create_source( + { + "source_type": "note", + "title": "Bike service source A", + "content_hash": "sha256:bike-a", + "domain": "personal", + "sensitivity": "private", + "captured_at": "2026-01-05T00:00:00Z", + } + ) + source_b = store.create_source( + { + "source_type": "note", + "title": "Bike service source B", + "content_hash": "sha256:bike-b", + "domain": "personal", + "sensitivity": "private", + "captured_at": "2026-01-06T00:00:00Z", + } + ) provenance = ( - ("record-1", "source-a", "chunk-a"), - ("record-2", "source-a", "chunk-a"), # restatement of the same captured turn - ("record-3", "source-b", "chunk-b"), + ("record-1", source_a["id"], "chunk-a"), + ("record-2", source_a["id"], "chunk-a"), # restatement of the same captured turn + ("record-3", source_b["id"], "chunk-b"), ) for memory_key, source_id, chunk_id in provenance: store.create_memory( From 89e13def51b508ce42155d7dcad9a310e7b95fbb Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 02:22:59 +0200 Subject: [PATCH 226/270] test: pin producer fixture event timestamps --- tests/integration/test_derived_labels_producers_postgres.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_derived_labels_producers_postgres.py b/tests/integration/test_derived_labels_producers_postgres.py index 832a28d20..7248d2482 100644 --- a/tests/integration/test_derived_labels_producers_postgres.py +++ b/tests/integration/test_derived_labels_producers_postgres.py @@ -63,6 +63,7 @@ def seed_grid(app_url): rows.extend(((label, "memories", backing), (label, "beliefs", belief))) loop = store.create_open_loop({"title": marker + " loop Atlas", "description": marker + " loop text", "status": "open", "domain": "project", "sensitivity": "public", "due_at": "2026-10-04T12:00:00Z", + "opened_at": "2026-10-05T09:00:00Z", "metadata_json": {"project_scope": scope}}) rows.append((label, "open_loops", loop)) artifact = store.create_artifact({"artifact_type": "research_brief", "title": marker + " artifact Atlas", @@ -210,10 +211,11 @@ def test_input_selection_uses_effective_labels_including_the_owner_default_ceili prior_id = promoted["metadata_json"]["source_artifact_id"] derived_loop = store.create_open_loop({"title": "STALE_SOURCE_LOOP Atlas", "description": "STALE_SOURCE_LOOP secret", "source_id": str(source["id"]), "status": "open", "domain": "project", "sensitivity": "public", + "opened_at": "2026-10-05T09:00:00Z", "due_at": "2026-10-04T12:00:00Z", "metadata_json": {"project_scope": [alpha], "discovered_by": "vnext_daily_brief", "source_id": str(source["id"])}}) conn.execute("UPDATE sources SET sensitivity='confidential' WHERE id=%s", (source["id"],)) - conn.execute("UPDATE memories SET created_at='2026-10-05T09:00:00Z' WHERE id=%s", (copy["id"],)) + conn.execute("UPDATE memories SET created_at='2026-10-05T09:00:00Z', updated_at='2026-10-05T09:00:00Z', first_seen_at='2026-10-05T09:00:00Z', last_seen_at='2026-10-05T09:00:00Z' WHERE id=%s", (copy["id"],)) if producer == "staleness": conn.execute("UPDATE memories SET valid_to='2026-10-04T12:00:00Z'") assert store.get_memory(str(copy["id"]))["sensitivity"] == "public" From 44694b55a06bbc2f41d46c3e43032e2a2a0c417b Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 02:22:59 +0200 Subject: [PATCH 227/270] test: pin producer fixture event timestamps --- tests/integration/test_derived_labels_producers_postgres.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_derived_labels_producers_postgres.py b/tests/integration/test_derived_labels_producers_postgres.py index 832a28d20..7248d2482 100644 --- a/tests/integration/test_derived_labels_producers_postgres.py +++ b/tests/integration/test_derived_labels_producers_postgres.py @@ -63,6 +63,7 @@ def seed_grid(app_url): rows.extend(((label, "memories", backing), (label, "beliefs", belief))) loop = store.create_open_loop({"title": marker + " loop Atlas", "description": marker + " loop text", "status": "open", "domain": "project", "sensitivity": "public", "due_at": "2026-10-04T12:00:00Z", + "opened_at": "2026-10-05T09:00:00Z", "metadata_json": {"project_scope": scope}}) rows.append((label, "open_loops", loop)) artifact = store.create_artifact({"artifact_type": "research_brief", "title": marker + " artifact Atlas", @@ -210,10 +211,11 @@ def test_input_selection_uses_effective_labels_including_the_owner_default_ceili prior_id = promoted["metadata_json"]["source_artifact_id"] derived_loop = store.create_open_loop({"title": "STALE_SOURCE_LOOP Atlas", "description": "STALE_SOURCE_LOOP secret", "source_id": str(source["id"]), "status": "open", "domain": "project", "sensitivity": "public", + "opened_at": "2026-10-05T09:00:00Z", "due_at": "2026-10-04T12:00:00Z", "metadata_json": {"project_scope": [alpha], "discovered_by": "vnext_daily_brief", "source_id": str(source["id"])}}) conn.execute("UPDATE sources SET sensitivity='confidential' WHERE id=%s", (source["id"],)) - conn.execute("UPDATE memories SET created_at='2026-10-05T09:00:00Z' WHERE id=%s", (copy["id"],)) + conn.execute("UPDATE memories SET created_at='2026-10-05T09:00:00Z', updated_at='2026-10-05T09:00:00Z', first_seen_at='2026-10-05T09:00:00Z', last_seen_at='2026-10-05T09:00:00Z' WHERE id=%s", (copy["id"],)) if producer == "staleness": conn.execute("UPDATE memories SET valid_to='2026-10-04T12:00:00Z'") assert store.get_memory(str(copy["id"]))["sensitivity"] == "public" From 6990607c53a12caabcfb909da771dbee9a138da8 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 14:20:16 +0200 Subject: [PATCH 228/270] Correct derived label boundaries and pin handoff regressions --- apps/api/src/alicebot_api/cli/automation.py | 6 +- apps/api/src/alicebot_api/cli/capture.py | 4 +- apps/api/src/alicebot_api/cli/context.py | 2 +- apps/api/src/alicebot_api/cli/labels.py | 10 +- apps/api/src/alicebot_api/cli/runner.py | 5 + apps/api/src/alicebot_api/cli/smokes.py | 4 +- apps/api/src/alicebot_api/label_commands.py | 2 +- .../alicebot_api/mcp/evidence_artifacts.py | 4 +- apps/api/src/alicebot_api/mcp/memories.py | 4 +- apps/api/src/alicebot_api/mcp/projects.py | 8 +- apps/api/src/alicebot_api/mcp/retrieval.py | 3 + apps/api/src/alicebot_api/mcp/review.py | 4 +- apps/api/src/alicebot_api/mcp/runtime.py | 21 ++- apps/api/src/alicebot_api/mcp/synthesis.py | 33 +++- .../alicebot_api/routers/vnext_memories.py | 15 +- .../src/alicebot_api/routers/vnext_review.py | 11 +- .../src/alicebot_api/routers/workspaces.py | 2 + apps/api/src/alicebot_api/sqlite_schema.py | 7 +- apps/api/src/alicebot_api/sqlite_store.py | 7 +- .../src/alicebot_api/vnext_artifact_review.py | 22 ++- apps/api/src/alicebot_api/vnext_brain.py | 29 ++- .../api/src/alicebot_api/vnext_connections.py | 21 ++- .../src/alicebot_api/vnext_consolidation.py | 25 ++- .../src/alicebot_api/vnext_contradictions.py | 21 ++- .../src/alicebot_api/vnext_derived_labels.py | 60 +++++-- apps/api/src/alicebot_api/vnext_dogfooding.py | 2 + .../api/src/alicebot_api/vnext_label_guard.py | 167 +++++++++++++++++- .../src/alicebot_api/vnext_label_writes.py | 48 ++++- .../vnext_open_loop_references.py | 18 +- apps/api/src/alicebot_api/vnext_projects.py | 18 +- apps/api/src/alicebot_api/vnext_retrieval.py | 23 ++- apps/api/src/alicebot_api/vnext_scheduler.py | 1 + apps/api/src/alicebot_api/vnext_store.py | 23 ++- .../derived-labels-security-note-draft.md | 9 + docs/release/v0.20.0-release-notes.md | 4 +- docs/runbooks/disaster-recovery.md | 5 +- eval/scale/harness.py | 2 +- tests/integration/conftest.py | 7 +- .../integration/test_calendar_accounts_api.py | 3 +- ...nsolidation_report_sensitivity_postgres.py | 2 +- tests/integration/test_context_compile.py | 13 +- tests/integration/test_continuity_api.py | 15 +- .../integration/test_continuity_recall_api.py | 21 +-- .../test_continuity_resumption_api.py | 21 +-- tests/integration/test_continuity_store.py | 24 +-- .../test_credential_floor_every_door_api.py | 2 +- .../test_daily_brief_project_id_postgres.py | 6 +- .../test_derived_domain_postgres.py | 2 +- ...est_derived_labels_concurrency_postgres.py | 21 ++- .../test_derived_labels_producers_postgres.py | 6 +- ...est_derived_labels_propagation_postgres.py | 4 +- tests/integration/test_gmail_accounts_api.py | 8 +- .../test_label_handoff_contention_postgres.py | 79 +++++++++ .../test_label_handoff_malformed_postgres.py | 55 ++++++ .../test_label_handoff_migration_postgres.py | 61 +++++++ .../test_label_handoff_writeback_postgres.py | 62 +++++++ ...test_label_read_handoff_budget_postgres.py | 46 +++++ .../test_label_repair_handoff_cas_postgres.py | 33 ++++ tests/integration/test_memory_admission.py | 3 +- .../test_memory_review_labels_api.py | 29 ++- .../integration/test_migration_0087_retry.py | 3 - ...test_producer_handoff_crowding_postgres.py | 77 ++++++++ ..._producer_handoff_stale_inputs_postgres.py | 68 +++++++ .../integration/test_provider_runtime_api.py | 32 +--- tests/integration/test_proxy_execution_api.py | 8 +- ...est_scheduler_handoff_identity_postgres.py | 45 +++++ tests/integration/test_task_artifacts_api.py | 5 +- tests/integration/test_temporal_state_api.py | 7 +- .../test_temporal_state_mcp_cli.py | 7 +- .../test_vnext_consolidation_postgres.py | 2 +- .../test_vnext_live_workspace_api.py | 6 +- tests/performance/read_budget_probe.py | 62 +++++++ .../test_label_read_handoff_budget.py | 80 +++++++++ tests/unit/test_cli.py | 12 +- tests/unit/test_consolidation_report_label.py | 5 +- ...test_consolidation_report_names_sources.py | 3 +- tests/unit/test_daily_brief_project_id.py | 7 +- tests/unit/test_derived_domain_fence.py | 12 +- .../unit/test_derived_label_handoff_kernel.py | 43 +++++ tests/unit/test_derived_labels_docs.py | 2 +- tests/unit/test_entity_fence_mutations.py | 2 +- ...y_report_producer_labels_what_it_prints.py | 8 +- .../unit/test_expired_memories_everywhere.py | 4 +- tests/unit/test_label_door_registry.py | 1 + tests/unit/test_label_reader_stage_matrix.py | 2 +- .../test_label_repair_handoff_atomicity.py | 67 +++++++ tests/unit/test_list_door_inputs.py | 2 +- tests/unit/test_main.py | 8 +- tests/unit/test_mcp.py | 22 +-- .../unit/test_open_loop_handoff_spellings.py | 37 ++++ .../test_open_loop_references_read_fence.py | 2 +- tests/unit/test_producer_handoff_context.py | 71 ++++++++ tests/unit/test_producer_handoff_refill.py | 29 +++ tests/unit/test_search_goldens.py | 17 -- .../unit/test_source_scrub_handoff_passes.py | 70 ++++++++ tests/unit/test_source_search_query_bound.py | 4 +- tests/unit/test_vnext_brain.py | 30 ++-- tests/unit/test_vnext_connections.py | 20 +-- tests/unit/test_vnext_consolidation.py | 72 ++++---- tests/unit/test_vnext_contradictions.py | 12 +- tests/unit/test_vnext_main.py | 6 +- tests/unit/test_vnext_projects.py | 72 ++++---- tests/unit/test_vnext_rollups.py | 4 +- tests/unit/test_vnext_rollups_semantic.py | 4 +- tests/unit/test_workspaces_router_split.py | 8 +- 105 files changed, 1752 insertions(+), 481 deletions(-) create mode 100644 docs/release/derived-labels-security-note-draft.md create mode 100644 tests/integration/test_label_handoff_contention_postgres.py create mode 100644 tests/integration/test_label_handoff_malformed_postgres.py create mode 100644 tests/integration/test_label_handoff_migration_postgres.py create mode 100644 tests/integration/test_label_handoff_writeback_postgres.py create mode 100644 tests/integration/test_label_read_handoff_budget_postgres.py create mode 100644 tests/integration/test_label_repair_handoff_cas_postgres.py create mode 100644 tests/integration/test_producer_handoff_crowding_postgres.py create mode 100644 tests/integration/test_producer_handoff_stale_inputs_postgres.py create mode 100644 tests/integration/test_scheduler_handoff_identity_postgres.py create mode 100644 tests/performance/read_budget_probe.py create mode 100644 tests/performance/test_label_read_handoff_budget.py create mode 100644 tests/unit/test_derived_label_handoff_kernel.py create mode 100644 tests/unit/test_label_repair_handoff_atomicity.py create mode 100644 tests/unit/test_open_loop_handoff_spellings.py create mode 100644 tests/unit/test_producer_handoff_context.py create mode 100644 tests/unit/test_producer_handoff_refill.py create mode 100644 tests/unit/test_source_scrub_handoff_passes.py diff --git a/apps/api/src/alicebot_api/cli/automation.py b/apps/api/src/alicebot_api/cli/automation.py index 08699619a..35cd5001e 100644 --- a/apps/api/src/alicebot_api/cli/automation.py +++ b/apps/api/src/alicebot_api/cli/automation.py @@ -18,7 +18,7 @@ def _connection_finder_request_from_args(args: argparse.Namespace) -> ConnectionFinderRequest: - return ConnectionFinderRequest( + return ConnectionFinderRequest(agent_identity=None, query=getattr(args, "query", "") or "", domains=tuple(args.domain), projects=tuple(getattr(args, "project", ())), @@ -48,7 +48,7 @@ def _run_vnext_graph_neighborhood(ctx: CLIContext, args: argparse.Namespace) -> def _contradiction_finder_request_from_args(args: argparse.Namespace) -> ContradictionFinderRequest: - return ContradictionFinderRequest( + return ContradictionFinderRequest(agent_identity=None, query=getattr(args, "query", "") or "", domains=tuple(args.domain), projects=tuple(getattr(args, "project", ())), @@ -84,7 +84,7 @@ def _run_vnext_belief_state(ctx: CLIContext, args: argparse.Namespace) -> str: def _project_automation_request_from_args(args: argparse.Namespace) -> ProjectAutomationRequest: - return ProjectAutomationRequest( + return ProjectAutomationRequest(agent_identity=None, domains=tuple(args.domain), sensitivity_allowed=_vnext_sensitivity_allowed(args), project_id=getattr(args, "project_id", None), diff --git a/apps/api/src/alicebot_api/cli/capture.py b/apps/api/src/alicebot_api/cli/capture.py index cce5ef0e6..e8cc206f3 100644 --- a/apps/api/src/alicebot_api/cli/capture.py +++ b/apps/api/src/alicebot_api/cli/capture.py @@ -705,7 +705,7 @@ def _run_vnext_demo_load(ctx: CLIContext, args: argparse.Namespace) -> str: created_open_loop_ids.append(str(loop["id"])) daily = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest( + BrainArtifactRequest(agent_identity=None, domains=("project",), sensitivity_allowed=("public", "internal", "private", "unknown"), generated_for="2026-05-12", @@ -730,7 +730,7 @@ def _run_vnext_demo_load(ctx: CLIContext, args: argparse.Namespace) -> str: ) if project_id is not None: project_update = VNextProjectService(store).generate_project_update_candidate( - ProjectAutomationRequest( + ProjectAutomationRequest(agent_identity=None, domains=("project",), sensitivity_allowed=("public", "internal", "private", "unknown"), project_id=project_id, diff --git a/apps/api/src/alicebot_api/cli/context.py b/apps/api/src/alicebot_api/cli/context.py index 17d1c28af..30ff47f00 100644 --- a/apps/api/src/alicebot_api/cli/context.py +++ b/apps/api/src/alicebot_api/cli/context.py @@ -63,7 +63,7 @@ def _run_vnext_context_tree(ctx: CLIContext, args: argparse.Namespace) -> str: def _brain_artifact_request_from_args(args: argparse.Namespace) -> BrainArtifactRequest: - return BrainArtifactRequest( + return BrainArtifactRequest(agent_identity=None, domains=tuple(args.domain), projects=tuple(getattr(args, "project", ())), sensitivity_allowed=_vnext_sensitivity_allowed(args), diff --git a/apps/api/src/alicebot_api/cli/labels.py b/apps/api/src/alicebot_api/cli/labels.py index 93f290062..7fbc81df9 100644 --- a/apps/api/src/alicebot_api/cli/labels.py +++ b/apps/api/src/alicebot_api/cli/labels.py @@ -29,8 +29,8 @@ def _run_vnext_labels_check(ctx: CLIContext, args: object) -> str: print(f"labels check failed: {exc}") raise SystemExit(1) from exc text = format_label_check(below, unverified) - print(text) if below or unverified: + print(text) raise SystemExit(1) return text @@ -132,9 +132,11 @@ def _run_vnext_labels_repair(ctx: CLIContext, args: object) -> str: print(f"labels repair failed: {exc}") raise SystemExit(2) from exc except Exception as exc: - message = str(exc).lower() - if "lock" in message and "timeout" in message: - print("labels repair waited for the label lock and changed nothing") + from alicebot_api.vnext_label_writes import label_error_response + + answer = label_error_response(exc) + if answer is not None and answer[0] == 503: + print(f"{answer[1]}; HTTP 503; Retry-After: 2") raise SystemExit(3) from exc raise return f"labels repair updated {applied}" diff --git a/apps/api/src/alicebot_api/cli/runner.py b/apps/api/src/alicebot_api/cli/runner.py index 850f2c852..2ee596681 100644 --- a/apps/api/src/alicebot_api/cli/runner.py +++ b/apps/api/src/alicebot_api/cli/runner.py @@ -94,6 +94,11 @@ def main(argv: list[str] | None = None) -> int: TemporalStateValidationError, TrustedFactPromotionNotFoundError, ) as exc: + from alicebot_api.vnext_label_writes import label_error_response + answer = label_error_response(exc) + if answer is not None and answer[0] == 503: + _emit_cli_error(code="retryable", message=answer[1] + "; HTTP 503; Retry-After: 2") + return 3 not_found_errors = ( ContinuityLifecycleNotFoundError, ContinuityReviewNotFoundError, diff --git a/apps/api/src/alicebot_api/cli/smokes.py b/apps/api/src/alicebot_api/cli/smokes.py index 62a94eefc..25cd10df3 100644 --- a/apps/api/src/alicebot_api/cli/smokes.py +++ b/apps/api/src/alicebot_api/cli/smokes.py @@ -654,7 +654,7 @@ def _run_vnext_smoke_capture_to_brief(ctx: CLIContext, _args: argparse.Namespace source_fence=_SourceReadFence.unfenced() ) artifact = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest( + BrainArtifactRequest(agent_identity=None, domains=("project",), sensitivity_allowed=("private", "unknown"), generated_for="2026-05-11" ) ) @@ -818,7 +818,7 @@ def _run_vnext_smoke_operator_console(ctx: CLIContext, _args: argparse.Namespace actor_type="user", ) artifact = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest( + BrainArtifactRequest(agent_identity=None, domains=("project",), sensitivity_allowed=("private", "unknown"), generated_for="2026-05-12", diff --git a/apps/api/src/alicebot_api/label_commands.py b/apps/api/src/alicebot_api/label_commands.py index c780585d4..d45340f31 100644 --- a/apps/api/src/alicebot_api/label_commands.py +++ b/apps/api/src/alicebot_api/label_commands.py @@ -67,7 +67,7 @@ def run_labels(args) -> int: print(f" {event_id}") return 1 if below or unverified or raised_on_next_open else 0 try: - with sqlite_user_connection(db, args.user_id) as conn: + with sqlite_user_connection(db, args.user_id, repair_labels=False) as conn: changed = relabel_labels_sqlite(conn, explicit=True) except DerivedDomainRepairError as exc: print(f"labels repair failed: {exc}") diff --git a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py index 994e08c04..936d338e6 100644 --- a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py +++ b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py @@ -813,9 +813,9 @@ def _handle_alice_vnext_artifact_review(context: MCPRuntimeContext, arguments: M trace_id: str | None = None with _vnext_store_context(context) as store: store.lock_graph_mutation() - from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + from alicebot_api.vnext_artifact_review import lock_artifact_review_labels - acquire_exclusive_label_lock(store) + lock_artifact_review_labels(store, artifact_id=artifact_id, action=_parse_required_text(arguments, "action")) _target, actor_type, actor_id, decision = _authorize_vnext_artifact_target( store, identity=identity, diff --git a/apps/api/src/alicebot_api/mcp/memories.py b/apps/api/src/alicebot_api/mcp/memories.py index 5d470cae7..4f2cfa106 100644 --- a/apps/api/src/alicebot_api/mcp/memories.py +++ b/apps/api/src/alicebot_api/mcp/memories.py @@ -412,9 +412,7 @@ def redact_memory_flow( raise VNextMemoryCommitValidationError("reason is required to redact a memory") memory_service = VNextMemoryCommitService(store) memory_service.lock_supersession_graph() - from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock - - acquire_exclusive_label_lock(store) + store.lock_label_writes() memory = store.get_memory_for_redaction(memory_id) if memory is None: raise MemoryNotFoundError("memory was not found") diff --git a/apps/api/src/alicebot_api/mcp/projects.py b/apps/api/src/alicebot_api/mcp/projects.py index 06782c0dd..e567ec10c 100644 --- a/apps/api/src/alicebot_api/mcp/projects.py +++ b/apps/api/src/alicebot_api/mcp/projects.py @@ -45,7 +45,7 @@ def _project_request_from_arguments(arguments: Mapping[str, object]) -> ProjectA "private", "unknown", ) - return ProjectAutomationRequest( + return ProjectAutomationRequest(agent_identity=None, domains=_parse_string_list(arguments, "domains"), sensitivity_allowed=sensitivity_allowed, project_id=_parse_optional_text(arguments, "project_id"), @@ -113,6 +113,11 @@ def _handle_alice_project_update_review(context: MCPRuntimeContext, arguments: M actor_id: str | None = None trace_id: str | None = None with _vnext_store_context(context) as store: + from alicebot_api.vnext_artifact_review import lock_artifact_review_labels + lock_graph = getattr(store, "lock_graph_mutation", None) + if callable(lock_graph): + lock_graph() + lock_artifact_review_labels(store, artifact_id=artifact_id, action=_parse_required_text(arguments, "action")) _target, actor_type, actor_id, decision = _authorize_vnext_artifact_target( store, identity=identity, @@ -172,6 +177,7 @@ def _handle_alice_project_dashboard(context: MCPRuntimeContext, arguments: Mappi VNextProjectService(store).project_dashboard( project_id=_parse_required_text(arguments, "project_id"), sensitivity_allowed=decision.effective_sensitivity_allowed, + identity=_agent_identity_from_arguments(context, arguments), ) ) diff --git a/apps/api/src/alicebot_api/mcp/retrieval.py b/apps/api/src/alicebot_api/mcp/retrieval.py index ee9db7cb8..b29b951f6 100644 --- a/apps/api/src/alicebot_api/mcp/retrieval.py +++ b/apps/api/src/alicebot_api/mcp/retrieval.py @@ -225,6 +225,9 @@ def _handle_alice_recall(context: MCPRuntimeContext, arguments: Mapping[str, obj with _vnext_store_context(context) as store: # Reuse the hybrid retrieval stages (Postgres FTS + pgvector) that back # vNext context packs so recall and context packs rank identically. + lock = getattr(store, "lock_label_writes", None) + if callable(lock): + lock() service = VNextRetrievalService(store) # Recall always searches sources (include_sources only gates whether the # excerpts come back), so a query the source search cannot take is diff --git a/apps/api/src/alicebot_api/mcp/review.py b/apps/api/src/alicebot_api/mcp/review.py index ba82ed3c7..1a99da5ae 100644 --- a/apps/api/src/alicebot_api/mcp/review.py +++ b/apps/api/src/alicebot_api/mcp/review.py @@ -571,9 +571,7 @@ def _vnext_memory_correct(context: MCPRuntimeContext, arguments: Mapping[str, ob # approval activates a memory too, so it must not be a row-first # exception to the lifecycle mutation boundary. memory_service.lock_supersession_graph() - from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock - - acquire_exclusive_label_lock(store) + store.lock_label_writes() get_memory_for_update = getattr(store, "get_memory_for_update", None) memory = get_memory_for_update(memory_id) if callable(get_memory_for_update) else store.get_memory(memory_id) if memory is None: diff --git a/apps/api/src/alicebot_api/mcp/runtime.py b/apps/api/src/alicebot_api/mcp/runtime.py index 2267f5f84..d85204f45 100644 --- a/apps/api/src/alicebot_api/mcp/runtime.py +++ b/apps/api/src/alicebot_api/mcp/runtime.py @@ -60,6 +60,8 @@ def _store_context(context: MCPRuntimeContext): @contextmanager def _vnext_store_context(context: MCPRuntimeContext): + from alicebot_api.vnext_label_guard import label_read_scope + if _is_sqlite_backend(context): sqlite_path = _sqlite_path_from_url(context.database_url) with sqlite_user_connection(sqlite_path, context.user_id) as conn: @@ -72,10 +74,23 @@ def _vnext_store_context(context: MCPRuntimeContext): _SQLITE_DEFAULT_USER_EMAIL, _SQLITE_DEFAULT_USER_DISPLAY_NAME, ) - yield SQLiteVNextStore(conn, context.user_id) + store = SQLiteVNextStore(conn, context.user_id) + with label_read_scope(store): + yield store return - with user_connection(context.database_url, context.user_id) as conn: - yield PostgresVNextStore(conn) + try: + with user_connection(context.database_url, context.user_id) as conn: + store = PostgresVNextStore(conn) + with label_read_scope(store): + yield store + except Exception as exc: + from alicebot_api.vnext_label_writes import label_error_response + from .types import MCPInvalidRequestError + + answer = label_error_response(exc) + if answer is not None and answer[0] == 503: + raise MCPInvalidRequestError(answer[1] + "; HTTP 503; Retry-After: 2") from None + raise def _persist_vnext_deferred_embedding_inputs( diff --git a/apps/api/src/alicebot_api/mcp/synthesis.py b/apps/api/src/alicebot_api/mcp/synthesis.py index 028e7bbe8..890847bb2 100644 --- a/apps/api/src/alicebot_api/mcp/synthesis.py +++ b/apps/api/src/alicebot_api/mcp/synthesis.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Mapping +from dataclasses import replace from alicebot_api.project_view import ProjectView from alicebot_api.store import JsonObject from alicebot_api.vnext_brain import ( @@ -20,6 +21,7 @@ from .shared import ( MCPRuntimeContext, + _agent_identity_from_arguments, _json_object, _mcp_agent_policy_preflight, _parse_bool, @@ -40,7 +42,7 @@ def _brain_artifact_request_from_arguments(arguments: Mapping[str, object]) -> B "private", "unknown", ) - return BrainArtifactRequest( + return BrainArtifactRequest(agent_identity=None, domains=_parse_string_list(arguments, "domains"), projects=_parse_string_list(arguments, "project_scope") or _parse_string_list(arguments, "projects"), sensitivity_allowed=sensitivity_allowed, @@ -56,19 +58,34 @@ def _brain_artifact_request_from_arguments(arguments: Mapping[str, object]) -> B def _handle_alice_generate_daily_brief(context: MCPRuntimeContext, arguments: Mapping[str, object]) -> JsonObject: + request = _authorized_brain_request(context, arguments) with _vnext_store_context(context) as store: return _json_object( - VNextBrainService(store).generate_daily_brief(_brain_artifact_request_from_arguments(arguments)) + VNextBrainService(store).generate_daily_brief(request) ) def _handle_alice_generate_weekly_synthesis(context: MCPRuntimeContext, arguments: Mapping[str, object]) -> JsonObject: + request = _authorized_brain_request(context, arguments) with _vnext_store_context(context) as store: return _json_object( - VNextBrainService(store).generate_weekly_synthesis(_brain_artifact_request_from_arguments(arguments)) + VNextBrainService(store).generate_weekly_synthesis(request) ) +def _authorized_brain_request(context: MCPRuntimeContext, arguments: Mapping[str, object]) -> BrainArtifactRequest: + request = _brain_artifact_request_from_arguments(arguments) + decision = _mcp_agent_policy_preflight( + context, arguments, action="artifact.generate", domains=request.domains, + sensitivity_allowed=request.sensitivity_allowed, project_scope=request.projects, + project_view=ProjectView.unscoped(), + ) + identity = _agent_identity_from_arguments(context, arguments) + return replace(request, agent_identity=identity.to_record() if identity is not None else None, + domains=decision.effective_domains, projects=decision.effective_project_scope, + sensitivity_allowed=decision.effective_sensitivity_allowed) + + def _connection_request_from_arguments(arguments: Mapping[str, object]) -> ConnectionFinderRequest: sensitivity_allowed = _parse_string_list(arguments, "sensitivity_allowed") or ( "public", @@ -77,7 +94,7 @@ def _connection_request_from_arguments(arguments: Mapping[str, object]) -> Conne "unknown", ) auto_accept_threshold = _parse_optional_float(arguments, "auto_accept_threshold") - return ConnectionFinderRequest( + return ConnectionFinderRequest(agent_identity=None, query=_parse_optional_text(arguments, "query") or "", domains=_parse_string_list(arguments, "domains"), projects=_parse_string_list(arguments, "project_scope") or _parse_string_list(arguments, "projects"), @@ -99,7 +116,8 @@ def _handle_alice_generate_connections(context: MCPRuntimeContext, arguments: Ma project_scope=_parse_string_list(arguments, "project_scope") or _parse_string_list(arguments, "projects"), project_view=ProjectView.unscoped(), ) - request = ConnectionFinderRequest( + identity = _agent_identity_from_arguments(context, arguments) + request = ConnectionFinderRequest(agent_identity=identity.to_record() if identity is not None else None, query=request.query, domains=decision.effective_domains, projects=decision.effective_project_scope, @@ -143,7 +161,7 @@ def _contradiction_request_from_arguments(arguments: Mapping[str, object]) -> Co "private", "unknown", ) - return ContradictionFinderRequest( + return ContradictionFinderRequest(agent_identity=None, query=_parse_optional_text(arguments, "query") or "", domains=_parse_string_list(arguments, "domains"), projects=_parse_string_list(arguments, "project_scope") or _parse_string_list(arguments, "projects"), @@ -164,7 +182,8 @@ def _handle_alice_generate_contradictions(context: MCPRuntimeContext, arguments: project_scope=_parse_string_list(arguments, "project_scope") or _parse_string_list(arguments, "projects"), project_view=ProjectView.unscoped(), ) - request = ContradictionFinderRequest( + identity = _agent_identity_from_arguments(context, arguments) + request = ContradictionFinderRequest(agent_identity=identity.to_record() if identity is not None else None, query=request.query, domains=decision.effective_domains, projects=decision.effective_project_scope, diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index 2ace1800f..b09ec1446 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -816,11 +816,13 @@ def review_vnext_source(source_id: UUID, request: VNextSourceReviewRequest) -> J with user_connection(settings.database_url, request.user_id) as conn: store = PostgresVNextStore(conn) label_change = request.domain is not None or request.sensitivity is not None or request.project_id is not None + store.lock_graph_mutation() if label_change: - store.lock_graph_mutation() from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock acquire_exclusive_label_lock(store) + else: + store.lock_label_writes() existing = store.get_source(str(source_id)) if existing is None: return _vnext_public_error_response(status_code=404, detail="vNext source was not found") @@ -1068,17 +1070,18 @@ def review_vnext_memory( # graph boundary before the route takes any candidate/member row lock; # delegated service calls may safely reacquire the transaction lock. memory_service.lock_supersession_graph() - # A status-only review can also raise stale derived labels at the - # owner floor, so acquire the label lock before reading for update. - from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock - - acquire_exclusive_label_lock(store) label_change = ( request.domain is not None or request.sensitivity is not None or request.project_id is not None or action in {"private", "assign_project"} ) + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + if label_change: + acquire_exclusive_label_lock(store) + else: + store.lock_label_writes() preview = store.get_memory(str(memory_id)) if preview is None: return _vnext_public_error_response(status_code=404, detail="vNext memory was not found") diff --git a/apps/api/src/alicebot_api/routers/vnext_review.py b/apps/api/src/alicebot_api/routers/vnext_review.py index 6050c133b..348a9bb4a 100644 --- a/apps/api/src/alicebot_api/routers/vnext_review.py +++ b/apps/api/src/alicebot_api/routers/vnext_review.py @@ -1,4 +1,5 @@ from __future__ import annotations +from alicebot_api.vnext_label_writes import label_http_errors from uuid import UUID @@ -641,6 +642,7 @@ def get_vnext_artifact( ) @review_router.post("/v0/vnext/artifacts/{artifact_id}/review") +@label_http_errors def review_vnext_artifact( artifact_id: UUID, request: VNextArtifactReviewRequest, @@ -663,9 +665,9 @@ def review_vnext_artifact( store, request, user_id=request.user_id, authorization=authorization ) store.lock_graph_mutation() - from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + from alicebot_api.vnext_artifact_review import lock_artifact_review_labels - acquire_exclusive_label_lock(store) + lock_artifact_review_labels(store, artifact_id=str(artifact_id), action=request.action) _artifact, decision = _vnext_authorized_artifact( store=store, identity=identity, @@ -1013,6 +1015,7 @@ def generate_vnext_project_update_candidate( return JSONResponse(status_code=201, content=jsonable_encoder(payload)) @review_router.post("/v0/vnext/projects/update-candidates/{artifact_id}/review") +@label_http_errors def review_vnext_project_update_candidate( artifact_id: str, request: VNextProjectUpdateReviewRequest, @@ -1035,9 +1038,9 @@ def review_vnext_project_update_candidate( store, request, user_id=request.user_id, authorization=authorization ) store.lock_graph_mutation() - from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + from alicebot_api.vnext_artifact_review import lock_artifact_review_labels - acquire_exclusive_label_lock(store) + lock_artifact_review_labels(store, artifact_id=str(artifact_id), action=request.action) _artifact, decision = _vnext_authorized_artifact( store=store, identity=identity, diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index aff8f5f22..499da32ac 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -11,6 +11,7 @@ from alicebot_api.config import get_settings from alicebot_api.db import user_connection +from alicebot_api.vnext_label_guard import label_read_request from alicebot_api.local_workspace import ( ensure_local_workspace, get_local_workspace, @@ -85,6 +86,7 @@ def _workspace_rows(store: PostgresVNextStore, kind: str, rows: Sequence[Mapping ) +@label_read_request def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdentity | None = None) -> dict[str, object]: from alicebot_api.vnext_label_guard import LabelGuard, sensitivity_ceiling from alicebot_api.vnext_open_loop_references import withhold_unreadable_references diff --git a/apps/api/src/alicebot_api/sqlite_schema.py b/apps/api/src/alicebot_api/sqlite_schema.py index 3486da4ab..3d8c688a2 100644 --- a/apps/api/src/alicebot_api/sqlite_schema.py +++ b/apps/api/src/alicebot_api/sqlite_schema.py @@ -2027,7 +2027,7 @@ def _relabel_derived_labels(conn: sqlite3.Connection) -> None: except Exception as exc: conn.execute("ROLLBACK TO SAVEPOINT alice_derived_labels_v3") conn.execute("RELEASE SAVEPOINT alice_derived_labels_v3") - left = getattr(exc, "left", 0) + left = getattr(exc, "left", "an unknown number of") logging.getLogger(__name__).warning( "alice-memory: label repair did not run, %s rows left to the read check; run alice-memory labels check", left, @@ -2036,7 +2036,7 @@ def _relabel_derived_labels(conn: sqlite3.Connection) -> None: conn.execute("RELEASE SAVEPOINT alice_derived_labels_v3") -def bootstrap_sqlite_schema(conn: sqlite3.Connection) -> None: +def bootstrap_sqlite_schema(conn: sqlite3.Connection, *, repair_labels: bool = True) -> None: """Create or upgrade the vNext SQLite schema. Safe to call repeatedly.""" conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA foreign_keys=ON") @@ -2059,7 +2059,8 @@ def bootstrap_sqlite_schema(conn: sqlite3.Connection) -> None: # identifier on a tombstone (audit P1 #3); a no-op on healthy files. _repair_tombstone_lookup_value_holders(conn) _relabel_derived_domains(conn) - _relabel_derived_labels(conn) + if repair_labels: + _relabel_derived_labels(conn) # The redaction flag row must exist before the append-only triggers # reference it, and it must be OFF: a crashed process must never leave # a database file with redaction mode stuck open. diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index b43d456aa..4dde6f6b8 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -363,7 +363,7 @@ def ensure_sqlite_user( @contextmanager -def sqlite_user_connection(path: str | Path, user_id: UUID | str) -> Iterator[sqlite3.Connection]: +def sqlite_user_connection(path: str | Path, user_id: UUID | str, *, repair_labels: bool = True) -> Iterator[sqlite3.Connection]: """Open a bootstrapped SQLite connection wrapped in one transaction. Mirrors ``alicebot_api.db.user_connection`` semantics: dict rows, the @@ -376,7 +376,10 @@ def sqlite_user_connection(path: str | Path, user_id: UUID | str) -> Iterator[sq conn = sqlite3.connect(str(path)) conn.row_factory = _dict_row_factory try: - bootstrap_sqlite_schema(conn) + if repair_labels: + bootstrap_sqlite_schema(conn) + else: + bootstrap_sqlite_schema(conn, repair_labels=False) conn.commit() yield conn conn.commit() diff --git a/apps/api/src/alicebot_api/vnext_artifact_review.py b/apps/api/src/alicebot_api/vnext_artifact_review.py index b42178dbd..8d45baf76 100644 --- a/apps/api/src/alicebot_api/vnext_artifact_review.py +++ b/apps/api/src/alicebot_api/vnext_artifact_review.py @@ -40,9 +40,7 @@ def dispatch_vnext_artifact_review( lock_graph = getattr(store, "lock_graph_mutation", None) if callable(lock_graph): lock_graph() - from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock - - acquire_exclusive_label_lock(store) + lock_artifact_review_labels(store, artifact_id=artifact_id, action=action) target = store.get_artifact_for_update(artifact_id) if target is None: raise VNextQueueNotFoundError(f"artifact {artifact_id} was not found") @@ -75,6 +73,24 @@ def dispatch_vnext_artifact_review( ) +def lock_artifact_review_labels(store, *, artifact_id: str, action: str) -> None: + """Project acceptance changes labels; ordinary artifact review does not.""" + from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock + + if not callable(getattr(store, "lock_label_writes", None)): + return + getter = getattr(store, "get_artifact", None) + target = getter(artifact_id) if callable(getter) else None + # Older adapters expose only the locking read. Until that read establishes + # the type, acceptance may change a project label and needs exclusivity. + if (target is None or is_project_update_artifact(target)) and action in {"accept", "edit", "promote"}: + acquire_exclusive_label_lock(store) + else: + lock = getattr(store, "lock_label_writes", None) + if callable(lock): + lock() + + __all__ = [ "VNextArtifactReviewDispatchResult", "VNextArtifactReviewDispatchStore", diff --git a/apps/api/src/alicebot_api/vnext_brain.py b/apps/api/src/alicebot_api/vnext_brain.py index 3eff95522..2d83150a2 100644 --- a/apps/api/src/alicebot_api/vnext_brain.py +++ b/apps/api/src/alicebot_api/vnext_brain.py @@ -145,7 +145,7 @@ class BrainArtifactRequest: actor_id: str | None = None trace_id: str | None = None run_id: str | None = None - agent_identity: JsonObject | None = None + agent_identity: JsonObject | None = field(kw_only=True) policy_decision: JsonObject | None = None metadata_json: JsonObject = field(default_factory=dict) generation_mode: str = "deterministic" @@ -241,16 +241,15 @@ def _matches_report_scope( window_end: datetime, all_of: tuple[str, ...] | None = None, ) -> bool: - if projects: - if all_of is not None: - from alicebot_api.vnext_derived_labels import input_admitted - - if not input_admitted(kind, row, all_of): - return False - else: - row_scope = source_project_scope(row) if kind == "source" else resource_project_scope(row) - if not project_scopes_overlap(row_scope, projects): - return False + if all_of is not None: + from alicebot_api.vnext_derived_labels import input_admitted + + if not input_admitted(kind, row, all_of): + return False + elif projects: + row_scope = source_project_scope(row) if kind == "source" else resource_project_scope(row) + if not project_scopes_overlap(row_scope, projects): + return False event_time = _row_event_time(row, kind=kind) return event_time is not None and window_start <= event_time < window_end @@ -466,7 +465,7 @@ def __init__(self, store: VNextBrainStore) -> None: self.store = store def generate_daily_brief(self, request: BrainArtifactRequest | None = None) -> JsonObject: - request = request or BrainArtifactRequest() + request = request or BrainArtifactRequest(agent_identity=None, ) _validate_request(request) day = _parse_generated_for(request.generated_for) window_start, window_end = _report_window(day, days=1) @@ -657,7 +656,7 @@ def generate_daily_brief(self, request: BrainArtifactRequest | None = None) -> J return artifact def generate_weekly_synthesis(self, request: BrainArtifactRequest | None = None) -> JsonObject: - request = request or BrainArtifactRequest() + request = request or BrainArtifactRequest(agent_identity=None, ) _validate_request(request) day = _parse_generated_for(request.generated_for) week_label = _iso_week_label(day) @@ -1046,14 +1045,14 @@ def _create_candidate_open_loops( "description": f"Candidate open loop discovered in {_title(source, 'source')}.", "status": "open", "priority": "normal", - "source_id": source.get("id"), + "source_id": str(source["id"]), "project_id": _stored_open_loop_project_id(project_scope), "domain": source.get("domain", "unknown"), "sensitivity": source.get("sensitivity", "unknown"), "metadata_json": { "candidate": True, "discovered_by": "vnext_daily_brief", - "source_id": source.get("id"), + "source_id": str(source["id"]), "project_scope": list(project_scope), "automation_digest": automation_digest, "workflow_digest": workflow_digest, diff --git a/apps/api/src/alicebot_api/vnext_connections.py b/apps/api/src/alicebot_api/vnext_connections.py index fe51a6ffa..ee8717ec0 100644 --- a/apps/api/src/alicebot_api/vnext_connections.py +++ b/apps/api/src/alicebot_api/vnext_connections.py @@ -137,7 +137,7 @@ class ConnectionFinderRequest: actor_id: str | None = None trace_id: str | None = None run_id: str | None = None - agent_identity: JsonObject | None = None + agent_identity: JsonObject | None = field(kw_only=True) policy_decision: JsonObject | None = None metadata_json: JsonObject = field(default_factory=dict) generation_mode: str = "deterministic" @@ -199,12 +199,12 @@ def _supports_parameter(method: object, name: str) -> bool: def _matches_projects( row: JsonObject, projects: tuple[str, ...], *, source_row: bool, all_of: tuple[str, ...] | None = None ) -> bool: - if not projects: - return True if all_of is not None: from alicebot_api.vnext_derived_labels import input_admitted return input_admitted("source" if source_row else "memory", row, all_of) + if not projects: + return True row_scope = source_project_scope(row) if source_row else resource_project_scope(row) return project_scopes_overlap(row_scope, projects) @@ -219,11 +219,18 @@ def _project_scoped_search( source_rows: bool = False, all_of: tuple[str, ...] | None = None, ) -> list[JsonObject]: - if not projects: + if not projects and all_of is None: return list(method(limit=limit, **kwargs)) if _supports_parameter(method, project_parameter): - rows = method(limit=limit, **kwargs, **{project_parameter: projects}) - return [row for row in rows if _matches_projects(row, projects, source_row=source_rows, all_of=all_of)] + fetch_limit = limit + while True: + rows = method(limit=fetch_limit, **kwargs, **{project_parameter: projects}) + selected = [row for row in rows if _matches_projects(row, projects, source_row=source_rows, all_of=all_of)] + if all_of is None or len(selected) >= limit or len(rows) < fetch_limit: + return selected[:limit] + if fetch_limit >= MAX_LEGACY_PROJECT_SCOPE_ROWS: + raise VNextConnectionValidationError("locked input selection could not prove complete project scope") + fetch_limit = min(fetch_limit * 2, MAX_LEGACY_PROJECT_SCOPE_ROWS) rows = list(method(limit=MAX_LEGACY_PROJECT_SCOPE_ROWS + 1, **kwargs)) if len(rows) > MAX_LEGACY_PROJECT_SCOPE_ROWS: raise VNextConnectionValidationError("legacy connection store could not prove complete project scope") @@ -411,7 +418,7 @@ def __init__(self, store: VNextConnectionStore) -> None: self.store = store def generate_connection_report(self, request: ConnectionFinderRequest | None = None) -> JsonObject: - request = request or ConnectionFinderRequest() + request = request or ConnectionFinderRequest(agent_identity=None, ) _validate_request(request) domains = list(request.domains) if request.domains else None sensitivity_allowed = list(request.sensitivity_allowed) diff --git a/apps/api/src/alicebot_api/vnext_consolidation.py b/apps/api/src/alicebot_api/vnext_consolidation.py index b9a9d5f5d..803f7aa37 100644 --- a/apps/api/src/alicebot_api/vnext_consolidation.py +++ b/apps/api/src/alicebot_api/vnext_consolidation.py @@ -178,7 +178,7 @@ class MemoryConsolidationRequest: generated_by: str = "system" trace_id: str | None = None run_id: str | None = None - agent_identity: JsonObject | None = None + agent_identity: JsonObject | None = field(kw_only=True) policy_decision: JsonObject | None = None metadata_json: JsonObject = field(default_factory=dict) generation_mode: str = "deterministic" @@ -1112,7 +1112,7 @@ def _reinforced_preferences(self, clusters: list[list[JsonObject]]) -> list[Json # -- report --------------------------------------------------------------- def generate_memory_consolidation(self, request: MemoryConsolidationRequest | None = None) -> JsonObject: - request = request or MemoryConsolidationRequest() + request = request or MemoryConsolidationRequest(agent_identity=None, ) _validate_request(request) options = _clustering_options(request) # Parsed before any write so invalid roll-up options fail the run @@ -1215,6 +1215,27 @@ def generate_memory_consolidation(self, request: MemoryConsolidationRequest | No ] brain_charter = self._brain_charter() clusters_for_proposals = clustering.clusters[: options.max_clusters] + if all_of is not None: + from alicebot_api.vnext_source_fence import cited_source_ids + refs = [ref for members in clusters_for_proposals for ref in _member_source_refs(members)] + sources = sources_named_by_refs(self.store, refs) + admitted = {str(row["id"]) for row in admit_loaded( + self.store, kind="source", rows=sources, domains=domains, + sensitivity_allowed=sensitivity, projects=projects, all_of=all_of, + )} + refused = {str(row["id"]) for row in sources} - admitted + def readable_reference(ref): + parsed = cited_source_ids([ref]) + return not (parsed.named - admitted or parsed.incidental & refused) + # Only copies used to render this run change. Stored members and + # their provenance remain available to their authorized readers. + clusters_for_proposals = [[{ + **member, "metadata_json": { + **(member.get("metadata_json") or {}), + "source_refs": [ref for ref in (member.get("metadata_json") or {}).get("source_refs", []) + if readable_reference(ref)], + }, + } for member in members] for members in clusters_for_proposals] # The report copies the ``source_refs`` of each proposed cluster member as stored, and the candidate memories # copy them too. The refs are not dropped: they are the provenance. But the report is read behind its label # alone, so the label has to be at least as strict as every source they name. The list printed is made here, diff --git a/apps/api/src/alicebot_api/vnext_contradictions.py b/apps/api/src/alicebot_api/vnext_contradictions.py index 30543af4b..973838326 100644 --- a/apps/api/src/alicebot_api/vnext_contradictions.py +++ b/apps/api/src/alicebot_api/vnext_contradictions.py @@ -153,7 +153,7 @@ class ContradictionFinderRequest: actor_id: str | None = None trace_id: str | None = None run_id: str | None = None - agent_identity: JsonObject | None = None + agent_identity: JsonObject | None = field(kw_only=True) policy_decision: JsonObject | None = None metadata_json: JsonObject = field(default_factory=dict) generation_mode: str = "deterministic" @@ -218,12 +218,12 @@ def _supports_parameter(method: object, name: str) -> bool: def _matches_projects( row: JsonObject, projects: tuple[str, ...], *, source_row: bool, all_of: tuple[str, ...] | None = None ) -> bool: - if not projects: - return True if all_of is not None: from alicebot_api.vnext_derived_labels import input_admitted return input_admitted("source" if source_row else "memory", row, all_of) + if not projects: + return True row_scope = source_project_scope(row) if source_row else resource_project_scope(row) return project_scopes_overlap(row_scope, projects) @@ -238,11 +238,18 @@ def _project_scoped_search( source_rows: bool = False, all_of: tuple[str, ...] | None = None, ) -> list[JsonObject]: - if not projects: + if not projects and all_of is None: return list(method(limit=limit, **kwargs)) if _supports_parameter(method, project_parameter): - rows = method(limit=limit, **kwargs, **{project_parameter: projects}) - return [row for row in rows if _matches_projects(row, projects, source_row=source_rows, all_of=all_of)] + fetch_limit = limit + while True: + rows = method(limit=fetch_limit, **kwargs, **{project_parameter: projects}) + selected = [row for row in rows if _matches_projects(row, projects, source_row=source_rows, all_of=all_of)] + if all_of is None or len(selected) >= limit or len(rows) < fetch_limit: + return selected[:limit] + if fetch_limit >= MAX_LEGACY_PROJECT_SCOPE_ROWS: + raise VNextContradictionValidationError("locked input selection could not prove complete project scope") + fetch_limit = min(fetch_limit * 2, MAX_LEGACY_PROJECT_SCOPE_ROWS) rows = list(method(limit=MAX_LEGACY_PROJECT_SCOPE_ROWS + 1, **kwargs)) if len(rows) > MAX_LEGACY_PROJECT_SCOPE_ROWS: raise VNextContradictionValidationError("legacy contradiction store could not prove complete project scope") @@ -437,7 +444,7 @@ def __init__(self, store: VNextContradictionStore) -> None: self.store = store def generate_contradiction_report(self, request: ContradictionFinderRequest | None = None) -> JsonObject: - request = request or ContradictionFinderRequest() + request = request or ContradictionFinderRequest(agent_identity=None, ) _validate_request(request) domains = list(request.domains) if request.domains else None sensitivity_allowed = list(request.sensitivity_allowed) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index e9dae28b8..c661ed3f2 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -211,11 +211,31 @@ def _object(value: object) -> Mapping[str, object]: def _metadata(row: Mapping[str, object]) -> Mapping[str, object]: - return _object(row.get("metadata_json")) + return _object(_uuid_strings(_object(row.get("metadata_json")))) + + +def _uuid_strings(value: object) -> object: + """Database UUID objects and JSON strings name the same recorded input.""" + if isinstance(value, UUID): + return str(value) + if isinstance(value, Mapping): + return {key: _uuid_strings(child) for key, child in value.items()} + if isinstance(value, list): + return [_uuid_strings(child) for child in value] + return value def _nonempty_str(value: object) -> bool: - return isinstance(value, str) and bool(value.strip()) + return isinstance(value, UUID) or isinstance(value, str) and bool(value.strip()) + + +def _marker_in(value: object, choices: Iterable[str]) -> bool: + return isinstance(value, str) and value in choices + + +def _malformed_marker(kind: str, meta: Mapping[str, object]) -> bool: + key = "workflow" if kind == "artifact" else "candidate_kind" if kind == "memory" else None + return key is not None and key in meta and not isinstance(meta[key], str) def _scrubbed(row: Mapping[str, object]) -> bool: @@ -240,15 +260,17 @@ def is_derived(kind: object, row: Mapping[str, object]) -> bool: does not count, so a forged ``value.kind`` does not make the row derived. """ - if _metadata(row).get("redacted") is True: + meta = _metadata(row) + if meta.get("redacted") is True: return False name = canon_kind(kind) - meta = _metadata(row) if name in {"source", "belief"}: return False if name == "project": return "derived_from" in meta if name == "open_loop": + if "derived_from" in meta: + return True discovered = meta.get("discovered_by") if not _nonempty_str(discovered): return False @@ -256,14 +278,14 @@ def is_derived(kind: object, row: Mapping[str, object]) -> bool: if name == "artifact": if "derived_from" in meta: return True - if meta.get("workflow") in DERIVED_WORKFLOWS: + if _malformed_marker(name, meta) or _marker_in(meta.get("workflow"), DERIVED_WORKFLOWS): return True if str(row.get("artifact_type") or "") in DERIVED_ARTIFACT_TYPES: return True return meta.get("connector_name") == "agent_output" if isinstance(meta.get("consolidation"), Mapping): return True - if meta.get("candidate_kind") in {"memory_consolidation", "memory_rollup"}: + if _malformed_marker(name, meta) or _marker_in(meta.get("candidate_kind"), {"memory_consolidation", "memory_rollup"}): return True if meta.get("discovered_by") == "vnext_weekly_synthesis": return True @@ -294,7 +316,7 @@ def row_class(kind: object, row: Mapping[str, object]) -> str: or meta.get("discovered_by") == "vnext_weekly_synthesis" or meta.get("workflow") == "project_auto_update" or isinstance(meta.get("consolidation"), Mapping) - or meta.get("candidate_kind") in {"memory_consolidation", "memory_rollup"} + or _marker_in(meta.get("candidate_kind"), {"memory_consolidation", "memory_rollup"}) ): return "aggregate" return "copy" @@ -415,8 +437,7 @@ def _source_ids_from(value: object) -> tuple[str, set[str]]: elif value.strip() and not named: # A sentence is not a source id. cited_source_ids already kept the # explicit ones. A whole token that is not a uuid still counts. - if _plain_token(value): - named.add(identifier(value)) + pass elif isinstance(value, Mapping): for key, child in value.items(): key_text = key.lower() if isinstance(key, str) else "" @@ -448,7 +469,10 @@ def _source_token(value: str) -> str | None: return None if ":" in text or "/" in text: return None - return identifier(text) + try: + return str(UUID(text)) + except (ValueError, AttributeError, TypeError): + return None def _typed_refs(value: object) -> set[tuple[str, str]]: @@ -490,7 +514,16 @@ def _collect_metadata_ids(value: object, found: set[tuple[str, str]]) -> str: continue if key == "derived_from": continue - if key == "source_refs" or key in {"source_id", "source_ids"}: + if key in {"source_id", "source_ids"}: + if isinstance(child, (str, list)): + child_problem, source_ids = _source_ids_from(child) + problem = problem or child_problem + source_ids.update(item for item in _strings(child) if _plain_token(item)) + _add_ids(found, "source", source_ids) + else: + problem = problem or "malformed" + continue + if key == "source_refs": child_problem, source_ids = _source_ids_from(child) problem = problem or child_problem _add_ids(found, "source", source_ids) @@ -671,7 +704,8 @@ def dependency_record(kind: object, row: Mapping[str, object]) -> tuple[frozense return frozenset(), "" meta = _metadata(row) found: set[tuple[str, str]] = set() - problem = _collect_metadata_ids(meta, found) + problem = "malformed_marker" if _malformed_marker(canon_kind(kind), meta) else "" + problem = problem or _collect_metadata_ids(meta, found) if "derived_from" in meta: derived_problem, derived_deps = _derived_from_deps(meta.get("derived_from")) problem = problem or derived_problem @@ -695,7 +729,7 @@ def dependency_record(kind: object, row: Mapping[str, object]) -> tuple[frozense def _value_dependencies(row: Mapping[str, object]) -> set[tuple[str, str]]: - value = _object(row.get("value")) + value = _object(_uuid_strings(_object(row.get("value")))) found: set[tuple[str, str]] = set() if _nonempty_str(value.get("source_id")): found.add(("source", identifier(value.get("source_id")))) diff --git a/apps/api/src/alicebot_api/vnext_dogfooding.py b/apps/api/src/alicebot_api/vnext_dogfooding.py index cf7e5f118..46cd085e3 100644 --- a/apps/api/src/alicebot_api/vnext_dogfooding.py +++ b/apps/api/src/alicebot_api/vnext_dogfooding.py @@ -9,6 +9,7 @@ from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_repositories import JsonObject from alicebot_api.vnext_store import is_redacted_project_update_artifact +from alicebot_api.vnext_label_guard import label_read_request if TYPE_CHECKING: from alicebot_api.vnext_label_guard import LabelGuard @@ -172,6 +173,7 @@ class VNextDogfoodingService: def __init__(self, store: VNextDogfoodingStore) -> None: self.store = store + @label_read_request def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None, label_guard: LabelGuard | None = None) -> JsonObject: from alicebot_api.vnext_agent_control import ALL_SENSITIVITY from alicebot_api.vnext_label_guard import LabelGuard diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index e229bec93..b12ebcc9c 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -8,8 +8,11 @@ from __future__ import annotations from collections.abc import Mapping, Sequence -from dataclasses import dataclass, replace +from dataclasses import dataclass, replace, field from typing import Any, TypeVar +from contextvars import ContextVar +from functools import wraps +from contextlib import contextmanager from alicebot_api.vnext_agent_control import ( ALL_SENSITIVITY, @@ -21,6 +24,8 @@ HOP_BOUND, NODE_BOUND, canon_kind, + dependencies_of, + identifier, is_derived, input_admitted, settle_labels, @@ -33,6 +38,80 @@ _Row = TypeVar("_Row", bound=Mapping[str, object]) +def _row_label_key(kind: str, row: Mapping[str, object]) -> tuple: + return (kind, *(repr(row.get(field)) for field in ( + "id", "user_id", "domain", "sensitivity", "metadata_json", "value", "project_id", "source_id", "artifact_type", "project_scope", "project_floor", + ))) + + +@dataclass +class _RequestLabels: + nodes: dict = field(default_factory=dict) + labels: dict = field(default_factory=dict) + targets: dict = field(default_factory=dict) + counts: dict = field(default_factory=dict) + source_copies: dict = field(default_factory=dict) + row_sets: dict = field(default_factory=dict) + dependencies: dict = field(default_factory=dict) + source_admission: dict = field(default_factory=dict) + + def clear(self): + self.nodes.clear() + self.labels.clear() + self.targets.clear() + self.counts.clear() + self.source_copies.clear() + self.row_sets.clear() + self.dependencies.clear() + self.source_admission.clear() + + +_REQUEST_LABELS: ContextVar[tuple[Any, _RequestLabels] | None] = ContextVar("request_labels", default=None) + + +def invalidate_read_labels(store: Any) -> None: + current = _REQUEST_LABELS.get() + if current is not None and current[0] is store: + current[1].clear() + + +def request_row_cache(store: Any, namespace: str) -> dict | None: + """Raw rows may be reused only within the active request, before admission.""" + current = _REQUEST_LABELS.get() + if current is None or current[0] is not store: + return None + return current[1].row_sets.setdefault(namespace, {}) + + +@contextmanager +def label_read_scope(store): + current = _REQUEST_LABELS.get() + if current is not None and current[0] is store: + yield + return + token = _REQUEST_LABELS.set((store, _RequestLabels())) + try: + yield + finally: + _REQUEST_LABELS.reset(token) + + +def label_read_request(fn): + """Share ancestry only for one synchronous read, including nested services.""" + @wraps(fn) + def wrapped(first, *args, **kwargs): + store = getattr(first, "store", first) + # Keep current label writers outside the request's cached snapshot. + # These services read label tables and write only traces or telemetry. + if getattr(store, "conn", None) is not None: + lock = getattr(store, "lock_label_writes", None) + if callable(lock): + lock() + with label_read_scope(store): + return fn(first, *args, **kwargs) + return wrapped + + def _filters_admit_every( domains: Sequence[str] | None, sensitivity_allowed: Sequence[str] | None, @@ -56,6 +135,16 @@ class LabelGuard: projects: tuple[str, ...] = () all_of: tuple[str, ...] | None = None _nodes: dict[tuple[str, str], list[dict[str, object]]] | None = None + _request: _RequestLabels | None = None + + def _state(self) -> _RequestLabels: + current = _REQUEST_LABELS.get() + state = current[1] if current is not None and current[0] is self.store else self._request + if state is None: + state = _RequestLabels() + self._request = state + self._nodes = state.nodes + return state @classmethod def for_fence(cls, store: Any, fence: Any) -> LabelGuard: @@ -97,9 +186,29 @@ def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[ return row if not is_derived(kind, row): return row - nodes = self._collected(kind, row) - settled = settle_labels(nodes, on_cycle="unverified", max_hops=HOP_BOUND, max_nodes=NODE_BOUND) - label = settled.by_stored(kind, str(row.get("id") or ""), user_id=_GUARD_USER) + state = self._state() + # Distinct projections and stored aliases are settled separately. + # Only labels, never caller admission, are shared in this request. + key = _row_label_key(kind, row) + label = state.labels.get(key) + template = (key[0], *key[2:]) + if label is None: + label = state.source_copies.get(template) + if label is None: + nodes = self._collected(kind, row) + settled = settle_labels(nodes, on_cycle="unverified", max_hops=HOP_BOUND, max_nodes=NODE_BOUND) + label = settled.by_stored(kind, str(row.get("id") or ""), user_id=_GUARD_USER) + # Copies of the same original sources have identical effective + # labels. No derived parent, alias, missing parent or root cycle + # is allowed in this shortcut; those retain an independent walk. + refs = dependencies_of(kind, row) + if refs and all( + ref_kind == "source" and len(state.nodes.get((ref_kind, ref_id), [])) == 1 + and not is_derived(ref_kind, state.nodes[(ref_kind, ref_id)][0]) + for ref_kind, ref_id in refs + ): + state.source_copies[template] = label + state.labels[key] = label copy = dict(row) raw_metadata = copy.get("metadata_json") metadata = dict(raw_metadata) if isinstance(raw_metadata, Mapping) else {} @@ -129,12 +238,46 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: if not self.active: return [row for row in rows if isinstance(row, Mapping)] + state = self._state() + reader = getattr(self.store, "read_label_rows", None) + wanted: dict[str, set[str]] = {} + for row in rows: + if isinstance(row, Mapping): + state.targets[(kind, str(row.get("id")))] = row + dependency_key = (kind, *(repr(row.get(field)) for field in ( + "metadata_json", "value", "source_id", "source_artifact_id", "artifact_id", "memory_id", "artifact_type", + ))) + refs = state.dependencies.get(dependency_key) + if refs is None: + refs = dependencies_of(kind, row) + state.dependencies[dependency_key] = refs + for ref_kind, ref_id in refs: + if (ref_kind, ref_id) not in state.nodes: + wanted.setdefault(ref_kind, set()).add(ref_id) + if callable(reader): + for ref_kind, ids in wanted.items(): + for ref_id in ids: + state.nodes[(ref_kind, ref_id)] = [] + for found in reader(ref_kind, sorted(ids)): + canonical = identifier(found.get("id")) + if canonical in ids: + state.nodes[(ref_kind, canonical)].append(dict(found)) kept: list[_Row] = [] for row in rows: if not isinstance(row, Mapping): continue + key = _row_label_key(kind, row) + template = (key[0], *key[2:]) + admission_key = (template, self.domains, self.sensitivity_allowed, self.projects, self.all_of) + if template in state.source_copies and admission_key in state.source_admission: + if state.source_admission[admission_key]: + kept.append(row) + continue effective = self.effective_row(kind, row) - if isinstance(effective, Mapping) and self._admits_effective(effective, kind=kind): + admitted = isinstance(effective, Mapping) and self._admits_effective(effective, kind=kind) + if template in state.source_copies: + state.source_admission[admission_key] = admitted + if admitted: kept.append(row) return kept @@ -149,11 +292,16 @@ def readable_status_counts(self, kind: str) -> dict[str, int]: iterator = getattr(self.store, "iter_label_rows", None) if not callable(iterator): raise TypeError("readable counts require complete label enumeration") + state = self._state() + key = (kind, self.domains, self.sensitivity_allowed, self.projects, self.all_of) + if key in state.counts: + return dict(state.counts[key]) counts: dict[str, int] = {} for batch in iterator(kind): for row in self.admit_rows(kind, batch): status = str(row.get("status", "unknown")) counts[status] = counts.get(status, 0) + 1 + state.counts[key] = dict(counts) return counts def admit_related_rows(self, rows: Sequence[_Row], *, kind: str, field: str) -> list[_Row]: @@ -165,7 +313,11 @@ def admit_related_rows(self, rows: Sequence[_Row], *, kind: str, field: str) -> if not callable(reader): return [] ids = list(dict.fromkeys(str(row.get(field)) for row in rows if row.get(field))) - found = list(reader(kind, ids)) if ids else [] + state = self._state() + missing = [row_id for row_id in ids if (kind, row_id) not in state.targets] + for row in reader(kind, missing) if missing else []: + state.targets[(kind, str(row.get("id")))] = row + found = [state.targets[(kind, row_id)] for row_id in ids if (kind, row_id) in state.targets] admitted = {str(row.get("id")) for row in ( self.admit_beliefs(found) if kind == "belief" else self.admit_rows(kind, found) )} @@ -238,8 +390,7 @@ def _admits_effective(self, row: Mapping[str, object], *, kind: str) -> bool: return True def _collected(self, kind: str, row: Mapping[str, object]) -> list[dict[str, object]]: - if self._nodes is None: - self._nodes = {} + self._state() nodes, _exceeded = collect_label_rows( self.store, [{**dict(row), "kind": canon_kind(kind)}], max_nodes=NODE_BOUND, max_hops=HOP_BOUND, cache=self._nodes, user_id=_GUARD_USER, diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 68f905528..9558c7d19 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -55,7 +55,7 @@ class LabelLockOrderError(RuntimeError): RETRYABLE_DETAIL = ( - "the label change was not applied because another change was running; nothing was changed; try again" + "the action was not applied because another change was running; nothing was changed; try again" ) REFUSED_DETAIL = "the label change could not be applied to every dependent row; nothing was changed" @@ -77,6 +77,9 @@ def key(self) -> tuple[str, int]: def invalidate_capture_label_inputs(store: Any) -> None: + from alicebot_api.vnext_label_guard import invalidate_read_labels + + invalidate_read_labels(store) batch = _CAPTURE_LABEL_INPUTS.get() if batch is not None and batch.conn is getattr(store, "conn", None): batch.rows.clear() @@ -135,6 +138,9 @@ def takes_label_lock(fn: Any) -> Any: @wraps(fn) def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: + from alicebot_api.vnext_label_guard import invalidate_read_labels + + invalidate_read_labels(self) lock = getattr(self, "lock_label_writes", None) if callable(lock): lock(exclusive=False) @@ -292,6 +298,17 @@ def apply_insert_floor(store: Any, kind: str, payload: Mapping[str, object]) -> own["kind"] = kind own["id"] = own_id own["user_id"] = user_id + # A copy selected under an earlier source scope still contains that + # source's earlier text. Preserve its selected scope before rereading the + # current parent, which may already have moved to another project. + from alicebot_api.vnext_derived_labels import row_class + if row_class(kind, own) == "copy": + raw_meta = own.get("metadata_json") + selected_meta = dict(raw_meta) if isinstance(raw_meta, Mapping) else {} + selected_meta["project_floor"] = list(union_floor( + project_floor_shape(own)[1], [stored_scope(kind, own)], + )) + own["metadata_json"] = selected_meta batch = _current_capture_inputs(store) cache = batch.rows if batch is not None else None nodes, exceeded = collect_label_rows(store, [own], max_nodes=PROPAGATION_BOUND, cache=cache) @@ -600,6 +617,17 @@ def clamp_owner_patch( proposed_patch = dict(patch) if before is None or not is_derived(kind, before): return proposed_patch + # Status, content and audit metadata writes do not repair stored labels. + # Their readers still enforce the effective floor. An explicit label edit + # owns that repair and takes exclusive L before any row lock. + patch_meta = proposed_patch.get("metadata_json") + old_meta = before.get("metadata_json") + meta_changes_labels = isinstance(patch_meta, Mapping) and any( + key in patch_meta and patch_meta[key] != (old_meta.get(key) if isinstance(old_meta, Mapping) else None) + for key in ("project_scope", "project_floor") + ) + if not meta_changes_labels and not any(proposed_patch.get(key) is not None for key in ("domain", "sensitivity", "project_id")): + return proposed_patch proposed = dict(before) for key in ("domain", "sensitivity", "project_id"): if key in proposed_patch and proposed_patch[key] is not None: @@ -922,6 +950,24 @@ def label_error_response(exc: BaseException) -> tuple[int, str, str | None] | No return None +def label_http_errors(fn): + """Give review routes the same atomic refusal as the relabel route.""" + @wraps(fn) + def wrapped(*args, **kwargs): + try: + return fn(*args, **kwargs) + except Exception as exc: + answer = label_error_response(exc) + if answer is None: + raise + from fastapi.responses import JSONResponse + + status, detail, retry_after = answer + return JSONResponse(status_code=status, content={"detail": detail}, + headers={"Retry-After": retry_after} if retry_after else None) + return wrapped + + def remember_floor_event(store: Any, event: JsonObject | None, target_id: object) -> None: """Append an insert-floor event once the row id is known.""" diff --git a/apps/api/src/alicebot_api/vnext_open_loop_references.py b/apps/api/src/alicebot_api/vnext_open_loop_references.py index ea06a3650..595a45647 100644 --- a/apps/api/src/alicebot_api/vnext_open_loop_references.py +++ b/apps/api/src/alicebot_api/vnext_open_loop_references.py @@ -133,7 +133,7 @@ def withhold_unreadable_references( memory_ids.add(memory) referenced.add(memory) named = cited_source_ids(row.get("metadata_json")).named - metadata_ids.update(named) + metadata_ids.update(cited_source_ids(row.get("metadata_json")).every) referenced.update(named) _collect_ids(row.get("metadata_json"), metadata_ids, referenced, at_reference=False, depth=0) source_rows = _rows_by_id(store, sorted(source_ids | metadata_ids), bulk="get_sources_by_ids", single="get_source") @@ -376,17 +376,20 @@ def _scrub_text(text: str, *, withheld: frozenset[str]) -> object: return "".join(pieces) -def _scrub(value: object, *, depth: int, withheld: frozenset[str]) -> object: +def _scrub(value: object, *, depth: int, withheld: frozenset[str], at_reference: bool = False) -> object: if depth > _METADATA_MAX_DEPTH: return _DROPPED if isinstance(value, str): decoded = _json_container(value) if decoded is not None: - checked = _scrub(decoded, depth=depth + 1, withheld=withheld) + checked = _scrub(decoded, depth=depth + 1, withheld=withheld, at_reference=at_reference) if checked is _DROPPED: return _DROPPED return value if checked == decoded else json.dumps(checked) - return _scrub_text(value, withheld=withheld) + cut = _scrub_text(value, withheld=withheld) + if isinstance(cut, str) and cited_source_ids(cut).every & withheld: + return _DROPPED + return cut if isinstance(value, Mapping): output: dict[object, object] = {} for key, nested in value.items(): @@ -395,13 +398,16 @@ def _scrub(value: object, *, depth: int, withheld: frozenset[str]) -> object: new_key = _scrub_text(key, withheld=withheld) if new_key is _DROPPED: continue - new_value = _scrub(nested, depth=depth + 1, withheld=withheld) + if cited_source_ids(new_key).every & withheld: + continue + names_reference = isinstance(key, str) and key.lower() in _REFERENCE_KEYS | {"sources"} + new_value = _scrub(nested, depth=depth + 1, withheld=withheld, at_reference=at_reference or names_reference) if new_value is _DROPPED: continue output[new_key] = new_value return output if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): - items = (_scrub(nested, depth=depth + 1, withheld=withheld) for nested in value) + items = (_scrub(nested, depth=depth + 1, withheld=withheld, at_reference=at_reference) for nested in value) return [item for item in items if item is not _DROPPED] return value diff --git a/apps/api/src/alicebot_api/vnext_projects.py b/apps/api/src/alicebot_api/vnext_projects.py index ff928c6f4..552723a0d 100644 --- a/apps/api/src/alicebot_api/vnext_projects.py +++ b/apps/api/src/alicebot_api/vnext_projects.py @@ -217,7 +217,7 @@ class ProjectAutomationRequest: actor_id: str | None = None trace_id: str | None = None run_id: str | None = None - agent_identity: JsonObject | None = None + agent_identity: JsonObject | None = field(kw_only=True) policy_decision: JsonObject | None = None metadata_json: JsonObject = field(default_factory=dict) generation_mode: str = "deterministic" @@ -460,16 +460,16 @@ def _open_loop_candidates(source: JsonObject) -> list[JsonObject]: "title": title[:240], "description": f"Candidate {loop_type} discovered from {source_label}.", "priority": "high" if loop_type == "project_blocker" else "normal", - "source_id": source.get("id"), + "source_id": str(source["id"]), "domain": source.get("domain", "unknown"), "sensitivity": source.get("sensitivity", "unknown"), - "metadata_json": { + "metadata_json": with_derived_from({ "candidate": True, "loop_type": loop_type, "owner": owner, "source_captured_at": source.get("captured_at"), "discovered_by": "vnext_project_automation", - }, + }, {"sources": [source]}), } ) return candidates @@ -529,7 +529,7 @@ def is_project_update_candidate(artifact: JsonObject) -> bool: return is_project_update_artifact(artifact) def generate_project_update_candidate(self, request: ProjectAutomationRequest | None = None) -> JsonObject: - request = request or ProjectAutomationRequest() + request = request or ProjectAutomationRequest(agent_identity=None, ) _validate_request(request) project = self._resolve_project(request) domains = list(request.domains) if request.domains else None @@ -771,7 +771,7 @@ def _project_update_content( ) def extract_open_loops(self, request: ProjectAutomationRequest | None = None) -> list[JsonObject]: - request = request or ProjectAutomationRequest() + request = request or ProjectAutomationRequest(agent_identity=None, ) _validate_request(request) domains = list(request.domains) if request.domains else None sources = self.store.search_sources( @@ -862,8 +862,10 @@ def review_project_update( if callable(lock_graph): lock_graph() from alicebot_api.vnext_label_writes import acquire_exclusive_label_lock - - acquire_exclusive_label_lock(self.store) + if action in {"accept", "edit"}: + acquire_exclusive_label_lock(self.store) + else: + self.store.lock_label_writes() # The artifact is the review decision's serialization point. Every # accept/edit/reject path must inspect and transition the same locked # row so stale reviewers cannot split project, memory, and artifact diff --git a/apps/api/src/alicebot_api/vnext_retrieval.py b/apps/api/src/alicebot_api/vnext_retrieval.py index 4c8f51d80..350186e61 100644 --- a/apps/api/src/alicebot_api/vnext_retrieval.py +++ b/apps/api/src/alicebot_api/vnext_retrieval.py @@ -102,7 +102,7 @@ from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_grounding import compute_query_grounding from alicebot_api.vnext_json import json_safe -from alicebot_api.vnext_label_guard import LabelGuard, admit_loaded +from alicebot_api.vnext_label_guard import LabelGuard, admit_loaded, label_read_request from alicebot_api.vnext_lifecycle import RETIRED_STATUSES from alicebot_api.vnext_promotion_policy import memory_write_provenance from alicebot_api.vnext_project_scope import ( @@ -2714,20 +2714,28 @@ def _memories_by_ids( normalized_ids = tuple(dict.fromkeys(str(memory_id) for memory_id in memory_ids if memory_id)) if not normalized_ids: return {} + from alicebot_api.vnext_label_guard import request_row_cache + + cached = request_row_cache(self.store, "retrieval_memories") + requested_ids = tuple(item for item in normalized_ids if cached is None or item not in cached) bulk = getattr(self.store, "get_memories_by_ids", None) if callable(bulk): rows = [ row - for start in range(0, len(normalized_ids), MEMORY_ID_LOOKUP_BATCH_SIZE) - for row in bulk(normalized_ids[start : start + MEMORY_ID_LOOKUP_BATCH_SIZE]) + for start in range(0, len(requested_ids), MEMORY_ID_LOOKUP_BATCH_SIZE) + for row in bulk(requested_ids[start : start + MEMORY_ID_LOOKUP_BATCH_SIZE]) ] else: get_memory = getattr(self.store, "get_memory", None) rows = ( - [row for memory_id in normalized_ids if (row := get_memory(memory_id)) is not None] + [row for memory_id in requested_ids if (row := get_memory(memory_id)) is not None] if callable(get_memory) else [] ) + if cached is not None: + cached.update({item: None for item in requested_ids}) + cached.update({str(row.get("id")): row for row in rows}) + rows = [cached[item] for item in normalized_ids if cached.get(item) is not None] rows = admit_loaded( self.store, kind="memory", @@ -4052,6 +4060,7 @@ def search_source_excerpts( ) return excerpts, stage_record + @label_read_request def compile_context_pack(self, request: VNextRetrievalRequest, *, source_fence: SourceReadFence) -> JsonObject: """Compile one context pack for a caller. @@ -5415,6 +5424,11 @@ def _target_visible(event: JsonObject) -> bool: "scope_window_end", ) use_scoped_events = _supports_explicit_parameters(list_memory_events, scoped_event_parameters) + event_ceiling = ( + {"sensitivity_allowed": sensitivity_allowed} + if _supports_explicit_parameters(list_memory_events, ("sensitivity_allowed",)) + else {} + ) def _fetch_events(row_limit: int) -> tuple[list[JsonObject], str]: # The store applies project, person and time scope before its LIMIT. @@ -5434,6 +5448,7 @@ def _fetch_events(row_limit: int) -> tuple[list[JsonObject], str]: scope_person_memory_ids=tuple(sorted(person_linked_memory_ids)), scope_window_start=scope.window_start, scope_window_end=scope.window_end, + **event_ceiling, limit=row_limit, ) ), diff --git a/apps/api/src/alicebot_api/vnext_scheduler.py b/apps/api/src/alicebot_api/vnext_scheduler.py index ff5226724..8fa63dae5 100644 --- a/apps/api/src/alicebot_api/vnext_scheduler.py +++ b/apps/api/src/alicebot_api/vnext_scheduler.py @@ -1775,6 +1775,7 @@ def _generate_project_update_scan_artifact( policy_decision_value = metadata.get("policy_decision") return VNextProjectService(self.store).generate_project_update_candidate( ProjectAutomationRequest( + agent_identity=_logical_agent_identity(request.agent_identity), domains=request.domains, sensitivity_allowed=request.sensitivity_allowed, project_id=str(projects[0]["id"]), diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index a7d3671e1..079640a7d 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -492,7 +492,12 @@ def lock_label_writes(self, *, exclusive: bool = False) -> None: def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: """Narrow label rows for the insert floor. No text columns.""" - wanted = [str(item) for item in ids if str(item)] + wanted = [] + for item in ids: + try: + wanted.append(str(UUID(str(item)))) + except (ValueError, AttributeError, TypeError): + continue if not wanted: return [] table = { @@ -750,6 +755,7 @@ def list_memory_events( scope_person_memory_ids: tuple[str, ...] = (), scope_window_start: datetime | None = None, scope_window_end: datetime | None = None, + sensitivity_allowed: Sequence[str] | None = None, limit: int = 20, ) -> list[VNextRow]: """Return memory-targeted events with target scope applied pre-LIMIT.""" @@ -759,6 +765,9 @@ def list_memory_events( people_list = [str(value).strip().casefold() for value in scope_people if str(value).strip()] or None person_memory_ids = [str(value) for value in scope_person_memory_ids if str(value)] or None prefix_pattern = f"{event_type_prefix}%" if event_type_prefix is not None else None + from alicebot_api.vnext_derived_labels import SENSITIVITY_RANK + ceiling = max((SENSITIVITY_RANK.get(value, 0) for value in sensitivity_allowed or ()), default=0) + blocked = [value for value, rank in SENSITIVITY_RANK.items() if rank > ceiling] if sensitivity_allowed else None return self._fetch_all( f""" SELECT @@ -778,6 +787,15 @@ def list_memory_events( JOIN memories m ON e.target_type = 'memory' AND e.target_id = m.id::text + AND (%s::text[] IS NULL OR ( + NOT (m.sensitivity = ANY(%s::text[])) + AND NOT EXISTS ( + SELECT 1 FROM sources parent + WHERE parent.id::text = m.metadata_json->>'source_id' + AND m.metadata_json->>'redacted' IS DISTINCT FROM 'true' + AND parent.sensitivity = ANY(%s::text[]) + ) + )) AND e.user_id = m.user_id WHERE m.deleted_at IS NULL AND (%s::text IS NULL OR e.event_type LIKE %s) @@ -793,6 +811,9 @@ def list_memory_events( LIMIT %s """, ( + blocked, + blocked, + blocked, prefix_pattern, prefix_pattern, project_list, diff --git a/docs/release/derived-labels-security-note-draft.md b/docs/release/derived-labels-security-note-draft.md new file mode 100644 index 000000000..56fb85879 --- /dev/null +++ b/docs/release/derived-labels-security-note-draft.md @@ -0,0 +1,9 @@ +# Draft security note: derived labels + +Unreleased (on main, not in v0.20.0): the derived-label fixes tighten recorded-input inserts, restricted report inputs, exact reads, operator lists, and repair. This draft is for review before a release is tagged. + +Unreleased (on main, not in v0.20.0): four older operator or legacy readers remain outside that scope. A trusted key can still obtain confidential source titles and raw text through the source GET route, edge explanations through graph neighborhood, and aggregate label-check counts through doctor. Legacy `alice_vnext_review_items` can still list hidden derived memories for a declared restricted identity. These behaviors also occur on the baseline. The owner must decide whether to extend the boundary before tagging; the five corrected screens do not establish that every operator route observes the same ceiling. + +Unreleased (on main, not in v0.20.0): SQLite retirement uses the saved-quote reader's reference rules. Repeated JSON object keys are decoded with the last value retained, and quote subtrees are omitted by that reader. Retirement can consequently retain a loop that an older raw JSON scan would blank. The product does not write those two forms, but an imported or hand-edited row can contain them. + +Unreleased (on main, not in v0.20.0): capture overhead has only a few percentage points of headroom below the 15 percent budget. The earlier relabel measurements scale at roughly 2.4 ms per dependant over the measured sizes. Repeat those measurements on a quiet machine before tagging; the read budgets measure a separate path. diff --git a/docs/release/v0.20.0-release-notes.md b/docs/release/v0.20.0-release-notes.md index b70654ef1..c8a67ad40 100644 --- a/docs/release/v0.20.0-release-notes.md +++ b/docs/release/v0.20.0-release-notes.md @@ -1,7 +1,9 @@ # Alice v0.20.0 Release Notes -> **Correction (2026-10-05):** these notes do not say that a derived summary, report or copy kept the label its inputs had when it was made, or that a report could show a key bound to one project rows of other projects or with no project. Both are in v0.20.0 and are listed under known limitations. A derived row an owner lowered on purpose is raised once, by the first open, a restore or migration `20261005_0096`. +> **Correction (2026-10-05):** these notes do not say that a derived summary, report or copy kept the label its inputs had when it was made, or that a report could show a key bound to one project rows of other projects or with no project. Both are in v0.20.0 and are listed under known limitations. + +Unreleased (on main, not in v0.20.0): four older operator or legacy readers still expose withheld information: source GET returns titles and raw text, graph neighborhood returns edge explanations, doctor returns label-check counts, and legacy `alice_vnext_review_items` lists derived memories for a declared restricted identity. See the [draft security note](derived-labels-security-note-draft.md) for the remaining boundary and the SQLite retirement limits. **Take this release if you run the Postgres stack's HTTP API, use an embeddings endpoint (above all a hosted one), import ChatGPT or Markdown files, use the diff --git a/docs/runbooks/disaster-recovery.md b/docs/runbooks/disaster-recovery.md index 50a225c17..cf5028b9f 100644 --- a/docs/runbooks/disaster-recovery.md +++ b/docs/runbooks/disaster-recovery.md @@ -132,9 +132,10 @@ alice-memory export --db ~/.alice/memory.db \ alice-memory import --db ~/alice-restore-test/memory.db \ --in ~/alice-backups/alice.jsonl alice-memory reindex-embeddings --db ~/alice-restore-test/memory.db -alice-memory labels check --db ~/alice-restore-test/memory.db ``` +Unreleased (on main, not in v0.20.0): run `alice-memory labels check --db ~/alice-restore-test/memory.db` after the SQLite restore. + If the backup holds a credential and the source vault is gone, `--quarantine [,...]` is the owner's recovery path. It removes the credential from the named memory and from the records @@ -258,7 +259,7 @@ Against the restored database, verify: - the memory still has `embedding_vector` and a content-matching embedding signature; - application-role access works with RLS enabled and forced; -- `alicebot vnext labels check` lists no derived row below its inputs. +- Unreleased (on main, not in v0.20.0): `alicebot vnext labels check` lists no derived row below its inputs. If the restore is from an older release, follow [Upgrade v0.12.0 to current](upgrade-v0.12-to-current.md) in the restored diff --git a/eval/scale/harness.py b/eval/scale/harness.py index 73a5a65b3..653ad1560 100644 --- a/eval/scale/harness.py +++ b/eval/scale/harness.py @@ -505,7 +505,7 @@ def run_sweep() -> None: # materialization and clustering uses bounded float32 row blocks. cons_store = store if session.backend == "postgres" else ArtifactSinkStore(store) consolidation = VNextConsolidationService(cons_store, embedding_provider=provider) - cons_request = MemoryConsolidationRequest() + cons_request = MemoryConsolidationRequest(agent_identity=None, ) cons_notes: dict[str, object] = { "embedded_memory_hard_cap": MAX_EMBEDDED_MEMORIES_HARD_CAP, "artifact_persisted": session.backend == "postgres", diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 5d8d3667f..c6aded657 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -185,10 +185,7 @@ def assert_append_only_mutation_refused(conn, *, snapshot_sql, mutation_sql, par """Prove refusal under both forced RLS and the append-only trigger.""" before = conn.execute(snapshot_sql, params).fetchall() assert before, "the synthetic target must be visible before attempting its mutation" - try: + with pytest.raises(psycopg.Error, match="append-only"): with conn.transaction(): - changed = conn.execute(mutation_sql, params).rowcount - assert changed == 0, "an append-only row was changed" - except psycopg.Error as error: - assert "append-only" in str(error) + conn.execute(mutation_sql, params) assert conn.execute(snapshot_sql, params).fetchall() == before diff --git a/tests/integration/test_calendar_accounts_api.py b/tests/integration/test_calendar_accounts_api.py index f221aac06..708abcc89 100644 --- a/tests/integration/test_calendar_accounts_api.py +++ b/tests/integration/test_calendar_accounts_api.py @@ -13,7 +13,7 @@ from alicebot_api.routers import legacy_gated as legacy_gated_router from alicebot_api.config import Settings import alicebot_api.calendar as calendar_module -from alicebot_api.db import set_current_user, user_connection +from alicebot_api.db import user_connection from alicebot_api.store import ContinuityStore @@ -251,7 +251,6 @@ def test_calendar_account_endpoints_connect_list_detail_and_isolate( assert '"access_token":' not in json.dumps(detail_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_consolidation_report_sensitivity_postgres.py b/tests/integration/test_consolidation_report_sensitivity_postgres.py index f2d62aea4..904997028 100644 --- a/tests/integration/test_consolidation_report_sensitivity_postgres.py +++ b/tests/integration/test_consolidation_report_sensitivity_postgres.py @@ -280,7 +280,7 @@ def _consolidate_a_cluster_citing(database_url: str, user_id: UUID, *, source_fi ) store.update_memory_embedding(memory_id=str(member["id"]), vector=pad_embedding_vector([0.5, 0.1, 0.2])) artifact = VNextConsolidationService(store, embedding_provider=_OneVector()).generate_memory_consolidation( - MemoryConsolidationRequest(sensitivity_allowed=list(ALLOWED_WITH_CONFIDENTIAL)) + MemoryConsolidationRequest(agent_identity=None, sensitivity_allowed=list(ALLOWED_WITH_CONFIDENTIAL)) ) return artifact, source diff --git a/tests/integration/test_context_compile.py b/tests/integration/test_context_compile.py index 012996646..905b26da4 100644 --- a/tests/integration/test_context_compile.py +++ b/tests/integration/test_context_compile.py @@ -12,9 +12,8 @@ import alicebot_api.main as main_module from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings -from alicebot_api.db import set_current_user, user_connection +from alicebot_api.db import user_connection from alicebot_api.store import ContinuityStore -from tests.integration.conftest import assert_append_only_mutation_refused def invoke_compile_context(payload: dict[str, Any]) -> tuple[int, dict[str, Any]]: @@ -663,13 +662,9 @@ def test_compile_context_endpoint_persists_trace_and_trace_events(migrated_datab assert trace_events[-1]["payload"]["excluded_entity_edge_limit_count"] == 1 with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, user_id) - assert_append_only_mutation_refused( - conn, - snapshot_sql="SELECT * FROM trace_events WHERE trace_id = %s ORDER BY id", - mutation_sql="UPDATE trace_events SET kind = 'mutated' WHERE trace_id = %s", - params=(trace_id,), - ) + with conn.cursor() as cur: + with pytest.raises(psycopg.Error, match="append-only"): + cur.execute("UPDATE trace_events SET kind = 'mutated' WHERE trace_id = %s", (trace_id,)) def test_compile_context_prefers_updated_active_memory_within_same_transaction( diff --git a/tests/integration/test_continuity_api.py b/tests/integration/test_continuity_api.py index e3d45d5a7..1502722ba 100644 --- a/tests/integration/test_continuity_api.py +++ b/tests/integration/test_continuity_api.py @@ -12,7 +12,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings -from alicebot_api.db import set_current_user, user_connection +from alicebot_api.db import user_connection from alicebot_api.store import ContinuityStore @@ -103,14 +103,12 @@ def seed_user_with_continuity(database_url: str, *, email: str) -> dict[str, obj def set_thread_timestamps( admin_database_url: str, - user_id: UUID, *, thread_id: UUID, created_at: datetime, updated_at: datetime, ) -> None: - with psycopg.connect(admin_database_url) as conn: - set_current_user(conn, user_id) + with psycopg.connect(admin_database_url, autocommit=True) as conn: with conn.cursor() as cur: cur.execute( "UPDATE threads SET created_at = %s, updated_at = %s WHERE id = %s", @@ -120,15 +118,13 @@ def set_thread_timestamps( def set_session_timestamps( admin_database_url: str, - user_id: UUID, *, session_id: UUID, started_at: datetime, ended_at: datetime | None, created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url) as conn: - set_current_user(conn, user_id) + with psycopg.connect(admin_database_url, autocommit=True) as conn: with conn.cursor() as cur: cur.execute( "UPDATE sessions SET started_at = %s, ended_at = %s, created_at = %s WHERE id = %s", @@ -223,28 +219,24 @@ def test_thread_continuity_endpoints_create_list_detail_sessions_and_events( set_thread_timestamps( migrated_database_urls["admin"], - user_id=seeded["user_id"], thread_id=seeded["first_thread"]["id"], created_at=shared_created_at, updated_at=shared_created_at, ) set_thread_timestamps( migrated_database_urls["admin"], - user_id=seeded["user_id"], thread_id=seeded["second_thread"]["id"], created_at=shared_created_at, updated_at=shared_created_at, ) set_thread_timestamps( migrated_database_urls["admin"], - user_id=seeded["user_id"], thread_id=api_thread_id, created_at=newer_created_at, updated_at=newer_created_at, ) set_session_timestamps( migrated_database_urls["admin"], - user_id=seeded["user_id"], session_id=seeded["first_session"]["id"], started_at=first_session_start, ended_at=first_session_end, @@ -252,7 +244,6 @@ def test_thread_continuity_endpoints_create_list_detail_sessions_and_events( ) set_session_timestamps( migrated_database_urls["admin"], - user_id=seeded["user_id"], session_id=seeded["second_session"]["id"], started_at=second_session_start, ended_at=None, diff --git a/tests/integration/test_continuity_recall_api.py b/tests/integration/test_continuity_recall_api.py index 7d71d206c..d30bc10e6 100644 --- a/tests/integration/test_continuity_recall_api.py +++ b/tests/integration/test_continuity_recall_api.py @@ -12,7 +12,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import continuity as continuity_router from alicebot_api.config import Settings -from alicebot_api.db import set_current_user, user_connection +from alicebot_api.db import user_connection from alicebot_api.store import ContinuityStore @@ -70,13 +70,11 @@ def seed_user(database_url: str, *, email: str) -> UUID: def set_continuity_timestamps( admin_database_url: str, - user_id: UUID, *, continuity_object_id: UUID, created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url) as conn: - set_current_user(conn, user_id) + with psycopg.connect(admin_database_url, autocommit=True) as conn: with conn.cursor() as cur: cur.execute( "UPDATE continuity_objects SET created_at = %s, updated_at = %s WHERE id = %s", @@ -86,7 +84,6 @@ def set_continuity_timestamps( def set_continuity_lifecycle_flags( admin_database_url: str, - user_id: UUID, *, continuity_object_id: UUID, is_searchable: bool | None = None, @@ -104,8 +101,7 @@ def set_continuity_lifecycle_flags( return values.append(continuity_object_id) - with psycopg.connect(admin_database_url) as conn: - set_current_user(conn, user_id) + with psycopg.connect(admin_database_url, autocommit=True) as conn: with conn.cursor() as cur: cur.execute( f"UPDATE continuity_objects SET {', '.join(assignments)} WHERE id = %s", @@ -174,13 +170,11 @@ def test_continuity_recall_api_returns_provenance_backed_scoped_results( set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=primary_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=other_object["id"], created_at=datetime(2026, 3, 29, 9, 0, tzinfo=UTC), ) @@ -322,7 +316,6 @@ def test_continuity_recall_debug_api_persists_and_exposes_trace( set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=continuity_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) @@ -400,7 +393,6 @@ def test_continuity_resumption_debug_api_includes_underlying_retrieval_trace( set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=decision["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) @@ -494,19 +486,16 @@ def test_continuity_recall_api_prefers_confirmed_fresh_active_truth_over_superse set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=current_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=stale_object["id"], created_at=datetime(2026, 3, 20, 9, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=superseded_object["id"], created_at=datetime(2026, 3, 10, 8, 0, tzinfo=UTC), ) @@ -584,19 +573,16 @@ def test_continuity_recall_api_excludes_preserved_but_non_searchable_objects( set_continuity_lifecycle_flags( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=hidden_object["id"], is_searchable=False, ) set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=hidden_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=visible_object["id"], created_at=datetime(2026, 3, 29, 10, 5, tzinfo=UTC), ) @@ -651,7 +637,6 @@ def test_continuity_lifecycle_debug_endpoints_expose_flags( set_continuity_lifecycle_flags( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=continuity_object["id"], is_promotable=False, ) diff --git a/tests/integration/test_continuity_resumption_api.py b/tests/integration/test_continuity_resumption_api.py index 665d96d70..9397924ff 100644 --- a/tests/integration/test_continuity_resumption_api.py +++ b/tests/integration/test_continuity_resumption_api.py @@ -12,7 +12,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import continuity as continuity_router from alicebot_api.config import Settings -from alicebot_api.db import set_current_user, user_connection +from alicebot_api.db import user_connection from alicebot_api.store import ContinuityStore @@ -74,13 +74,11 @@ def seed_user(database_url: str, *, email: str) -> UUID: def set_continuity_timestamps( admin_database_url: str, - user_id: UUID, *, continuity_object_id: UUID, created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url) as conn: - set_current_user(conn, user_id) + with psycopg.connect(admin_database_url, autocommit=True) as conn: with conn.cursor() as cur: cur.execute( "UPDATE continuity_objects SET created_at = %s, updated_at = %s WHERE id = %s", @@ -90,13 +88,11 @@ def set_continuity_timestamps( def set_continuity_lifecycle_flags( admin_database_url: str, - user_id: UUID, *, continuity_object_id: UUID, is_promotable: bool, ) -> None: - with psycopg.connect(admin_database_url) as conn: - set_current_user(conn, user_id) + with psycopg.connect(admin_database_url, autocommit=True) as conn: with conn.cursor() as cur: cur.execute( "UPDATE continuity_objects SET is_promotable = %s WHERE id = %s", @@ -180,25 +176,21 @@ def test_continuity_resumption_api_returns_required_sections( set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=decision_object["id"], created_at=datetime(2026, 3, 29, 9, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=waiting_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=next_object["id"], created_at=datetime(2026, 3, 29, 10, 5, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=latest_decision_object["id"], created_at=datetime(2026, 3, 29, 10, 10, tzinfo=UTC), ) @@ -271,7 +263,6 @@ def test_continuity_resumption_api_returns_explicit_empty_states( set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=continuity_object["id"], created_at=datetime(2026, 3, 29, 9, 0, tzinfo=UTC), ) @@ -385,20 +376,17 @@ def test_continuity_resumption_api_selects_latest_sections_beyond_recall_limit( for index, continuity_object_id in enumerate(historical_object_ids): set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=continuity_object_id, created_at=base_time + timedelta(minutes=index), ) set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=latest_decision_object["id"], created_at=base_time + timedelta(minutes=200), ) set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=latest_next_action_object["id"], created_at=base_time + timedelta(minutes=201), ) @@ -477,19 +465,16 @@ def test_continuity_resumption_api_uses_promotable_facts_by_default_with_overrid set_continuity_lifecycle_flags( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=fact_object["id"], is_promotable=False, ) set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=fact_object["id"], created_at=datetime(2026, 3, 29, 10, 0, tzinfo=UTC), ) set_continuity_timestamps( migrated_database_urls["admin"], - user_id=user_id, continuity_object_id=decision_object["id"], created_at=datetime(2026, 3, 29, 10, 5, tzinfo=UTC), ) diff --git a/tests/integration/test_continuity_store.py b/tests/integration/test_continuity_store.py index fd91100fc..956156388 100644 --- a/tests/integration/test_continuity_store.py +++ b/tests/integration/test_continuity_store.py @@ -9,7 +9,6 @@ from alicebot_api.db import set_current_user, user_connection from alicebot_api.store import ContinuityStore -from tests.integration.conftest import assert_append_only_mutation_refused def test_thread_session_and_event_persistence(migrated_database_urls): @@ -41,13 +40,12 @@ def test_thread_session_and_event_persistence(migrated_database_urls): assert events[0]["payload"]["text"] == "hello" with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, user_id) - assert_append_only_mutation_refused( - conn, - snapshot_sql="SELECT * FROM events WHERE id = %s", - mutation_sql="UPDATE events SET kind = 'message.mutated' WHERE id = %s", - params=(first_event['id'],), - ) + with pytest.raises(psycopg.Error, match="append-only"): + with conn.cursor() as cur: + cur.execute( + "UPDATE events SET kind = 'message.mutated' WHERE id = %s", + (first_event["id"],), + ) def test_event_deletes_are_rejected_at_database_level(migrated_database_urls): @@ -61,13 +59,9 @@ def test_event_deletes_are_rejected_at_database_level(migrated_database_urls): event = store.append_event(thread["id"], session["id"], "message.user", {"text": "keep"}) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, user_id) - assert_append_only_mutation_refused( - conn, - snapshot_sql="SELECT * FROM events WHERE id = %s", - mutation_sql='DELETE FROM events WHERE id = %s', - params=(event['id'],), - ) + with pytest.raises(psycopg.Error, match="append-only"): + with conn.cursor() as cur: + cur.execute("DELETE FROM events WHERE id = %s", (event["id"],)) def test_continuity_rls_blocks_cross_user_access(migrated_database_urls): diff --git a/tests/integration/test_credential_floor_every_door_api.py b/tests/integration/test_credential_floor_every_door_api.py index fe3d782ee..a3a5acbb6 100644 --- a/tests/integration/test_credential_floor_every_door_api.py +++ b/tests/integration/test_credential_floor_every_door_api.py @@ -662,7 +662,7 @@ def test_round2_c2_project_update_accept_refuses_a_credential_state(migrated_dat } ) candidate = VNextProjectService(store).generate_project_update_candidate( - ProjectAutomationRequest(project_id=project_id, domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id=project_id, domains=("project",)) ) artifact_id = str(candidate["id"]) memory_id = str(candidate["metadata_json"]["candidate_memory_id"]) # type: ignore[index] diff --git a/tests/integration/test_daily_brief_project_id_postgres.py b/tests/integration/test_daily_brief_project_id_postgres.py index 81c25dd2f..3089f8733 100644 --- a/tests/integration/test_daily_brief_project_id_postgres.py +++ b/tests/integration/test_daily_brief_project_id_postgres.py @@ -57,11 +57,14 @@ def test_daily_brief_keeps_a_free_form_project_out_of_the_uuid_column(migrated_d actor_type="user", ) project_id = str(project["id"]) + second = store.create_project({"name": "Second", "slug": "second", "domain": "project", "sensitivity": "internal"}) + second_id = str(second["id"]) + _source(store, projects=[project_id, second_id], title="Two UUID projects", line="TODO: publish both uuid projects") _source(store, projects=["Alice"], title="Named project", line="TODO: publish the named note") _source(store, projects=[project_id], title="Uuid project", line="TODO: publish the uuid note") _source(store, projects=["Alice", "Bob"], title="Two projects", line="TODO: publish the pair") VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest(sensitivity_allowed=ALL_SENSITIVITY, discover_open_loops=True) + BrainArtifactRequest(agent_identity=None, sensitivity_allowed=ALL_SENSITIVITY, discover_open_loops=True) ) loops = store.list_open_loops( status="open", @@ -72,3 +75,4 @@ def test_daily_brief_keeps_a_free_form_project_out_of_the_uuid_column(migrated_d assert by_scope[("Alice",)] == "" assert by_scope[(project_id,)] == project_id assert by_scope[("Alice", "Bob")] == "" + assert by_scope[tuple(sorted((project_id, second_id)))] == "" diff --git a/tests/integration/test_derived_domain_postgres.py b/tests/integration/test_derived_domain_postgres.py index 87ba9ed76..7c7ac9328 100644 --- a/tests/integration/test_derived_domain_postgres.py +++ b/tests/integration/test_derived_domain_postgres.py @@ -74,7 +74,7 @@ def test_postgres_derived_domain_upgrade_and_generation(database_urls): } assert store.get_artifact(str(report["id"]))["domain"] == "health" fresh = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest(sensitivity_allowed=ALL_SENSITIVITY, discover_open_loops=False) + BrainArtifactRequest(agent_identity=None, sensitivity_allowed=ALL_SENSITIVITY, discover_open_loops=False) ) assert fresh["domain"] == "health" # The data-only downgrade retains safe labels, and repeating the upgrade is harmless. diff --git a/tests/integration/test_derived_labels_concurrency_postgres.py b/tests/integration/test_derived_labels_concurrency_postgres.py index 793e87f57..499188612 100644 --- a/tests/integration/test_derived_labels_concurrency_postgres.py +++ b/tests/integration/test_derived_labels_concurrency_postgres.py @@ -50,7 +50,7 @@ def test_a_report_built_from_stale_inputs_is_floored_at_insert(label_harness, mo start = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0) stale = deepcopy( VNextBrainService(store)._load_inputs( - BrainArtifactRequest(generated_for=today()), window_start=start, window_end=start + timedelta(days=1) + BrainArtifactRequest(agent_identity=None, generated_for=today()), window_start=start, window_end=start + timedelta(days=1) ) ) assert str(source["id"]) in [str(row["id"]) for row in stale[0]] @@ -59,7 +59,7 @@ def test_a_report_built_from_stale_inputs_is_floored_at_insert(label_harness, mo status, body, _ = h.request( "POST", "/v0/vnext/artifacts/generate/daily-brief", - payload={"options": {"generated_for": today(), "discover_open_loops": False}}, + payload={"options": {"generated_for": today(), "discover_open_loops": True}}, key=reader, ) assert status == 201, body @@ -67,6 +67,11 @@ def test_a_report_built_from_stale_inputs_is_floored_at_insert(label_harness, mo # The 201 is the read made at selection; the durable row has the current floor. with h.store() as store: assert_raised(store.get_artifact(str(body["id"]))) + loops = store.list_open_loops(status=None, sensitivity_allowed=["confidential", "regulated"], limit=50) + assert loops + for loop in loops: + assert_raised(loop) + assert "project_floor" in loop["metadata_json"] assert h.request("GET", f"/v0/vnext/artifacts/{body['id']}", key=reader)[0] == 403 @@ -76,7 +81,7 @@ def test_a_relabel_waits_for_an_open_generation_and_then_labels_its_report(label response = [] with h.store() as store: report = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest(generated_for=today(), discover_open_loops=False) + BrainArtifactRequest(agent_identity=None, generated_for=today(), discover_open_loops=False) ) thread, failures = _thread( lambda: response.append(h.relabel("source", source["id"], domain="health", sensitivity="confidential")) @@ -116,7 +121,7 @@ def generate(): with h.store() as store: generated.append( VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest(generated_for=today(), generation_mode="model_backed") + BrainArtifactRequest(agent_identity=None, generated_for=today(), generation_mode="model_backed") ) ) @@ -158,7 +163,7 @@ def test_a_scheduler_plan_staged_before_a_relabel_is_floored_at_publish(label_ha with h.store() as store: staged = _StagedSchedulerStore(store) report = VNextBrainService(staged).generate_daily_brief( - BrainArtifactRequest(generated_for=today(), discover_open_loops=False) + BrainArtifactRequest(agent_identity=None, generated_for=today(), discover_open_loops=False) ) plan = staged.plan(report) assert h.relabel("source", source["id"], domain="health", sensitivity="confidential")[0] == 200 @@ -261,11 +266,11 @@ def test_every_entry_point_that_locks_a_row_and_a_relabel_never_deadlock(label_h source_id=str(source["id"]), patch={"metadata_json": {"project_scope": [str(project["id"])]}} ) artifact = VNextProjectService(store).generate_project_update_candidate( - ProjectAutomationRequest(project_id=str(project["id"])) + ProjectAutomationRequest(agent_identity=None, project_id=str(project["id"])) ) else: artifact = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest(generated_for=today(), discover_open_loops=False) + BrainArtifactRequest(agent_identity=None, generated_for=today(), discover_open_loops=False) ) row_locked, release = Event(), Event() @@ -367,7 +372,7 @@ def test_each_row_locker_holds_the_shared_label_lock(label_harness, method): with h.store() as store: project = store.create_project({"name": "Synthetic locker", "slug": "synthetic-locker"}) artifact = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest(generated_for=today(), discover_open_loops=False) + BrainArtifactRequest(agent_identity=None, generated_for=today(), discover_open_loops=False) ) row_id = { "get_artifact_for_update": artifact["id"], diff --git a/tests/integration/test_derived_labels_producers_postgres.py b/tests/integration/test_derived_labels_producers_postgres.py index 7248d2482..31900be44 100644 --- a/tests/integration/test_derived_labels_producers_postgres.py +++ b/tests/integration/test_derived_labels_producers_postgres.py @@ -221,7 +221,7 @@ def test_input_selection_uses_effective_labels_including_the_owner_default_ceili assert store.get_memory(str(copy["id"]))["sensitivity"] == "public" assert store.get_memory(str(promoted["id"]))["sensitivity"] == "public" _, trusted = create_agent_key(store, user_id=user_id, agent_id="trusted-alpha", permission_profile="trusted_local_agent", project_scope=alpha) - owner = VNextBrainService(store).generate_daily_brief(BrainArtifactRequest(generated_for="2026-10-05", + owner = VNextBrainService(store).generate_daily_brief(BrainArtifactRequest(agent_identity=None, generated_for="2026-10-05", source_limit=50, memory_limit=50, artifact_limit=50, open_loop_limit=50, discover_open_loops=False, create_candidate_memories=False)) bound, _ = generate(producer, user_id, alpha, trusted) @@ -245,7 +245,7 @@ def test_the_owner_keeps_the_cross_project_brief_and_bound_keys_cannot_read_it(m user_id, alpha, beta, rows, alpha_key, beta_key, unbound = seed_grid(app_url) with user_connection(app_url, user_id) as conn: store = PostgresVNextStore(conn) - owner = VNextBrainService(store).generate_daily_brief(BrainArtifactRequest(generated_for="2026-10-05", + owner = VNextBrainService(store).generate_daily_brief(BrainArtifactRequest(agent_identity=None, generated_for="2026-10-05", source_limit=50, memory_limit=50, artifact_limit=50, open_loop_limit=50, discover_open_loops=False, create_candidate_memories=False)) text = json.dumps(owner, default=str) @@ -293,7 +293,7 @@ def embed_batch(self, texts): monkeypatch.setattr(store, "list_memory_ids_with_embeddings", lambda ids: set(ids)) service = VNextConsolidationService(store, embedding_provider=RecordingProvider()) service._cluster_memories(domains=None, sensitivity=["public", "internal", "private", "unknown"], - projects=(alpha,), all_of=(alpha,), options=_clustering_options(MemoryConsolidationRequest())) + projects=(alpha,), all_of=(alpha,), options=_clustering_options(MemoryConsolidationRequest(agent_identity=None, ))) text = json.dumps(printed) assert printed assert "SENTINEL_ALPHA memory" in text diff --git a/tests/integration/test_derived_labels_propagation_postgres.py b/tests/integration/test_derived_labels_propagation_postgres.py index 237a50fe4..f8cbd5396 100644 --- a/tests/integration/test_derived_labels_propagation_postgres.py +++ b/tests/integration/test_derived_labels_propagation_postgres.py @@ -75,7 +75,7 @@ def _build_chain(h, *, source=None, project=None, label=("project", "public")): for row in copies ] brain = VNextBrainService(store) - request = BrainArtifactRequest( + request = BrainArtifactRequest(agent_identity=None, generated_for=today(), projects=(str(project["id"]),), sensitivity_allowed=("public", "internal", "private", "confidential", "regulated", "unknown"), @@ -88,7 +88,7 @@ def _build_chain(h, *, source=None, project=None, label=("project", "public")): artifact_id=str(weekly["id"]), actor_type="user", actor_id=str(h.user_id), trace_id=None, run_id=None ) update = VNextProjectService(store, defer_embeddings=True).generate_project_update_candidate( - ProjectAutomationRequest(project_id=str(project["id"]), sensitivity_allowed=request.sensitivity_allowed) + ProjectAutomationRequest(agent_identity=None, project_id=str(project["id"]), sensitivity_allowed=request.sensitivity_allowed) ) VNextProjectService(store, defer_embeddings=True).review_project_update( artifact_id=str(update["id"]), action="accept", actor_type="user" diff --git a/tests/integration/test_gmail_accounts_api.py b/tests/integration/test_gmail_accounts_api.py index 73d407c70..0869c5f61 100644 --- a/tests/integration/test_gmail_accounts_api.py +++ b/tests/integration/test_gmail_accounts_api.py @@ -14,7 +14,7 @@ from alicebot_api.routers import legacy_gated as legacy_gated_router from alicebot_api.config import Settings import alicebot_api.gmail as gmail_module -from alicebot_api.db import set_current_user, user_connection +from alicebot_api.db import user_connection from alicebot_api.store import ContinuityStore @@ -273,7 +273,6 @@ def test_gmail_account_endpoints_connect_list_detail_and_isolate( assert '"client_secret":' not in json.dumps(create_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -496,7 +495,6 @@ def fake_fetch_gmail_message_raw_bytes(*, access_token: str, **_kwargs) -> bytes assert '"client_secret":' not in json.dumps(ingest_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -638,7 +636,6 @@ def fake_fetch_gmail_message_raw_bytes(*, access_token: str, **_kwargs) -> bytes assert '"client_secret":' not in json.dumps(ingest_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -760,7 +757,6 @@ def fail_fetch(**_kwargs): assert store.list_task_artifacts_for_task(owner["task_id"]) == [] with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -895,7 +891,6 @@ def fail_fetch(**_kwargs): ) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( "DELETE FROM gmail_account_credentials WHERE gmail_account_id = %s", @@ -969,7 +964,6 @@ def test_gmail_message_ingestion_endpoint_rejects_missing_external_secret_withou ) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_label_handoff_contention_postgres.py b/tests/integration/test_label_handoff_contention_postgres.py new file mode 100644 index 000000000..d32a3780c --- /dev/null +++ b/tests/integration/test_label_handoff_contention_postgres.py @@ -0,0 +1,79 @@ +"""Ordinary reviews share the label lock; label changes give a retryable refusal.""" +import time + +import pytest + +from alicebot_api.cli.automation import _run_vnext_artifact_review +from alicebot_api.cli.models import CLIContext +from alicebot_api.config import Settings +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext, MCPInvalidRequestError +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService +from tests.integration.derived_labels_postgres_support import label_harness, today + + +@pytest.mark.parametrize("entry", ["memory", "artifact", "mcp_memory", "mcp_artifact", "project_reject"]) +def test_non_label_review_succeeds_while_shared_label_lock_is_held(label_harness, entry): + h = label_harness + source = h.source() + memory = h.memory(source=source) + with h.store() as store: + if entry == "project_reject": + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + project = store.create_project({"name": "Synthetic acceptance", "slug": "synthetic"}) + store.update_source(source_id=str(source["id"]), patch={"metadata_json": {"project_scope": [str(project["id"])]}}) + artifact = VNextProjectService(store).generate_project_update_candidate(ProjectAutomationRequest(agent_identity=None, project_id=str(project["id"]))) + else: + artifact = VNextBrainService(store).generate_daily_brief(BrainArtifactRequest(agent_identity=None, generated_for=today(), discover_open_loops=False)) + context = MCPRuntimeContext(database_url=h.urls["app"], user_id=h.user_id) + with h.store() as holder: + holder.lock_label_writes() + started = time.monotonic() + if entry == "memory": + result = h.request("POST", f"/v0/vnext/memories/{memory['id']}/review", payload={"action": "accept"}) + assert result[0] == 200, result + elif entry == "mcp_memory": + result = call_mcp_tool(context, name="alice_memory_correct", arguments={"review_item_id": str(memory["id"]), "action": "approve"}) + elif entry == "mcp_artifact": + result = call_mcp_tool(context, name="alice_vnext_artifact_review", arguments={"artifact_id": str(artifact["id"]), "action": "accept"}) + else: + path = f"/v0/vnext/projects/update-candidates/{artifact['id']}/review" if entry == "project_reject" else f"/v0/vnext/artifacts/{artifact['id']}/review" + result = h.request("POST", path, payload={"action": "reject" if entry == "project_reject" else "accept"}) + assert result[0] == 200, result + assert time.monotonic() - started < 2.0 + # Keep the shared grant live beyond the exclusive wait bound. The + # action above completed while that grant still existed. + time.sleep(max(0, 3.1 - (time.monotonic() - started))) + + +@pytest.mark.parametrize("entry", ["memory", "artifact", "project", "mcp_project"]) +def test_label_changing_review_times_out_with_retry_after(label_harness, entry): + h = label_harness + source = h.source() + memory = h.memory(source=source) + with h.store() as store: + store.lock_graph_mutation() + store.lock_label_writes(exclusive=True) + project = store.create_project({"name": "Synthetic acceptance", "slug": "synthetic"}) + store.update_source(source_id=str(source["id"]), patch={"metadata_json": {"project_scope": [str(project["id"])]}}) + artifact = VNextProjectService(store).generate_project_update_candidate(ProjectAutomationRequest(agent_identity=None, project_id=str(project["id"]))) + context = MCPRuntimeContext(database_url=h.urls["app"], user_id=h.user_id) + before = h.snapshot() + with h.store() as holder: + holder.lock_label_writes() + started = time.monotonic() + if entry == "mcp_project": + with pytest.raises(MCPInvalidRequestError, match="HTTP 503; Retry-After: 2"): + call_mcp_tool(context, name="alice_project_update_review", arguments={"artifact_id": str(artifact["id"]), "action": "accept"}) + else: + path = f"/v0/vnext/memories/{memory['id']}/review" if entry == "memory" else ( + f"/v0/vnext/artifacts/{artifact['id']}/review" if entry == "artifact" else f"/v0/vnext/projects/update-candidates/{artifact['id']}/review") + payload = {"action": "edit", "sensitivity": "confidential"} if entry == "memory" else {"action": "accept"} + status, body, headers = h.request("POST", path, payload=payload) + assert status == 503, body + assert headers[b"retry-after"] == b"2" + assert "nothing was changed" in body["detail"] + assert 2.5 <= time.monotonic() - started < 4.0 + assert h.snapshot() == before diff --git a/tests/integration/test_label_handoff_malformed_postgres.py b/tests/integration/test_label_handoff_malformed_postgres.py new file mode 100644 index 000000000..962851d2a --- /dev/null +++ b/tests/integration/test_label_handoff_malformed_postgres.py @@ -0,0 +1,55 @@ +"""Corrupt recorded identifiers restrict reads without crashing PostgreSQL.""" +from uuid import uuid4 + +import pytest + +from alicebot_api.db import user_connection +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.derived_labels_postgres_support import label_harness + + +@pytest.mark.parametrize("key", ["source_refs", "derived_from", "source_id", "artifact_id"]) +def test_non_uuid_record_does_not_crash_get_or_recall(label_harness, key): + h = label_harness + project = str(uuid4()) + if key == "source_refs": + metadata = {key: ["meeting-notes"]} + elif key == "derived_from": + metadata = {key: {"v": 1, "sources": ["meeting-notes"], "memories": [], "artifacts": [], "open_loops": [], "beliefs": [], + "counts": {"sources": 1, "memories": 0, "artifacts": 0, "open_loops": 0, "beliefs": 0}}} + else: + metadata = {key: "meeting-notes"} + with h.store() as store: + store.create_project({"id": project, "name": "Alpha", "slug": "alpha"}) + artifact = store.create_artifact({"artifact_type": "daily_brief", "title": "Synthetic malformed report", + "content_markdown": "Synthetic malformed report", "sensitivity": "public", "domain": "project", + "metadata_json": {"workflow": "daily_brief", "project_scope": [project], **metadata}}) + memory = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Synthetic malformed memory", + "status": "active", "sensitivity": "public", "domain": "project", + "metadata_json": {"candidate_kind": "memory_rollup", "project_scope": [project], **metadata}}) + assert store.read_label_rows("source", ["meeting-notes"]) == [] + reader = h.key("trusted_local_agent") + bound = h.key("admin_agent", project=project) + owner = h.key("admin_agent") + assert h.request("GET", f"/v0/vnext/artifacts/{artifact['id']}", key=owner)[0] == 200 + for key_value in (reader, bound): + status, body, _ = h.request("GET", f"/v0/vnext/artifacts/{artifact['id']}", key=key_value) + # A free-text source_refs entry is not a dependency. The remaining + # explicitly typed or canonical records cannot be resolved. + if key != "source_refs": + assert status == 403, body + if key_value == bound: + assert "derived_labels_unverified" in str(body), body + if key != "source_refs": + from alicebot_api.routers import vnext_memories + from uuid import UUID + response = vnext_memories.get_vnext_memory_audit(UUID(str(memory["id"])), h.user_id, + authorization=f"Bearer {bound}") + assert response.status_code == 403, response.body + assert b"derived_labels_unverified" in response.body + context = MCPRuntimeContext(database_url=h.urls["app"], user_id=h.user_id) + result = call_mcp_tool(context, name="alice_recall", arguments={"query": "Synthetic malformed", "agent_id": "synthetic-reader", "permission_profile": "read_only_agent"}) + if key != "source_refs": + assert str(memory["id"]) not in str(result) diff --git a/tests/integration/test_label_handoff_migration_postgres.py b/tests/integration/test_label_handoff_migration_postgres.py new file mode 100644 index 000000000..5c5e3cb20 --- /dev/null +++ b/tests/integration/test_label_handoff_migration_postgres.py @@ -0,0 +1,61 @@ +"""Malformed restored metadata passes 0096 under a restricted table owner.""" +from urllib.parse import quote, urlsplit, urlunsplit +from uuid import uuid4 +import json + +from alembic import command +import psycopg +from psycopg import sql + +from alicebot_api.db import user_connection, close_connection_pools +from alicebot_api.migrations import make_alembic_config +from alicebot_api.store import ContinuityStore +from alicebot_api.vnext_label_writes import without_insert_floor +from alicebot_api.vnext_store import PostgresVNextStore + +TABLES = ("sources", "memories", "open_loops", "generated_artifacts", "beliefs", "event_log", "projects") + + +def test_0096_malformed_marker_is_unverified_and_not_repaired(migrated_database_urls): + urls = migrated_database_urls + command.downgrade(make_alembic_config(urls["admin"]), "20261004_0095") + user = uuid4() + with without_insert_floor(), user_connection(urls["app"], user) as conn: + ContinuityStore(conn).create_user(user, f"malformed-{user}@example.invalid", "Synthetic") + store = PostgresVNextStore(conn) + memory = store.create_memory({"memory_key": "malformed", "canonical_text": "Synthetic malformed marker", + "domain": "project", "sensitivity": "public", "status": "active", "metadata_json": {"candidate_kind": ["memory_rollup"]}}) + artifact = store.create_artifact({"artifact_type": "research_brief", "title": "Synthetic malformed marker", + "content_markdown": "Synthetic malformed marker", "domain": "project", "sensitivity": "public", "metadata_json": {"workflow": {"broken": 1}}}) + role, password = "alice_handoff_owner_" + uuid4().hex[:12], uuid4().hex + parsed = urlsplit(urls["admin"]) + owner_url = urlunsplit((parsed.scheme, f"{quote(role)}:{quote(password)}@" + parsed.netloc.rsplit("@", 1)[-1], parsed.path, parsed.query, parsed.fragment)) + with psycopg.connect(urls["admin"], autocommit=True) as conn: + original = conn.execute("SELECT current_user").fetchone()[0] + conn.execute(sql.SQL("CREATE ROLE {} LOGIN PASSWORD {} NOSUPERUSER NOBYPASSRLS").format(sql.Identifier(role), sql.Literal(password))) + for table in TABLES: + conn.execute(sql.SQL("ALTER TABLE {} OWNER TO {}").format(sql.Identifier(table), sql.Identifier(role))) + conn.execute(sql.SQL("GRANT USAGE ON SCHEMA public, app TO {}").format(sql.Identifier(role))) + conn.execute(sql.SQL("GRANT SELECT, UPDATE, INSERT, DELETE ON alembic_version TO {}").format(sql.Identifier(role))) + try: + with psycopg.connect(owner_url) as conn: + assert conn.execute("SELECT rolsuper,rolbypassrls FROM pg_roles WHERE rolname=current_user").fetchone() == (False, False) + command.upgrade(make_alembic_config(owner_url), "head") + with psycopg.connect(owner_url) as conn: + assert all(row[0] for row in conn.execute("SELECT relforcerowsecurity FROM pg_class WHERE relname=ANY(%s)", (list(TABLES),))) + with user_connection(urls["app"], user) as conn: + from alicebot_api.vnext_label_repair import classify_stored_labels + store = PostgresVNextStore(conn) + mem, art = store.get_memory(str(memory["id"])), store.get_artifact(str(artifact["id"])) + assert mem["sensitivity"] == art["sensitivity"] == "public" + below, unverified = classify_stored_labels({"memories": [mem], "generated_artifacts": [art]}) + assert not below + assert set(unverified["malformed_marker"]) == {str(memory["id"]), str(artifact["id"])} + assert conn.execute("SELECT count(*) AS n FROM event_log WHERE event_type LIKE '%%.labels_raised'").fetchone()["n"] == 0 + finally: + close_connection_pools() + with psycopg.connect(urls["admin"], autocommit=True) as conn: + for table in TABLES: + conn.execute(sql.SQL("ALTER TABLE {} OWNER TO {}").format(sql.Identifier(table), sql.Identifier(original))) + conn.execute(sql.SQL("DROP OWNED BY {}").format(sql.Identifier(role))) + conn.execute(sql.SQL("DROP ROLE {}").format(sql.Identifier(role))) diff --git a/tests/integration/test_label_handoff_writeback_postgres.py b/tests/integration/test_label_handoff_writeback_postgres.py new file mode 100644 index 000000000..b3bba9b12 --- /dev/null +++ b/tests/integration/test_label_handoff_writeback_postgres.py @@ -0,0 +1,62 @@ +"""A plain source review cannot write back an earlier project assignment.""" +from threading import Event +from uuid import uuid4 + +import pytest + +from alicebot_api.routers import vnext_memories as router +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.derived_labels_postgres_support import label_harness, join_thread +from tests.integration.test_derived_labels_concurrency_postgres import _thread + + +@pytest.mark.parametrize("strict", [False, True]) +def test_plain_source_review_serializes_before_project_move(label_harness, monkeypatch, strict): + from alicebot_api import vnext_label_writes + monkeypatch.setattr(vnext_label_writes, "STRICT_LOCK_ORDER", strict) + h = label_harness + alpha, beta = str(uuid4()), str(uuid4()) + with h.store() as store: + for project, name in ((alpha, "Alpha"), (beta, "Beta")): + store.create_project({"id": project, "name": name, "slug": name.lower()}) + source = h.source(scope=(alpha,)) + read, release = Event(), Event() + original = PostgresVNextStore.get_source + def pause_first_read(self, source_id): + row = original(self, source_id) + if not read.is_set(): + read.set() + assert release.wait(5) + return row + monkeypatch.setattr(PostgresVNextStore, "get_source", pause_first_read) + responses = [] + review, review_failures = _thread(lambda: responses.append(router.review_vnext_source( + source["id"], router.VNextSourceReviewRequest(user_id=h.user_id, action="review")))) + assert read.wait(2) + move, move_failures = _thread(lambda: responses.append(router.review_vnext_source( + source["id"], router.VNextSourceReviewRequest(user_id=h.user_id, action="assign_project", project_id=beta)))) + try: + h.wait_relabel() + assert review.is_alive() and move.is_alive() + finally: + release.set() + join_thread(review, review_failures) + join_thread(move, move_failures) + assert all(response.status_code == 200 for response in responses) + with h.store() as store: + assert store.get_source(str(source["id"]))["metadata_json"]["project_scope"] == [beta] + + +def test_event_prefilter_preserves_visible_and_noncontiguous_ceiling_controls(label_harness): + h = label_harness + public = h.memory() + private_source = h.source(sensitivity="confidential") + hidden = h.memory(source=private_source) + with h.store() as store: + public_events = store.list_memory_events(sensitivity_allowed=["public"], limit=100) + assert str(public["id"]) in {str(event["target_id"]) for event in public_events} + assert str(hidden["id"]) not in {str(event["target_id"]) for event in public_events} + # This is a conservative SQL filter. A gap in the allowed set is left + # to effective admission because a derived label can increase into it. + all_events = store.list_memory_events(sensitivity_allowed=["public", "regulated"], limit=100) + assert str(hidden["id"]) in {str(event["target_id"]) for event in all_events} diff --git a/tests/integration/test_label_read_handoff_budget_postgres.py b/tests/integration/test_label_read_handoff_budget_postgres.py new file mode 100644 index 000000000..eb8873670 --- /dev/null +++ b/tests/integration/test_label_read_handoff_budget_postgres.py @@ -0,0 +1,46 @@ +"""Native 5,000-row read budgets, compared with a supplied main checkout.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest +import psycopg + +from tests.integration.derived_labels_postgres_support import label_harness +from tests.performance.test_label_read_handoff_budget import seed_copies + + +def probe(repo, backend, location, user): + script = Path(__file__).resolve().parents[1] / "performance/read_budget_probe.py" + completed = subprocess.run([sys.executable, str(script), str(repo), backend, str(location), str(user)], + capture_output=True, text=True, check=True, timeout=120) + return json.loads(completed.stdout.strip().splitlines()[-1]) + + +@pytest.mark.parametrize("hidden", [False, True]) +def test_postgres_five_thousand_native_read_budgets(label_harness, hidden): + h = label_harness + with h.store() as store: + source, _ = seed_copies(store, postgres=True) + if not hidden: + store.conn.execute("UPDATE sources SET sensitivity='public' WHERE id=%s", (source["id"],)) + with psycopg.connect(h.urls["admin"], autocommit=True) as conn: + conn.execute("ANALYZE memories") + conn.execute("ANALYZE event_log") + conn.execute("ANALYZE sources") + repo = Path(__file__).resolve().parents[2] + head = probe(repo, "postgres", h.urls["app"], h.user_id) + print("PostgreSQL hidden=" + str(hidden) + " head=" + json.dumps(head)) + assert head["workspace"]["median"] <= 1.0, head + assert head["dogfooding"]["median"] <= 1.0, head + main = os.environ.get("ALICE_READ_MAIN_CHECKOUT") + if main: + baseline = probe(main, "postgres", h.urls["app"], h.user_id) + print("PostgreSQL main=" + json.dumps(baseline)) + for action in ("pack", "recall"): + assert head[action]["median"] <= 2 * baseline[action]["median"] + .1, (action, head, baseline) + else: + for action in ("pack", "recall"): + assert head[action]["median"] <= .15, (action, head) diff --git a/tests/integration/test_label_repair_handoff_cas_postgres.py b/tests/integration/test_label_repair_handoff_cas_postgres.py new file mode 100644 index 000000000..9544a6200 --- /dev/null +++ b/tests/integration/test_label_repair_handoff_cas_postgres.py @@ -0,0 +1,33 @@ +"""A writer from an older binary cannot be overwritten by a repair plan.""" +from types import SimpleNamespace + +import pytest + +from alicebot_api.cli import labels +from alicebot_api.db import user_connection +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.test_derived_labels_migration_postgres import seed_stale + + +def test_repair_compare_and_set_rejects_a_raw_concurrent_raise(migrated_database_urls, monkeypatch): + urls = migrated_database_urls + user, rows = seed_stale(urls) + # Locate the first planned memory rather than depending on the fixture's return shape. + original = labels.plan_label_repairs + changed_id = [] + + def race(tables): + plan = original(tables) + target = next(item for item in plan if item[0] == "memories") + changed_id.append(target[2]) + with user_connection(urls["app"], user) as conn: + conn.execute("UPDATE memories SET sensitivity='regulated' WHERE id=%s::uuid", (target[2],)) + return plan + + monkeypatch.setattr(labels, "plan_label_repairs", race) + with pytest.raises(SystemExit) as refused: + labels._run_vnext_labels_repair(SimpleNamespace(database_url=urls["app"], user_id=user), None) + assert refused.value.code == 2 + with user_connection(urls["app"], user) as conn: + assert PostgresVNextStore(conn).get_memory(changed_id[0])["sensitivity"] == "regulated" + assert conn.execute("SELECT count(*) AS n FROM event_log WHERE event_type LIKE '%%.labels_raised'").fetchone()["n"] == 0 diff --git a/tests/integration/test_memory_admission.py b/tests/integration/test_memory_admission.py index 202f597a1..31a04e735 100644 --- a/tests/integration/test_memory_admission.py +++ b/tests/integration/test_memory_admission.py @@ -11,7 +11,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings -from alicebot_api.db import set_current_user, user_connection +from alicebot_api.db import user_connection from alicebot_api.store import ContinuityStore @@ -240,7 +240,6 @@ def test_admit_memory_endpoint_persists_add_update_and_delete_revisions( assert revisions[2]["new_value"] is None with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, user_id) with conn.cursor() as cur: with pytest.raises(psycopg.Error, match="append-only"): cur.execute( diff --git a/tests/integration/test_memory_review_labels_api.py b/tests/integration/test_memory_review_labels_api.py index ab80e4ad8..288fc03b6 100644 --- a/tests/integration/test_memory_review_labels_api.py +++ b/tests/integration/test_memory_review_labels_api.py @@ -13,10 +13,9 @@ from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings from alicebot_api.contracts import MemoryCandidateInput -from alicebot_api.db import set_current_user, user_connection +from alicebot_api.db import user_connection from alicebot_api.memory import admit_memory_candidate from alicebot_api.store import ContinuityStore -from tests.integration.conftest import assert_append_only_mutation_refused def invoke_request( @@ -297,22 +296,20 @@ def test_memory_review_labels_reject_update_and_delete_at_database_level(migrate ) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(seeded["user_id"])) - assert_append_only_mutation_refused( - conn, - snapshot_sql="SELECT * FROM memory_review_labels WHERE id = %s", - mutation_sql="UPDATE memory_review_labels SET label = 'incorrect' WHERE id = %s", - params=(label['id'],), - ) + with pytest.raises(psycopg.Error, match="append-only"): + with conn.cursor() as cur: + cur.execute( + "UPDATE memory_review_labels SET label = 'incorrect' WHERE id = %s", + (label["id"],), + ) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(seeded["user_id"])) - assert_append_only_mutation_refused( - conn, - snapshot_sql="SELECT * FROM memory_review_labels WHERE id = %s", - mutation_sql='DELETE FROM memory_review_labels WHERE id = %s', - params=(label['id'],), - ) + with pytest.raises(psycopg.Error, match="append-only"): + with conn.cursor() as cur: + cur.execute( + "DELETE FROM memory_review_labels WHERE id = %s", + (label["id"],), + ) def test_memory_review_label_endpoints_enforce_per_user_isolation_and_not_found_behavior( diff --git a/tests/integration/test_migration_0087_retry.py b/tests/integration/test_migration_0087_retry.py index caeb0a8a5..4a1c2800d 100644 --- a/tests/integration/test_migration_0087_retry.py +++ b/tests/integration/test_migration_0087_retry.py @@ -5,7 +5,6 @@ import pytest from alicebot_api.migrations import make_alembic_config -from tests.integration.test_migrations import _set_fixture_identity _PARTIALLY_COMMITTED_SCHEMA = ( @@ -91,7 +90,6 @@ def test_0087_retries_after_committed_ddl_and_invalid_concurrent_unique_index( user_id = "00000000-0000-0000-0000-000000008701" with psycopg.connect(database_url) as conn: - _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: for statement in _PARTIALLY_COMMITTED_SCHEMA: cur.execute(statement) @@ -131,7 +129,6 @@ def test_0087_retries_after_committed_ddl_and_invalid_concurrent_unique_index( # catalog row. This models an operator correcting the build cause before # rerunning the still-unapplied Alembic revision. with psycopg.connect(database_url) as conn: - _set_fixture_identity(conn, user_id=user_id) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_producer_handoff_crowding_postgres.py b/tests/integration/test_producer_handoff_crowding_postgres.py new file mode 100644 index 000000000..3a71a6fd2 --- /dev/null +++ b/tests/integration/test_producer_handoff_crowding_postgres.py @@ -0,0 +1,77 @@ +"""Filtering precedes limits, and locked readers refill crowded store pages.""" +import json +from uuid import uuid4 + +import pytest + +from alicebot_api.db import user_connection +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.test_derived_labels_producers_postgres import seed_grid, wire_database +from tests.integration.test_memory_mutations_api import invoke_request + + +@pytest.mark.parametrize("producer", ["daily-brief", "connections", "contradictions"]) +def test_bound_producer_uses_admitted_memories_behind_newer_shared_rows(migrated_database_urls, monkeypatch, producer): + url = migrated_database_urls["app"] + wire_database(monkeypatch, url) + user, alpha, beta, _, key, *_ = seed_grid(url) + own = [] + selected = [] + if producer != "daily-brief": + from alicebot_api import vnext_connections, vnext_contradictions + module = vnext_connections if producer == "connections" else vnext_contradictions + original = module._find_candidates + def observe(**kwargs): + selected.extend(str(row["id"]) for row in kwargs.get("memories", kwargs.get("new_items", []))) + return original(**kwargs) + monkeypatch.setattr(module, "_find_candidates", observe) + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + for shared, count in ((False, 3), (True, 30)): + for index in range(count): + row = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Atlas synthetic launch evidence " + str(index), + "status": "active", "domain": "project", "sensitivity": "public", "memory_type": "episode", + "metadata_json": {"project_scope": [alpha, beta] if shared else [alpha]}}) + conn.execute("UPDATE memories SET created_at=%s, updated_at=%s, first_seen_at=%s, last_seen_at=%s WHERE id=%s", ( + "2026-10-05T11:00:00Z" if shared else "2026-10-05T10:00:00Z", + "2026-10-05T11:00:00Z" if shared else "2026-10-05T10:00:00Z", + "2026-10-05T11:00:00Z" if shared else "2026-10-05T10:00:00Z", + "2026-10-05T11:00:00Z" if shared else "2026-10-05T10:00:00Z", row["id"])) + if not shared: + own.append(str(row["id"])) + status, body = invoke_request("POST", "/v0/vnext/artifacts/generate/" + producer, + payload={"user_id": str(user), "scope": {"projects": [alpha]}, "options": { + "generated_for": "2026-10-05", "memory_limit": 3, "max_connections": 2, "max_contradictions": 2, "discover_open_loops": False}}, + headers={"authorization": "Bearer " + key}) + assert status == 201, body + printed_inputs = body["metadata_json"]["derived_from"]["memories"] if producer == "daily-brief" else selected + assert set(own) <= set(printed_inputs), (own, printed_inputs) + + +def test_locked_daily_refills_sources_and_loops(migrated_database_urls, monkeypatch): + url = migrated_database_urls["app"] + wire_database(monkeypatch, url) + user, alpha, beta, _, key, *_ = seed_grid(url) + own = {"sources": [], "open_loops": []} + with user_connection(url, user) as conn: + store = PostgresVNextStore(conn) + for shared, count in ((False, 3), (True, 30)): + for index in range(count): + scope = [alpha, beta] if shared else [alpha] + source = store.create_source({"source_type": "note", "title": "Synthetic refill " + str(index), + "content_hash": str(uuid4()), "captured_at": "2026-10-05T11:00:00Z" if shared else "2026-10-05T10:00:00Z", + "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": scope, "raw_text": "Synthetic refill evidence"}}) + loop = store.create_open_loop({"title": "Synthetic refill " + str(index), "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": scope}}) + conn.execute("UPDATE open_loops SET created_at=%s, opened_at=%s WHERE id=%s", ("2026-10-05T11:00:00Z" if shared else "2026-10-05T10:00:00Z", "2026-10-05T11:00:00Z" if shared else "2026-10-05T10:00:00Z", loop["id"])) + if not shared: + own["sources"].append(str(source["id"])) + own["open_loops"].append(str(loop["id"])) + status, body = invoke_request("POST", "/v0/vnext/artifacts/generate/daily-brief", + payload={"user_id": str(user), "scope": {"projects": [alpha]}, "options": { + "generated_for": "2026-10-05", "source_limit": 5, "open_loop_limit": 5, "discover_open_loops": False}}, + headers={"authorization": "Bearer " + key}) + assert status == 201, body + record = body["metadata_json"]["derived_from"] + for kind, ids in own.items(): + assert set(ids) <= set(record[kind]), json.dumps(record) diff --git a/tests/integration/test_producer_handoff_stale_inputs_postgres.py b/tests/integration/test_producer_handoff_stale_inputs_postgres.py new file mode 100644 index 000000000..3fd0561be --- /dev/null +++ b/tests/integration/test_producer_handoff_stale_inputs_postgres.py @@ -0,0 +1,68 @@ +"""Real producers floor UUID-bearing snapshots after a committed source move.""" +from copy import deepcopy +from uuid import UUID, uuid4 + +import pytest + +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from alicebot_api.vnext_derived_labels import is_derived +from alicebot_api.vnext_label_repair import label_gap_counts +from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService +from alicebot_api.vnext_source_regeneration import regenerate_source_inputs +from alicebot_api.vnext_store import PostgresVNextStore +from tests.integration.derived_labels_postgres_support import label_harness, today + + +@pytest.mark.parametrize("producer", ["brief", "project", "regenerate"]) +def test_real_producer_payloads_and_stale_uuid_source_floor(label_harness, monkeypatch, producer): + h = label_harness + alpha, beta = str(uuid4()), str(uuid4()) + with h.store() as store: + for project, name in ((alpha, "Alpha"), (beta, "Beta")): + store.create_project({"id": project, "name": name, "slug": name.lower()}) + source = h.source(scope=(alpha,), text="I prefer synthetic blue ink.\nTODO: Review synthetic draft") + assert isinstance(source["id"], UUID) + with h.store() as store: + store.create_source_chunk({"source_id": str(source["id"]), "chunk_index": 0, + "text": source["metadata_json"]["raw_text"]}) + stale = deepcopy(source) + if producer == "brief": + from datetime import UTC, datetime, timedelta + start = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0) + inputs = deepcopy(VNextBrainService(store)._load_inputs( + BrainArtifactRequest(agent_identity=None, generated_for=today()), + window_start=start, window_end=start + timedelta(days=1))) + from alicebot_api.routers import vnext_memories + moved = vnext_memories.review_vnext_source(source["id"], vnext_memories.VNextSourceReviewRequest( + user_id=h.user_id, action="assign_project", sensitivity="confidential", domain="health", + project_id=beta, confirm_label_hide=True)) + assert moved.status_code == 200, moved.body + with h.store() as store: + assert store.get_source(str(source["id"]))["metadata_json"]["project_scope"] == [beta] + observed = [] + for kind, method in (("memory", "create_memory"), ("open_loop", "create_open_loop"), ("artifact", "create_artifact")): + original = getattr(PostgresVNextStore, method) + def spy(self, payload, *args, _kind=kind, _original=original, **kwargs): + assert is_derived(_kind, payload), (_kind, payload) + observed.append(_kind) + return _original(self, payload, *args, **kwargs) + monkeypatch.setattr(PostgresVNextStore, method, spy) + with h.store() as store: + if producer == "brief": + monkeypatch.setattr(VNextBrainService, "_load_inputs", lambda *_args, **_kwargs: deepcopy(inputs)) + VNextBrainService(store).generate_daily_brief(BrainArtifactRequest( + agent_identity=None, generated_for=today(), discover_open_loops=True)) + elif producer == "project": + monkeypatch.setattr(store, "search_sources", lambda **_kwargs: [deepcopy(stale)]) + VNextProjectService(store).extract_open_loops(ProjectAutomationRequest(agent_identity=None, project_id=alpha)) + else: + regenerate_source_inputs(store, stale) + assert "open_loop" in observed + loops = store.list_open_loops(status=None, sensitivity_allowed=["confidential"], limit=50) + assert loops + for loop in loops: + assert loop["sensitivity"] == "confidential" + assert loop["domain"] == "health" + assert set(loop["metadata_json"]["project_floor"]) == {alpha, beta}, (producer, alpha, beta, loop) + assert str(source["id"]) in loop["metadata_json"]["derived_from"]["sources"] + assert label_gap_counts(store) == (0, 0) diff --git a/tests/integration/test_provider_runtime_api.py b/tests/integration/test_provider_runtime_api.py index 63adf3ce3..27b5dc408 100644 --- a/tests/integration/test_provider_runtime_api.py +++ b/tests/integration/test_provider_runtime_api.py @@ -15,7 +15,7 @@ import alicebot_api.main as main_module from alicebot_api.config import Settings, WorkspaceProviderConfig -from alicebot_api.db import set_current_user, set_current_user_account, user_connection +from alicebot_api.db import user_connection from alicebot_api.provider_configuration import provider_config_fingerprint from alicebot_api.public_errors import UPSTREAM_FAILURE from alicebot_api.provider_secrets import decode_provider_secret_ref, resolve_provider_api_key @@ -148,8 +148,6 @@ def _bootstrap_local_workspace(email: str) -> tuple[str, str, str]: def _seed_thread_for_user(*, admin_db_url: str, user_id: str, email: str) -> str: thread_id = str(uuid4()) with psycopg.connect(admin_db_url) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -844,8 +842,6 @@ def test_openai_compatible_registration_still_works(migrated_database_urls, monk assert any(record["url"] == "https://provider.example/v1/models" for record in captured_requests) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -933,8 +929,6 @@ def test_openai_compatible_no_auth_update_omits_auth_for_test_and_runtime( assert len(captured_requests) == 4 assert all("authorization" not in {str(key).lower() for key in record["headers"]} for record in captured_requests) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( "SELECT auth_mode, api_key FROM model_providers WHERE id = %s AND workspace_id = %s", @@ -971,8 +965,6 @@ def test_provider_update_uses_atomic_cas_and_hides_stale_capability( migrated_database_urls["admin"], row_factory=psycopg.rows.dict_row, ) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) store = ContinuityStore(conn) original = store.get_model_provider_for_workspace_optional( provider_id=provider_id, @@ -1025,8 +1017,6 @@ def test_provider_update_uses_atomic_cas_and_hides_stale_capability( migrated_database_urls["admin"], row_factory=psycopg.rows.dict_row, ) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) lost_update = ContinuityStore(conn).update_model_provider( provider_id=provider_id, workspace_id=UUID(workspace_id), @@ -1470,8 +1460,6 @@ def test_provider_invocation_telemetry_persists_for_test_and_runtime( assert provider_secret not in caplog.text with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -1692,8 +1680,6 @@ def fake_urlopen(request, timeout, enforce_public_peer): assert test_payload["result"]["usage"]["total_tokens"] == 14 with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -1752,8 +1738,6 @@ def fake_urlopen(request, timeout, enforce_public_peer): provider_id = register_payload["provider"]["id"] with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -1793,8 +1777,6 @@ def fake_urlopen(request, timeout, enforce_public_peer): assert updated_payload["capabilities"]["snapshot"]["azure_auth_mode"] == ("azure_ad_token") with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2015,8 +1997,6 @@ def private_dns_getaddrinfo(hostname: str, port, type=0, proto=0): } with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2039,7 +2019,7 @@ def test_provider_test_and_runtime_reject_disallowed_target_without_outbound( user_id, workspace_id, user_account_id = _bootstrap_local_workspace("provider-security-blocked-runtime@example.com") urlopen_call_count = 0 - def fake_urlopen(_request, timeout, enforce_public_peer): + def fake_urlopen(_request, _timeout): nonlocal urlopen_call_count urlopen_call_count += 1 raise AssertionError("outbound request should not be attempted for blocked targets") @@ -2062,8 +2042,6 @@ def fake_urlopen(_request, timeout, enforce_public_peer): provider_id = register_payload["provider"]["id"] with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2139,8 +2117,6 @@ def test_provider_rejects_userinfo_and_redacts_legacy_rows( legacy_provider_id: str with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2259,8 +2235,6 @@ def fake_urlopen(request, timeout, enforce_public_peer): assert sensitive_detail not in json.dumps(test_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ @@ -2298,8 +2272,6 @@ def fake_urlopen(request, timeout, enforce_public_peer): assert sensitive_detail not in json.dumps(runtime_payload) with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, UUID(user_id)) - set_current_user_account(conn, UUID(user_id)) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_proxy_execution_api.py b/tests/integration/test_proxy_execution_api.py index 68909d9d9..24ca106e8 100644 --- a/tests/integration/test_proxy_execution_api.py +++ b/tests/integration/test_proxy_execution_api.py @@ -11,7 +11,7 @@ import alicebot_api.main as main_module from alicebot_api.routers import legacy_gated as legacy_gated_router from alicebot_api.config import Settings -from alicebot_api.db import set_current_user, user_connection +from alicebot_api.db import user_connection from alicebot_api.store import ContinuityStore @@ -167,13 +167,11 @@ def create_execution_budget( def set_execution_executed_at( admin_database_url: str, - user_id: UUID, *, execution_id: UUID, executed_at_sql: str, ) -> None: with psycopg.connect(admin_database_url) as conn: - set_current_user(conn, user_id) conn.execute( f"UPDATE tool_executions SET executed_at = {executed_at_sql} WHERE id = %s", (execution_id,), @@ -183,13 +181,11 @@ def set_execution_executed_at( def set_approval_request_thread_id( admin_database_url: str, - user_id: UUID, *, approval_id: UUID, request_thread_id: str, ) -> None: with psycopg.connect(admin_database_url) as conn: - set_current_user(conn, user_id) conn.execute( """ UPDATE approvals @@ -855,7 +851,6 @@ def test_execute_approved_proxy_endpoint_fail_closes_when_runtime_context_is_inv set_approval_request_thread_id( migrated_database_urls["admin"], - user_id=owner["user_id"], approval_id=UUID(create_payload["approval"]["id"]), request_thread_id="not-a-uuid", ) @@ -1322,7 +1317,6 @@ def test_execute_approved_proxy_endpoint_excludes_old_window_history_and_keeps_c set_execution_executed_at( migrated_database_urls["admin"], - user_id=owner["user_id"], execution_id=owner_first_execution_id, executed_at_sql="clock_timestamp() - interval '2 hours'", ) diff --git a/tests/integration/test_scheduler_handoff_identity_postgres.py b/tests/integration/test_scheduler_handoff_identity_postgres.py new file mode 100644 index 000000000..df1ad49f6 --- /dev/null +++ b/tests/integration/test_scheduler_handoff_identity_postgres.py @@ -0,0 +1,45 @@ +"""Every scheduler workflow keeps a bound key's input admission.""" +import json + +import pytest + +from tests.integration.test_derived_labels_producers_postgres import seed_grid, wire_database +from tests.integration.test_memory_mutations_api import invoke_request + + +@pytest.mark.parametrize("workflow", ["daily_brief", "weekly_synthesis", "connection_report", "contradiction_report", "memory_consolidation", "staleness_sweep", "open_loop_review", "project_update_scan"]) +@pytest.mark.parametrize("scoped", [True, False]) +def test_scheduler_never_drops_the_bound_identity(migrated_database_urls, monkeypatch, workflow, scoped): + app_url = migrated_database_urls["app"] + wire_database(monkeypatch, app_url) + user, alpha, beta, rows, key, _, _ = seed_grid(app_url) + status, body = invoke_request("POST", f"/v0/vnext/scheduler/workflows/{workflow}/run-now", + payload={"user_id": str(user), "scope": {"projects": [alpha]} if scoped else {}, + "options": {"generated_for": "2026-10-05", "reference_time": "2026-10-05T12:00:00Z", "source_limit": 50, "memory_limit": 50, "max_items": 50}}, + headers={"authorization": f"Bearer {key}"}) + assert status == 201, body + text = json.dumps(body, default=str) + for label, _, row in rows: + if label != "alpha": + assert str(row["id"]) not in text + assert f"SENTINEL_{label.upper()}" not in text + + +@pytest.mark.parametrize("workflow", ["connection_report", "contradiction_report"]) +def test_core_mcp_automation_keeps_the_bound_identity(migrated_database_urls, monkeypatch, workflow): + from alicebot_api.mcp.registry import call_mcp_tool + from alicebot_api.mcp.types import MCPRuntimeContext + url = migrated_database_urls["app"] + wire_database(monkeypatch, url) + user, alpha, beta, rows, key, _, _ = seed_grid(url) + monkeypatch.setenv("ALICE_AGENT_API_KEY", key) + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + context = MCPRuntimeContext(database_url=url, user_id=user) + response = call_mcp_tool(context, name="alice_vnext_generate_artifact", arguments={ + "workflow_type": workflow, "query": "Atlas", + "project_scope": [alpha]}) + text = json.dumps(response, default=str) + for label, _, row in rows: + if label != "alpha": + assert str(row["id"]) not in text + assert f"SENTINEL_{label.upper()}" not in text diff --git a/tests/integration/test_task_artifacts_api.py b/tests/integration/test_task_artifacts_api.py index fdf78c113..e5092e9e0 100644 --- a/tests/integration/test_task_artifacts_api.py +++ b/tests/integration/test_task_artifacts_api.py @@ -18,7 +18,7 @@ from alicebot_api.routers import memories_legacy as memories_legacy_router from alicebot_api.config import Settings from alicebot_api.artifacts import TASK_ARTIFACT_CHUNK_RETRIEVAL_MATCHING_RULE -from alicebot_api.db import set_current_user, user_connection +from alicebot_api.db import user_connection from alicebot_api.store import ContinuityStore @@ -1742,7 +1742,6 @@ def test_task_artifact_ingestion_enforces_rooted_workspace_paths( assert register_status == 201 with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -1822,7 +1821,6 @@ def test_task_artifact_docx_ingestion_enforces_rooted_workspace_paths( assert register_status == 201 with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ @@ -1902,7 +1900,6 @@ def test_task_artifact_rfc822_ingestion_enforces_rooted_workspace_paths( assert register_status == 201 with psycopg.connect(migrated_database_urls["admin"]) as conn: - set_current_user(conn, owner["user_id"]) with conn.cursor() as cur: cur.execute( """ diff --git a/tests/integration/test_temporal_state_api.py b/tests/integration/test_temporal_state_api.py index fd81b2c3a..0a7bc76f0 100644 --- a/tests/integration/test_temporal_state_api.py +++ b/tests/integration/test_temporal_state_api.py @@ -14,7 +14,7 @@ from alicebot_api.routers import continuity as continuity_router from alicebot_api.config import Settings from alicebot_api.contracts import MemoryCandidateInput -from alicebot_api.db import set_current_user, user_connection +from alicebot_api.db import user_connection from alicebot_api.memory import admit_memory_candidate from alicebot_api.store import ContinuityStore @@ -69,15 +69,13 @@ async def send(message: dict[str, object]) -> None: def _set_temporal_timestamps( admin_database_url: str, - user_id: UUID, *, entity_id: UUID, edge_id: UUID, entity_created_at: datetime, edge_created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url) as conn: - set_current_user(conn, user_id) + with psycopg.connect(admin_database_url, autocommit=True) as conn: with conn.cursor() as cur: cur.execute( "UPDATE entities SET created_at = %s WHERE id = %s", @@ -152,7 +150,6 @@ def seed_temporal_entity_graph(database_url: str, admin_database_url: str) -> di midpoint = add_at + (update_at - add_at) / 2 _set_temporal_timestamps( admin_database_url, - user_id=user_id, entity_id=entity["id"], edge_id=edge["id"], entity_created_at=add_at - timedelta(seconds=1), diff --git a/tests/integration/test_temporal_state_mcp_cli.py b/tests/integration/test_temporal_state_mcp_cli.py index 1e434a508..5550497f4 100644 --- a/tests/integration/test_temporal_state_mcp_cli.py +++ b/tests/integration/test_temporal_state_mcp_cli.py @@ -12,7 +12,7 @@ import psycopg from alicebot_api.contracts import MemoryCandidateInput -from alicebot_api.db import set_current_user, user_connection +from alicebot_api.db import user_connection from alicebot_api.memory import admit_memory_candidate from alicebot_api.store import ContinuityStore @@ -137,15 +137,13 @@ def _call_tool(client: MCPClient, *, name: str, arguments: dict[str, object]) -> def _set_temporal_timestamps( admin_database_url: str, - user_id: UUID, *, entity_id: UUID, edge_id: UUID, entity_created_at: datetime, edge_created_at: datetime, ) -> None: - with psycopg.connect(admin_database_url) as conn: - set_current_user(conn, user_id) + with psycopg.connect(admin_database_url, autocommit=True) as conn: with conn.cursor() as cur: cur.execute("UPDATE entities SET created_at = %s WHERE id = %s", (entity_created_at, entity_id)) cur.execute( @@ -216,7 +214,6 @@ def seed_temporal_entity_graph(database_url: str, admin_database_url: str) -> di midpoint = add_at + (update_at - add_at) / 2 _set_temporal_timestamps( admin_database_url, - user_id=user_id, entity_id=entity["id"], edge_id=edge["id"], entity_created_at=add_at - timedelta(seconds=1), diff --git a/tests/integration/test_vnext_consolidation_postgres.py b/tests/integration/test_vnext_consolidation_postgres.py index 1b125b2aa..73605f264 100644 --- a/tests/integration/test_vnext_consolidation_postgres.py +++ b/tests/integration/test_vnext_consolidation_postgres.py @@ -95,7 +95,7 @@ def test_rollup_candidate_round_trips_and_acceptance_promotes_it( ) artifact = VNextConsolidationService(store).generate_memory_consolidation( - MemoryConsolidationRequest() + MemoryConsolidationRequest(agent_identity=None, ) ) rollups = artifact["metadata_json"]["rollups"] assert rollups["enabled"] is True diff --git a/tests/integration/test_vnext_live_workspace_api.py b/tests/integration/test_vnext_live_workspace_api.py index 73653f62d..106a35193 100644 --- a/tests/integration/test_vnext_live_workspace_api.py +++ b/tests/integration/test_vnext_live_workspace_api.py @@ -310,7 +310,7 @@ def test_project_update_true_redaction_scrubs_the_role_separated_coupled_graph( ) service = VNextProjectService(store) candidate = service.generate_project_update_candidate( - ProjectAutomationRequest( + ProjectAutomationRequest(agent_identity=None, project_id=project_id, domains=("project",), metadata_json={"redaction_test_secret": sentinel}, @@ -814,7 +814,7 @@ def test_project_update_terminal_replay_survives_authorized_true_redaction( ) service = VNextProjectService(store) candidate = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id=project_id, domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id=project_id, domains=("project",)) ) artifact_id = str(candidate["id"]) reviewed = service.review_project_update(artifact_id=artifact_id, action=action) @@ -1036,7 +1036,7 @@ def test_project_update_terminal_replay_rejects_competing_postgres_decision_with ) service = VNextProjectService(store) candidate = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id=project_id, domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id=project_id, domains=("project",)) ) artifact_id = str(candidate["id"]) reviewed = service.review_project_update(artifact_id=artifact_id, action=action) diff --git a/tests/performance/read_budget_probe.py b/tests/performance/read_budget_probe.py new file mode 100644 index 000000000..afebad0ba --- /dev/null +++ b/tests/performance/read_budget_probe.py @@ -0,0 +1,62 @@ +"""Measure native read paths on the same synthetic population.""" +import json, os, sys, time, statistics, inspect +from pathlib import Path +repo, backend, location, user = sys.argv[1:5] +sys.path[:0] = [str(Path(repo)/"apps/api/src"), repo] +from contextlib import contextmanager +from uuid import UUID +from alicebot_api.db import user_connection +from alicebot_api.sqlite_store import sqlite_user_connection, SQLiteVNextStore +from alicebot_api.vnext_store import PostgresVNextStore +from alicebot_api.vnext_retrieval import VNextRetrievalService, VNextRetrievalRequest +from alicebot_api.vnext_agent_control import AgentIdentity +from alicebot_api.vnext_source_fence import SourceReadFence +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext +@contextmanager +def store_context(): + if backend == "sqlite": + with sqlite_user_connection(location, user) as conn: + yield SQLiteVNextStore(conn, user) + else: + with user_connection(location, UUID(user)) as conn: + yield PostgresVNextStore(conn) +identity = AgentIdentity(agent_id="budget", permission_profile="trusted_local_agent") +ceiling = ("public", "internal", "private", "unknown") +def pack(): + with store_context() as store: + return VNextRetrievalService(store, embedding_provider=None, reranker_provider=None).compile_context_pack( + VNextRetrievalRequest(query="synthetic budget observation", sensitivity_allowed=ceiling), + source_fence=SourceReadFence.for_identity(identity)) +def recall(): + return call_mcp_tool(MCPRuntimeContext(database_url="sqlite:///"+location if backend=="sqlite" else location, user_id=UUID(user)), + name="alice_recall", arguments={"query":"synthetic budget observation", "agent_id":"budget", + "permission_profile":"trusted_local_agent"}) +actions = {"pack":pack, "recall":recall} +if backend == "postgres": + from alicebot_api.routers.workspaces import _vnext_workspace_payload + from alicebot_api.vnext_dogfooding import VNextDogfoodingService + def workspace(): + with store_context() as store: + kwargs = {"identity":identity} if "identity" in inspect.signature(_vnext_workspace_payload).parameters else {} + return _vnext_workspace_payload(store, **kwargs) + def dogfooding(): + with store_context() as store: + service=VNextDogfoodingService(store) + kwargs={"sensitivity_allowed":ceiling} if "sensitivity_allowed" in inspect.signature(service.dashboard).parameters else {} + return service.dashboard(**kwargs) + actions.update(workspace=workspace,dogfooding=dogfooding) +times={} +for name, action in actions.items(): + if name == "pack" and os.environ.get("ALICE_BUDGET_PROFILE_PATH"): + import cProfile + profile = cProfile.Profile() + profile.runcall(action) + profile.dump_stats(os.environ["ALICE_BUDGET_PROFILE_PATH"] + "." + Path(repo).name) + else: + action() + samples=[] + for _ in range(5): + start=time.perf_counter(); action(); samples.append(time.perf_counter()-start) + times[name]={"median":statistics.median(samples), "samples":samples} +print(json.dumps(times)) diff --git a/tests/performance/test_label_read_handoff_budget.py b/tests/performance/test_label_read_handoff_budget.py new file mode 100644 index 000000000..7be0ad391 --- /dev/null +++ b/tests/performance/test_label_read_handoff_budget.py @@ -0,0 +1,80 @@ +"""Read budgets use 5,000 copies and preserve complete effective admission.""" +import json +import time +import os +from pathlib import Path +from uuid import uuid4 + +from alicebot_api.onramp import bootstrap_database +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_agent_control import AgentIdentity +from alicebot_api.vnext_label_guard import LabelGuard, label_read_request +from alicebot_api.vnext_retrieval import VNextRetrievalRequest, VNextRetrievalService +from alicebot_api.vnext_source_fence import SourceReadFence + +USER = "11111111-1111-4111-8111-111111111111" +CEILING = ("public", "internal", "private", "unknown") + + +def seed_copies(store, count=5000, *, postgres=False): + source = store.create_source({"source_type": "note", "title": "Private input", "content_hash": str(uuid4()), + "domain": "project", "sensitivity": "confidential"}) + meta = json.dumps({"source_id": str(source["id"])}) + ids = [str(uuid4()) for _ in range(count)] + values = [(row_id, USER if not postgres else str(source["user_id"]), "copy." + row_id, "synthetic budget observation", meta) for row_id in ids] + if postgres: + sql = "INSERT INTO memories(id,user_id,memory_key,canonical_text,metadata_json,value,source_event_ids,status,domain,sensitivity) VALUES(%s::uuid,%s::uuid,%s,%s,%s::jsonb,'{}','{}','active','project','public')" + with store.conn.cursor() as cur: + cur.executemany(sql, values) + else: + store.conn.executemany("INSERT INTO memories(id,user_id,memory_key,canonical_text,metadata_json,value,source_event_ids,status,domain,sensitivity) VALUES(?,?,?,?,?,'{}','[]','active','project','public')", values) + events = [(str(uuid4()), str(source["user_id"]), row_id) for row_id in ids] + if postgres: + with store.conn.cursor() as cur: + cur.executemany("INSERT INTO event_log(id,user_id,target_id,event_type,actor_type,target_type,payload_json) VALUES(%s::uuid,%s::uuid,%s::uuid,'memory.created','system','memory','{}')", events) + else: + store.conn.executemany("INSERT INTO event_log(id,user_id,target_id,event_type,actor_type,target_type,payload_json) VALUES(?,?,?,'memory.created','system','memory','{}')", events) + return source, ids + + +def run_pack(store): + service = VNextRetrievalService(store, embedding_provider=None, reranker_provider=None) + identity = AgentIdentity(agent_id="budget", permission_profile="trusted_local_agent") + return service.compile_context_pack(VNextRetrievalRequest(query="synthetic budget observation", sensitivity_allowed=CEILING), + source_fence=SourceReadFence.for_identity(identity)) + + +@label_read_request +def count_twice(store): + first = LabelGuard.for_filters(store, (), CEILING).readable_status_counts("memory") + second = LabelGuard.for_filters(store, (), CEILING).readable_status_counts("memory") + assert first == second == {} + + +def test_sqlite_five_thousand_derived_rows_budget(tmp_path): + path = tmp_path / "vault.db" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.invalid") + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source, ids = seed_copies(store) + started = time.perf_counter() + count_twice(store) + counted = time.perf_counter() - started + assert counted <= 1.0, counted + started = time.perf_counter() + pack = run_pack(store) + elapsed = time.perf_counter() - started + assert not any(row_id in str(pack) for row_id in ids) + print(f"SQLite 5000: counted={counted:.4f}s pack={elapsed:.4f}s") + assert elapsed <= 0.15, elapsed + from tests.integration.test_label_read_handoff_budget_postgres import probe + head = probe(Path(__file__).resolve().parents[2], "sqlite", path, USER) + main = os.environ.get("ALICE_READ_MAIN_CHECKOUT") + if main: + baseline = probe(main, "sqlite", path, USER) + print("SQLite head=" + json.dumps(head) + " main=" + json.dumps(baseline)) + for action in ("pack", "recall"): + assert head[action]["median"] <= 2 * baseline[action]["median"] + .1, (action, head, baseline) + else: + assert head["pack"]["median"] <= .15, head + assert head["recall"]["median"] <= .15, head diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index db8f918a7..d5c612003 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -556,7 +556,7 @@ def get_memory_for_update(self, memory_id: str) -> dict[str, object] | None: def get_memory_for_redaction(self, memory_id: str) -> dict[str, object] | None: assert self.graph_locked - assert self.labels_exclusive + assert "shared_labels" in self.lock_calls or "exclusive_labels" in self.lock_calls self.lock_calls.append("redaction_row") return self.get_memory(memory_id) @@ -2552,7 +2552,7 @@ def fake_vnext_store_context(_ctx): ) first = json.loads(cli_module._run_vnext_memory_redact(context, args)) - assert store.lock_calls[:3] == ["graph", "exclusive_labels", "redaction_row"] + assert store.lock_calls[:3] == ["graph", "shared_labels", "redaction_row"] assert store.conn.lock_timeout == "0" assert first["status"] == "redacted" assert first["forgotten_first"] is True @@ -2561,7 +2561,7 @@ def fake_vnext_store_context(_ctx): previous_lock_count = len(store.lock_calls) second = json.loads(cli_module._run_vnext_memory_redact(context, args)) - assert store.lock_calls[previous_lock_count:][:3] == ["graph", "exclusive_labels", "redaction_row"] + assert store.lock_calls[previous_lock_count:][:3] == ["graph", "shared_labels", "redaction_row"] assert store.conn.lock_timeout == "0" assert second["status"] == "redacted" assert second["forgotten_first"] is False @@ -2701,7 +2701,7 @@ def _cli_project_update_review_fixture() -> tuple[FakeVNextCliStore, dict[str, o } ) artifact = cli_module.VNextProjectService(store).generate_project_update_candidate( - cli_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + cli_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) return store, artifact @@ -2777,7 +2777,7 @@ def test_cli_generic_memory_mutations_cannot_strand_pending_project_update_candi assert (store.projects, store.memories, store.artifacts, store.revisions) == state_before assert [event.get("event_type") for event in store.events] == event_types_before if operation == "redact": - assert store.lock_calls[-3:] == ["graph", "exclusive_labels", "redaction_row"] + assert store.lock_calls[-3:] == ["graph", "shared_labels", "redaction_row"] assert store.conn.lock_timeout == "0" @@ -2815,7 +2815,7 @@ def _apply_supported_cli_memory_lifecycle( def _accept_later_cli_project_update(store: FakeVNextCliStore, *, first_artifact_id: str) -> None: service = cli_module.VNextProjectService(store) later = service.generate_project_update_candidate( - cli_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + cli_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) assert later["id"] != first_artifact_id service.review_project_update( diff --git a/tests/unit/test_consolidation_report_label.py b/tests/unit/test_consolidation_report_label.py index 6bee7bb46..c74972f06 100644 --- a/tests/unit/test_consolidation_report_label.py +++ b/tests/unit/test_consolidation_report_label.py @@ -43,6 +43,7 @@ def _no_embedding_provider(monkeypatch): def _run(store, **request) -> dict: + request.setdefault("agent_identity", None) request.setdefault("sensitivity_allowed", EVERYTHING) return VNextConsolidationService(store, embedding_provider=None).generate_memory_consolidation( MemoryConsolidationRequest(**request) @@ -101,7 +102,7 @@ def test_cluster_rows_and_roll_up_rows_are_labelled_together() -> None: row["sensitivity"] = cluster_sensitivity if str(row["id"]) in cluster_ids else rollup_sensitivity artifact = VNextConsolidationService( store, embedding_provider=MappedEmbeddingProvider(mapping) - ).generate_memory_consolidation(MemoryConsolidationRequest(sensitivity_allowed=EVERYTHING)) + ).generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, sensitivity_allowed=EVERYTHING)) assert artifact["metadata_json"]["consolidation"]["cluster_membership"], "the run must have a cluster" assert len(artifact["metadata_json"]["rollups"]["proposals"]) == 1, "and a roll-up card" @@ -234,7 +235,7 @@ def test_a_row_the_cluster_and_a_roll_up_line_both_name_counts_once_for_the_doma row["domain"] = "health" if str(row["id"]) in cluster_ids else "legal" artifact = VNextConsolidationService( store, embedding_provider=MappedEmbeddingProvider(mapping) - ).generate_memory_consolidation(MemoryConsolidationRequest(sensitivity_allowed=EVERYTHING)) + ).generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, sensitivity_allowed=EVERYTHING)) assert artifact["metadata_json"]["consolidation"]["cluster_membership"] assert any( diff --git a/tests/unit/test_consolidation_report_names_sources.py b/tests/unit/test_consolidation_report_names_sources.py index 9d8d09848..9830f3523 100644 --- a/tests/unit/test_consolidation_report_names_sources.py +++ b/tests/unit/test_consolidation_report_names_sources.py @@ -92,6 +92,7 @@ def _source(store, **fields) -> dict: def _run(store, mapping, **request) -> dict: + request.setdefault("agent_identity", None) request.setdefault("sensitivity_allowed", EVERYTHING) return VNextConsolidationService( store, embedding_provider=MappedEmbeddingProvider(mapping) @@ -446,7 +447,7 @@ def test_a_source_that_shares_a_memory_id_never_stands_in_for_the_memory_label(t "metadata_json": {"source_refs": [f"source:{source_id}"]}, }) report = VNextConsolidationService(store, embedding_provider=MappedEmbeddingProvider(mapping)).generate_memory_consolidation( - MemoryConsolidationRequest(sensitivity_allowed=list(ALL_SENSITIVITY), propose_rollups=False, + MemoryConsolidationRequest(agent_identity=None, sensitivity_allowed=list(ALL_SENSITIVITY), propose_rollups=False, create_candidate_memories=False)) assert source_id in json.dumps(report["metadata_json"]) and protected["canonical_text"] in report["content_markdown"] assert (report["domain"], report["sensitivity"]) == ("project", "confidential") diff --git a/tests/unit/test_daily_brief_project_id.py b/tests/unit/test_daily_brief_project_id.py index 451404160..632c3a8cc 100644 --- a/tests/unit/test_daily_brief_project_id.py +++ b/tests/unit/test_daily_brief_project_id.py @@ -32,13 +32,16 @@ def test_a_free_form_name_is_not_a_stored_project_id() -> None: assert _stored_open_loop_project_id(("Alice",)) is None assert _stored_open_loop_project_id(("prj_0123456789abcdef",)) is None assert _stored_open_loop_project_id(("Alice", "Bob")) is None - canonical = "11111111-1111-1111-1111-111111111111" + assert _stored_open_loop_project_id(()) is None + canonical = "abcdef01-2345-6789-abcd-ef0123456789" + assert _stored_open_loop_project_id((canonical, "abcdef02-2345-6789-abcd-ef0123456789")) is None assert _stored_open_loop_project_id((canonical.upper(),)) == canonical + assert _stored_open_loop_project_id((canonical.replace("-", "").upper(),)) == canonical assert _stored_open_loop_project_id((str(UUID(canonical)),)) == canonical store = _Store() VNextBrainService(store)._create_candidate_open_loops( - BrainArtifactRequest(), + BrainArtifactRequest(agent_identity=None, ), [("publish the note", _source(["Alice"]))], workflow_digest="digest", ) diff --git a/tests/unit/test_derived_domain_fence.py b/tests/unit/test_derived_domain_fence.py index 5ae6ea97c..687533d3c 100644 --- a/tests/unit/test_derived_domain_fence.py +++ b/tests/unit/test_derived_domain_fence.py @@ -160,7 +160,7 @@ def test_consolidation_report_includes_rollup_input_domains(monkeypatch): members = _seed_game_memories(store) for index, row in enumerate(store.memories): row['domain'] = 'health' if index % 2 else 'project' - artifact = VNextConsolidationService(store, embedding_provider=None).generate_memory_consolidation(MemoryConsolidationRequest()) + artifact = VNextConsolidationService(store, embedding_provider=None).generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) assert artifact['metadata_json']['rollups']['proposals'] assert artifact['domain'] == 'health' @@ -183,7 +183,7 @@ def test_unrestricted_rollup_only_report_keeps_its_domain_and_reads_the_same(mon monkeypatch.delenv('ALICE_EMBEDDINGS_BASE_URL', raising=False) store = FakeConsolidationStore() _seed_game_memories(store) - artifact = VNextConsolidationService(store, embedding_provider=None).generate_memory_consolidation(MemoryConsolidationRequest()) + artifact = VNextConsolidationService(store, embedding_provider=None).generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) assert artifact['metadata_json']['rollups']['proposals'] # The domain is as before. The sensitivity is now the label of the internal inputs the report names. Every profile # reads `internal` exactly as it reads the `unknown` the report carried before, so no reader gains or loses it. @@ -204,7 +204,7 @@ def test_every_report_producer_retains_restricted_inputs(workflow): from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService store = _seed_store() store.sources[0]['domain'] = 'health' - artifact = getattr(VNextBrainService(store), 'generate_' + workflow)(BrainArtifactRequest(generated_for='2026-05-10')) + artifact = getattr(VNextBrainService(store), 'generate_' + workflow)(BrainArtifactRequest(agent_identity=None, generated_for='2026-05-10')) if workflow == 'weekly_synthesis': assert store.memories[-1]['domain'] == 'health' assert store.memories[-1]['metadata_json']['input_summary']['source_ids'] @@ -213,19 +213,19 @@ def test_every_report_producer_retains_restricted_inputs(workflow): from alicebot_api.vnext_connections import ConnectionFinderRequest, VNextConnectionService store = _seed_store() store.sources[0]['domain'] = 'health' - artifact = VNextConnectionService(store).generate_connection_report(ConnectionFinderRequest()) + artifact = VNextConnectionService(store).generate_connection_report(ConnectionFinderRequest(agent_identity=None, )) elif workflow == 'contradiction_report': from tests.unit.test_vnext_contradictions import _seed_store from alicebot_api.vnext_contradictions import ContradictionFinderRequest, VNextContradictionService store = _seed_store() store.sources[0]['domain'] = 'health' - artifact = VNextContradictionService(store).generate_contradiction_report(ContradictionFinderRequest()) + artifact = VNextContradictionService(store).generate_contradiction_report(ContradictionFinderRequest(agent_identity=None, )) elif workflow == 'project_update': from tests.unit.test_vnext_projects import _seed_store from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService store = _seed_store() store.sources[0]['domain'] = 'health' - artifact = VNextProjectService(store).generate_project_update_candidate(ProjectAutomationRequest(project_id='project-1')) + artifact = VNextProjectService(store).generate_project_update_candidate(ProjectAutomationRequest(agent_identity=None, project_id='project-1')) assert store.memories[artifact['metadata_json']['candidate_memory_id']]['domain'] == 'health' else: from tests.unit.test_vnext_scheduler import _staleness_store diff --git a/tests/unit/test_derived_label_handoff_kernel.py b/tests/unit/test_derived_label_handoff_kernel.py new file mode 100644 index 000000000..def4b35f9 --- /dev/null +++ b/tests/unit/test_derived_label_handoff_kernel.py @@ -0,0 +1,43 @@ +"""Regressions from the October 6 derived-label review.""" +from uuid import UUID + +import pytest + +from alicebot_api.vnext_derived_labels import MARKER_KEYS, dependency_record, is_derived, row_class, with_derived_from +from alicebot_api.vnext_label_repair import classify_stored_labels, plan_label_repairs + +SID = "abcdef01-2345-6789-abcd-ef0123456789" + + +@pytest.mark.parametrize("kind", ["memory", "open_loop", "artifact"]) +@pytest.mark.parametrize("key", ["source_id", "source_artifact_id", "artifact_id"]) +def test_uuid_objects_in_server_records_are_dependencies(kind, key): + meta = {"discovered_by": "vnext_daily_brief", "workflow": "daily_brief", "derived_from": {}, key: UUID(SID)} + row = {"id": "copy", "source_id": UUID(SID), "metadata_json": meta} + assert is_derived(kind, row) + deps, problem = dependency_record(kind, row) + assert not problem + assert ("source" if key == "source_id" else "artifact", SID) in deps + + +def test_open_loop_derived_from_is_a_sufficient_marker(): + row = {"id": "loop", "metadata_json": with_derived_from({}, {"sources": [{"id": UUID(SID)}]})} + assert is_derived("open_loop", row) + assert dependency_record("open_loop", row) == (frozenset({("source", SID)}), "") + + +@pytest.mark.parametrize("kind", ["memory", "artifact", "open_loop", "project", "source", "belief"]) +@pytest.mark.parametrize("shape", [[], {}, 12, None]) +def test_planner_is_total_over_malformed_marker_shapes(kind, shape): + for key in MARKER_KEYS: + row = {"kind": kind, "id": "row", "user_id": "user", "domain": "project", "sensitivity": "public", + "metadata_json": {key: shape}} + row_class(kind, row) + table = {"artifact": "generated_artifacts", "memory": "memories", "open_loop": "open_loops", "project": "projects", "source": "sources", "belief": "beliefs"}[kind] + plan_label_repairs({table: [row]}) + classify_stored_labels({table: [row]}) + + +def test_free_text_source_refs_are_not_database_identifiers(): + row = {"metadata_json": {"candidate_kind": "memory_consolidation", "source_refs": ["meeting-notes"]}} + assert dependency_record("memory", row) == (frozenset(), "") diff --git a/tests/unit/test_derived_labels_docs.py b/tests/unit/test_derived_labels_docs.py index 50f538065..d25be896e 100644 --- a/tests/unit/test_derived_labels_docs.py +++ b/tests/unit/test_derived_labels_docs.py @@ -87,7 +87,7 @@ def test_the_pages_name_the_v3_repair_and_the_seven_table_migration() -> None: notes = _text("docs/release/v0.20.0-release-notes.md") assert "> **Correction (2026-10-05):**" in notes assert "listed under known limitations" in notes - assert "raised once" in notes + assert "Unreleased (on main, not in v0.20.0):" in _text("docs/runbooks/disaster-recovery.md") changelog = _text("CHANGELOG.md") assert "keeps such a row at its old label until a restore" not in changelog diff --git a/tests/unit/test_entity_fence_mutations.py b/tests/unit/test_entity_fence_mutations.py index 2040c0089..88bb932e5 100644 --- a/tests/unit/test_entity_fence_mutations.py +++ b/tests/unit/test_entity_fence_mutations.py @@ -25,7 +25,7 @@ def kill(owner, name, before, after, check): original = getattr(owner, name) - function = original.fget if isinstance(original, property) else original + function = inspect.unwrap(original.fget if isinstance(original, property) else original) source = textwrap.dedent(inspect.getsource(function)) assert before in source, f'mutation no longer matches: {name}: {before}' namespace = dict(function.__globals__) diff --git a/tests/unit/test_every_report_producer_labels_what_it_prints.py b/tests/unit/test_every_report_producer_labels_what_it_prints.py index f0f71e44b..9d3095640 100644 --- a/tests/unit/test_every_report_producer_labels_what_it_prints.py +++ b/tests/unit/test_every_report_producer_labels_what_it_prints.py @@ -68,7 +68,7 @@ def rows(current) -> list[dict]: def run(current) -> dict: service = VNextBrainService(current) - request = BrainArtifactRequest(generated_for="2026-05-10", sensitivity_allowed=EVERYTHING) + request = BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", sensitivity_allowed=EVERYTHING) return getattr(service, f"generate_{workflow}")(request) return store, rows, run @@ -84,7 +84,7 @@ def _connections() -> tuple[object, Callable, Callable]: store, lambda current: [*current.sources, *current.memories], lambda current: VNextConnectionService(current).generate_connection_report( - ConnectionFinderRequest(sensitivity_allowed=EVERYTHING) + ConnectionFinderRequest(agent_identity=None, sensitivity_allowed=EVERYTHING) ), ) @@ -97,7 +97,7 @@ def _contradictions() -> tuple[object, Callable, Callable]: store, lambda current: [*current.sources, *current.memories, *current.beliefs.values()], lambda current: VNextContradictionService(current).generate_contradiction_report( - ContradictionFinderRequest(sensitivity_allowed=EVERYTHING) + ContradictionFinderRequest(agent_identity=None, sensitivity_allowed=EVERYTHING) ), ) @@ -110,7 +110,7 @@ def _project_update() -> tuple[object, Callable, Callable]: store, lambda current: [*current.projects.values(), *current.sources, *current.memories.values()], lambda current: VNextProjectService(current).generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", sensitivity_allowed=EVERYTHING) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", sensitivity_allowed=EVERYTHING) ), ) diff --git a/tests/unit/test_expired_memories_everywhere.py b/tests/unit/test_expired_memories_everywhere.py index dd13430ea..088e9f092 100644 --- a/tests/unit/test_expired_memories_everywhere.py +++ b/tests/unit/test_expired_memories_everywhere.py @@ -851,7 +851,7 @@ def test_consolidation_sends_the_text_of_open_memories_only_and_counts_only_them _seed_embedded_then_expire(store, ("e", "f")) seeded = len(server.texts) artifact = VNextConsolidationService(_ArtifactShim(store)).generate_memory_consolidation( # type: ignore[arg-type] - MemoryConsolidationRequest(metadata_json={"consolidation_options": {"max_embedded_memories": 4}}) + MemoryConsolidationRequest(agent_identity=None, metadata_json={"consolidation_options": {"max_embedded_memories": 4}}) ) assert _received(server, seeded) == ["a", "b", "c", "d"] counts = artifact["metadata_json"]["input_counts"] @@ -886,7 +886,7 @@ def count_memories(self, *, status=None, domains=None, sensitivity_allowed=None, seeded = len(server.texts) adapter = OldAdapter(store) assert "include_expired" not in inspect.signature(adapter.list_memories).parameters - VNextConsolidationService(adapter).generate_memory_consolidation(MemoryConsolidationRequest()) # type: ignore[arg-type] + VNextConsolidationService(adapter).generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) # type: ignore[arg-type] assert _received(server, seeded) == ["a", "b", "c", "d"] diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py index 6aebf88d8..5d285b3fd 100644 --- a/tests/unit/test_label_door_registry.py +++ b/tests/unit/test_label_door_registry.py @@ -119,6 +119,7 @@ } NOT_A_DOOR = { + "vnext_artifact_review.py:lock_artifact_review_labels": "write lock classification only; the adapter authorizes before the dispatcher mutates", "routers/vnext_memories.py:regenerate_vnext_source": "operator-only regeneration rejects every profile except owner and unbound admin before the source lookup; real-profile rejection tests pin this gate", "mcp/evidence_artifacts.py:_handle_alice_vnext_review_items": "legacy review list has no policy check", "vnext_projects.py:VNextProjectService.review_project_update": "write path; the route authorizes before this mutation", diff --git a/tests/unit/test_label_reader_stage_matrix.py b/tests/unit/test_label_reader_stage_matrix.py index d695cdea6..59ee6ba55 100644 --- a/tests/unit/test_label_reader_stage_matrix.py +++ b/tests/unit/test_label_reader_stage_matrix.py @@ -93,7 +93,7 @@ def _stage(stage, store): tree = VNextContextTreeService(store).build_tree(ContextTreeRequest(domains=("project",), sensitivity_allowed=tuple(CEILING))) return next(root["children"] for root in tree["roots"] if root["id"] == "root:" + stage.removeprefix("context_")) if stage == "project_resolution": - try: return [VNextProjectService(store)._resolve_project(ProjectAutomationRequest(domains=("project",), sensitivity_allowed=tuple(CEILING)))] + try: return [VNextProjectService(store)._resolve_project(ProjectAutomationRequest(agent_identity=None, domains=("project",), sensitivity_allowed=tuple(CEILING)))] except VNextProjectValidationError: return [] if stage == "dashboard_lists": result = VNextProjectService(store).project_dashboard(project_id=ALPHA, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent")) diff --git a/tests/unit/test_label_repair_handoff_atomicity.py b/tests/unit/test_label_repair_handoff_atomicity.py new file mode 100644 index 000000000..249047527 --- /dev/null +++ b/tests/unit/test_label_repair_handoff_atomicity.py @@ -0,0 +1,67 @@ +"""Exercise repair rollback through the actual SQLite open wrapper.""" +import json +import sqlite3 + +from alicebot_api import vnext_label_repair as repair +from alicebot_api.sqlite_store import sqlite_user_connection +from alicebot_api.vnext_label_repair import REPAIR_STATE_KEY, plan_label_repairs +from tests.unit.test_sqlite_label_repair_v3 import _vault_with_a_public_copy, USER + + +def snapshot(path): + with sqlite3.connect(path) as conn: + return (conn.execute("SELECT id,domain,sensitivity,metadata_json FROM memories ORDER BY id").fetchall(), + conn.execute("SELECT id FROM event_log WHERE event_type LIKE '%.labels_raised'").fetchall(), + conn.execute("SELECT value FROM alice_schema_state WHERE key=?", (REPAIR_STATE_KEY,)).fetchall()) + + +def test_failed_real_open_rolls_back_rows_events_and_stamp(tmp_path, monkeypatch): + path = tmp_path / "vault.db" + _vault_with_a_public_copy(path, monkeypatch) + before = snapshot(path) + original = repair.require_changed + calls = 0 + + def interrupt(changed, table, row_id): + nonlocal calls + original(changed, table, row_id) + calls += 1 + raise RuntimeError("synthetic interruption after the first label update") + + with monkeypatch.context() as patch: + patch.setattr(repair, "require_changed", interrupt) + with sqlite_user_connection(path, USER): + pass + assert calls == 1 + assert snapshot(path) == before + with sqlite_user_connection(path, USER): + pass + rows, events, stamp = snapshot(path) + assert rows[0][2] == "confidential" + assert len(events) == 1 + assert stamp + + +def test_unverified_rows_are_not_in_the_repair_plan(): + rows = [{"id": "dangling", "user_id": "user", "domain": "project", "sensitivity": "public", + "metadata_json": {"source_id": "abcdef01-2345-6789-abcd-ef0123456789"}}, + {"id": "incomplete", "user_id": "user", "domain": "project", "sensitivity": "public", + "metadata_json": {"candidate_kind": "memory_rollup"}}] + assert plan_label_repairs({"memories": rows}) == [] + + +def test_malformed_marker_restore_succeeds_without_repair(tmp_path, monkeypatch, capsys): + from alicebot_api.onramp import main + + path = tmp_path / "vault.db" + memory_id = _vault_with_a_public_copy(path, monkeypatch) + with sqlite3.connect(path) as conn: + conn.execute("UPDATE memories SET metadata_json=? WHERE id=?", (json.dumps({"candidate_kind": ["memory_rollup"]}), memory_id)) + exported = tmp_path / "backup.jsonl" + assert main(["export", "--db", str(path), "--user-id", USER, "--out", str(exported)]) == 0 + target = tmp_path / "restored.db" + assert main(["import", "--db", str(target), "--user-id", USER, "--in", str(exported)]) == 0, capsys.readouterr() + with sqlite_user_connection(target, USER) as conn: + tables = repair._load_tables(conn) + assert plan_label_repairs(tables) == [] + assert repair.classify_stored_labels(tables)[1] == {"malformed_marker": [memory_id]} diff --git a/tests/unit/test_list_door_inputs.py b/tests/unit/test_list_door_inputs.py index ab8366d94..470b0d03b 100644 --- a/tests/unit/test_list_door_inputs.py +++ b/tests/unit/test_list_door_inputs.py @@ -56,7 +56,7 @@ def read_label_rows(kind: str, ids: list[str]) -> list[dict[str, object]]: store.read_label_rows = read_label_rows # type: ignore[attr-defined] artifact = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest(generated_for="2026-05-10", domains=("project",)) + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", domains=("project",)) ) assert "SENTINEL confidential fact" not in artifact["content_markdown"] derived = artifact["metadata_json"]["derived_from"] diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 7899762b3..93352efbc 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5325,16 +5325,16 @@ class FakeStore: def lock_label_writes(self, *, exclusive: bool = False) -> None: assert transaction_depth == 1 - assert exclusive is True + assert exclusive is False assert lock_calls == ["graph"] - lock_calls.append("exclusive_labels") + lock_calls.append("shared_labels") def get_memory(self, _memory_id: str): return memory def get_memory_for_update(self, _memory_id: str): assert transaction_depth == 1 - assert lock_calls == ["graph", "exclusive_labels"] + assert lock_calls == ["graph", "shared_labels"] lock_calls.append("row") return memory @@ -5389,7 +5389,7 @@ def fake_persist(**kwargs) -> None: assert response.status_code == 200 assert calls == ["refresh", "embedding"] - assert lock_calls == ["graph", "exclusive_labels", "row"] + assert lock_calls == ["graph", "shared_labels", "row"] def test_vnext_consolidation_defers_embedding_until_primary_transaction_closes(monkeypatch) -> None: diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index f73db46df..a1db5d432 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -3647,7 +3647,7 @@ def _seed_pending_project_update_mcp_candidate( classifier: str, ) -> tuple[str, str]: artifact = mcp_tools_module.VNextProjectService(store).generate_project_update_candidate( - mcp_tools_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + mcp_tools_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_id = str(artifact["id"]) artifact_metadata = artifact["metadata_json"] @@ -3929,7 +3929,7 @@ def test_vnext_artifact_review_locks_and_authorizes_persisted_scope(monkeypatch, def test_generic_mcp_artifact_review_preserves_applied_project_update_state(monkeypatch, legacy_tools_enabled) -> None: store = FakeVNextMCPStore() artifact = mcp_tools_module.VNextProjectService(store).generate_project_update_candidate( - mcp_tools_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + mcp_tools_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_id = str(artifact["id"]) metadata = artifact["metadata_json"] @@ -3997,7 +3997,7 @@ def _apply_supported_mcp_memory_lifecycle( def _accept_later_mcp_project_update(store: FakeVNextMCPStore, *, first_artifact_id: str) -> None: service = mcp_tools_module.VNextProjectService(store) later = service.generate_project_update_candidate( - mcp_tools_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + mcp_tools_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) assert later["id"] != first_artifact_id service.review_project_update( @@ -4114,7 +4114,7 @@ def test_mcp_project_update_review_rejects_forced_terminal_status_without_mutati ) -> None: store = FakeVNextMCPStore() artifact = mcp_tools_module.VNextProjectService(store).generate_project_update_candidate( - mcp_tools_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + mcp_tools_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact["status"] = forced_status artifact_id = str(artifact["id"]) @@ -4140,7 +4140,7 @@ def test_mcp_project_update_review_rejects_terminal_clone_after_true_redaction_w ) -> None: store = FakeVNextMCPStore() artifact = mcp_tools_module.VNextProjectService(store).generate_project_update_candidate( - mcp_tools_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + mcp_tools_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_id = str(artifact["id"]) _patch_vnext_store(monkeypatch, store) @@ -4174,7 +4174,7 @@ def test_mcp_project_update_review_keeps_consistent_terminal_outcomes_idempotent ) -> None: store = FakeVNextMCPStore() artifact = mcp_tools_module.VNextProjectService(store).generate_project_update_candidate( - mcp_tools_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + mcp_tools_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_id = str(artifact["id"]) _patch_vnext_store(monkeypatch, store) @@ -4207,7 +4207,7 @@ def test_mcp_project_update_terminal_replay_rejects_every_coupled_competing_deci ) -> None: store = FakeVNextMCPStore() artifact = mcp_tools_module.VNextProjectService(store).generate_project_update_candidate( - mcp_tools_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + mcp_tools_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_id = str(artifact["id"]) _patch_vnext_store(monkeypatch, store) @@ -4234,7 +4234,7 @@ def test_mcp_accepted_project_update_replay_survives_supported_memory_lifecycle( ) -> None: store = FakeVNextMCPStore() artifact = mcp_tools_module.VNextProjectService(store).generate_project_update_candidate( - mcp_tools_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + mcp_tools_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_id = str(artifact["id"]) _patch_vnext_store(monkeypatch, store) @@ -4258,7 +4258,7 @@ def test_mcp_accepted_project_update_replay_preserves_a_genuine_later_project_up ) -> None: store = FakeVNextMCPStore() artifact = mcp_tools_module.VNextProjectService(store).generate_project_update_candidate( - mcp_tools_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + mcp_tools_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_id = str(artifact["id"]) _patch_vnext_store(monkeypatch, store) @@ -4278,7 +4278,7 @@ def test_mcp_accepted_project_update_replay_preserves_a_genuine_later_project_up def test_dedicated_mcp_project_review_schema_attributes_payload_agent(monkeypatch, legacy_tools_enabled) -> None: store = FakeVNextMCPStore() artifact = mcp_tools_module.VNextProjectService(store).generate_project_update_candidate( - mcp_tools_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + mcp_tools_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_id = str(artifact["id"]) _patch_vnext_store(monkeypatch, store) @@ -4323,7 +4323,7 @@ def test_dedicated_mcp_project_review_schema_attributes_payload_agent(monkeypatc def test_generic_memory_rejection_is_blocked_before_project_artifact_review(monkeypatch, legacy_tools_enabled) -> None: store = FakeVNextMCPStore() artifact = mcp_tools_module.VNextProjectService(store).generate_project_update_candidate( - mcp_tools_module.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + mcp_tools_module.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_id = str(artifact["id"]) artifact_metadata = artifact["metadata_json"] diff --git a/tests/unit/test_open_loop_handoff_spellings.py b/tests/unit/test_open_loop_handoff_spellings.py new file mode 100644 index 000000000..5855aac6d --- /dev/null +++ b/tests/unit/test_open_loop_handoff_spellings.py @@ -0,0 +1,37 @@ +"""The canonical reader must find no refused id in a loop response.""" +import pytest + +from alicebot_api.vnext_source_fence import cited_source_ids +from tests.unit.test_open_loop_references_read_fence import world, vault # noqa: F401 + + +@pytest.mark.parametrize("position", ["source_ids", "id", "ref", "mapping_key", "nested_key"]) +@pytest.mark.parametrize("shape", ["arabic_list", "fullwidth_sentence", "hyphens_list", "space_zero", "tab_zero", "newline_zero"]) +def test_real_key_withholds_every_named_spelling(world, shape, position): + hidden, admitted = world.sources["beta"], world.sources["own"] + if shape.endswith("zero"): + # UUID accepts leading whitespace in place of a zero through int(hex, 16). + hidden = "0447f7bb-8123-4567-8912-abcdefabcdef" + original = world.sources["beta"] + world.vault.sql("UPDATE sources SET id = ? WHERE id = ?", (hidden, original)) + whitespace = {"space_zero": " ", "tab_zero": "\t", "newline_zero": "\n"}[shape] + value = whitespace + hidden.replace("-", "")[1:] + else: + digits = "٠١٢٣٤٥٦٧٨٩" if shape == "arabic_list" else "０１２３４５６７８９" + encoded = hidden.translate(str.maketrans("0123456789", digits)) + if shape == "hyphens_list": + encoded = hidden.replace("-", "") + encoded = encoded[:3] + "-" + encoded[3:19] + "-" + encoded[19:] + value = f"source:{encoded}, source:{admitted}" if shape != "fullwidth_sentence" else f"See source:{encoded} for details" + reference = ({value: "control"} if position == "mapping_key" else + {"source_ids": {value: "control"}} if position == "nested_key" else {position: value}) + metadata = {"project_scope": ["alpha"], **reference, "kept": "control"} + assert hidden in cited_source_ids(metadata).named + world._plant("handoff", metadata=metadata) + loop_id = world.loops["handoff"] + world.vault.sql("DELETE FROM open_loops WHERE id != ?", (loop_id,)) + owner = world.vault.wire("alice_open_loops", {"status": "all", "limit": 100}, key=None) + assert owner["payload"]["items"][0]["metadata_json"] == metadata + item = world.list_items("alpha_project")[0] + assert hidden not in cited_source_ids(item["metadata_json"]).named + assert item["metadata_json"]["kept"] == "control" diff --git a/tests/unit/test_open_loop_references_read_fence.py b/tests/unit/test_open_loop_references_read_fence.py index 7d5edb180..abcd55d62 100644 --- a/tests/unit/test_open_loop_references_read_fence.py +++ b/tests/unit/test_open_loop_references_read_fence.py @@ -1293,7 +1293,7 @@ def test_the_reference_keys_include_every_key_the_reverse_lookup_of_a_source_rea ("list_open_loops", "vnext_scheduler.py", "_generate_open_loop_review_artifact"): _FENCED, ("list_open_loops_referencing_source", "routers/_vnext_shared.py", "_vnext_load_source_trace"): _FENCED, ("project_dashboard", "cli/automation.py", "_run_vnext_project_dashboard"): _OPERATOR, - ("project_dashboard", "mcp/projects.py", "_handle_alice_project_dashboard"): _OWNER, + ("project_dashboard", "mcp/projects.py", "_handle_alice_project_dashboard"): _SENSITIVITY, ("project_dashboard", "routers/vnext_projects.py", "get_vnext_project_dashboard"): _SENSITIVITY, ("project_dashboard", "routers/workspaces.py", "_vnext_workspace_payload"): _FENCED, ("review_open_loop", "cli/automation.py", "_run_vnext_open_loop_review"): _OPERATOR, diff --git a/tests/unit/test_producer_handoff_context.py b/tests/unit/test_producer_handoff_context.py new file mode 100644 index 000000000..872408a14 --- /dev/null +++ b/tests/unit/test_producer_handoff_context.py @@ -0,0 +1,71 @@ +"""Independent controls for group floors and optional report context.""" +from copy import deepcopy +from uuid import uuid4 +import pytest + +from alicebot_api.vnext_consolidation import _ClusteringOutcome +from alicebot_api.vnext_rollups import RollupOptions, VNextRollupService +from tests.unit.test_group_scope_sqlite import seed_members +from tests.unit.test_consolidation_report_names_sources import SourceReadingStore, _run, _cluster_citing, _source + + +def test_rollup_partitions_same_scope_members_by_their_different_floors(): + store = SourceReadingStore() + first = seed_members(store) + rows = [] + for floor in ("alpha", "beta"): + for member in first: + row = deepcopy(member) + row["id"] = str(uuid4()) + row["metadata_json"].update(project_scope=[], project_floor=[floor], source_artifact_id=str(uuid4())) + rows.append(row) + groups, _, gate, _, _ = VNextRollupService(store)._group_members( + rows, options=RollupOptions(), exclude_member_id_sets=[]) + assert gate["partition_count"] == 2 + assert groups + for group in groups: + assert len({tuple(row["metadata_json"]["project_floor"]) for row in group.members}) == 1 + + +def test_locked_consolidation_context_drops_shared_events_and_ratings(monkeypatch): + store = SourceReadingStore() + store.events = [{"id": str(uuid4()), "event_type": "memory.updated", "sensitivity": "internal", + "metadata_json": {"project_scope": scope}} for scope in (["alpha"], ["alpha", "beta"])] + store.list_events = lambda **kwargs: list(store.events) + store.list_artifact_quality_ratings = lambda **kwargs: [ + {"id": str(uuid4()), "sensitivity": "internal", "metadata_json": {"project_scope": scope}} + for scope in (["alpha"], ["alpha", "beta"])] + from alicebot_api.vnext_consolidation import VNextConsolidationService + monkeypatch.setattr(VNextConsolidationService, "_cluster_memories", lambda *args, **kwargs: _ClusteringOutcome()) + artifact = _run(store, {}, propose_rollups=False, + agent_identity={"project_scope_locked": True, "project_scope": ["alpha"]}) + assert "0 artifacts, 1 events, 1 ratings" in artifact["content_markdown"] + + +@pytest.mark.parametrize("template", ["source:{}", "See {} for notes", "https://example.test/sources/{}"]) +def test_locked_consolidation_copies_only_admitted_source_references(template): + store = SourceReadingStore() + own = _source(store, metadata_json={"project_scope": ["alpha"]}) + hidden = _source(store, metadata_json={"project_scope": ["beta"]}) + mapping, members = _cluster_citing(store, lambda index: [f"source:{own['id']}", template.format(hidden['id'])]) + for row in store.memories: + row["metadata_json"]["project_scope"] = ["alpha"] + before = deepcopy(store.memories) + artifact = _run(store, mapping, propose_rollups=False, + agent_identity={"project_scope_locked": True, "project_scope": ["alpha"]}) + assert str(own["id"]) in str(artifact) + assert str(hidden["id"]) not in str(artifact) + for row in store.memories[len(before):]: + assert str(hidden["id"]) not in str(row) + assert store.memories[:len(before)] == before + + +def test_locked_consolidation_preserves_external_urls_with_unknown_incidental_ids(): + store = SourceReadingStore() + external = "https://example.test/tasks/" + str(uuid4()) + mapping, members = _cluster_citing(store, lambda index: [external]) + for row in store.memories: + row["metadata_json"]["project_scope"] = ["alpha"] + artifact = _run(store, mapping, propose_rollups=False, + agent_identity={"project_scope_locked": True, "project_scope": ["alpha"]}) + assert external in artifact["metadata_json"]["source_refs"] diff --git a/tests/unit/test_producer_handoff_refill.py b/tests/unit/test_producer_handoff_refill.py new file mode 100644 index 000000000..5b9ebd8f0 --- /dev/null +++ b/tests/unit/test_producer_handoff_refill.py @@ -0,0 +1,29 @@ +"""A scoped adapter's overlap prefix does not establish locked completeness.""" +from datetime import UTC, datetime, timedelta + +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from tests.unit.test_vnext_brain import InMemoryVNextBrainStore + + +def test_locked_brief_refills_a_crowded_adapter_prefix(): + class Store(InMemoryVNextBrainStore): + def search_sources(self, *, query, domains=None, sensitivity_allowed=None, limit=8, + scope_projects=(), scope_window_start=None, scope_window_end=None): + return self.sources[:limit] + + def list_open_loops(self, *, status="open", domains=None, sensitivity_allowed=None, limit=8, + scope_projects=(), scope_window_start=None, scope_window_end=None): + return self.open_loops[:limit] + + store = Store() + rows = [{"id": str(index), "domain": "project", "sensitivity": "public", "created_at": "2026-10-05T09:00:00Z", + "captured_at": "2026-10-05T09:00:00Z", "metadata_json": {"project_scope": ["alpha", "beta"] if index < 30 else ["alpha"]}} + for index in range(33)] + store.sources = rows + store.open_loops = rows + start = datetime(2026, 10, 5, tzinfo=UTC) + selected = VNextBrainService(store)._load_inputs(BrainArtifactRequest( + agent_identity={"project_scope_locked": True, "project_scope": ["alpha"]}, projects=("alpha",), + source_limit=5, open_loop_limit=5, generated_for="2026-10-05"), window_start=start, window_end=start + timedelta(days=1)) + assert [row["id"] for row in selected[0]] == ["30", "31", "32"] + assert [row["id"] for row in selected[2]] == ["30", "31", "32"] diff --git a/tests/unit/test_search_goldens.py b/tests/unit/test_search_goldens.py index add771338..01d967883 100644 --- a/tests/unit/test_search_goldens.py +++ b/tests/unit/test_search_goldens.py @@ -90,23 +90,6 @@ def test_a_scenario_equals_its_golden(computed: dict[str, dict[str, object]], su """ expected = _scenarios(surface)[name] - # The owner's replacement ruling adds a warning for a changed path even - # with replacement off. Keep the frozen fixture and every other byte. - if surface == "import_receipt" and name == "edited_reimport": - expected = {**expected, "receipt": {**expected["receipt"], - "changed_files_count": 1, - "replacement_hint": "Use --supersede --dry-run to preview replacement, then --supersede to apply it.", - }} - # Derived inserts now persist the empty project floor. The sources-first - # budget prices that stored metadata before compact projection, adding five - # tokens; every returned content field and the frozen fixture stay pinned. - if surface == "context_pack" and name == "sources_first": - result = expected["result"] - report = result["token_report"] - expected = {**expected, "result": {**result, "token_report": {**report, - "token_estimate": report["token_estimate"] + 5, - "full_pack_serialized_token_estimate": report["full_pack_serialized_token_estimate"] + 5, - }}} actual = computed[surface].get(name) assert actual == expected, _explain(surface, name, expected, actual) diff --git a/tests/unit/test_source_scrub_handoff_passes.py b/tests/unit/test_source_scrub_handoff_passes.py new file mode 100644 index 000000000..d3e8b9f34 --- /dev/null +++ b/tests/unit/test_source_scrub_handoff_passes.py @@ -0,0 +1,70 @@ +"""An empty cached answer is not permission to scan every source again.""" +import json +from uuid import UUID + +import pytest + +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_stores.sqlite import source_retirement +from tests.unit.test_source_scrub_loop_references import _command, _create_loop +from tests.unit.test_source_supersede import run_import +from tests.unit.test_importer_per_file_savepoint import USER_ID, _folder, _read, _vault + + +@pytest.mark.parametrize("loop_count", [0, 1]) +@pytest.mark.parametrize("operation", ["prune", "supersede", "dry_run"]) +def test_many_sources_empty_answers_use_one_loop_pass(tmp_path, monkeypatch, capsys, loop_count, operation): + db = _vault(tmp_path) + folder = _folder(tmp_path, **{f"note{i}": f"The older synthetic statement {i}." for i in range(12)}) + ids = run_import(db, folder).source_ids + for i in range(12): + (folder / f"note{i}.md").write_text(f"The newer synthetic statement {i}.") + if operation == "prune": + run_import(db, folder, supersede=True) + with sqlite_user_connection(db, USER_ID) as conn: + if loop_count: + _create_loop(SQLiteVNextStore(conn, USER_ID), "only-loop", metadata={"source_id": ids[0]}) + calls = [] + original = source_retirement._user_open_loops + + def count(store): + calls.append(store.user_id) + return original(store) + + monkeypatch.setattr(source_retirement, "_user_open_loops", count) + if operation == "prune": + assert _command(db, "prune", "--superseded") == 2 + preview = json.loads(capsys.readouterr().out)["would_delete"] + assert len(calls) == 1 + assert sum(row["open_loops"] for row in preview) == loop_count + calls.clear() + assert _command(db, "prune", "--superseded", "--yes") == 0 + receipt = json.loads(capsys.readouterr().out)["deleted"] + assert len(calls) == 1 + assert sum(row["open_loops"] for row in receipt) == loop_count + else: + result = run_import(db, folder, supersede=True, dry_run=operation == "dry_run") + assert len(result.superseded) == 12 + assert len(calls) == 1 + + +def test_reverse_lookup_limit_order_and_canonical_source_id(tmp_path): + db = _vault(tmp_path) + sid = run_import(db, _folder(tmp_path, note="Synthetic source.")).source_ids[0] + with sqlite_user_connection(db, USER_ID) as conn: + store = SQLiteVNextStore(conn, USER_ID) + old = _create_loop(store, "old", metadata={"source_id": sid}) + new = _create_loop(store, "new", metadata={"source_id": sid}) + conn.execute("UPDATE open_loops SET opened_at='2020-01-01T00:00:00Z', created_at='2020-01-01T00:00:00Z' WHERE id=?", (old,)) + conn.execute("UPDATE open_loops SET opened_at='2021-01-01T00:00:00Z', created_at='2021-01-01T00:00:00Z' WHERE id=?", (new,)) + assert [row["id"] for row in store.list_open_loops_referencing_source(source_id=UUID(sid).hex.upper(), limit=1)] == [new] + with pytest.raises(ValueError): + store.list_open_loops_referencing_source(source_id=sid, limit=0) + + +def test_plain_import_does_not_scan_loops(tmp_path, monkeypatch): + db = _vault(tmp_path) + def refuse(_store): + raise AssertionError("plain import scanned open loops") + monkeypatch.setattr(source_retirement, "_user_open_loops", refuse) + assert run_import(db, _folder(tmp_path, note="Synthetic source.")).imported_count == 1 diff --git a/tests/unit/test_source_search_query_bound.py b/tests/unit/test_source_search_query_bound.py index ea7ad9d00..597807483 100644 --- a/tests/unit/test_source_search_query_bound.py +++ b/tests/unit/test_source_search_query_bound.py @@ -546,11 +546,11 @@ def search_sources(self, **kwargs: object) -> list[dict[str, object]]: with pytest.raises(SourceSearchQueryTooLarge): VNextContradictionService(store).generate_contradiction_report( # type: ignore[arg-type] - ContradictionFinderRequest(query=query, sensitivity_allowed=sensitivity) + ContradictionFinderRequest(agent_identity=None, query=query, sensitivity_allowed=sensitivity) ) with pytest.raises(SourceSearchQueryTooLarge): VNextConnectionService(store).generate_connection_report( # type: ignore[arg-type] - ConnectionFinderRequest(query=query, sensitivity_allowed=sensitivity) + ConnectionFinderRequest(agent_identity=None, query=query, sensitivity_allowed=sensitivity) ) with pytest.raises(SourceSearchQueryTooLarge): VNextContextTreeService(TreeStore(store)).build_tree( # type: ignore[arg-type] diff --git a/tests/unit/test_vnext_brain.py b/tests/unit/test_vnext_brain.py index 5f496e59c..b1069994f 100644 --- a/tests/unit/test_vnext_brain.py +++ b/tests/unit/test_vnext_brain.py @@ -141,7 +141,7 @@ def test_daily_brief_generates_dated_reviewable_artifact_with_sources_and_open_l store = _seed_store() artifact = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest(generated_for="2026-05-10", domains=("project",)) + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", domains=("project",)) ) assert artifact["artifact_type"] == "daily_brief" @@ -176,7 +176,7 @@ def test_daily_brief_respects_sensitivity_filtering() -> None: ) artifact = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest( + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", domains=("project",), sensitivity_allowed=("public", "private"), @@ -191,7 +191,7 @@ def test_weekly_synthesis_creates_candidate_memory_without_auto_promotion() -> N store = _seed_store() artifact = VNextBrainService(store).generate_weekly_synthesis( - BrainArtifactRequest(generated_for="2026-05-10", domains=("project",)) + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", domains=("project",)) ) assert artifact["artifact_type"] == "weekly_synthesis" @@ -211,7 +211,7 @@ def test_weekly_synthesis_can_skip_candidate_memory_creation() -> None: store = _seed_store() artifact = VNextBrainService(store).generate_weekly_synthesis( - BrainArtifactRequest( + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", domains=("project",), create_candidate_memories=False, @@ -226,7 +226,7 @@ def test_daily_brief_model_backed_mode_stores_provider_metadata_and_review_only_ store = _seed_store() artifact = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest( + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", domains=("project",), generation_mode="model_backed", @@ -250,7 +250,7 @@ def test_weekly_synthesis_model_backed_mode_keeps_candidate_memory_reviewable() store = _seed_store() artifact = VNextBrainService(store).generate_weekly_synthesis( - BrainArtifactRequest( + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", domains=("project",), generation_mode="model_backed", @@ -284,14 +284,14 @@ def test_daily_and_weekly_reports_use_true_half_open_time_windows() -> None: ) daily = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest( + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", domains=("project",), discover_open_loops=False, ) ) weekly = VNextBrainService(store).generate_weekly_synthesis( - BrainArtifactRequest( + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", domains=("project",), create_candidate_memories=False, @@ -334,7 +334,7 @@ def test_daily_brief_applies_project_scope_before_bounded_legacy_limit() -> None ) artifact = VNextBrainService(store).generate_daily_brief( - BrainArtifactRequest( + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", domains=("project",), projects=("project-a",), @@ -365,14 +365,14 @@ def source(source_id: str, canonical_scope: list[str]) -> dict[str, object]: scoped_store = InMemoryVNextBrainStore() scoped_store.sources.extend((source("empty-source", []), source("real-source", ["real"]))) real = VNextBrainService(scoped_store).generate_daily_brief( - BrainArtifactRequest( + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", projects=("real",), discover_open_loops=False, ) ) stale = VNextBrainService(scoped_store).generate_daily_brief( - BrainArtifactRequest( + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10", projects=("stale",), discover_open_loops=False, @@ -385,7 +385,7 @@ def source(source_id: str, canonical_scope: list[str]) -> dict[str, object]: daily_store = InMemoryVNextBrainStore() daily_store.sources.extend((source("empty-source", []), source("real-source", ["real"]))) daily = VNextBrainService(daily_store).generate_daily_brief( - BrainArtifactRequest(generated_for="2026-05-10") + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10") ) loops_by_source = {row["source_id"]: row for row in daily_store.open_loops} @@ -396,7 +396,7 @@ def source(source_id: str, canonical_scope: list[str]) -> dict[str, object]: weekly_store = InMemoryVNextBrainStore() weekly_store.sources.extend((source("empty-source", []), source("real-source", ["real"]))) weekly = VNextBrainService(weekly_store).generate_weekly_synthesis( - BrainArtifactRequest(generated_for="2026-05-10") + BrainArtifactRequest(agent_identity=None, generated_for="2026-05-10") ) assert weekly["metadata_json"]["project_scope"] == ["real"] @@ -407,7 +407,7 @@ def test_brain_request_validation_rejects_bad_dates_and_limits() -> None: service = VNextBrainService(InMemoryVNextBrainStore()) with pytest.raises(VNextBrainValidationError, match="generated_for"): - service.generate_daily_brief(BrainArtifactRequest(generated_for="10-05-2026")) + service.generate_daily_brief(BrainArtifactRequest(agent_identity=None, generated_for="10-05-2026")) with pytest.raises(VNextBrainValidationError, match="source_limit"): - service.generate_weekly_synthesis(BrainArtifactRequest(source_limit=0)) + service.generate_weekly_synthesis(BrainArtifactRequest(agent_identity=None, source_limit=0)) diff --git a/tests/unit/test_vnext_connections.py b/tests/unit/test_vnext_connections.py index 45f6a26ff..b4b7ea415 100644 --- a/tests/unit/test_vnext_connections.py +++ b/tests/unit/test_vnext_connections.py @@ -188,7 +188,7 @@ def test_connection_report_creates_candidate_edges_artifact_and_logs_each_edge() store = _seed_store() artifact = VNextConnectionService(store).generate_connection_report( - ConnectionFinderRequest(domains=("project",), max_connections=2) + ConnectionFinderRequest(agent_identity=None, domains=("project",), max_connections=2) ) assert artifact["artifact_type"] == "connection_report" @@ -220,7 +220,7 @@ def test_connection_report_filters_sensitivity_and_can_auto_accept_high_confiden ) artifact = VNextConnectionService(store).generate_connection_report( - ConnectionFinderRequest( + ConnectionFinderRequest(agent_identity=None, domains=("project",), sensitivity_allowed=("public", "private"), max_connections=3, @@ -261,7 +261,7 @@ def test_connection_report_enforces_project_scope_before_candidate_limits() -> N ) artifact = VNextConnectionService(store).generate_connection_report( - ConnectionFinderRequest( + ConnectionFinderRequest(agent_identity=None, domains=("project",), projects=("project-a",), max_connections=2, @@ -304,7 +304,7 @@ def test_connection_report_source_filter_honors_embedded_canonical_envelope() -> ) artifact = VNextConnectionService(store).generate_connection_report( - ConnectionFinderRequest(projects=("real",), max_connections=2) + ConnectionFinderRequest(agent_identity=None, projects=("real",), max_connections=2) ) assert artifact["metadata_json"]["source_ids"] == ["source-real"] @@ -316,7 +316,7 @@ def test_connection_report_model_backed_mode_preserves_candidate_edges_and_metad store = _seed_store() artifact = VNextConnectionService(store).generate_connection_report( - ConnectionFinderRequest( + ConnectionFinderRequest(agent_identity=None, domains=("project",), max_connections=2, generation_mode="model_backed", @@ -365,7 +365,7 @@ def test_connection_report_logical_retry_replays_artifact_and_edges() -> None: def test_connection_edge_review_and_graph_neighborhood() -> None: store = _seed_store() service = VNextConnectionService(store) - service.generate_connection_report(ConnectionFinderRequest(domains=("project",), max_connections=1)) + service.generate_connection_report(ConnectionFinderRequest(agent_identity=None, domains=("project",), max_connections=1)) accepted = service.review_edge(edge_id="edge-1", action="accept") neighborhood = service.graph_neighborhood(target_id="source-1") @@ -385,7 +385,7 @@ def test_connection_request_validation() -> None: service = VNextConnectionService(InMemoryVNextConnectionStore()) with pytest.raises(VNextConnectionValidationError, match="max_connections"): - service.generate_connection_report(ConnectionFinderRequest(max_connections=0)) + service.generate_connection_report(ConnectionFinderRequest(agent_identity=None, max_connections=0)) with pytest.raises(VNextConnectionValidationError, match="edge review action"): service.review_edge(edge_id="edge-1", action="ship") @@ -397,7 +397,7 @@ def test_connection_edges_carry_event_time_from_the_source() -> None: store.sources[0]["captured_at"] = "2026-07-01T00:00:00Z" VNextConnectionService(store).generate_connection_report( - ConnectionFinderRequest(domains=("project",), max_connections=2) + ConnectionFinderRequest(agent_identity=None, domains=("project",), max_connections=2) ) edge = store.edges["edge-1"] @@ -412,7 +412,7 @@ def test_connection_edges_fall_back_to_captured_at_then_now_for_event_time() -> captured_only = _seed_store() captured_only.sources[0]["captured_at"] = "2026-07-01T00:00:00Z" VNextConnectionService(captured_only).generate_connection_report( - ConnectionFinderRequest(domains=("project",), max_connections=1) + ConnectionFinderRequest(agent_identity=None, domains=("project",), max_connections=1) ) edge = captured_only.edges["edge-1"] assert edge["observed_at"] == "2026-07-01T00:00:00Z" @@ -423,7 +423,7 @@ def test_connection_edges_fall_back_to_captured_at_then_now_for_event_time() -> # fallback is noted on the edge so as-of readers can tell them apart. bare = _seed_store() VNextConnectionService(bare).generate_connection_report( - ConnectionFinderRequest(domains=("project",), max_connections=1) + ConnectionFinderRequest(agent_identity=None, domains=("project",), max_connections=1) ) edge = bare.edges["edge-1"] assert isinstance(edge["observed_at"], str) and edge["observed_at"].endswith("Z") diff --git a/tests/unit/test_vnext_consolidation.py b/tests/unit/test_vnext_consolidation.py index 56a15615f..c501f4d44 100644 --- a/tests/unit/test_vnext_consolidation.py +++ b/tests/unit/test_vnext_consolidation.py @@ -402,20 +402,20 @@ def _service(store, mapping) -> VNextConsolidationService: def test_invalid_similarity_threshold_is_rejected() -> None: service = VNextConsolidationService(FakeConsolidationStore(), embedding_provider=None) with pytest.raises(VNextConsolidationValidationError): - service.generate_memory_consolidation(MemoryConsolidationRequest(similarity_threshold=1.5)) + service.generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, similarity_threshold=1.5)) with pytest.raises(VNextConsolidationValidationError): - service.generate_memory_consolidation(MemoryConsolidationRequest(similarity_threshold=0.0)) + service.generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, similarity_threshold=0.0)) def test_invalid_metadata_option_overrides_are_rejected() -> None: service = VNextConsolidationService(FakeConsolidationStore(), embedding_provider=None) with pytest.raises(VNextConsolidationValidationError): service.generate_memory_consolidation( - MemoryConsolidationRequest(metadata_json={"consolidation_options": {"similarity_threshold": "high"}}) + MemoryConsolidationRequest(agent_identity=None, metadata_json={"consolidation_options": {"similarity_threshold": "high"}}) ) with pytest.raises(VNextConsolidationValidationError): service.generate_memory_consolidation( - MemoryConsolidationRequest(metadata_json={"consolidation_options": {"max_embedded_memories": 50000}}) + MemoryConsolidationRequest(agent_identity=None, metadata_json={"consolidation_options": {"max_embedded_memories": 50000}}) ) @@ -423,7 +423,7 @@ def test_max_embedded_memories_request_field_cannot_exceed_hard_cap() -> None: service = VNextConsolidationService(FakeConsolidationStore(), embedding_provider=None) with pytest.raises(VNextConsolidationValidationError): service.generate_memory_consolidation( - MemoryConsolidationRequest(max_embedded_memories=MAX_EMBEDDED_MEMORIES_HARD_CAP + 1) + MemoryConsolidationRequest(agent_identity=None, max_embedded_memories=MAX_EMBEDDED_MEMORIES_HARD_CAP + 1) ) @@ -443,7 +443,7 @@ def test_near_duplicates_produce_one_dedup_candidate_with_correct_members() -> N near_dups, distinct = _seed_six_memories(store, mapping) provider = MappedEmbeddingProvider(mapping) artifact = VNextConsolidationService(store, embedding_provider=provider).generate_memory_consolidation( - MemoryConsolidationRequest() + MemoryConsolidationRequest(agent_identity=None, ) ) candidates = _consolidation_candidates(store) @@ -521,7 +521,7 @@ def test_consolidation_never_admits_a_single_link_bridge_chain() -> None: ] artifact = _service(store, mapping).generate_memory_consolidation( - MemoryConsolidationRequest( + MemoryConsolidationRequest(agent_identity=None, similarity_threshold=0.70, create_candidate_memories=False, propose_rollups=False, @@ -551,7 +551,7 @@ def test_similarity_work_uses_float32_blocks_instead_of_a_dense_matrix() -> None ) artifact = _service(store, mapping).generate_memory_consolidation( - MemoryConsolidationRequest( + MemoryConsolidationRequest(agent_identity=None, max_embedded_memories=member_count, create_candidate_memories=False, propose_rollups=False, @@ -571,8 +571,8 @@ def test_rerun_with_same_input_set_creates_no_duplicate_candidate() -> None: mapping: dict[str, list[float]] = {} _seed_six_memories(store, mapping) service = _service(store, mapping) - first = service.generate_memory_consolidation(MemoryConsolidationRequest()) - second = service.generate_memory_consolidation(MemoryConsolidationRequest()) + first = service.generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) + second = service.generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) candidates = _consolidation_candidates(store) assert len(candidates) == 1 @@ -626,13 +626,13 @@ def find_artifact_by_workflow_digest( } ) service = _service(store, mapping) - request = MemoryConsolidationRequest(propose_rollups=False, max_clusters=20) + request = MemoryConsolidationRequest(agent_identity=None, propose_rollups=False, max_clusters=20) first = service.generate_memory_consolidation(request) candidate_count = len(_consolidation_candidates(store)) second = service.generate_memory_consolidation(request) changed = service.generate_memory_consolidation( - MemoryConsolidationRequest(propose_rollups=False, max_clusters=19) + MemoryConsolidationRequest(agent_identity=None, propose_rollups=False, max_clusters=19) ) assert second["id"] == first["id"] @@ -646,7 +646,7 @@ def test_without_embedding_provider_clustering_is_skipped_review_only() -> None: mapping: dict[str, list[float]] = {} _seed_six_memories(store, mapping) service = VNextConsolidationService(store, embedding_provider=None) - artifact = service.generate_memory_consolidation(MemoryConsolidationRequest()) + artifact = service.generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) assert _consolidation_candidates(store) == [] assert "no_embedding_provider_configured" in artifact["metadata_json"]["consolidation"]["skipped"] @@ -666,7 +666,7 @@ def list_memory_ids_with_embeddings(self, ids) -> set[str]: artifact = VNextConsolidationService( store, embedding_provider=MappedEmbeddingProvider(mapping) - ).generate_memory_consolidation(MemoryConsolidationRequest()) + ).generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) skipped = artifact["metadata_json"]["consolidation"]["skipped"] assert "embedding_presence_read_failed" in skipped @@ -686,7 +686,7 @@ def embed_batch(self, texts) -> list[list[float]]: artifact = VNextConsolidationService( store, embedding_provider=FailingProvider(mapping) - ).generate_memory_consolidation(MemoryConsolidationRequest()) + ).generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) skipped = artifact["metadata_json"]["consolidation"]["skipped"] assert "embedding_provider_failed" in skipped @@ -707,7 +707,7 @@ def search_memories_vector(self, **kwargs): # type: ignore[override] near_dups, _distinct = _seed_six_memories(store, mapping) provider = MappedEmbeddingProvider(mapping) artifact = VNextConsolidationService(store, embedding_provider=provider).generate_memory_consolidation( - MemoryConsolidationRequest() + MemoryConsolidationRequest(agent_identity=None, ) ) candidates = _consolidation_candidates(store) @@ -738,7 +738,7 @@ def test_memories_without_stored_embeddings_are_excluded() -> None: ) provider = MappedEmbeddingProvider(mapping) artifact = VNextConsolidationService(store, embedding_provider=provider).generate_memory_consolidation( - MemoryConsolidationRequest() + MemoryConsolidationRequest(agent_identity=None, ) ) candidates = _consolidation_candidates(store) assert len(candidates) == 1 @@ -756,7 +756,7 @@ def test_cap_bound_is_applied_and_logged(caplog: pytest.LogCaptureFixture) -> No _seed_six_memories(store, mapping) with caplog.at_level(logging.INFO, logger="alicebot_api.vnext_consolidation"): artifact = _service(store, mapping).generate_memory_consolidation( - MemoryConsolidationRequest(max_embedded_memories=2) + MemoryConsolidationRequest(agent_identity=None, max_embedded_memories=2) ) assert artifact["metadata_json"]["consolidation"]["bounded"] is True assert artifact["metadata_json"]["input_counts"]["active_memories"] == 6 @@ -782,7 +782,7 @@ def _forbidden(*args, **kwargs): monkeypatch.setattr(np, "triu", _forbidden) monkeypatch.setattr(np, "where", _forbidden) artifact = _service(store, mapping).generate_memory_consolidation( - MemoryConsolidationRequest(propose_rollups=False) + MemoryConsolidationRequest(agent_identity=None, propose_rollups=False) ) assert artifact["metadata_json"]["input_counts"]["clusters"] == 1 @@ -796,7 +796,7 @@ def test_threshold_override_via_metadata_json_options() -> None: mapping: dict[str, list[float]] = {} _seed_six_memories(store, mapping) artifact = _service(store, mapping).generate_memory_consolidation( - MemoryConsolidationRequest(metadata_json={"consolidation_options": {"similarity_threshold": 0.9999}}) + MemoryConsolidationRequest(agent_identity=None, metadata_json={"consolidation_options": {"similarity_threshold": 0.9999}}) ) assert _consolidation_candidates(store) == [] assert artifact["metadata_json"]["consolidation"]["similarity_threshold"] == 0.9999 @@ -820,7 +820,7 @@ def test_near_duplicate_clusters_are_partitioned_by_exact_project_scope() -> Non store, embedding_provider=MappedEmbeddingProvider(mapping), ).generate_memory_consolidation( - MemoryConsolidationRequest(min_cluster_size=2, propose_rollups=False) + MemoryConsolidationRequest(agent_identity=None, min_cluster_size=2, propose_rollups=False) ) candidates = _consolidation_candidates(store) @@ -866,7 +866,7 @@ def test_project_scoped_consolidation_filters_decoys_before_corpus_limit() -> No } artifact = _service(store, mapping).generate_memory_consolidation( - MemoryConsolidationRequest( + MemoryConsolidationRequest(agent_identity=None, projects=("project-a",), max_embedded_memories=2, min_cluster_size=2, @@ -904,7 +904,7 @@ def test_preference_cluster_spanning_three_sources_is_reported_review_only() -> source_event_ids=[f"event-{index}"], source_id=f"source-{index}", ) - artifact = _service(store, mapping).generate_memory_consolidation(MemoryConsolidationRequest()) + artifact = _service(store, mapping).generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) reinforced = artifact["metadata_json"]["consolidation"]["reinforced_preferences"] assert len(reinforced) == 1 assert reinforced[0]["distinct_source_count"] >= 3 @@ -931,7 +931,7 @@ def test_distinct_event_ids_do_not_count_as_independent_sources() -> None: ) artifact = _service(store, mapping).generate_memory_consolidation( - MemoryConsolidationRequest(propose_rollups=False) + MemoryConsolidationRequest(agent_identity=None, propose_rollups=False) ) assert artifact["metadata_json"]["consolidation"]["reinforced_preferences"] == [] @@ -941,7 +941,7 @@ def test_non_preference_cluster_is_not_reported_as_reinforced() -> None: store = FakeConsolidationStore() mapping: dict[str, list[float]] = {} _seed_six_memories(store, mapping) # memory_type semantic - artifact = _service(store, mapping).generate_memory_consolidation(MemoryConsolidationRequest()) + artifact = _service(store, mapping).generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) assert artifact["metadata_json"]["consolidation"]["reinforced_preferences"] == [] @@ -962,7 +962,7 @@ def test_model_backed_merge_uses_stub_provider_and_records_provenance() -> None: merge_provider=stub, ) artifact = service.generate_memory_consolidation( - MemoryConsolidationRequest( + MemoryConsolidationRequest(agent_identity=None, generation_mode="model_backed", model_route_mode="cloud_allowed", model_provider="mock", @@ -990,7 +990,7 @@ def test_model_backed_with_deterministic_route_falls_back_to_dedup() -> None: near_dups, _ = _seed_six_memories(store, mapping) service = _service(store, mapping) service.generate_memory_consolidation( - MemoryConsolidationRequest(generation_mode="model_backed", sensitivity_allowed=("public", "internal")) + MemoryConsolidationRequest(agent_identity=None, generation_mode="model_backed", sensitivity_allowed=("public", "internal")) ) candidates = _consolidation_candidates(store) assert len(candidates) == 1 @@ -1010,7 +1010,7 @@ def test_model_backed_approval_required_route_fails_before_any_writes() -> None: memory_count = len(store.memories) with pytest.raises(VNextModelIntelligenceError): _service(store, mapping).generate_memory_consolidation( - MemoryConsolidationRequest( + MemoryConsolidationRequest(agent_identity=None, generation_mode="model_backed", model_route_mode="cloud_requires_approval", sensitivity_allowed=("public", "internal"), @@ -1025,7 +1025,7 @@ def test_create_candidate_memories_false_lists_proposals_without_writes() -> Non mapping: dict[str, list[float]] = {} _seed_six_memories(store, mapping) artifact = _service(store, mapping).generate_memory_consolidation( - MemoryConsolidationRequest(create_candidate_memories=False) + MemoryConsolidationRequest(agent_identity=None, create_candidate_memories=False) ) assert _consolidation_candidates(store) == [] proposals = artifact["metadata_json"]["consolidation"]["proposals"] @@ -1039,7 +1039,7 @@ def test_rollup_pass_skips_groups_covered_by_near_duplicate_clusters() -> None: store = FakeConsolidationStore() mapping: dict[str, list[float]] = {} _seed_six_memories(store, mapping) - artifact = _service(store, mapping).generate_memory_consolidation(MemoryConsolidationRequest()) + artifact = _service(store, mapping).generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) rollups = artifact["metadata_json"]["rollups"] assert rollups["enabled"] is True @@ -1085,7 +1085,7 @@ def test_rollup_quality_gate_drops_junk_groups_and_is_disclosed() -> None: } ) artifact = VNextConsolidationService(store, embedding_provider=None).generate_memory_consolidation( - MemoryConsolidationRequest() + MemoryConsolidationRequest(agent_identity=None, ) ) rollups = artifact["metadata_json"]["rollups"] @@ -1102,7 +1102,7 @@ def test_propose_rollups_false_discloses_disabled_state() -> None: mapping: dict[str, list[float]] = {} _seed_six_memories(store, mapping) artifact = _service(store, mapping).generate_memory_consolidation( - MemoryConsolidationRequest(propose_rollups=False) + MemoryConsolidationRequest(agent_identity=None, propose_rollups=False) ) assert artifact["metadata_json"]["rollups"] == {"enabled": False} assert artifact["metadata_json"]["input_counts"]["rollup_proposals"] == 0 @@ -1118,7 +1118,7 @@ def test_invalid_rollup_options_fail_before_any_writes() -> None: memory_count = len(store.memories) with pytest.raises(VNextRollupValidationError): _service(store, mapping).generate_memory_consolidation( - MemoryConsolidationRequest(metadata_json={"rollup_options": {"min_members": 1}}) + MemoryConsolidationRequest(agent_identity=None, metadata_json={"rollup_options": {"min_members": 1}}) ) assert len(store.memories) == memory_count assert store.artifacts == [] @@ -1214,7 +1214,7 @@ def test_live_sqlite_smoke_clusters_near_duplicates_idempotently() -> None: near_dup_ids = sorted(str(row["id"]) for row in rows[:3]) service = VNextConsolidationService(store, embedding_provider=MappedEmbeddingProvider(mapping)) - artifact = service.generate_memory_consolidation(MemoryConsolidationRequest()) + artifact = service.generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) candidates = _consolidation_candidates(sqlite_store) assert len(candidates) == 1 @@ -1230,7 +1230,7 @@ def test_live_sqlite_smoke_clusters_near_duplicates_idempotently() -> None: assert self_distance is not None and self_distance < 1e-6 # Idempotent rerun: same input set, no duplicate candidate. - second = service.generate_memory_consolidation(MemoryConsolidationRequest()) + second = service.generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) assert len(_consolidation_candidates(sqlite_store)) == 1 assert second["metadata_json"]["candidate_memory_ids"] == artifact["metadata_json"]["candidate_memory_ids"] assert second["metadata_json"]["consolidation"]["proposals"][0]["candidate_state"] == "existing" @@ -1288,7 +1288,7 @@ def test_accepting_the_dedup_candidate_executes_supersessions_on_live_sqlite(mon rows.append(row) service = VNextConsolidationService(store, embedding_provider=MappedEmbeddingProvider(mapping)) - service.generate_memory_consolidation(MemoryConsolidationRequest()) + service.generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) candidates = _consolidation_candidates(sqlite_store) assert len(candidates) == 1 candidate_id = str(candidates[0]["id"]) diff --git a/tests/unit/test_vnext_contradictions.py b/tests/unit/test_vnext_contradictions.py index ee16b6443..23bf85847 100644 --- a/tests/unit/test_vnext_contradictions.py +++ b/tests/unit/test_vnext_contradictions.py @@ -219,7 +219,7 @@ def test_contradiction_report_creates_candidate_edges_and_preserves_beliefs() -> store = _seed_store() artifact = VNextContradictionService(store).generate_contradiction_report( - ContradictionFinderRequest(domains=("project",), max_contradictions=2) + ContradictionFinderRequest(agent_identity=None, domains=("project",), max_contradictions=2) ) assert artifact["artifact_type"] == "contradiction_report" @@ -257,7 +257,7 @@ def test_contradiction_report_filters_sensitivity_and_distinguishes_nuance() -> } artifact = VNextContradictionService(store).generate_contradiction_report( - ContradictionFinderRequest( + ContradictionFinderRequest(agent_identity=None, domains=("project",), sensitivity_allowed=("public", "private"), max_contradictions=2, @@ -323,7 +323,7 @@ def test_contradiction_report_enforces_project_scope_for_inputs_and_beliefs() -> } artifact = VNextContradictionService(store).generate_contradiction_report( - ContradictionFinderRequest( + ContradictionFinderRequest(agent_identity=None, domains=("project",), projects=("project-a",), max_contradictions=4, @@ -367,7 +367,7 @@ def test_contradiction_report_source_filter_honors_embedded_canonical_envelope() ] artifact = VNextContradictionService(store).generate_contradiction_report( - ContradictionFinderRequest(projects=("real",), max_contradictions=2) + ContradictionFinderRequest(agent_identity=None, projects=("real",), max_contradictions=2) ) assert artifact["metadata_json"]["source_ids"] == ["source-real"] @@ -377,7 +377,7 @@ def test_contradiction_report_model_backed_mode_records_source_grounded_metadata store = _seed_store() artifact = VNextContradictionService(store).generate_contradiction_report( - ContradictionFinderRequest( + ContradictionFinderRequest(agent_identity=None, domains=("project",), max_contradictions=2, generation_mode="model_backed", @@ -445,7 +445,7 @@ def test_contradiction_validation_errors() -> None: service = VNextContradictionService(InMemoryVNextContradictionStore()) with pytest.raises(VNextContradictionValidationError, match="max_contradictions"): - service.generate_contradiction_report(ContradictionFinderRequest(max_contradictions=0)) + service.generate_contradiction_report(ContradictionFinderRequest(agent_identity=None, max_contradictions=0)) with pytest.raises(VNextContradictionValidationError, match="belief review action"): service.review_belief(belief_id="belief-1", action="delete") diff --git a/tests/unit/test_vnext_main.py b/tests/unit/test_vnext_main.py index 7274a8041..49717ed0c 100644 --- a/tests/unit/test_vnext_main.py +++ b/tests/unit/test_vnext_main.py @@ -3003,7 +3003,7 @@ def test_generic_artifact_review_preserves_applied_project_update_state(monkeypa _install_fake_vnext_store(monkeypatch, store) user_id = uuid4() artifact = VNextProjectService(store).generate_project_update_candidate( - vnext_automation.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + vnext_automation.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_id = str(artifact["id"]) candidate_memory_id = str(artifact["metadata_json"]["candidate_memory_id"]) @@ -3052,7 +3052,7 @@ def _http_project_update_review_fixture() -> tuple[FakeVNextStore, dict[str, obj }, } artifact = VNextProjectService(store).generate_project_update_candidate( - vnext_automation.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + vnext_automation.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) return store, artifact @@ -3109,7 +3109,7 @@ def _apply_supported_http_memory_lifecycle( def _accept_later_http_project_update(store: FakeVNextStore, *, first_artifact_id: str) -> None: service = VNextProjectService(store) later = service.generate_project_update_candidate( - vnext_automation.ProjectAutomationRequest(project_id="project-1", domains=("project",)) + vnext_automation.ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) assert later["id"] != first_artifact_id service.review_project_update( diff --git a/tests/unit/test_vnext_projects.py b/tests/unit/test_vnext_projects.py index eec4242b1..d8844d3f6 100644 --- a/tests/unit/test_vnext_projects.py +++ b/tests/unit/test_vnext_projects.py @@ -406,7 +406,7 @@ def test_project_update_candidate_creates_reviewable_artifact_and_candidate_memo store = _seed_store() artifact = VNextProjectService(store).generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) assert artifact["artifact_type"] == "project_update" @@ -422,7 +422,7 @@ def test_project_update_candidate_model_backed_mode_is_review_only_and_source_gr store = _seed_store() artifact = VNextProjectService(store).generate_project_update_candidate( - ProjectAutomationRequest( + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",), generation_mode="model_backed", @@ -479,7 +479,7 @@ def test_project_automation_and_dashboard_never_mix_same_domain_projects() -> No } artifact = VNextProjectService(store).generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) dashboard = VNextProjectService(store).project_dashboard(project_id="project-1") @@ -521,7 +521,7 @@ def unscoped_sources(**kwargs) -> list[dict[str, object]]: store.search_sources = unscoped_sources # type: ignore[method-assign] service = VNextProjectService(store) - request = ProjectAutomationRequest(project_id="project-1", domains=("project",)) + request = ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) artifact = service.generate_project_update_candidate(request) source_metadata = store.sources[1]["metadata_json"] @@ -539,7 +539,7 @@ def unscoped_sources(**kwargs) -> list[dict[str, object]]: def test_direct_project_workflows_are_idempotent_for_unchanged_evidence() -> None: store = _seed_store() service = VNextProjectService(store) - request = ProjectAutomationRequest(project_id="project-1", domains=("project",)) + request = ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) first_artifact = service.generate_project_update_candidate(request) first_loops = service.extract_open_loops(request) @@ -592,7 +592,7 @@ def test_a_candidate_loop_description_never_holds_the_id_of_its_source() -> None } ) service = VNextProjectService(store) - request = ProjectAutomationRequest(project_id="project-1", domains=("project",)) + request = ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) first = service.extract_open_loops(request) descriptions: dict[str, list[object]] = {} @@ -605,7 +605,7 @@ def test_a_candidate_loop_description_never_holds_the_id_of_its_source() -> None assert descriptions[untitled] == ["Candidate task discovered from a source with no title."] assert set(descriptions) == {"source-1", untitled} for loop in first: - assert untitled not in json.dumps({key: value for key, value in loop.items() if key != "source_id"}) + assert untitled not in json.dumps({key: loop.get(key) for key in ("title", "description", "resolution_note")}) second = service.extract_open_loops(request) assert [row["id"] for row in second] == [row["id"] for row in first] assert len(store.open_loops) == len(first) @@ -616,10 +616,10 @@ def test_project_update_digest_changes_when_behavior_config_changes() -> None: service = VNextProjectService(store) first = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",), max_items=8) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",), max_items=8) ) second = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",), max_items=7) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",), max_items=7) ) assert second["id"] != first["id"] @@ -630,7 +630,7 @@ def test_accepting_project_update_updates_project_promotes_memory_and_appends_re store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) reviewed = service.review_project_update( @@ -671,7 +671,7 @@ def test_accepting_project_update_updates_project_promotes_memory_and_appends_re def test_central_artifact_review_dispatches_project_updates_to_coupled_lifecycle(action: str) -> None: store = _seed_store() artifact = VNextProjectService(store).generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) metadata = artifact["metadata_json"] assert isinstance(metadata, dict) @@ -700,7 +700,7 @@ def test_central_artifact_review_dispatches_project_updates_to_coupled_lifecycle def test_central_artifact_review_does_not_treat_project_update_promote_as_generic_promotion() -> None: store = _seed_store() artifact = VNextProjectService(store).generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) metadata = artifact["metadata_json"] assert isinstance(metadata, dict) @@ -719,7 +719,7 @@ def test_central_artifact_review_does_not_treat_project_update_promote_as_generi def test_central_artifact_review_dispatch_propagates_candidate_supersession_guard() -> None: store = _seed_store() artifact = VNextProjectService(store).generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_metadata = artifact["metadata_json"] assert isinstance(artifact_metadata, dict) @@ -743,7 +743,7 @@ def test_project_update_scope_linkage_compares_canonical_project_identity() -> N store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_metadata = artifact["metadata_json"] assert isinstance(artifact_metadata, dict) @@ -776,7 +776,7 @@ def test_project_update_review_rejects_superseded_candidate_markers_without_muta store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_id = str(artifact["id"]) artifact_metadata = artifact["metadata_json"] @@ -845,7 +845,7 @@ def test_project_update_review_rejects_every_candidate_linkage_mismatch_without_ store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_id = str(artifact["id"]) artifact_metadata = artifact["metadata_json"] @@ -908,7 +908,7 @@ def test_project_update_review_refreshes_content_derived_indexes() -> None: store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) memory_id = str(artifact["metadata_json"]["candidate_memory_id"]) memory = store.memories[memory_id] @@ -936,7 +936,7 @@ def test_project_update_review_can_defer_embedding_until_after_commit() -> None: store = _seed_store() service = VNextProjectService(store, defer_embeddings=True) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) service.review_project_update( @@ -955,7 +955,7 @@ def test_rejecting_project_update_logs_rejection_without_updating_project() -> N store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) reviewed = service.review_project_update(artifact_id=str(artifact["id"]), action="reject") @@ -976,7 +976,7 @@ def test_rejecting_project_update_requires_memory_key_before_any_mutation(memory store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact_metadata = artifact["metadata_json"] assert isinstance(artifact_metadata, dict) @@ -995,7 +995,7 @@ def test_terminal_project_update_replay_uses_one_coupled_event_lookup(action: st store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) terminal = service.review_project_update(artifact_id=str(artifact["id"]), action=action) metadata = terminal["metadata_json"] @@ -1020,7 +1020,7 @@ def test_project_update_forced_terminal_status_fails_closed_without_mutation( store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) artifact["status"] = forced_status artifact_id = str(artifact["id"]) @@ -1112,7 +1112,7 @@ def test_project_update_terminal_consistency_requires_every_immutable_evidence_l store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) terminal = service.review_project_update(artifact_id=str(artifact["id"]), action=action) artifact_id = str(terminal["id"]) @@ -1303,7 +1303,7 @@ def test_project_update_terminal_replay_rejects_clone_after_authorized_true_reda store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) terminal = service.review_project_update(artifact_id=str(artifact["id"]), action="accept") _redact_project_update_terminal_evidence(store, terminal=terminal) @@ -1326,7 +1326,7 @@ def test_project_update_terminal_replay_allows_repeated_creation_rows_for_one_ar store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) terminal = service.review_project_update(artifact_id=str(artifact["id"]), action="accept") creation_event = next( @@ -1451,7 +1451,7 @@ def test_project_update_terminal_replay_rejects_every_coupled_competing_decision store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) terminal = service.review_project_update(artifact_id=str(artifact["id"]), action=action) _append_conflicting_project_update_decision(store, terminal=terminal, conflict=conflict) @@ -1471,7 +1471,7 @@ def test_accepted_project_update_terminal_replay_ignores_supported_memory_lifecy store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) terminal = service.review_project_update(artifact_id=str(artifact["id"]), action="accept") terminal_metadata = terminal["metadata_json"] @@ -1507,11 +1507,11 @@ def test_accepted_project_update_replay_survives_a_genuine_later_project_update( store = _seed_store() service = VNextProjectService(store) first = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) accepted_first = service.review_project_update(artifact_id=str(first["id"]), action="accept") second = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) assert second["id"] != first["id"] service.review_project_update( @@ -1531,7 +1531,7 @@ def test_project_update_consistent_terminal_outcome_remains_idempotent_without_m store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) terminal = service.review_project_update(artifact_id=str(artifact["id"]), action=action) project_before = deepcopy(store.projects) @@ -1552,7 +1552,7 @@ def test_accepted_project_update_cannot_later_be_rejected() -> None: store = _seed_store() service = VNextProjectService(store) artifact = service.generate_project_update_candidate( - ProjectAutomationRequest(project_id="project-1", domains=("project",)) + ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",)) ) accepted = service.review_project_update(artifact_id=str(artifact["id"]), action="accept") @@ -1566,7 +1566,7 @@ def test_open_loop_extraction_and_review_support_source_owner_and_filters() -> N store = _seed_store() service = VNextProjectService(store) - loops = service.extract_open_loops(ProjectAutomationRequest(project_id="project-1", domains=("project",))) + loops = service.extract_open_loops(ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",))) snoozed = service.review_open_loop(loop_id="loop-1", action="snooze", due_at="2026-05-12T09:00:00Z") closed = service.review_open_loop(loop_id="loop-2", action="close", resolution_note="Decision captured.") dashboard = service.project_dashboard(project_id="project-1") @@ -1586,7 +1586,7 @@ def test_open_loop_extraction_and_review_support_source_owner_and_filters() -> N def test_dashboard_withholds_loops_after_their_source_becomes_unreadable(parent_change) -> None: store = _seed_store() service = VNextProjectService(store) - loops = service.extract_open_loops(ProjectAutomationRequest(project_id="project-1", domains=("project",))) + loops = service.extract_open_loops(ProjectAutomationRequest(agent_identity=None, project_id="project-1", domains=("project",))) assert service.project_dashboard(project_id="project-1")["counts"]["open_loops"] == 2 if parent_change == "missing": store.sources.clear() @@ -1603,13 +1603,13 @@ def test_project_service_validation_errors() -> None: service = VNextProjectService(InMemoryVNextProjectStore()) with pytest.raises(VNextProjectValidationError, match="max_items"): - service.extract_open_loops(ProjectAutomationRequest(max_items=0)) + service.extract_open_loops(ProjectAutomationRequest(agent_identity=None, max_items=0)) with pytest.raises(VNextProjectValidationError, match="no active project"): - service.generate_project_update_candidate(ProjectAutomationRequest()) + service.generate_project_update_candidate(ProjectAutomationRequest(agent_identity=None, )) store = _seed_store() service = VNextProjectService(store) - artifact = service.generate_project_update_candidate(ProjectAutomationRequest(project_id="project-1")) + artifact = service.generate_project_update_candidate(ProjectAutomationRequest(agent_identity=None, project_id="project-1")) with pytest.raises(VNextProjectValidationError, match="edited_current_state"): service.review_project_update(artifact_id=str(artifact["id"]), action="edit") diff --git a/tests/unit/test_vnext_rollups.py b/tests/unit/test_vnext_rollups.py index 2fc3fe5d9..1150dd312 100644 --- a/tests/unit/test_vnext_rollups.py +++ b/tests/unit/test_vnext_rollups.py @@ -1630,7 +1630,7 @@ def create_artifact(self, artifact: JsonObject, *, actor_type: str = "system") - shim = ArtifactShim(store) members = _seed_live_game_memories(store) artifact = VNextConsolidationService(shim, embedding_provider=None).generate_memory_consolidation( - MemoryConsolidationRequest() + MemoryConsolidationRequest(agent_identity=None, ) ) candidates = _rollup_candidates(store) @@ -1667,7 +1667,7 @@ def create_artifact(self, artifact: JsonObject, *, actor_type: str = "system") - _seed_live_game_memories(store) artifact = VNextConsolidationService(ArtifactShim(store), embedding_provider=None).generate_memory_consolidation( - MemoryConsolidationRequest(propose_rollups=False) + MemoryConsolidationRequest(agent_identity=None, propose_rollups=False) ) assert _rollup_candidates(store) == [] assert artifact["metadata_json"]["rollups"] == {"enabled": False} diff --git a/tests/unit/test_vnext_rollups_semantic.py b/tests/unit/test_vnext_rollups_semantic.py index 4e2ba5d45..5265c3b27 100644 --- a/tests/unit/test_vnext_rollups_semantic.py +++ b/tests/unit/test_vnext_rollups_semantic.py @@ -747,7 +747,7 @@ def test_consolidation_workflow_runs_semantic_tier_and_discloses() -> None: artifact = VNextConsolidationService( store, embedding_provider=(provider := MappedEmbeddingProvider(mapping)) - ).generate_memory_consolidation(MemoryConsolidationRequest()) + ).generate_memory_consolidation(MemoryConsolidationRequest(agent_identity=None, )) rollups_metadata = artifact["metadata_json"]["rollups"] assert rollups_metadata["enabled"] is True @@ -782,7 +782,7 @@ def test_consolidation_without_provider_keeps_rollups_dormant() -> None: mapping: dict[str, list[float]] = {} _seed(store, mapping, KITCHEN_SPECS, KITCHEN_VECTORS) artifact = VNextConsolidationService(store, embedding_provider=None).generate_memory_consolidation( - MemoryConsolidationRequest() + MemoryConsolidationRequest(agent_identity=None, ) ) rollups_metadata = artifact["metadata_json"]["rollups"] assert rollups_metadata["enabled"] is True diff --git a/tests/unit/test_workspaces_router_split.py b/tests/unit/test_workspaces_router_split.py index 74d5cf205..583136566 100644 --- a/tests/unit/test_workspaces_router_split.py +++ b/tests/unit/test_workspaces_router_split.py @@ -86,10 +86,10 @@ # Re-pin 2026-10-06: workspace reads authenticate the protected identity and # admit rows through effective labels before totals or dashboard disclosure. EXPECTED_ROUTE_AST_SHA256 = "b99f1435de67d9499819acb9ed7ed61b588a3fb0ff037e782eeca070b39af742" -EXPECTED_SUPPORT_AST_SHA256 = "052f564a10f2f32c090f857142eda105f62d5e44b698d3e7475506c422a5ca47" +EXPECTED_SUPPORT_AST_SHA256 = "000d3a03afeff74c5c7fbca8bfc5195553de4fec6dd962ea94aa9b7bfb58ad90" EXPECTED_ROUTE_NAME_MANIFEST_SHA256 = "225c57c08bd8314156c56352dd1c53ffed3f556ce285c666dd6fca125115d0b4" EXPECTED_OPERATION_MANIFEST_SHA256 = "c320979b62d7ee8de244fe38bde5bf3761a4f9d76f76bf3cd8576c30fce9857e" -EXPECTED_IMPORT_MANIFEST_SHA256 = "d8887934cacc6a4e5d52f080dae3c2c0d0cdf23d1518a4060a885a8c7f209c0c" +EXPECTED_IMPORT_MANIFEST_SHA256 = "e8c18d6831ca012b55b22f46c9b2151d62575773a2452ef0d0f2e869cabc8abb" EXPECTED_CARRIER_NAMES_SHA256 = "2c109fc234a05dd8f44e4c34bee49e797fbb5e49e92413391541a7e504da328b" # Re-pinned 2026-10-02 (DB-005, legacy /v0 routes). One definition changed, # found by a per-definition AST diff against the previous pin: @@ -140,7 +140,7 @@ } EXPECTED_SUPPORT_NODE_SHA256 = { "_vnext_status_counts": "0bf0ed228a14bd648a9d18fcd5f99ebf8c585bd29f4b5e81e1df17fe0201fd15", - "_vnext_workspace_payload": "c2d41b35c27496c469fd70ebdf6bba01d526dac24ed00bddf983cf6dee57022a", + "_vnext_workspace_payload": "762ce1dc4c0d85bb48a56730d5c9ced74230c87a5b50ec1e1cd65cd25851ba96", "_workspace_rows": "070bdfbd1eae10608bd8208b08367e1c0ea10e2064f03ad5a84121a190ed4cf0", "_workspace_event_visible": "8343c060909326a5cb69fa6f671ac62f160630ecf989f04d78e792ea74c0ea90", } @@ -489,7 +489,7 @@ def test_workspace_import_direction_pruning_timing_and_runtime_identities_are_ex main_definitions = _top_level_definitions(main_tree) router_imports = _import_manifest(router_tree) - assert len(router_imports) == 38 + assert len(router_imports) == 39 assert hashlib.sha256(json.dumps(router_imports, separators=(",", ":")).encode()).hexdigest() == ( EXPECTED_IMPORT_MANIFEST_SHA256 ) From 6e90658a7a0357c26cb6f67167ea6c0fa667a213 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 14:20:45 +0200 Subject: [PATCH 229/270] Regenerate search expectations and document remaining review boundaries --- CHANGELOG.md | 2 ++ tests/unit/fixtures_search_goldens.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de7b75f40..a640b3f62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Unreleased (on main, not in v0.20.0): the October 6 derived-label correction normalizes server UUIDs, carries bound scheduler identity, withholds encoded references and handles malformed marker shapes. Plain reviews share the label lock; label-changing reviews retain the bounded retryable refusal. Read labels and distinct parents are cached within one request. Context-pack rows dropped by effective admission are absent from the later stage's excluded counts and warnings. The search golden was regenerated through its recorder. Four older operator or legacy readers remain outside the corrected ceiling, as disclosed in the draft security note. SQLite retirement deliberately follows the saved-quote reader for repeated keys and quote subtrees; an older raw scan blanked some such rows that it now retains. + - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings, including JSON escapes and repeated keys. Lookup and removal both decode JSON text under source and memory references and inside trace fields. Admitted references and surrounding values stay, and the owner receives live references unchanged. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. - Unreleased (on main, not in v0.20.0): the derived-row pages state the three-part label, who reads an unverified row, and that the five operator screens apply the caller's sensitivity ceiling. The v0.20.0 notes omitted both limits. No migration is required. - Unreleased (on main, not in v0.20.0): PostgreSQL migration `20261005_0096` raises a stored derived row to the labels of its inputs, and SQLite does the same on the next open and before a restore is published. The open pass can leave a vault open with its completion key unstamped after a failed repair; explicit repair and restore refuse the whole change on failure. `alicebot vnext labels` and `alice-memory labels` check and repair the same rows, including rows added after an earlier repair. The doctors print how many derived rows are below their inputs or unverified, as a warning, and report failed checks as unavailable. v0.20.0 left the stored label where it was written. Migration `20261005_0096` must run before serving requests as the NOSUPERUSER NOBYPASSRLS table owner. diff --git a/tests/unit/fixtures_search_goldens.py b/tests/unit/fixtures_search_goldens.py index e8f6477cf..e4db19700 100644 --- a/tests/unit/fixtures_search_goldens.py +++ b/tests/unit/fixtures_search_goldens.py @@ -17,7 +17,7 @@ { "brief": { "computed_from": { - "commit": "040a2a10bd959e38295c0805875e4def1a8b465f", + "commit": "6990607c53a12caabcfb909da771dbee9a138da8", "tag": "v0.20.0" }, "scenarios": { @@ -43,7 +43,7 @@ }, "commit_result": { "computed_from": { - "commit": "040a2a10bd959e38295c0805875e4def1a8b465f", + "commit": "6990607c53a12caabcfb909da771dbee9a138da8", "tag": "v0.20.0" }, "scenarios": { @@ -1219,7 +1219,7 @@ }, "context_pack": { "computed_from": { - "commit": "040a2a10bd959e38295c0805875e4def1a8b465f", + "commit": "6990607c53a12caabcfb909da771dbee9a138da8", "tag": "v0.20.0" }, "scenarios": { @@ -2048,13 +2048,13 @@ "policy_decision" ], "full_pack_excluded_token_estimate": 1775, - "full_pack_serialized_token_estimate": 4545, + "full_pack_serialized_token_estimate": 4550, "is_transport_cap": false, "scope": "content_sections", "serialized_token_estimate": 1531, "serialized_token_estimate_scope": "compact_mcp_tool_payload", "token_budget": 8000, - "token_estimate": 2770, + "token_estimate": 2775, "truncated": false }, "trace_id": "policy-", @@ -2065,7 +2065,7 @@ }, "import_receipt": { "computed_from": { - "commit": "040a2a10bd959e38295c0805875e4def1a8b465f", + "commit": "6990607c53a12caabcfb909da771dbee9a138da8", "tag": "v0.20.0" }, "scenarios": { @@ -2136,11 +2136,13 @@ ], "exit_code": 0, "receipt": { + "changed_files_count": 1, "duplicate_count": 4, "error_code": null, "errors": [], "failed_count": 0, "imported_count": 1, + "replacement_hint": "Use --supersede --dry-run to preview replacement, then --supersede to apply it.", "skipped_count": 0, "skipped_credential_items": [], "skipped_credentials": 0, @@ -2427,7 +2429,7 @@ }, "recall": { "computed_from": { - "commit": "040a2a10bd959e38295c0805875e4def1a8b465f", + "commit": "6990607c53a12caabcfb909da771dbee9a138da8", "tag": "v0.20.0" }, "scenarios": { From 2cc78fe2a6cea3c887e02c3d1b7806626b3e89f2 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 14:24:45 +0200 Subject: [PATCH 230/270] Preserve selected source scopes at their producer boundary --- apps/api/src/alicebot_api/vnext_brain.py | 1 + .../src/alicebot_api/vnext_label_writes.py | 11 --------- apps/api/src/alicebot_api/vnext_projects.py | 1 + .../alicebot_api/vnext_source_regeneration.py | 1 + ...label_handoff_legacy_dashboard_postgres.py | 24 +++++++++++++++++++ ...est_scheduler_handoff_identity_postgres.py | 13 ++++++---- 6 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 tests/integration/test_label_handoff_legacy_dashboard_postgres.py diff --git a/apps/api/src/alicebot_api/vnext_brain.py b/apps/api/src/alicebot_api/vnext_brain.py index 2d83150a2..1e198cabb 100644 --- a/apps/api/src/alicebot_api/vnext_brain.py +++ b/apps/api/src/alicebot_api/vnext_brain.py @@ -1054,6 +1054,7 @@ def _create_candidate_open_loops( "discovered_by": "vnext_daily_brief", "source_id": str(source["id"]), "project_scope": list(project_scope), + "project_floor": list(source_project_scope(source)), "automation_digest": automation_digest, "workflow_digest": workflow_digest, }, diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 9558c7d19..7dd394b8b 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -298,17 +298,6 @@ def apply_insert_floor(store: Any, kind: str, payload: Mapping[str, object]) -> own["kind"] = kind own["id"] = own_id own["user_id"] = user_id - # A copy selected under an earlier source scope still contains that - # source's earlier text. Preserve its selected scope before rereading the - # current parent, which may already have moved to another project. - from alicebot_api.vnext_derived_labels import row_class - if row_class(kind, own) == "copy": - raw_meta = own.get("metadata_json") - selected_meta = dict(raw_meta) if isinstance(raw_meta, Mapping) else {} - selected_meta["project_floor"] = list(union_floor( - project_floor_shape(own)[1], [stored_scope(kind, own)], - )) - own["metadata_json"] = selected_meta batch = _current_capture_inputs(store) cache = batch.rows if batch is not None else None nodes, exceeded = collect_label_rows(store, [own], max_nodes=PROPAGATION_BOUND, cache=cache) diff --git a/apps/api/src/alicebot_api/vnext_projects.py b/apps/api/src/alicebot_api/vnext_projects.py index 552723a0d..debf4ea8a 100644 --- a/apps/api/src/alicebot_api/vnext_projects.py +++ b/apps/api/src/alicebot_api/vnext_projects.py @@ -468,6 +468,7 @@ def _open_loop_candidates(source: JsonObject) -> list[JsonObject]: "loop_type": loop_type, "owner": owner, "source_captured_at": source.get("captured_at"), + "project_floor": list(source_project_scope(source)), "discovered_by": "vnext_project_automation", }, {"sources": [source]}), } diff --git a/apps/api/src/alicebot_api/vnext_source_regeneration.py b/apps/api/src/alicebot_api/vnext_source_regeneration.py index 589d72010..e49fd16f2 100644 --- a/apps/api/src/alicebot_api/vnext_source_regeneration.py +++ b/apps/api/src/alicebot_api/vnext_source_regeneration.py @@ -33,6 +33,7 @@ def regenerate_source_inputs(store: Any, source: JsonObject) -> JsonObject: "source_chunk_index": candidate.source_chunk_index, "extraction_rule": candidate.extraction_rule, "project_scope": list(scope), + "project_floor": list(scope), "regeneration_id": generation, **({"provenance_role": candidate.provenance_role, "assertion_class": candidate.assertion_class} if candidate.provenance_role is not None else {}), }, {"sources": [source]}) diff --git a/tests/integration/test_label_handoff_legacy_dashboard_postgres.py b/tests/integration/test_label_handoff_legacy_dashboard_postgres.py new file mode 100644 index 000000000..7ca8e6dff --- /dev/null +++ b/tests/integration/test_label_handoff_legacy_dashboard_postgres.py @@ -0,0 +1,24 @@ +"""Both legacy dashboard aliases honor a declared reader's ceiling.""" +import pytest + +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError +from alicebot_api.vnext_derived_labels import with_derived_from +from tests.integration.derived_labels_postgres_support import label_harness + + +@pytest.mark.parametrize("tool", ["alice_vnext_project_dashboard", "alice_project_dashboard"]) +@pytest.mark.parametrize("profile", ["read_only_agent", "trusted_local_agent"]) +def test_legacy_dashboard_refuses_confidential_derived_project(label_harness, tool, profile): + h = label_harness + source = h.source(sensitivity="confidential") + with h.store() as store: + project = store.create_project({"name": "Synthetic secret project", "slug": "synthetic", + "domain": "project", "sensitivity": "public", + "metadata_json": with_derived_from({}, {"sources": [source]})}) + context = MCPRuntimeContext(database_url=h.urls["app"], user_id=h.user_id) + owner = call_mcp_tool(context, name=tool, arguments={"project_id": str(project["id"])}) + assert "Synthetic secret project" in str(owner) + with pytest.raises(MCPToolError): + call_mcp_tool(context, name=tool, arguments={"project_id": str(project["id"]), + "agent_id": "synthetic-reader", "permission_profile": profile}) diff --git a/tests/integration/test_scheduler_handoff_identity_postgres.py b/tests/integration/test_scheduler_handoff_identity_postgres.py index df1ad49f6..cce99c5ba 100644 --- a/tests/integration/test_scheduler_handoff_identity_postgres.py +++ b/tests/integration/test_scheduler_handoff_identity_postgres.py @@ -26,16 +26,21 @@ def test_scheduler_never_drops_the_bound_identity(migrated_database_urls, monkey @pytest.mark.parametrize("workflow", ["connection_report", "contradiction_report"]) -def test_core_mcp_automation_keeps_the_bound_identity(migrated_database_urls, monkeypatch, workflow): - from alicebot_api.mcp.registry import call_mcp_tool +def test_mcp_generation_adapter_keeps_a_resolved_bound_identity(migrated_database_urls, monkeypatch, workflow): + from alicebot_api.db import user_connection + from alicebot_api.vnext_store import PostgresVNextStore + from alicebot_api.vnext_agent_keys import resolve_agent_identity + from alicebot_api.mcp.capture_automation import _handle_alice_vnext_generate_artifact from alicebot_api.mcp.types import MCPRuntimeContext url = migrated_database_urls["app"] wire_database(monkeypatch, url) user, alpha, beta, rows, key, _, _ = seed_grid(url) monkeypatch.setenv("ALICE_AGENT_API_KEY", key) monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") - context = MCPRuntimeContext(database_url=url, user_id=user) - response = call_mcp_tool(context, name="alice_vnext_generate_artifact", arguments={ + with user_connection(url, user) as conn: + identity = resolve_agent_identity(PostgresVNextStore(conn), user_id=user, raw_key=key, payload={}) + context = MCPRuntimeContext(database_url=url, user_id=user, agent_identity=identity, agent_identity_resolved=True) + response = _handle_alice_vnext_generate_artifact(context, arguments={ "workflow_type": workflow, "query": "Atlas", "project_scope": [alpha]}) text = json.dumps(response, default=str) From b52be2c83717278864dc0d1e52da183b087ed562 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 14:25:44 +0200 Subject: [PATCH 231/270] Pin shared label locks at ordinary exact review entrypoints --- .../test_derived_labels_exact_entrypoints_postgres.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_derived_labels_exact_entrypoints_postgres.py b/tests/integration/test_derived_labels_exact_entrypoints_postgres.py index 476991fea..765e0a2ad 100644 --- a/tests/integration/test_derived_labels_exact_entrypoints_postgres.py +++ b/tests/integration/test_derived_labels_exact_entrypoints_postgres.py @@ -66,7 +66,9 @@ def test_exact_entrypoint_checks_effective_floor(migrated_database_urls, monkeyp def checked(store, *args, _original=original, **kwargs): locks = held_label_locks(store) - assert locks == (True, True, True), (door, locks) + # Accept, approve and redact change status/content only. + # They still hold S and L before the row lock, with L shared. + assert locks == (True, True, False), (door, locks) mutation_locks.append(locks) return _original(store, *args, **kwargs) From b27ca6d38d1c9cd9606cfbb5718547aa78183d98 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 14:31:54 +0200 Subject: [PATCH 232/270] Clarify retirement limits without changing release history --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a640b3f62..d46013990 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ ## Unreleased -- Unreleased (on main, not in v0.20.0): the October 6 derived-label correction normalizes server UUIDs, carries bound scheduler identity, withholds encoded references and handles malformed marker shapes. Plain reviews share the label lock; label-changing reviews retain the bounded retryable refusal. Read labels and distinct parents are cached within one request. Context-pack rows dropped by effective admission are absent from the later stage's excluded counts and warnings. The search golden was regenerated through its recorder. Four older operator or legacy readers remain outside the corrected ceiling, as disclosed in the draft security note. SQLite retirement deliberately follows the saved-quote reader for repeated keys and quote subtrees; an older raw scan blanked some such rows that it now retains. -- Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings, including JSON escapes and repeated keys. Lookup and removal both decode JSON text under source and memory references and inside trace fields. Admitted references and surrounding values stay, and the owner receives live references unchanged. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. +- Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings, including JSON escapes. Repeated keys and quote subtrees follow the limits in the draft security note. Lookup and removal both decode JSON text under source and memory references and inside trace fields. Admitted references and surrounding values stay, and the owner receives live references unchanged. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. +- Unreleased (on main, not in v0.20.0): the October 6 derived-label correction normalizes server UUIDs, carries bound scheduler identity, withholds encoded references and handles malformed marker shapes. Plain reviews share the label lock; label-changing reviews retain the bounded retryable refusal. Read labels and distinct parents are cached within one request. Context-pack rows dropped by effective admission are absent from the later stage's excluded counts and warnings. The search golden was regenerated through its recorder. Four older operator or legacy readers remain outside the corrected ceiling, as disclosed in the draft security note. SQLite retirement deliberately follows the saved-quote reader for repeated keys and quote subtrees; an older raw scan blanked some such rows that it now retains. - Unreleased (on main, not in v0.20.0): the derived-row pages state the three-part label, who reads an unverified row, and that the five operator screens apply the caller's sensitivity ceiling. The v0.20.0 notes omitted both limits. No migration is required. - Unreleased (on main, not in v0.20.0): PostgreSQL migration `20261005_0096` raises a stored derived row to the labels of its inputs, and SQLite does the same on the next open and before a restore is published. The open pass can leave a vault open with its completion key unstamped after a failed repair; explicit repair and restore refuse the whole change on failure. `alicebot vnext labels` and `alice-memory labels` check and repair the same rows, including rows added after an earlier repair. The doctors print how many derived rows are below their inputs or unverified, as a warning, and report failed checks as unavailable. v0.20.0 left the stored label where it was written. Migration `20261005_0096` must run before serving requests as the NOSUPERUSER NOBYPASSRLS table owner. - Unreleased (on main, not in v0.20.0): recall, context packs, resume, recent decisions, the session brief, the review queue, open-loop lists, and the workspace, dashboard, context tree, and dogfooding lists drop a loaded row whose effective label is outside the request. The artifact list, the source trace, the project list, the dashboard project row, and the belief state route apply the caller's sensitivity ceiling, including titles, ids, and counts. Resume, recent decisions, the session brief, audit, and explain apply the target row's current effective label, so an excluded key does not learn of a label change. The producer input readers also drop an input whose effective label is outside the request before it appears in report text. v0.20.0 returned rows by their stored labels and could copy a public row with confidential inputs into a new report. No migration is required. From 36794025e41902708f7640dc87f52ea568e03039 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 14:38:21 +0200 Subject: [PATCH 233/270] Check regenerated memory floors in the stale source race --- .../test_producer_handoff_stale_inputs_postgres.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_producer_handoff_stale_inputs_postgres.py b/tests/integration/test_producer_handoff_stale_inputs_postgres.py index 3fd0561be..7b8abb6d6 100644 --- a/tests/integration/test_producer_handoff_stale_inputs_postgres.py +++ b/tests/integration/test_producer_handoff_stale_inputs_postgres.py @@ -56,7 +56,12 @@ def spy(self, payload, *args, _kind=kind, _original=original, **kwargs): monkeypatch.setattr(store, "search_sources", lambda **_kwargs: [deepcopy(stale)]) VNextProjectService(store).extract_open_loops(ProjectAutomationRequest(agent_identity=None, project_id=alpha)) else: - regenerate_source_inputs(store, stale) + regenerated = regenerate_source_inputs(store, stale) + assert regenerated["memory_ids"] + for memory in store.read_label_rows("memory", regenerated["memory_ids"]): + assert memory["domain"] == "health" + assert memory["sensitivity"] == "confidential" + assert set(memory["metadata_json"]["project_floor"]) == {alpha, beta} assert "open_loop" in observed loops = store.list_open_loops(status=None, sensitivity_allowed=["confidential"], limit=50) assert loops From 822ad9a60598cf484acf8960442bee4d29f19ff0 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 14:42:02 +0200 Subject: [PATCH 234/270] Preserve read compatibility and published release corrections --- apps/api/src/alicebot_api/mcp/runtime.py | 1 + .../src/alicebot_api/vnext_consolidation.py | 19 ++++++++++++------- apps/api/src/alicebot_api/vnext_projects.py | 4 +++- docs/release/v0.20.0-release-notes.md | 2 +- tests/unit/test_cli.py | 3 ++- ...oop_ids_every_spelling_and_after_delete.py | 17 +++++++++-------- tests/unit/test_per_project_policy_views.py | 3 +++ 7 files changed, 31 insertions(+), 18 deletions(-) diff --git a/apps/api/src/alicebot_api/mcp/runtime.py b/apps/api/src/alicebot_api/mcp/runtime.py index d85204f45..3909b1c10 100644 --- a/apps/api/src/alicebot_api/mcp/runtime.py +++ b/apps/api/src/alicebot_api/mcp/runtime.py @@ -62,6 +62,7 @@ def _store_context(context: MCPRuntimeContext): def _vnext_store_context(context: MCPRuntimeContext): from alicebot_api.vnext_label_guard import label_read_scope + store: SQLiteVNextStore | PostgresVNextStore if _is_sqlite_backend(context): sqlite_path = _sqlite_path_from_url(context.database_url) with sqlite_user_connection(sqlite_path, context.user_id) as conn: diff --git a/apps/api/src/alicebot_api/vnext_consolidation.py b/apps/api/src/alicebot_api/vnext_consolidation.py index 803f7aa37..ce32883d9 100644 --- a/apps/api/src/alicebot_api/vnext_consolidation.py +++ b/apps/api/src/alicebot_api/vnext_consolidation.py @@ -1229,13 +1229,18 @@ def readable_reference(ref): return not (parsed.named - admitted or parsed.incidental & refused) # Only copies used to render this run change. Stored members and # their provenance remain available to their authorized readers. - clusters_for_proposals = [[{ - **member, "metadata_json": { - **(member.get("metadata_json") or {}), - "source_refs": [ref for ref in (member.get("metadata_json") or {}).get("source_refs", []) - if readable_reference(ref)], - }, - } for member in members] for members in clusters_for_proposals] + def readable_member(member: JsonObject) -> JsonObject: + metadata = member.get("metadata_json") + if not isinstance(metadata, dict): + return member + refs = metadata.get("source_refs") + if not isinstance(refs, list): + return member + return {**member, "metadata_json": { + **metadata, "source_refs": [ref for ref in refs if readable_reference(ref)], + }} + clusters_for_proposals = [[readable_member(member) for member in members] + for members in clusters_for_proposals] # The report copies the ``source_refs`` of each proposed cluster member as stored, and the candidate memories # copy them too. The refs are not dropped: they are the provenance. But the report is read behind its label # alone, so the label has to be at least as strict as every source they name. The list printed is made here, diff --git a/apps/api/src/alicebot_api/vnext_projects.py b/apps/api/src/alicebot_api/vnext_projects.py index debf4ea8a..01dbd1511 100644 --- a/apps/api/src/alicebot_api/vnext_projects.py +++ b/apps/api/src/alicebot_api/vnext_projects.py @@ -866,7 +866,9 @@ def review_project_update( if action in {"accept", "edit"}: acquire_exclusive_label_lock(self.store) else: - self.store.lock_label_writes() + lock_labels = getattr(self.store, "lock_label_writes", None) + if callable(lock_labels): + lock_labels() # The artifact is the review decision's serialization point. Every # accept/edit/reject path must inspect and transition the same locked # row so stale reviewers cannot split project, memory, and artifact diff --git a/docs/release/v0.20.0-release-notes.md b/docs/release/v0.20.0-release-notes.md index c8a67ad40..59888d6c8 100644 --- a/docs/release/v0.20.0-release-notes.md +++ b/docs/release/v0.20.0-release-notes.md @@ -3,7 +3,7 @@ > **Correction (2026-10-05):** these notes do not say that a derived summary, report or copy kept the label its inputs had when it was made, or that a report could show a key bound to one project rows of other projects or with no project. Both are in v0.20.0 and are listed under known limitations. -Unreleased (on main, not in v0.20.0): four older operator or legacy readers still expose withheld information: source GET returns titles and raw text, graph neighborhood returns edge explanations, doctor returns label-check counts, and legacy `alice_vnext_review_items` lists derived memories for a declared restricted identity. See the [draft security note](derived-labels-security-note-draft.md) for the remaining boundary and the SQLite retirement limits. +> **Correction (2026-10-06):** four older operator or legacy readers still expose withheld information: source GET returns titles and raw text, graph neighborhood returns edge explanations, doctor returns label-check counts, and legacy `alice_vnext_review_items` lists derived memories for a declared restricted identity. These limits also occur in v0.20.0. See the [draft security note](derived-labels-security-note-draft.md) for the current boundary and SQLite retirement limits. **Take this release if you run the Postgres stack's HTTP API, use an embeddings endpoint (above all a hosted one), import ChatGPT or Markdown files, use the diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index d5c612003..a65c70825 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -455,9 +455,10 @@ def lock_graph_mutation(self) -> None: self.lock_calls.append("graph") def lock_label_writes(self, *, exclusive: bool = False) -> None: - assert self.graph_locked if exclusive: + assert self.graph_locked assert self.conn.lock_timeout == "3s" + # A read can hold shared L alone. Label-changing writes take S first. self.labels_exclusive |= exclusive self.lock_calls.append("exclusive_labels" if exclusive else "shared_labels") diff --git a/tests/unit/test_open_loop_ids_every_spelling_and_after_delete.py b/tests/unit/test_open_loop_ids_every_spelling_and_after_delete.py index 7e5e9d618..51be34da2 100644 --- a/tests/unit/test_open_loop_ids_every_spelling_and_after_delete.py +++ b/tests/unit/test_open_loop_ids_every_spelling_and_after_delete.py @@ -693,13 +693,10 @@ def test_a_value_that_is_only_an_id_with_hyphens_in_other_places_is_read_as_an_i assert out[0]["source_id"] == odd and out[0]["metadata_json"] is metadata -def test_an_id_with_hyphens_in_other_places_inside_longer_text_is_not_read_and_is_returned_as_stored() -> None: - """The limit the module docstring, the docs and the CHANGELOG state: inside text only the hyphenated layout (8, 4, 4, - 4 and 12 digits) and a run of 32 digits are read. The 32 digits with hyphens in groups of four, in a URL or a - sentence, are not read, under a reference key or under another key. Reading them was the cause of the false - positive above, and reading them is not what the tower spec asks for. - - Mutation: let hyphens stand anywhere among the 32 digits in ``_BARE_ID`` (the values are then cut). +def test_an_id_with_irregular_hyphens_is_withheld_when_the_reference_reader_names_it() -> None: + """The shared reference reader names irregular hyphens in prose, while an + unrelated URL remains opaque. After text scrubbing, its parser is the final + authority on whether a refused id remains in the returned projection. """ store, identifier = _state("source", "refused") @@ -710,7 +707,11 @@ def test_an_id_with_hyphens_in_other_places_inside_longer_text_is_not_read_and_i } out = withhold_unreadable_references(store, [_loop(source_id=identifier, metadata_json=metadata)], fence=_FENCE) assert out[0]["source_id"] is None - assert out[0]["metadata_json"] == metadata + assert out[0]["metadata_json"] == { + "source_refs": [f"https://example.test/{odd}"], + "evidence": {"all": []}, + } + assert metadata["source_refs"] == [f"see {odd} now", f"https://example.test/{odd}"] @pytest.mark.parametrize("dense", ["a-" * 20000, "ab-" * 20000, "0123-4567-89ab-cdef-" * 2000, "-".join(["0f"] * 30000)]) diff --git a/tests/unit/test_per_project_policy_views.py b/tests/unit/test_per_project_policy_views.py index de19209b0..add929dd3 100644 --- a/tests/unit/test_per_project_policy_views.py +++ b/tests/unit/test_per_project_policy_views.py @@ -65,6 +65,9 @@ ("mcp/scheduler.py", "_handle_alice_vnext_scheduler_resume", "_policy_checked"): 1, ("mcp/synthesis.py", "_handle_alice_generate_connections", "_mcp_agent_policy_preflight"): 1, ("mcp/synthesis.py", "_handle_alice_generate_contradictions", "_mcp_agent_policy_preflight"): 1, + # The legacy brain adapters preflight here and pass the resulting identity + # and scope to the producer, which applies all-of admission. + ("mcp/synthesis.py", "_authorized_brain_request", "_mcp_agent_policy_preflight"): 1, } ) From 7b8bb12d29c53c1f733f319aa039e3cde612a771 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 14:45:38 +0200 Subject: [PATCH 235/270] Compare the stored project scope as an unordered set --- tests/integration/test_daily_brief_project_id_postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_daily_brief_project_id_postgres.py b/tests/integration/test_daily_brief_project_id_postgres.py index 3089f8733..c7730cd52 100644 --- a/tests/integration/test_daily_brief_project_id_postgres.py +++ b/tests/integration/test_daily_brief_project_id_postgres.py @@ -71,7 +71,7 @@ def test_daily_brief_keeps_a_free_form_project_out_of_the_uuid_column(migrated_d sensitivity_allowed=list(ALL_SENSITIVITY), limit=20, ) - by_scope = {tuple(row["metadata_json"]["project_scope"]): str(row.get("project_id") or "") for row in loops} + by_scope = {tuple(sorted(row["metadata_json"]["project_scope"])): str(row.get("project_id") or "") for row in loops} assert by_scope[("Alice",)] == "" assert by_scope[(project_id,)] == project_id assert by_scope[("Alice", "Bob")] == "" From 9dfcd20a945deab20a6fbdc0858d1a3ac092933d Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 14:57:45 +0200 Subject: [PATCH 236/270] Pin the source map security patch required by the dependency audit --- apps/web/package.json | 1 + apps/web/pnpm-lock.yaml | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/web/package.json b/apps/web/package.json index f4e3cf95c..accc5ec82 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -55,6 +55,7 @@ "nanoid": "3.3.18", "postcss": "8.5.26", "sharp": "0.35.4", + "source-map-js": "1.2.2", "vite": "6.4.3", "ws": "8.21.0" }, diff --git a/apps/web/pnpm-lock.yaml b/apps/web/pnpm-lock.yaml index 8c98fe4ca..c79da48a7 100644 --- a/apps/web/pnpm-lock.yaml +++ b/apps/web/pnpm-lock.yaml @@ -12,6 +12,7 @@ overrides: nanoid: 3.3.18 postcss: 8.5.26 sharp: 0.35.4 + source-map-js: 1.2.2 vite: 6.4.3 ws: 8.21.0 @@ -2330,8 +2331,8 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} - source-map-js@1.2.1: - resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + source-map-js@1.2.2: + resolution: {integrity: sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==} engines: {node: '>=0.10.0'} stable-hash@0.0.5: @@ -3789,7 +3790,7 @@ snapshots: css-tree@3.2.1: dependencies: mdn-data: 2.27.1 - source-map-js: 1.2.1 + source-map-js: 1.2.2 css.escape@1.5.1: {} @@ -4661,7 +4662,7 @@ snapshots: dependencies: '@babel/parser': 7.29.7 '@babel/types': 7.29.7 - source-map-js: 1.2.1 + source-map-js: 1.2.2 make-dir@4.0.0: dependencies: @@ -4847,7 +4848,7 @@ snapshots: dependencies: nanoid: 3.3.18 picocolors: 1.1.1 - source-map-js: 1.2.1 + source-map-js: 1.2.2 prelude-ls@1.2.1: {} @@ -5080,7 +5081,7 @@ snapshots: signal-exit@4.1.0: {} - source-map-js@1.2.1: {} + source-map-js@1.2.2: {} stable-hash@0.0.5: {} From 76de360a2ab28f36c3a6ee3a1b447701762337f9 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 15:02:47 +0200 Subject: [PATCH 237/270] Pin the patched worker pool required by the full dependency audit --- apps/web/package.json | 1 + apps/web/pnpm-lock.yaml | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/web/package.json b/apps/web/package.json index accc5ec82..3a21d9c14 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -56,6 +56,7 @@ "postcss": "8.5.26", "sharp": "0.35.4", "source-map-js": "1.2.2", + "tinypool": "2.1.2", "vite": "6.4.3", "ws": "8.21.0" }, diff --git a/apps/web/pnpm-lock.yaml b/apps/web/pnpm-lock.yaml index c79da48a7..5681eb734 100644 --- a/apps/web/pnpm-lock.yaml +++ b/apps/web/pnpm-lock.yaml @@ -13,6 +13,7 @@ overrides: postcss: 8.5.26 sharp: 0.35.4 source-map-js: 1.2.2 + tinypool: 2.1.2 vite: 6.4.3 ws: 8.21.0 @@ -2437,9 +2438,9 @@ packages: resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} engines: {node: '>=12.0.0'} - tinypool@1.1.1: - resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} - engines: {node: ^18.0.0 || >=20.0.0} + tinypool@2.1.2: + resolution: {integrity: sha512-9YodfrxS9g9IbFr/KOjE5bAeJ0p61n3bW6mqvy0jtoeKd1kTW1Cxm0oulm6KX2lyM9Gl6WIe8nEbY7LWv5ZJww==} + engines: {node: ^20.0.0 || >=22.0.0} tinyrainbow@2.0.0: resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} @@ -5204,7 +5205,7 @@ snapshots: fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 - tinypool@1.1.1: {} + tinypool@2.1.2: {} tinyrainbow@2.0.0: {} @@ -5389,7 +5390,7 @@ snapshots: tinybench: 2.9.0 tinyexec: 0.3.2 tinyglobby: 0.2.16 - tinypool: 1.1.1 + tinypool: 2.1.2 tinyrainbow: 2.0.0 vite: 6.4.3(@types/node@26.1.2) vite-node: 3.2.4(@types/node@26.1.2) From 1947386282e5557f7c1a78e7d5040686efa9cdd6 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 23:16:59 +0200 Subject: [PATCH 238/270] Fix reassignment retries, weekly floors and reader budgets --- .github/workflows/tests.yml | 13 ++ CHANGELOG.md | 4 +- apps/api/src/alicebot_api/main.py | 2 +- .../alicebot_api/mcp/evidence_artifacts.py | 25 ++- .../alicebot_api/routers/vnext_memories.py | 20 ++- apps/api/src/alicebot_api/sqlite_store.py | 61 +++++++- apps/api/src/alicebot_api/vnext_brain.py | 10 +- apps/api/src/alicebot_api/vnext_connectors.py | 14 +- .../src/alicebot_api/vnext_derived_labels.py | 82 +++++++++- .../src/alicebot_api/vnext_label_closure.py | 3 + .../api/src/alicebot_api/vnext_label_guard.py | 117 ++++++++++---- apps/api/src/alicebot_api/vnext_label_sql.py | 22 +++ .../vnext_open_loop_references.py | 4 +- apps/api/src/alicebot_api/vnext_store.py | 28 +++- .../vnext_stores/postgres/graph_open_loops.py | 4 +- docs/alpha/known-limitations.md | 2 +- .../derived-labels-security-note-draft.md | 6 +- docs/release/v0.20.0-release-notes.md | 2 +- tests/integration/conftest.py | 10 -- .../test_label_handoff_contention_postgres.py | 17 ++- .../test_label_round2_producers_postgres.py | 70 +++++++++ .../test_label_round2_read_budget_postgres.py | 18 +++ ...est_label_round2_reader_fences_postgres.py | 97 ++++++++++++ tests/performance/round2_budget_probe.py | 74 +++++++++ .../test_label_round2_read_budget.py | 89 +++++++++++ tests/unit/test_label_count_partition.py | 43 ++++++ tests/unit/test_label_dependency_cache.py | 143 ++++++++++++++++++ tests/unit/test_label_door_registry.py | 4 +- .../unit/test_source_scrub_handoff_passes.py | 10 +- 29 files changed, 916 insertions(+), 78 deletions(-) create mode 100644 apps/api/src/alicebot_api/vnext_label_sql.py create mode 100644 tests/integration/test_label_round2_producers_postgres.py create mode 100644 tests/integration/test_label_round2_read_budget_postgres.py create mode 100644 tests/integration/test_label_round2_reader_fences_postgres.py create mode 100644 tests/performance/round2_budget_probe.py create mode 100644 tests/performance/test_label_round2_read_budget.py create mode 100644 tests/unit/test_label_count_partition.py create mode 100644 tests/unit/test_label_dependency_cache.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 010d9c6c5..a01911cf2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2304,6 +2304,12 @@ jobs: --health-retries 20 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Paired read-budget baseline (main at the round-two handoff) + if: matrix.integration_check == 'Integration tests (Postgres + pgvector, role separation)' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: 48873b038013f4cf548099fcc4610a150972eedd + path: .read-budget-baseline - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" @@ -2329,6 +2335,7 @@ jobs: env: DATABASE_URL: postgresql://alicebot_app:ci@localhost:5432/alicebot DATABASE_ADMIN_URL: postgresql://alicebot_admin:ci@localhost:5432/alicebot + ALICE_READ_MAIN_CHECKOUT: ${{ github.workspace }}/.read-budget-baseline # --timeout=180 names one test that runs that long. The cancelled # jobs were slow database setup, not a gap near 180s, so that flag # would not have fired. --session-timeout=1500 is checked between @@ -2336,6 +2343,12 @@ jobs: # job cancel. run: ALICE_LEGACY_SURFACES=1 ./.venv/bin/python -m pytest tests/integration -q -p no:cacheprovider --durations=20 --timeout=180 --session-timeout=1500 + - name: SQLite varied-parent read budgets against the same baseline + if: matrix.integration_check == 'Integration tests (Postgres + pgvector, role separation)' + env: + ALICE_READ_MAIN_CHECKOUT: ${{ github.workspace }}/.read-budget-baseline + run: ./.venv/bin/python -m pytest tests/performance/test_label_round2_read_budget.py -q --timeout=180 --session-timeout=240 + - name: Default-surface core round-trip if: matrix.integration_check == 'Default surface integration smoke (Postgres)' env: diff --git a/CHANGELOG.md b/CHANGELOG.md index d46013990..2c60e2031 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,11 @@ ## Unreleased +- Unreleased (on main, not in v0.20.0): October 6 round-two corrections recover a per-user open-loop digest after a project reassignment or stale duplicate extraction, and weekly candidates record the report inputs before insert so their stored floor matches the weekly artifact in the same transaction. Read-time labels reuse verified dependency signatures within one request; original rows are counted in SQL, and the derived partition still receives the kernel guard. PostgreSQL event filtering uses an indexed guarded UUID comparison. SQLite filters definitely hidden direct parents before its event limit, then applies the full effective-label guard. CI compares unique-metadata, varied-parent and repaired fixtures against pinned main 48873b03. No new migration is required. + - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings, including JSON escapes. Repeated keys and quote subtrees follow the limits in the draft security note. Lookup and removal both decode JSON text under source and memory references and inside trace fields. Admitted references and surrounding values stay, and the owner receives live references unchanged. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. -- Unreleased (on main, not in v0.20.0): the October 6 derived-label correction normalizes server UUIDs, carries bound scheduler identity, withholds encoded references and handles malformed marker shapes. Plain reviews share the label lock; label-changing reviews retain the bounded retryable refusal. Read labels and distinct parents are cached within one request. Context-pack rows dropped by effective admission are absent from the later stage's excluded counts and warnings. The search golden was regenerated through its recorder. Four older operator or legacy readers remain outside the corrected ceiling, as disclosed in the draft security note. SQLite retirement deliberately follows the saved-quote reader for repeated keys and quote subtrees; an older raw scan blanked some such rows that it now retains. +- Unreleased (on main, not in v0.20.0): the October 6 derived-label correction normalizes server UUIDs, carries bound scheduler identity, withholds encoded references and handles malformed marker shapes. Plain reviews share the label lock; label-changing reviews retain the bounded retryable refusal. Read labels and distinct parents are cached within one request. Context-pack rows dropped by effective admission are absent from the later stage's excluded counts and warnings. The search golden was regenerated through its recorder. Source GET and the legacy review list now apply the full caller fence. Graph edge explanations remain outside that ceiling, and the doctor label counts are new in this set, as disclosed in the draft security note. The sources_first golden token estimates each rise by 5 because derived inserts persist empty project_scope and project_floor. SQLite retirement deliberately follows the saved-quote reader for repeated keys and quote subtrees; an older raw scan blanked some such rows that it now retains. - Unreleased (on main, not in v0.20.0): the derived-row pages state the three-part label, who reads an unverified row, and that the five operator screens apply the caller's sensitivity ceiling. The v0.20.0 notes omitted both limits. No migration is required. - Unreleased (on main, not in v0.20.0): PostgreSQL migration `20261005_0096` raises a stored derived row to the labels of its inputs, and SQLite does the same on the next open and before a restore is published. The open pass can leave a vault open with its completion key unstamped after a failed repair; explicit repair and restore refuse the whole change on failure. `alicebot vnext labels` and `alice-memory labels` check and repair the same rows, including rows added after an earlier repair. The doctors print how many derived rows are below their inputs or unverified, as a warning, and report failed checks as unavailable. v0.20.0 left the stored label where it was written. Migration `20261005_0096` must run before serving requests as the NOSUPERUSER NOBYPASSRLS table owner. - Unreleased (on main, not in v0.20.0): recall, context packs, resume, recent decisions, the session brief, the review queue, open-loop lists, and the workspace, dashboard, context tree, and dogfooding lists drop a loaded row whose effective label is outside the request. The artifact list, the source trace, the project list, the dashboard project row, and the belief state route apply the caller's sensitivity ceiling, including titles, ids, and counts. Resume, recent decisions, the session brief, audit, and explain apply the target row's current effective label, so an excluded key does not learn of a label change. The producer input readers also drop an input whose effective label is outside the request before it appears in report text. v0.20.0 returned rows by their stored labels and could copy a public row with confidential inputs into a new report. No migration is required. diff --git a/apps/api/src/alicebot_api/main.py b/apps/api/src/alicebot_api/main.py index 1378f61a4..b75e7706d 100644 --- a/apps/api/src/alicebot_api/main.py +++ b/apps/api/src/alicebot_api/main.py @@ -701,6 +701,7 @@ async def receive() -> dict[str, object]: _VNEXT_ROUTE_LOCAL_POLICY = frozenset( { + ("GET", "/v0/vnext/sources/{source_id}"), ("POST", "/v0/vnext/sources"), ("POST", "/v0/vnext/agents/ingest-output"), ("POST", "/v0/vnext/artifacts/{artifact_id}/insight-feedback"), @@ -762,7 +763,6 @@ async def receive() -> dict[str, object]: ("GET", "/v0/vnext/scheduler/runs"), ("GET", "/v0/vnext/scheduler/status"), ("GET", "/v0/vnext/settings/brain-charter"), - ("GET", "/v0/vnext/sources/{source_id}"), ("GET", "/v0/vnext/traces/sources/{source_id}"), ("GET", "/v0/vnext/workspace"), ("PATCH", "/v0/vnext/connectors/{connector_name}/config"), diff --git a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py index 936d338e6..feab58e69 100644 --- a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py +++ b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py @@ -734,12 +734,27 @@ def _handle_alice_vnext_memory_audit(context: MCPRuntimeContext, arguments: Mapp def _handle_alice_vnext_review_items(context: MCPRuntimeContext, arguments: Mapping[str, object]) -> JsonObject: + from alicebot_api.vnext_label_guard import LabelGuard, label_read_scope + identity = _agent_identity_from_arguments(context, arguments) + fence = SourceReadFence.for_identity(identity) + limit = _parse_int(arguments, key="limit", default=20, minimum=1, maximum=100) with _vnext_store_context(context) as store: - items = [ - row - for row in store.list_memories(status=None) - if str(row.get("status")) in {"candidate", "needs_review", "private_only"} - ][: _parse_int(arguments, key="limit", default=20, minimum=1, maximum=100)] + lock = getattr(store, "lock_label_writes", None) + if callable(lock): + lock() + with label_read_scope(store): + guard = LabelGuard.for_fence(store, fence) + items = [] + prefix = max(50, limit) + while True: + rows = store.list_memories(status=None, limit=prefix) + items = [row for row in rows + if str(row.get("status")) in {"candidate", "needs_review", "private_only"} + and isinstance(effective := guard.effective_row("memory", row), Mapping) + and fence.admits_memory(effective)][:limit] + if len(items) >= limit or len(rows) < prefix: + break + prefix *= 2 return _json_object({"items": items, "count": len(items)}) diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index b09ec1446..587ce8208 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -764,11 +764,23 @@ def run_vnext_doctor(request: VNextDoctorRunRequest) -> JSONResponse: @source_review_router.get("/v0/vnext/sources/{source_id}") -def get_vnext_source(source_id: UUID, user_id: UUID) -> JSONResponse: +def get_vnext_source(source_id: UUID, user_id: UUID, authorization: str | None = Header(default=None)) -> JSONResponse: + from alicebot_api.vnext_label_guard import effective_row_for_fence + from alicebot_api.vnext_source_fence import SourceReadFence settings = get_settings() - - with user_connection(settings.database_url, user_id) as conn: - payload = PostgresVNextStore(conn).get_source(str(source_id)) + try: + with user_connection(settings.database_url, user_id) as conn: + store = PostgresVNextStore(conn) + identity = resolve_protected_agent_identity( + store, user_id=user_id, + raw_key=agent_key_from_authorization(authorization if isinstance(authorization, str) else None), payload={}) + payload = store.get_source(str(source_id)) + if payload is not None and not SourceReadFence.for_identity(identity).admits( + effective_row_for_fence(store, identity, "source", payload) + ): + payload = None + except AgentKeyAuthenticationError as exc: + return _vnext_agent_auth_error_response(exc) if payload is None: return JSONResponse(status_code=404, content={"detail": f"vNext source {source_id} was not found"}) diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index 4dde6f6b8..6fb3fada9 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -390,6 +390,30 @@ def sqlite_user_connection(path: str | Path, user_id: UUID | str, *, repair_labe conn.close() +def _direct_source_hint(raw: object) -> str | None: + """A direct parent for a conservative prefilter, never an admission grant. + + Decode like the store and kernel, including JSON strings and duplicate or + escaped keys. Aliases the exact SQLite index cannot find fall through to + the complete read-time guard. + """ + try: + metadata = json.loads(raw) if isinstance(raw, str) else raw + if isinstance(metadata, str): + metadata = json.loads(metadata) + except (ValueError, TypeError): + return None + if not isinstance(metadata, Mapping) or metadata.get("redacted") is True: + return None + source_id = metadata.get("source_id") + if not isinstance(source_id, str) or not source_id.strip(): + return None + try: + return str(UUID(source_id)) + except (ValueError, TypeError): + return source_id + + class SQLiteVNextStore: """SQLite-backed vNext repository facade for the second-brain kernel.""" @@ -403,6 +427,7 @@ def __init__(self, conn: sqlite3.Connection, user_id: UUID | str): self.user_id = str(user_id) _ensure_embedding_content_sha256_sqlite(self.conn) _ensure_project_scope_identity_sqlite(self.conn) + self.conn.create_function("alice_direct_source_hint", 1, _direct_source_hint, deterministic=True) def lock_label_writes(self, *, exclusive: bool = False) -> None: """The SQLite writer lock is the label lock. Begin it when none is open.""" @@ -446,7 +471,25 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: (self.user_id, *wanted, *canonical), ) - def iter_label_rows(self, kind: str, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + def count_original_label_statuses(self, kind: str, *, domains=(), sensitivity_allowed=()) -> dict[str, int]: + """Count the definitely-original, unscoped partition using stored labels.""" + from alicebot_api.vnext_label_sql import original_label_sql + table = {"source": "sources", "memory": "memories", "open_loop": "open_loops"}[kind] + status = "status" if kind != "source" else "'unknown'" + predicate = original_label_sql(kind, sqlite=True) + live = " AND deleted_at IS NULL" if kind in {"source", "memory"} else "" + params = [self.user_id] + where = "user_id=? AND " + predicate + live + if domains: + where += " AND (domain IN (" + ",".join("?" for _ in domains) + ") OR domain='unknown')" + params.extend(domains) + if sensitivity_allowed: + where += " AND sensitivity IN (" + ",".join("?" for _ in sensitivity_allowed) + ")" + params.extend(sensitivity_allowed) + rows = self._fetch_all(f"SELECT {status} AS status, COUNT(*) AS count FROM {table} WHERE {where} GROUP BY {status}", tuple(params)) + return {str(row["status"]): int(cast(int, row["count"])) for row in rows} + + def iter_label_rows(self, kind: str, *, batch_size: int = 500, derived_only: bool = False) -> Iterator[list[VNextRow]]: """Complete counted population, in narrow tenant-bound keyset batches.""" if batch_size < 1: @@ -460,6 +503,9 @@ def iter_label_rows(self, kind: str, *, batch_size: int = 200) -> Iterator[list[ elif kind == "open_loop": extra = ", status, project_id, source_id, memory_id" live = " AND deleted_at IS NULL" if kind in {"source", "memory"} else "" + from alicebot_api.vnext_label_sql import original_label_sql + if derived_only: + live += " AND NOT COALESCE(" + original_label_sql(kind, sqlite=True) + ", FALSE)" after = "" while True: rows = self._fetch_all( @@ -693,6 +739,7 @@ def list_memory_events( scope_person_memory_ids: tuple[str, ...] = (), scope_window_start: datetime | None = None, scope_window_end: datetime | None = None, + sensitivity_allowed: Sequence[str] | None = None, limit: int = 20, ) -> list[VNextRow]: """Memory events whose target row matches scope before LIMIT.""" @@ -706,6 +753,17 @@ def list_memory_events( if event_type_prefix is not None: prefix_sql = " AND e.event_type LIKE ?" params.append(f"{event_type_prefix}%") + from alicebot_api.vnext_derived_labels import SENSITIVITY_RANK + ceiling = max((SENSITIVITY_RANK.get(value, 0) for value in sensitivity_allowed or ()), default=0) + blocked = [value for value, rank in SENSITIVITY_RANK.items() if rank > ceiling] if sensitivity_allowed else [] + label_sql = "" + if blocked: + marks = self._placeholders(blocked) + label_sql = f""" AND m.sensitivity NOT IN ({marks}) AND NOT EXISTS ( + SELECT 1 FROM sources parent WHERE parent.user_id=m.user_id + AND parent.id=alice_direct_source_hint(m.metadata_json) + AND parent.sensitivity IN ({marks}))""" + params.extend((*blocked, *blocked)) params.extend(project_params) people_sql = "" if people or person_ids: @@ -746,6 +804,7 @@ def list_memory_events( WHERE e.user_id = ? AND m.deleted_at IS NULL {prefix_sql} + {label_sql} {project_sql} {people_sql} {window_sql} diff --git a/apps/api/src/alicebot_api/vnext_brain.py b/apps/api/src/alicebot_api/vnext_brain.py index 1e198cabb..0be961d5e 100644 --- a/apps/api/src/alicebot_api/vnext_brain.py +++ b/apps/api/src/alicebot_api/vnext_brain.py @@ -728,6 +728,7 @@ def generate_weekly_synthesis(self, request: BrainArtifactRequest | None = None) sources, memories, open_loops, + artifacts, workflow_digest=workflow_digest, ) all_rows = [*sources, *memories, *open_loops, *artifacts] @@ -1083,6 +1084,7 @@ def _create_weekly_candidate_memories( sources: list[JsonObject], memories: list[JsonObject], open_loops: list[JsonObject], + artifacts: list[JsonObject], *, workflow_digest: str, ) -> list[JsonObject]: @@ -1105,12 +1107,12 @@ def _create_weekly_candidate_memories( "confidence": 0.6, "canonical_text": insight, "summary": insight, - "domain": _artifact_domain(request, [*sources, *memories, *open_loops]), - "sensitivity": _highest_sensitivity([*sources, *memories, *open_loops]), + "domain": _artifact_domain(request, [*sources, *memories, *open_loops, *artifacts]), + "sensitivity": _highest_sensitivity([*sources, *memories, *open_loops, *artifacts]), "metadata_json": { "candidate": True, "discovered_by": "vnext_weekly_synthesis", - "input_summary": _input_summary(sources=sources, memories=memories, open_loops=open_loops, artifacts=[]), + "input_summary": _input_summary(sources=sources, memories=memories, open_loops=open_loops, artifacts=artifacts), "generated_by": request.generated_by, "agent_identity": request.agent_identity, "scheduler_run_id": request.run_id if request.generated_by == "scheduler" else None, @@ -1121,7 +1123,7 @@ def _create_weekly_candidate_memories( } memory_payload["metadata_json"] = with_derived_from( cast(dict, memory_payload["metadata_json"]), - {"sources": sources, "memories": memories, "open_loops": open_loops}, + {"sources": sources, "memories": memories, "open_loops": open_loops, "artifacts": artifacts}, ) upsert_memory = getattr(self.store, "upsert_memory_by_key", None) if callable(upsert_memory): diff --git a/apps/api/src/alicebot_api/vnext_connectors.py b/apps/api/src/alicebot_api/vnext_connectors.py index c0b2e49b5..86911b8ad 100644 --- a/apps/api/src/alicebot_api/vnext_connectors.py +++ b/apps/api/src/alicebot_api/vnext_connectors.py @@ -1505,8 +1505,20 @@ def connector_health(self, connector_name: str) -> JsonObject: } def connector_health_all(self) -> JsonObject: + from copy import deepcopy + from alicebot_api.vnext_label_guard import request_row_cache + + # Workspace, dogfooding and doctor render the same connector snapshot + # in one guarded request. Keep this raw census request-local, with the + # same store/write invalidation as label input rows. + cache = request_row_cache(self.store, "connector_health_all") + if cache is not None and "result" in cache: + return deepcopy(cache["result"]) items = [self.connector_health(definition.name) for definition in list_connector_definitions()] - return {"items": items, "count": len(items), "order": [str(item["connector_name"]) for item in items]} + result: JsonObject = {"items": items, "count": len(items), "order": [str(item["connector_name"]) for item in items]} + if cache is not None: + cache["result"] = deepcopy(result) + return result def set_connector_secret(self, connector_name: str, *, secret_ref: str, secret_value: str) -> JsonObject: definition = get_connector_definition(connector_name) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index c661ed3f2..861b431c7 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -13,6 +13,8 @@ import json from collections import deque from collections.abc import Iterable, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar from dataclasses import dataclass, replace from uuid import UUID from typing import Any, TypeVar, overload @@ -210,12 +212,39 @@ def _object(value: object) -> Mapping[str, object]: return value if isinstance(value, Mapping) else {} +_READ_METADATA: ContextVar[dict | None] = ContextVar("label_read_metadata", default=None) + + +@contextmanager +def label_metadata_cache(cache: dict): + """Reuse pure decoding only inside a guarded store snapshot. + + The caller owns invalidation on label writes and rollback. Strong raw + references prevent object-id reuse; exiting the request drops every entry. + """ + token = _READ_METADATA.set(cache) + try: + yield + finally: + _READ_METADATA.reset(token) + + def _metadata(row: Mapping[str, object]) -> Mapping[str, object]: - return _object(_uuid_strings(_object(row.get("metadata_json")))) + raw = row.get("metadata_json") + cache = _READ_METADATA.get() + cached = cache.get(id(raw)) if cache is not None else None + if cached is not None and cached[0] is raw: + return cached[1] + normalized = _object(_uuid_strings(_object(raw))) + if cache is not None: + cache[id(raw)] = (raw, normalized) + return normalized def _uuid_strings(value: object) -> object: """Database UUID objects and JSON strings name the same recorded input.""" + if value is None or type(value) in (str, int, float, bool): + return value if isinstance(value, UUID): return str(value) if isinstance(value, Mapping): @@ -868,6 +897,57 @@ def _node_label(kind: str, row: Mapping[str, object]) -> SettledLabel: ) +def dependency_syntax_key(kind: str, row: Mapping[str, object]) -> tuple: + """Memoize parsing without allowing incidental scalar metadata to split it. + + Keep every marker, reference, count, scope alias and value dependency. Walk + unknown containers as the dependency parser does; their scalar text cannot + name an input. This is only a parsing key, never a settled label or grant. + """ + relevant = MARKER_KEYS | _ID_KIND.keys() | { + "redacted", "scrubbed", "project_scope", "project_floor", "project_id", "project", "projects", + "scope_json", "metadata_json", "agent_identity", "agentic_memory", "consolidation", + } + + def metadata_key(value: object) -> tuple: + if isinstance(value, Mapping): + parts: list[tuple[str, object]] = [] + for key, child in value.items(): + if not isinstance(key, str): + continue + if key in relevant: + parts.append((key, repr(child))) + elif isinstance(child, (Mapping, list)): + nested = metadata_key(child) + if nested: + parts.append((key, nested)) + return tuple(sorted(parts)) + if isinstance(value, list): + return tuple(nested for child in value if isinstance(child, (Mapping, list)) + and (nested := metadata_key(child))) + return () + + fields = ("user_id", "domain", "sensitivity", "value", "project_id", "project", "projects", "scope_json", + "source_id", "artifact_type", "project_scope", "project_floor") + return (canon_kind(kind), isinstance(row.get("metadata_json"), Mapping), metadata_key(_metadata(row)), + *((field in row, repr(row.get(field))) for field in fields)) + + +def dependency_label_signature(kind: str, row: Mapping[str, object]) -> tuple: + """Fields that determine a root label, excluding identity and incidental text. + + A caller may share a verified settlement only when the root is outside its + cached ancestry and no implicit per-candidate parent rule is involved. + Structural errors are part of the signature, never normalized into validity. + """ + name = canon_kind(kind) + deps, problem = dependency_record(name, row) + label = _node_label(name, row) + return (name, deps, problem, label.row_class, label.stored_domain, + label.stored_sensitivity, label.stored_scope, label.stored_floor, + label.carries_scope, str(row.get("user_id") or "")) + + def _copy_scope(stored: tuple[str, ...], parents: Sequence[SettledLabel]) -> tuple[str, ...]: live = [item for item in parents if item.carries_scope] if not parents: diff --git a/apps/api/src/alicebot_api/vnext_label_closure.py b/apps/api/src/alicebot_api/vnext_label_closure.py index fc7a59719..c72435e13 100644 --- a/apps/api/src/alicebot_api/vnext_label_closure.py +++ b/apps/api/src/alicebot_api/vnext_label_closure.py @@ -38,6 +38,9 @@ def collect_label_rows( exceeded = True continue seen.add(stored_key) + # Tag and alias-raise only this walk's copy. Cached raw rows keep + # their tenant and stored label across origins in the same request. + row = dict(row) row["kind"] = kind if user_id is not None: row["user_id"] = user_id diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index b12ebcc9c..ee9c4f029 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -25,6 +25,9 @@ NODE_BOUND, canon_kind, dependencies_of, + dependency_label_signature, + dependency_syntax_key, + label_metadata_cache, identifier, is_derived, input_admitted, @@ -39,8 +42,8 @@ def _row_label_key(kind: str, row: Mapping[str, object]) -> tuple: - return (kind, *(repr(row.get(field)) for field in ( - "id", "user_id", "domain", "sensitivity", "metadata_json", "value", "project_id", "source_id", "artifact_type", "project_scope", "project_floor", + return (kind, *((field in row, repr(row.get(field))) for field in ( + "id", "user_id", "domain", "sensitivity", "metadata_json", "value", "project_id", "project", "projects", "scope_json", "source_id", "artifact_type", "project_scope", "project_floor", ))) @@ -50,20 +53,30 @@ class _RequestLabels: labels: dict = field(default_factory=dict) targets: dict = field(default_factory=dict) counts: dict = field(default_factory=dict) - source_copies: dict = field(default_factory=dict) + dependency_labels: dict = field(default_factory=dict) + dependency_ancestry: dict = field(default_factory=dict) + signatures: dict = field(default_factory=dict) + parsed_signatures: dict = field(default_factory=dict) row_sets: dict = field(default_factory=dict) dependencies: dict = field(default_factory=dict) source_admission: dict = field(default_factory=dict) + normalized_metadata: dict = field(default_factory=dict) + row_keys: dict = field(default_factory=dict) def clear(self): self.nodes.clear() self.labels.clear() self.targets.clear() self.counts.clear() - self.source_copies.clear() + self.dependency_labels.clear() + self.dependency_ancestry.clear() + self.signatures.clear() + self.parsed_signatures.clear() self.row_sets.clear() self.dependencies.clear() self.source_admission.clear() + self.normalized_metadata.clear() + self.row_keys.clear() _REQUEST_LABELS: ContextVar[tuple[Any, _RequestLabels] | None] = ContextVar("request_labels", default=None) @@ -89,9 +102,11 @@ def label_read_scope(store): if current is not None and current[0] is store: yield return - token = _REQUEST_LABELS.set((store, _RequestLabels())) + state = _RequestLabels() + token = _REQUEST_LABELS.set((store, state)) try: - yield + with label_metadata_cache(state.normalized_metadata): + yield finally: _REQUEST_LABELS.reset(token) @@ -189,25 +204,34 @@ def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[ state = self._state() # Distinct projections and stored aliases are settled separately. # Only labels, never caller admission, are shared in this request. - key = _row_label_key(kind, row) + key = self._key(kind, row) label = state.labels.get(key) - template = (key[0], *key[2:]) - if label is None: - label = state.source_copies.get(template) + template = self._signature(kind, row, key=key) + root = (canon_kind(kind), identifier(row.get("id"))) + if label is None and root not in state.dependency_ancestry.get(template, ()): + label = state.dependency_labels.get(template) if label is None: nodes = self._collected(kind, row) settled = settle_labels(nodes, on_cycle="unverified", max_hops=HOP_BOUND, max_nodes=NODE_BOUND) label = settled.by_stored(kind, str(row.get("id") or ""), user_id=_GUARD_USER) - # Copies of the same original sources have identical effective - # labels. No derived parent, alias, missing parent or root cycle - # is allowed in this shortcut; those retain an independent walk. - refs = dependencies_of(kind, row) - if refs and all( - ref_kind == "source" and len(state.nodes.get((ref_kind, ref_id), [])) == 1 - and not is_derived(ref_kind, state.nodes[(ref_kind, ref_id)][0]) - for ref_kind, ref_id in refs - ): - state.source_copies[template] = label + ancestry = frozenset( + [*(ref for node in nodes for ref in dependencies_of(str(node["kind"]), node)), + *((str(node["kind"]), identifier(node.get("id"))) for node in nodes[1:]), + *(("memory", identifier(node["memory_id"])) for node in nodes + if node.get("kind") == "belief" and node.get("memory_id"))] + ) + implicit_parent = False + for node in nodes: + metadata = node.get("metadata_json") + if node.get("kind") == "artifact" and isinstance(metadata, Mapping) and metadata.get("candidate_memory_ids"): + implicit_parent = True + break + # Complete verified ancestry has already passed the per-root hop, + # node, alias and missing-parent checks. Only another root outside + # that ancestry can reuse this same dependency signature. + if not label.unverified and root not in ancestry and not implicit_parent: + state.dependency_labels[template] = label + state.dependency_ancestry[template] = ancestry state.labels[key] = label copy = dict(row) raw_metadata = copy.get("metadata_json") @@ -244,13 +268,9 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: for row in rows: if isinstance(row, Mapping): state.targets[(kind, str(row.get("id")))] = row - dependency_key = (kind, *(repr(row.get(field)) for field in ( - "metadata_json", "value", "source_id", "source_artifact_id", "artifact_id", "memory_id", "artifact_type", - ))) - refs = state.dependencies.get(dependency_key) - if refs is None: - refs = dependencies_of(kind, row) - state.dependencies[dependency_key] = refs + if not is_derived(kind, row): + continue + refs = self._signature(kind, row, key=self._key(kind, row))[1] for ref_kind, ref_id in refs: if (ref_kind, ref_id) not in state.nodes: wanted.setdefault(ref_kind, set()).add(ref_id) @@ -266,16 +286,22 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: for row in rows: if not isinstance(row, Mapping): continue - key = _row_label_key(kind, row) - template = (key[0], *key[2:]) + if not is_derived(kind, row): + if self._admits_effective(row, kind=kind): + kept.append(row) + continue + key = self._key(kind, row) + template = self._signature(kind, row, key=key) admission_key = (template, self.domains, self.sensitivity_allowed, self.projects, self.all_of) - if template in state.source_copies and admission_key in state.source_admission: + root = (canon_kind(kind), identifier(row.get("id"))) + reusable = template in state.dependency_labels and root not in state.dependency_ancestry.get(template, ()) + if reusable and admission_key in state.source_admission: if state.source_admission[admission_key]: kept.append(row) continue effective = self.effective_row(kind, row) admitted = isinstance(effective, Mapping) and self._admits_effective(effective, kind=kind) - if template in state.source_copies: + if reusable: state.source_admission[admission_key] = admitted if admitted: kept.append(row) @@ -296,8 +322,14 @@ def readable_status_counts(self, kind: str) -> dict[str, int]: key = (kind, self.domains, self.sensitivity_allowed, self.projects, self.all_of) if key in state.counts: return dict(state.counts[key]) - counts: dict[str, int] = {} - for batch in iterator(kind): + plain_counter = getattr(self.store, "count_original_label_statuses", None) + if callable(getattr(type(self.store), "count_original_label_statuses", None)) and callable(plain_counter) and not self.projects and self.all_of is None: + counts = plain_counter(kind, domains=self.domains, sensitivity_allowed=self.sensitivity_allowed) + batches = iterator(kind, derived_only=True) + else: + counts = {} + batches = iterator(kind) + for batch in batches: for row in self.admit_rows(kind, batch): status = str(row.get("status", "unknown")) counts[status] = counts.get(status, 0) + 1 @@ -397,6 +429,25 @@ def _collected(self, kind: str, row: Mapping[str, object]) -> list[dict[str, obj ) return nodes + def _key(self, kind: str, row: Mapping[str, object]) -> tuple: + current = _REQUEST_LABELS.get() + if current is None or current[0] is not self.store: + return _row_label_key(kind, row) + keys = current[1].row_keys + raw_key = (kind, id(row)) + if raw_key not in keys or keys[raw_key][0] is not row: + keys[raw_key] = (row, _row_label_key(kind, row)) + return keys[raw_key][1] + + def _signature(self, kind: str, row: Mapping[str, object], *, key: tuple) -> tuple: + state = self._state() + if key not in state.signatures: + syntax = dependency_syntax_key(kind, row) + if syntax not in state.parsed_signatures: + state.parsed_signatures[syntax] = dependency_label_signature(kind, row) + state.signatures[key] = state.parsed_signatures[syntax] + return state.signatures[key] + def admit_loaded( store: Any, diff --git a/apps/api/src/alicebot_api/vnext_label_sql.py b/apps/api/src/alicebot_api/vnext_label_sql.py new file mode 100644 index 000000000..bbb671e86 --- /dev/null +++ b/apps/api/src/alicebot_api/vnext_label_sql.py @@ -0,0 +1,22 @@ +"""Conservative SQL partition: anything uncertain still reaches the kernel.""" +from alicebot_api.vnext_derived_labels import MARKER_KEYS, DERIVED_ARTIFACT_TYPES + + +def original_label_sql(kind: str, *, sqlite: bool = False) -> str: + """Only rows definitely original by is_derived may be counted in SQL. + + Scalar metadata, redacted derived markers and legacy shapes remain in the + Python partition. JSON text is not reinterpreted by the SQL partition. + """ + if kind == "source": + return "TRUE" + markers = ",".join("'" + key + "'" for key in sorted(MARKER_KEYS)) + if sqlite: + safe = "CASE WHEN json_valid(metadata_json) THEN metadata_json ELSE 'null' END" + predicate = f"(json_type({safe}) = 'object' AND NOT EXISTS (SELECT 1 FROM json_each({safe}) WHERE key IN ({markers})))" + else: + predicate = f"(jsonb_typeof(metadata_json) = 'object' AND NOT (metadata_json ?| ARRAY[{markers}]::text[]))" + if kind == "artifact": + types = ",".join("'" + value + "'" for value in sorted(DERIVED_ARTIFACT_TYPES)) + predicate = f"({predicate} AND artifact_type NOT IN ({types}))" + return predicate diff --git a/apps/api/src/alicebot_api/vnext_open_loop_references.py b/apps/api/src/alicebot_api/vnext_open_loop_references.py index 595a45647..530ea3cc9 100644 --- a/apps/api/src/alicebot_api/vnext_open_loop_references.py +++ b/apps/api/src/alicebot_api/vnext_open_loop_references.py @@ -29,7 +29,9 @@ git sha or a 64-digit digest is never cut into an id, and neither is the id of a row the reader may read when hex digits follow it after a hyphen (``-20261003``). A string that is only an id is read as the link writer reads it (``UUID()``, which also ignores hyphens in other places). Inside longer text only ASCII hex digits in those two - layouts are read, and an id with its hyphens in other places, split or otherwise encoded is not an id to this scan. + layouts are read by that scan. A second, source-only pass uses the canonical saved-quote parser and withholds + strings naming refused sources in its wider spellings, including JSON escapes. Irregular or non-ASCII memory-prefixed + ids in longer text and whitespace-based source URI spellings can remain; see the draft security note. The hyphenated layout is read with no boundary, so hex digits glued to an id in that layout can form a second window that names no row: a hyphen and groups of 4, 4, 4 and 12 digits right after an id, or groups of 8, 4, 4 and 4 digits and a hyphen right before a 32-digit id. Under a reference key that window is cut, and with it part of an id the diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 079640a7d..ce302282e 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -535,7 +535,21 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: (wanted,), ) - def iter_label_rows(self, kind: str, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + def count_original_label_statuses(self, kind: str, *, domains=(), sensitivity_allowed=()) -> dict[str, int]: + """Count the definitely-original, unscoped partition using stored labels.""" + from alicebot_api.vnext_label_sql import original_label_sql + table = {"source": "sources", "memory": "memories", "open_loop": "open_loops", "artifact": "generated_artifacts", "project": "projects"}[kind] + status = "status" if kind != "source" else "'unknown'" + predicate = original_label_sql(kind, sqlite=False) + live = " AND deleted_at IS NULL" if kind in {"source", "memory"} else "" + rows = self._fetch_all(f"SELECT {status} AS status, COUNT(*) AS count FROM {table} WHERE {predicate}{live} " + "AND (%s::text[] IS NULL OR domain=ANY(%s::text[]) OR domain='unknown') " + "AND (%s::text[] IS NULL OR sensitivity=ANY(%s::text[])) " + "GROUP BY 1", (list(domains) or None, list(domains) or None, + list(sensitivity_allowed) or None, list(sensitivity_allowed) or None)) + return {str(row["status"]): int(cast(int, row["count"])) for row in rows} + + def iter_label_rows(self, kind: str, *, batch_size: int = 1000, derived_only: bool = False) -> Iterator[list[VNextRow]]: """Complete counted population, in narrow keyset batches under tenant RLS.""" if batch_size < 1: @@ -554,6 +568,9 @@ def iter_label_rows(self, kind: str, *, batch_size: int = 200) -> Iterator[list[ elif kind == "project": extra = ", status" live = " AND deleted_at IS NULL" if kind in {"source", "memory"} else "" + from alicebot_api.vnext_label_sql import original_label_sql + if derived_only: + live += " AND NOT COALESCE(" + original_label_sql(kind, sqlite=False) + ", FALSE)" after: str | None = None while True: rows = self._fetch_all( @@ -568,7 +585,7 @@ def iter_label_rows(self, kind: str, *, batch_size: int = 200) -> Iterator[list[ yield rows after = str(rows[-1]["id"]) - def iter_label_events(self, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + def iter_label_events(self, *, batch_size: int = 1000) -> Iterator[list[VNextRow]]: """Complete event targets for readable counts, without event payloads.""" if batch_size < 1: @@ -585,7 +602,7 @@ def iter_label_events(self, *, batch_size: int = 200) -> Iterator[list[VNextRow] yield rows after = str(rows[-1]["id"]) - def iter_label_ratings(self, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + def iter_label_ratings(self, *, batch_size: int = 1000) -> Iterator[list[VNextRow]]: """Complete rating targets for counts, without feedback text.""" if batch_size < 1: @@ -791,7 +808,10 @@ def list_memory_events( NOT (m.sensitivity = ANY(%s::text[])) AND NOT EXISTS ( SELECT 1 FROM sources parent - WHERE parent.id::text = m.metadata_json->>'source_id' + WHERE parent.id = CASE + WHEN m.metadata_json->>'source_id' ~* '^(?:[0-9a-f]{{32}}|[0-9a-f]{{8}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{12}})$' + THEN (m.metadata_json->>'source_id')::uuid + END AND m.metadata_json->>'redacted' IS DISTINCT FROM 'true' AND parent.sensitivity = ANY(%s::text[]) ) diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py b/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py index d20c0a152..13e44ce91 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/graph_open_loops.py @@ -924,10 +924,10 @@ def upsert_open_loop_by_automation_digest( try: return self.create_open_loop(record, actor_type=actor_type) except ContinuityStoreInvariantError: + # The unique digest belongs to the user, not the mutable project or + # person columns. Label propagation may have emptied those columns. existing = self.find_open_loop_by_automation_digest( digest=normalized_digest, - project_id=str(loop["project_id"]) if loop.get("project_id") is not None else None, - person_id=str(loop["person_id"]) if loop.get("person_id") is not None else None, ) if existing is None: raise diff --git a/docs/alpha/known-limitations.md b/docs/alpha/known-limitations.md index 1eacc7e17..a516c6694 100644 --- a/docs/alpha/known-limitations.md +++ b/docs/alpha/known-limitations.md @@ -75,7 +75,7 @@ Open in v0.20.0, with the detail in the [v0.20.0 release notes](../release/v0.20 - in v0.20.0 a key bound to one project can attach a source it cannot read through `alice_memory_commit` `source_refs`, the `provenance` of `alice_memory_correct` and, over HTTP on Postgres, `POST /v0/vnext/open-loops`, and a saved quote stays readable after its source is reclassified. Unreleased (on main, not in v0.20.0): those doors answer `not_found` (404 over HTTP) for a source or memory the caller may not read and the readers of a saved quote and of an open loop ask the reader's own fence again, but a link that passes the fence of an `admin_agent` writer still makes `alice_explain` of that memory fail for the keys of a project with a lower ceiling (see [Cited sources](mcp-tools.md#cited-sources) and [Saved quotes](mcp-tools.md#saved-quotes)) - the cited-source fence does not cover everything yet. Unreleased (on main, not in v0.20.0): memory proposals and the agent-output ingest store their `source_refs` as given, the owner dependency trace lists a memory by the first id of a multi-id ref only, `provenance_count` still counts a withheld link, and a memory or open loop saved before the fix keeps its link or id, so `alice_explain` still fails for such a memory; the rest is under [Cited sources](mcp-tools.md#cited-sources), [Saved quotes](mcp-tools.md#saved-quotes) and in the [CHANGELOG](../../CHANGELOG.md) - Unreleased (on main, not in v0.20.0): on SQLite, `import-markdown --supersede` matches resolved paths (not moved folders), and `sources delete` and `prune` keep source events, hashes, memory text and backups, and blank an open loop that names the source in any spelling the saved-quote reader names, in one pass. v0.20.0 kept loops using other spellings, and freed space keeps text until the [VACUUM step](../integrations/importers.md#list-delete-and-prune-sqlite-sources). -- in v0.20.0 a derived summary, report or copy keeps the label its inputs had when it was made, so relabelling an input leaves it readable by keys the input now refuses, and a report can show a key bound to one project the titles, ids or text of rows from other projects or with no project. Unreleased (on main, not in v0.20.0): a derived row follows its inputs when they are relabelled and stays as strict as they have been, a report with an input that has no project is global and keeps every input's project, and a row whose recorded inputs cannot be found is read only by the owner and an unbound admin key until it is regenerated, and the five screens apply the sensitivity ceiling, counting rows the caller may read. See [Derived row domains](mcp-tools.md#derived-row-domains) +- in v0.20.0 a derived summary, report or copy keeps the label its inputs had when it was made, so relabelling an input leaves it readable by keys the input now refuses, and a report can show a key bound to one project the titles, ids or text of rows from other projects or with no project. Unreleased (on main, not in v0.20.0): a derived row follows its inputs when they are relabelled and stays as strict as they have been, a report with an input that has no project is global and keeps every input's project, and a row whose recorded inputs cannot be found is read only by the owner and an unbound admin key until it is regenerated, and the five screens apply the sensitivity ceiling, counting rows the caller may read. See [Derived row domains](mcp-tools.md#derived-row-domains) Graph edge explanations and the new doctor label counts remain outside that ceiling. Bounded retrieval, consolidation and staleness reads can under-fill; imported irregular memory references can retain ids. See the [draft security note](../release/derived-labels-security-note-draft.md) for these limits and reverse-lookup cost. What v0.19.0 and v0.19.2 limited and v0.20.0 fixed or narrowed (request body size, the `Host` check, provider redirects, local-folder reads, the memory id fence, deep backup JSON, the lone surrogate turn, the query size bounds and the data directory variable) is recorded in the [v0.19.2 release notes](../release/v0.19.2-release-notes.md), the [v0.20.0 release notes](../release/v0.20.0-release-notes.md) and the [CHANGELOG](../../CHANGELOG.md). diff --git a/docs/release/derived-labels-security-note-draft.md b/docs/release/derived-labels-security-note-draft.md index 56fb85879..de1a0fe0c 100644 --- a/docs/release/derived-labels-security-note-draft.md +++ b/docs/release/derived-labels-security-note-draft.md @@ -2,8 +2,12 @@ Unreleased (on main, not in v0.20.0): the derived-label fixes tighten recorded-input inserts, restricted report inputs, exact reads, operator lists, and repair. This draft is for review before a release is tagged. -Unreleased (on main, not in v0.20.0): four older operator or legacy readers remain outside that scope. A trusted key can still obtain confidential source titles and raw text through the source GET route, edge explanations through graph neighborhood, and aggregate label-check counts through doctor. Legacy `alice_vnext_review_items` can still list hidden derived memories for a declared restricted identity. These behaviors also occur on the baseline. The owner must decide whether to extend the boundary before tagging; the five corrected screens do not establish that every operator route observes the same ceiling. +Unreleased (on main, not in v0.20.0): source GET and legacy `alice_vnext_review_items` now apply the caller's domain, sensitivity and locked project fence, including effective labels of recorded inputs. Graph neighborhood still returns edge explanations outside that ceiling; that behavior predates this set. Doctor's aggregate derived-label check counts are new in this set, absent from v0.20.0 and baseline main 48873b03, and remain visible to trusted keys. The owner should review that new count exposure separately before tagging. The five corrected screens and these two readers do not establish that every operator route observes the same ceiling. Unreleased (on main, not in v0.20.0): SQLite retirement uses the saved-quote reader's reference rules. Repeated JSON object keys are decoded with the last value retained, and quote subtrees are omitted by that reader. Retirement can consequently retain a loop that an older raw JSON scan would blank. The product does not write those two forms, but an imported or hand-edited row can contain them. Unreleased (on main, not in v0.20.0): capture overhead has only a few percentage points of headroom below the 15 percent budget. The earlier relabel measurements scale at roughly 2.4 ms per dependant over the measured sizes. Repeat those measurements on a quiet machine before tagging; the read budgets measure a separate path. + +Unreleased (on main, not in v0.20.0): bounded retrieval, consolidation clustering and staleness sweeps can under-fill after restricted admission when newer hidden or shared rows consume their candidate window. Defaults are 2,000 embedded memories and 500 staleness rows. Complete effective-label counts use the full population, but retrieval completeness needs a separate refill and ranking change. + +Unreleased (on main, not in v0.20.0): open-loop metadata withholding follows canonical saved-quote parsing for source ids. Its wider pass is source-only: irregular or non-ASCII memory-prefixed ids inside longer text, `alice://sources/` whitespace forms and equivalent imported spellings can retain the id. The product does not write these forms. Unknown trace ids and the loop's free-text columns remain as stored. At 20,000 loops the canonical reverse lookup costs about ten times the earlier scan in the external fixture; SQL text prefiltering is deferred because it must retain JSON escapes and every canonical parser spelling. diff --git a/docs/release/v0.20.0-release-notes.md b/docs/release/v0.20.0-release-notes.md index 59888d6c8..279883aad 100644 --- a/docs/release/v0.20.0-release-notes.md +++ b/docs/release/v0.20.0-release-notes.md @@ -3,7 +3,7 @@ > **Correction (2026-10-05):** these notes do not say that a derived summary, report or copy kept the label its inputs had when it was made, or that a report could show a key bound to one project rows of other projects or with no project. Both are in v0.20.0 and are listed under known limitations. -> **Correction (2026-10-06):** four older operator or legacy readers still expose withheld information: source GET returns titles and raw text, graph neighborhood returns edge explanations, doctor returns label-check counts, and legacy `alice_vnext_review_items` lists derived memories for a declared restricted identity. These limits also occur in v0.20.0. See the [draft security note](derived-labels-security-note-draft.md) for the current boundary and SQLite retirement limits. +> **Correction (2026-10-06):** three older operator or legacy readers expose withheld information in v0.20.0: source GET returns titles and raw text, graph neighborhood returns edge explanations, and legacy `alice_vnext_review_items` lists derived memories for a declared restricted identity. Unreleased corrections now fence source GET and the legacy review list. Doctor's derived-label check counts are new in this set, absent from v0.20.0 and baseline main 48873b03. See the [draft security note](derived-labels-security-note-draft.md) for the current boundary and SQLite retirement limits. **Take this release if you run the Postgres stack's HTTP API, use an embeddings endpoint (above all a hosted one), import ChatGPT or Markdown files, use the diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c6aded657..7e0a3be98 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -179,13 +179,3 @@ def lock_label_fixture(store) -> None: store.lock_graph_mutation() acquire_exclusive_label_lock(store) - - -def assert_append_only_mutation_refused(conn, *, snapshot_sql, mutation_sql, params) -> None: - """Prove refusal under both forced RLS and the append-only trigger.""" - before = conn.execute(snapshot_sql, params).fetchall() - assert before, "the synthetic target must be visible before attempting its mutation" - with pytest.raises(psycopg.Error, match="append-only"): - with conn.transaction(): - conn.execute(mutation_sql, params) - assert conn.execute(snapshot_sql, params).fetchall() == before diff --git a/tests/integration/test_label_handoff_contention_postgres.py b/tests/integration/test_label_handoff_contention_postgres.py index d32a3780c..c8b3952db 100644 --- a/tests/integration/test_label_handoff_contention_postgres.py +++ b/tests/integration/test_label_handoff_contention_postgres.py @@ -1,9 +1,11 @@ """Ordinary reviews share the label lock; label changes give a retryable refusal.""" import time +from argparse import Namespace import pytest from alicebot_api.cli.automation import _run_vnext_artifact_review +from alicebot_api.cli.memories import _run_vnext_memory_redact from alicebot_api.cli.models import CLIContext from alicebot_api.config import Settings from alicebot_api.mcp.registry import call_mcp_tool @@ -13,7 +15,7 @@ from tests.integration.derived_labels_postgres_support import label_harness, today -@pytest.mark.parametrize("entry", ["memory", "artifact", "mcp_memory", "mcp_artifact", "project_reject"]) +@pytest.mark.parametrize("entry", ["memory", "artifact", "mcp_memory", "mcp_artifact", "project_reject", "source", "http_redact", "mcp_redact", "cli_redact"]) def test_non_label_review_succeeds_while_shared_label_lock_is_held(label_harness, entry): h = label_harness source = h.source() @@ -34,6 +36,19 @@ def test_non_label_review_succeeds_while_shared_label_lock_is_held(label_harness if entry == "memory": result = h.request("POST", f"/v0/vnext/memories/{memory['id']}/review", payload={"action": "accept"}) assert result[0] == 200, result + elif entry == "source": + result = h.request("POST", f"/v0/vnext/sources/{source['id']}/review", payload={"action": "review"}) + assert result[0] == 200, result + elif entry == "http_redact": + result = h.request("POST", "/v0/vnext/memories/redact", payload={"memory_id": str(memory["id"]), "reason": "synthetic regression"}) + assert result[0] == 200, result + elif entry == "mcp_redact": + result = call_mcp_tool(context, name="alice_memory_manage", arguments={"action": "redact", "memory_id": str(memory["id"]), "reason": "synthetic regression"}) + assert result["status"] == "redacted", result + elif entry == "cli_redact": + result = _run_vnext_memory_redact(CLIContext(settings=Settings(database_url=h.urls["app"]), database_url=h.urls["app"], user_id=h.user_id), + Namespace(memory_id=str(memory["id"]), reason="synthetic regression", agent_id=None)) + assert '"redacted"' in result, result elif entry == "mcp_memory": result = call_mcp_tool(context, name="alice_memory_correct", arguments={"review_item_id": str(memory["id"]), "action": "approve"}) elif entry == "mcp_artifact": diff --git a/tests/integration/test_label_round2_producers_postgres.py b/tests/integration/test_label_round2_producers_postgres.py new file mode 100644 index 000000000..b54aab281 --- /dev/null +++ b/tests/integration/test_label_round2_producers_postgres.py @@ -0,0 +1,70 @@ +"""Real round-two producer lifecycle regressions.""" +from copy import deepcopy +from uuid import uuid4 + +import pytest + +from alicebot_api.routers import vnext_memories +from alicebot_api.vnext_brain import BrainArtifactRequest, VNextBrainService +from alicebot_api.vnext_label_repair import label_gap_counts +from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService +from tests.integration.derived_labels_postgres_support import label_harness, today + + +def projects(h): + ids = [str(uuid4()), str(uuid4())] + with h.store() as store: + for identifier, name in zip(ids, ("Round2 Alpha", "Round2 Beta"), strict=True): + store.create_project({"id": identifier, "name": name, "slug": name.lower().replace(" ", "-")}) + return ids + + +def move(h, source, project): + result = vnext_memories.review_vnext_source(source["id"], vnext_memories.VNextSourceReviewRequest( + user_id=h.user_id, action="assign_project", project_id=project, confirm_label_hide=True)) + assert result.status_code == 200, result.body + + +@pytest.mark.parametrize("stale", [False, True]) +def test_digest_replays_after_project_moves(label_harness, monkeypatch, stale): + h = label_harness + alpha, beta = projects(h) + source = h.source(scope=(alpha,), text="TODO: Round trip task") + request = ProjectAutomationRequest(agent_identity=None, project_id=alpha) + with h.store() as store: + VNextProjectService(store).extract_open_loops(request) + initial = store.list_open_loops(status=None, limit=20) + assert len(initial) == 1 + move(h, source, beta) + if not stale: + move(h, source, alpha) + with h.store() as store: + existing = store.get_open_loop(str(initial[0]["id"])) + assert existing["project_id"] is None + if stale: + monkeypatch.setattr(store, "search_sources", lambda **_kwargs: [deepcopy(source)]) + VNextProjectService(store).extract_open_loops(request) + VNextProjectService(store).extract_open_loops(request) + rows = store.list_open_loops(status=None, limit=20) + assert [str(row["id"]) for row in rows] == [str(initial[0]["id"])] + assert set(rows[0]["metadata_json"]["project_floor"]) == {alpha, beta} + assert label_gap_counts(store) == (0, 0) + + +def test_weekly_candidate_inherits_report_inputs_before_commit(label_harness): + h = label_harness + alpha, beta = projects(h) + source = h.source(scope=(alpha,)) + h.memory(scope=(alpha,)) + with h.store() as store: + VNextBrainService(store).generate_daily_brief(BrainArtifactRequest(agent_identity=None, projects=(alpha,), generated_for=today())) + move(h, source, beta) + with h.store() as store: + artifact = VNextBrainService(store).generate_weekly_synthesis(BrainArtifactRequest( + agent_identity=None, projects=(alpha,), generated_for=today(), create_candidate_memories=True)) + candidates = store.read_label_rows("memory", artifact["metadata_json"]["candidate_memory_ids"]) + assert candidates + assert set(artifact["metadata_json"]["project_floor"]) == {alpha, beta} + for candidate in candidates: + assert set(candidate["metadata_json"]["project_floor"]) == {alpha, beta} + assert label_gap_counts(store) == (0, 0) diff --git a/tests/integration/test_label_round2_read_budget_postgres.py b/tests/integration/test_label_round2_read_budget_postgres.py new file mode 100644 index 000000000..4bbe7d6d7 --- /dev/null +++ b/tests/integration/test_label_round2_read_budget_postgres.py @@ -0,0 +1,18 @@ +"""Varied and plain native reads use the same database at both revisions.""" +import pytest +import psycopg + +from tests.integration.derived_labels_postgres_support import label_harness +from tests.performance.test_label_round2_read_budget import seed_varied, assert_budgets + + +@pytest.mark.parametrize("case", ["varied", "repaired", "plain"]) +def test_postgres_round2_read_budgets(label_harness, case): + h = label_harness + with h.store() as store: + keys = seed_varied(store, postgres=True, repaired=case == "repaired", plain=case == "plain", + source_count=1000 if case == "plain" else 300) + with psycopg.connect(h.urls["admin"], autocommit=True) as conn: + for table in ("sources", "memories", "event_log"): + conn.execute("ANALYZE " + table) + assert_budgets("postgres", h.urls["app"], h.user_id, keys) diff --git a/tests/integration/test_label_round2_reader_fences_postgres.py b/tests/integration/test_label_round2_reader_fences_postgres.py new file mode 100644 index 000000000..1383cf3ab --- /dev/null +++ b/tests/integration/test_label_round2_reader_fences_postgres.py @@ -0,0 +1,97 @@ +"""The two added reader doors preserve owner and admitted controls.""" +from uuid import uuid4 + +import pytest + +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext +from tests.integration.derived_labels_postgres_support import label_harness + + +@pytest.mark.parametrize("profile,bound", [("trusted_local_agent", False), ("admin_agent", False), ("trusted_local_agent", True), ("admin_agent", True)]) +def test_source_get_uses_the_callers_entire_fence(label_harness, profile, bound): + h = label_harness + alpha, beta = str(uuid4()), str(uuid4()) + with h.store() as store: + for identifier in (alpha, beta): + store.create_project({"id": identifier, "name": identifier, "slug": identifier}) + visible = h.source(scope=(alpha,)) + private = h.source(scope=(alpha,), sensitivity="confidential") + other = h.source(scope=(beta,)) + # The owner's control precedes key provisioning, as the real key gate requires. + for source in (visible, private, other): + status, body, _ = h.request("GET", "/v0/vnext/sources/" + str(source["id"])) + assert status == 200 and str(body["id"]) == str(source["id"]) + key = h.key(profile, project=alpha if bound else None) + status, body, _ = h.request("GET", "/v0/vnext/sources/" + str(visible["id"]), key=key) + assert status == 200 and str(body["id"]) == str(visible["id"]) + for source, permitted in ((private, profile == "admin_agent"), (other, not bound)): + status, body, _ = h.request("GET", "/v0/vnext/sources/" + str(source["id"]), key=key) + assert status == (200 if permitted else 404), (profile, bound, body) + if not permitted: + assert body == {"detail": f"vNext source {source['id']} was not found"} + + +@pytest.mark.parametrize("profile,nested", [("trusted_local_agent", False), ("trusted_local_agent", True), ("admin_agent", False), ("read_only_agent", True)]) +def test_legacy_review_list_fences_effective_candidates(label_harness, profile, nested): + h = label_harness + alpha, beta = "round2-alpha", "round2-beta" + hidden_source = h.source(scope=(alpha,), sensitivity="confidential") + with h.store() as store: + # Raw legacy copy: stored public, effective confidential. + hidden = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Hidden synthetic candidate", + "status": "candidate", "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": [alpha]}}) + store.conn.execute("UPDATE memories SET metadata_json=metadata_json || %s::jsonb WHERE id=%s", + ('{"source_id":"' + str(hidden_source["id"]) + '"}', hidden["id"])) + other = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Other synthetic candidate", "status": "candidate", + "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [beta]}}) + visible = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Visible synthetic candidate", "status": "candidate", + "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [alpha]}}) + context = MCPRuntimeContext(database_url=h.urls["app"], user_id=h.user_id) + owner = call_mcp_tool(context, name="alice_vnext_review_items", arguments={"limit": 50}) + assert {str(row["id"]) for row in owner["items"]} == {str(row["id"]) for row in (visible, hidden, other)} + identity = {"agent_id": "round2-reader", "permission_profile": profile} + if nested: + identity["project_scope"] = [alpha] + arguments = {"agent_identity": identity} if nested else identity + if nested: + # Nested identity is a handler compatibility form, outside the flat + # advertised tool schema. Exercise that boundary directly. + from alicebot_api.mcp.evidence_artifacts import _handle_alice_vnext_review_items + result = _handle_alice_vnext_review_items(context, {"limit": 50, **arguments}) + else: + result = call_mcp_tool(context, name="alice_vnext_review_items", arguments={"limit": 50, **arguments}) + ids = {str(row["id"]) for row in result["items"]} + expected = {str(visible["id"])} + # A declared local scope is a request hint. Only resolved locked identity + # imposes a key binding; the additional test below covers that form. + expected.add(str(other["id"])) + if profile == "admin_agent": + expected.add(str(hidden["id"])) + assert ids == expected, (profile, result) + assert result["count"] == len(expected) + + +@pytest.mark.parametrize("profile", ["trusted_local_agent", "admin_agent"]) +def test_legacy_review_list_honors_resolved_locked_identity_and_refills(label_harness, profile): + from dataclasses import replace + from alicebot_api.vnext_agent_control import AgentIdentity + h = label_harness + alpha, beta = "round2-alpha", "round2-beta" + with h.store() as store: + visible = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Visible synthetic candidate", "status": "candidate", + "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [alpha]}}) + for _ in range(55): + store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Other synthetic candidate", "status": "candidate", + "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [beta]}}) + # A stale public copy whose effective floor also needs beta. + store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Floor-bound synthetic candidate", "status": "candidate", + "domain": "project", "sensitivity": "public", "metadata_json": {"project_scope": [alpha], "project_floor": [beta]}}) + identity = AgentIdentity(agent_id="resolved-reader", permission_profile=profile, project_scope=(alpha,), project_scope_locked=True) + context = MCPRuntimeContext(database_url=h.urls["app"], user_id=h.user_id, agent_identity=identity, agent_identity_resolved=True) + result = call_mcp_tool(context, name="alice_vnext_review_items", arguments={"limit": 1}) + assert [str(row["id"]) for row in result["items"]] == [str(visible["id"])] + assert result["count"] == 1 + owner = call_mcp_tool(replace(context, agent_identity=None), name="alice_vnext_review_items", arguments={"limit": 50}) + assert owner["count"] == 50 diff --git a/tests/performance/round2_budget_probe.py b/tests/performance/round2_budget_probe.py new file mode 100644 index 000000000..fe3715303 --- /dev/null +++ b/tests/performance/round2_budget_probe.py @@ -0,0 +1,74 @@ +"""Interpreted by either revision; exercise actual MCP and HTTP readers.""" +import importlib.util +import json +import os +from pathlib import Path +import statistics +import sys +import time +from uuid import UUID + +repo, backend, location, user, profile, key = sys.argv[1:7] +sys.path[:0] = [str(Path(repo) / "apps/api/src"), repo] +os.environ["DATABASE_URL"] = location +os.environ["ALICE_AGENT_API_KEY"] = key +os.environ["ALICE_MCP_FULL_TOOLS"] = "1" +os.environ["ALICE_LEGACY_SURFACES"] = "1" +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext + +context = MCPRuntimeContext(database_url=location, user_id=UUID(user)) +arguments = {"query": "synthetic budget observation"} + + +def pack(): + return call_mcp_tool(context, name="alice_context_pack", arguments=arguments) + + +def recall(): + return call_mcp_tool(context, name="alice_recall", arguments=arguments) + + +actions = {"pack": pack, "recall": recall} +if backend == "postgres": + from alicebot_api.config import Settings + from alicebot_api import main + from alicebot_api.routers import workspaces, vnext_memories + settings = Settings(database_url=location) + for module in (main, workspaces, vnext_memories): + module.get_settings = lambda: settings + # Reuse the synthetic ASGI transport from the current test tree while + # importing the selected revision's app and stores. + support = Path(__file__).resolve().parents[1] / "integration/derived_labels_postgres_support.py" + spec = importlib.util.spec_from_file_location("round2_transport", support) + transport = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = transport + spec.loader.exec_module(transport) + + def read(path): + status, body, _headers = transport.invoke("GET", path, user_id=user, key=key) + assert status == 200, (status, body) + return body + + actions.update(workspace=lambda: read("/v0/vnext/workspace"), + dogfooding=lambda: read("/v0/vnext/dogfooding")) + +results = {} +for name, action in actions.items(): + action() + wall, cpu = [], [] + for _ in range(5): + wall_start, cpu_start = time.perf_counter(), time.process_time() + action() + wall.append(time.perf_counter() - wall_start) + cpu.append(time.process_time() - cpu_start) + results[name] = {"minimum_wall": min(wall), "minimum_cpu": min(cpu), + "median_wall": statistics.median(wall), "wall": wall, "cpu": cpu} + if os.environ.get("ALICE_ROUND2_PROFILE_DIR"): + import cProfile + output = Path(os.environ["ALICE_ROUND2_PROFILE_DIR"]) + output.mkdir(parents=True, exist_ok=True) + profiler = cProfile.Profile() + profiler.runcall(action) + profiler.dump_stats(str(output / f"{Path(repo).name}-{profile}-{name}.prof")) +print(json.dumps({"profile": profile, "times": results})) diff --git a/tests/performance/test_label_round2_read_budget.py b/tests/performance/test_label_round2_read_budget.py new file mode 100644 index 000000000..273052bd2 --- /dev/null +++ b/tests/performance/test_label_round2_read_budget.py @@ -0,0 +1,89 @@ +"""Varied parent and metadata fixtures expose per-row label work.""" +import json +import os +from pathlib import Path +import subprocess +import sys +from uuid import uuid4 + +import pytest + +from alicebot_api.onramp import bootstrap_database +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_derived_labels import with_derived_from + +USER = "11111111-1111-4111-8111-111111111111" + + +def seed_varied(store, *, postgres=False, count=5000, source_count=300, mixed=True, repaired=False, plain=False, all_hidden=False): + sources = [store.create_source({"source_type": "note", "title": "Synthetic budget input", + "content_hash": str(uuid4()), "domain": "project", + "sensitivity": "confidential" if all_hidden or i % 2 else "public"}) for i in range(source_count)] + ids = [str(uuid4()) for _ in range(count)] + values = [] + for i, row_id in enumerate(ids): + parent_index = i % source_count + source_id = str(sources[parent_index]["id"]) + metadata = {"observation_index": i, "project_scope": [], "project_floor": []} + if not plain: + metadata["source_id"] = source_id + if mixed and i >= source_count: + shape = (i // source_count) % 4 + if shape == 1: + metadata.update(workflow="project_auto_update") + metadata = with_derived_from(metadata, {"sources": [sources[parent_index]]}) + elif shape == 2: + metadata.update(candidate_kind="memory_consolidation", consolidation={"cluster_member_ids": [ids[parent_index]]}) + elif shape == 3: + metadata.update(discovered_by="vnext_weekly_synthesis") + metadata = with_derived_from(metadata, {"sources": [sources[parent_index]], "memories": [{"id": ids[parent_index]}]}) + sensitivity = "confidential" if repaired and not plain and (all_hidden or parent_index % 2) else "public" + values.append((row_id, str(store.user_id) if not postgres else str(sources[0]["user_id"]), + "budget." + row_id, "synthetic budget observation " + str(i), json.dumps(metadata), sensitivity)) + if postgres: + with store.conn.cursor() as cur: + cur.executemany("INSERT INTO memories(id,user_id,memory_key,canonical_text,metadata_json,sensitivity,value,source_event_ids,status,domain) VALUES(%s::uuid,%s::uuid,%s,%s,%s::jsonb,%s,'{}','{}','active','project')", values) + cur.executemany("INSERT INTO event_log(id,user_id,target_id,event_type,actor_type,target_type,payload_json) VALUES(%s::uuid,%s::uuid,%s::uuid,'memory.created','system','memory','{}')", [(str(uuid4()), values[0][1], row_id) for row_id in ids]) + else: + store.conn.executemany("INSERT INTO memories(id,user_id,memory_key,canonical_text,metadata_json,sensitivity,value,source_event_ids,status,domain) VALUES(?,?,?,?,?,?,'{}','[]','active','project')", values) + store.conn.executemany("INSERT INTO event_log(id,user_id,target_id,event_type,actor_type,target_type,payload_json) VALUES(?,?,?,'memory.created','system','memory','{}')", [(str(uuid4()), values[0][1], row_id) for row_id in ids]) + keys = {} + for profile in ("trusted_local_agent", "admin_agent"): + _, keys[profile] = create_agent_key(store, user_id=store.user_id if not postgres else sources[0]["user_id"], + agent_id="budget-" + profile, permission_profile=profile) + return keys + + +def probe(repo, backend, location, user, profile, key): + script = Path(__file__).with_name("round2_budget_probe.py") + completed = subprocess.run([sys.executable, str(script), str(repo), backend, str(location), str(user), profile, key], + capture_output=True, text=True, timeout=180) + assert completed.returncode == 0, completed.stderr + return json.loads(completed.stdout.strip().splitlines()[-1])["times"] + + +def assert_budgets(backend, location, user, keys): + repo = Path(__file__).resolve().parents[2] + main = os.environ.get("ALICE_READ_MAIN_CHECKOUT") + assert main, "round-two budgets require the exact paired main checkout" + for profile, key in keys.items(): + baseline = probe(main, backend, location, user, profile, key) + head = probe(repo, backend, location, user, profile, key) + print(json.dumps({"store": backend, "profile": profile, "main": baseline, "head": head})) + for action in ("pack", "recall"): + for clock in ("minimum_wall", "minimum_cpu"): + assert head[action][clock] <= 2 * baseline[action][clock] + .1, (action, clock, head, baseline) + for action in ("workspace", "dogfooding") if backend == "postgres" else (): + assert head[action]["minimum_wall"] <= 1, (action, head) + + +@pytest.mark.parametrize("case,repaired", [("mixed", False), ("mixed", True), ("many-hidden", False), ("many-hidden", True), ("one-hidden", False)]) +def test_sqlite_varied_read_budget(tmp_path, case, repaired): + path = tmp_path / "round2.db" + bootstrap_database(path, user_id=USER, user_email="budget@example.invalid") + with sqlite_user_connection(path, USER) as conn: + keys = seed_varied(SQLiteVNextStore(conn, USER), repaired=repaired, + source_count=1 if case == "one-hidden" else 3000 if case == "many-hidden" else 300, + all_hidden=case != "mixed", mixed=case == "mixed") + assert_budgets("sqlite", "sqlite:///" + str(path), USER, keys) diff --git a/tests/unit/test_label_count_partition.py b/tests/unit/test_label_count_partition.py new file mode 100644 index 000000000..8ca321a50 --- /dev/null +++ b/tests/unit/test_label_count_partition.py @@ -0,0 +1,43 @@ +"""SQL shortcuts cannot put a derived or uncertain row in the original partition.""" +import json +import sqlite3 + +import pytest + +from alicebot_api.sqlite_store import _direct_source_hint +from alicebot_api.vnext_derived_labels import MARKER_KEYS, DERIVED_ARTIFACT_TYPES, is_derived +from alicebot_api.vnext_label_sql import original_label_sql + + +@pytest.mark.parametrize("kind", ["source", "memory", "open_loop", "artifact", "project"]) +def test_sqlite_original_partition_is_conservative_for_legacy_and_marker_shapes(kind): + conn = sqlite3.connect(":memory:") + conn.execute("CREATE TABLE rows(metadata_json TEXT, artifact_type TEXT)") + shapes = [{}, {"observation": 1}, [], None, "text", json.dumps({"source_id": "legacy-parent"})] + shapes += [{key: value} for key in MARKER_KEYS for value in (None, "", "id", [], {}, True, 3)] + shapes += [{"redacted": True, "source_id": "parent"}, {"derived_from": {"sources": ["parent"], "counts": {"sources": 1}}}] + types = ["other", *sorted(DERIVED_ARTIFACT_TYPES)] if kind == "artifact" else ["other"] + predicate = original_label_sql(kind, sqlite=True) + for metadata in shapes: + for artifact_type in types: + raw = json.dumps(metadata) + conn.execute("DELETE FROM rows") + conn.execute("INSERT INTO rows VALUES(?,?)", (raw, artifact_type)) + original = conn.execute(f"SELECT {predicate} FROM rows").fetchone()[0] + if original: + assert not is_derived(kind, {"metadata_json": metadata, "artifact_type": artifact_type}), (kind, metadata, artifact_type) + conn.close() + + +@pytest.mark.parametrize("metadata,expected", [ + ('{"source_id":"hidden","source_id":"visible"}', "visible"), + ('{"source_\\u0069d":"hidden"}', "hidden"), + (json.dumps(json.dumps({"source_id": "hidden"})), "hidden"), + ('{"source_id":"hidden","redacted":true}', None), + ('{"source_id":"hidden","redacted":"true"}', "hidden"), + ('{"source_id":["hidden"]}', None), + ('not JSON', None), + ('{"source_id":"11111111111141118111111111111111"}', "11111111-1111-4111-8111-111111111111"), +]) +def test_sqlite_parent_hint_matches_store_and_kernel_decoding(metadata, expected): + assert _direct_source_hint(metadata) == expected diff --git a/tests/unit/test_label_dependency_cache.py b/tests/unit/test_label_dependency_cache.py new file mode 100644 index 000000000..24592687f --- /dev/null +++ b/tests/unit/test_label_dependency_cache.py @@ -0,0 +1,143 @@ +"""Dependency reuse preserves canonical kernel outcomes and request boundaries.""" +from copy import deepcopy +import json +from dataclasses import replace +from uuid import UUID + +import pytest + +from alicebot_api.vnext_agent_control import ALL_SENSITIVITY +from alicebot_api.vnext_label_guard import LabelGuard, invalidate_read_labels, label_read_scope +from alicebot_api.vnext_derived_labels import identifier, with_derived_from + +SOURCE = "11111111-1111-4111-8111-111111111111" +PARENT = "22222222-2222-4222-8222-222222222222" +ROOT = "33333333-3333-4333-8333-333333333333" + + +def row(row_id, metadata=None, **extra): + return {"id": row_id, "domain": "project", "sensitivity": "public", "metadata_json": metadata or {}, **extra} + + +class Store: + def __init__(self, rows): + self.rows = rows + + def read_label_rows(self, kind, ids): + return [value for name, value in self.rows if name == kind and identifier(value["id"]) in ids] + + +def stamp(kind, *ids): + return with_derived_from({}, {kind: [{"id": value} for value in ids]})["derived_from"] + + +def effective(guard, kind, value): + result = guard.effective_row(kind, value) + return tuple(result.get(key) for key in ("domain", "sensitivity", "project_scope", "project_floor", "unverified")) + + +def test_unique_metadata_and_text_share_one_settlement_but_never_admission(monkeypatch): + import alicebot_api.vnext_label_guard as module + source = row(SOURCE, {"project_scope": ["alpha"]}, sensitivity="confidential") + store = Store([("source", source)]) + calls = [] + original = module.settle_labels + def counted(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + monkeypatch.setattr(module, "settle_labels", counted) + with label_read_scope(store): + restricted = LabelGuard.for_filters(store, (), ("public",)) + copies = [row(str(UUID(int=i + 10)), {"source_id": SOURCE, "observation": i}, canonical_text=str(i)) for i in range(30)] + assert restricted.admit_rows("memory", copies) == [] + assert len(calls) == 1 + broad = replace(restricted, sensitivity_allowed=ALL_SENSITIVITY) + assert broad.admit_rows("memory", copies) == copies + assert len(calls) == 1 + source["sensitivity"] = "public" + invalidate_read_labels(store) + assert restricted.admit_rows("memory", copies) == copies + assert len(calls) == 2 + source["sensitivity"] = "confidential" + with label_read_scope(store): + assert restricted.admit_rows("memory", copies) == [] + assert len(calls) == 3 + + +@pytest.mark.parametrize("variant", ["self", "ancestor", "alias", "belief", "malformed", "missing", "floor", "class"]) +def test_reused_signature_agrees_with_fresh_kernel_for_boundary_variants(variant): + source = row(SOURCE) + parent = row(PARENT, {"source_id": SOURCE}) + first = row(ROOT, {"source_id": SOURCE, "observation": 1}) + second = row("44444444-4444-4444-8444-444444444444", {"source_id": SOURCE, "observation": 2}) + rows = [("source", source), ("memory", parent)] + if variant == "self": + first["metadata_json"] = second["metadata_json"] = {"derived_from": stamp("memories", second["id"])} + elif variant in {"ancestor", "alias"}: + first["metadata_json"] = second["metadata_json"] = {"derived_from": stamp("memories", PARENT)} + second["id"] = PARENT.upper() if variant == "alias" else PARENT + elif variant == "belief": + belief = row("55555555-5555-4555-8555-555555555555", memory_id=second["id"]) + first["metadata_json"] = second["metadata_json"] = {"derived_from": stamp("beliefs", belief["id"])} + rows.extend([("belief", belief), ("memory", row(second["id"]))]) + elif variant == "malformed": + second["metadata_json"]["derived_from"] = {"sources": [SOURCE], "counts": {"sources": 99}} + elif variant == "missing": + second["metadata_json"]["source_id"] = "missing-source" + elif variant == "floor": + second["metadata_json"]["project_floor"] = ["beta"] + elif variant == "class": + second["metadata_json"]["workflow"] = "project_auto_update" + if variant == "self": + rows.append(("memory", row(second["id"]))) + store = Store(rows) + guard = LabelGuard(store, active=True) + with label_read_scope(store): + assert effective(guard, "memory", first)[-1] is False + reused = effective(guard, "memory", second) + fresh = effective(LabelGuard(store, active=True), "memory", deepcopy(second)) + assert reused == fresh + + +def test_implicit_weekly_candidate_parent_is_not_shared_between_candidate_ids(): + weekly = row("66666666-6666-4666-8666-666666666666", + {"candidate_memory_ids": [ROOT], "derived_from": stamp("sources", SOURCE)}, artifact_type="weekly_synthesis") + first = row(ROOT, {"discovered_by": "vnext_weekly_synthesis", "source_artifact_id": weekly["id"]}) + second = row(PARENT, deepcopy(first["metadata_json"])) + store = Store([("artifact", weekly), ("source", row(SOURCE, sensitivity="confidential"))]) + guard = LabelGuard(store, active=True) + with label_read_scope(store): + effective(guard, "memory", first) + reused = effective(guard, "memory", second) + assert reused == effective(LabelGuard(store, active=True), "memory", second) + + +@pytest.mark.parametrize("change", ["nested_parent", "encoded_ref", "counts", "nested_scope", "floor_presence", "value_parent", "encoded_scope"]) +def test_parsing_memo_preserves_non_incidental_fields(change): + source = row(SOURCE, {"project_scope": ["alpha"]}) + hidden = row(PARENT, {"project_scope": ["beta"]}, sensitivity="confidential") + first = row(ROOT, {"source_id": SOURCE}) + second = row("44444444-4444-4444-8444-444444444444", deepcopy(first["metadata_json"])) + if change == "nested_parent": + second["metadata_json"]["evidence"] = {"source_id": PARENT} + elif change == "encoded_ref": + second["metadata_json"]["source_refs"] = ['{"source_id":"' + PARENT + '"}'] + elif change == "counts": + second["metadata_json"]["input_counts"] = {"sources": 99} + second["metadata_json"]["workflow"] = "project_auto_update" + elif change == "nested_scope": + second["metadata_json"]["agentic_memory"] = {"project_scope": ["beta"]} + elif change == "floor_presence": + first["metadata_json"]["project_floor"] = second["metadata_json"]["project_floor"] = ["beta"] + second["project_floor"] = None + elif change == "encoded_scope": + first["metadata_json"]["project_scope"] = ["alpha"] + second["metadata_json"] = json.dumps(first["metadata_json"]) + elif change == "value_parent": + second["value"] = {"source_id": PARENT} + store = Store([("source", source), ("source", hidden)]) + with label_read_scope(store): + guard = LabelGuard(store, active=True) + effective(guard, "memory", first) + reused = effective(guard, "memory", second) + assert reused == effective(LabelGuard(store, active=True), "memory", second) diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py index 5d285b3fd..98c03c502 100644 --- a/tests/unit/test_label_door_registry.py +++ b/tests/unit/test_label_door_registry.py @@ -55,6 +55,8 @@ # function -> helper that holds the guard call, or None when the function calls it DOORS = { + "routers/vnext_memories.py:get_vnext_source": None, + "mcp/evidence_artifacts.py:_handle_alice_vnext_review_items": None, "routers/_vnext_shared.py:_vnext_authorized_artifact": None, "vnext_source_fence.py:resolve_attachable_memory_id": None, "vnext_open_loop_references.py:withhold_unreadable_references": None, @@ -121,7 +123,6 @@ NOT_A_DOOR = { "vnext_artifact_review.py:lock_artifact_review_labels": "write lock classification only; the adapter authorizes before the dispatcher mutates", "routers/vnext_memories.py:regenerate_vnext_source": "operator-only regeneration rejects every profile except owner and unbound admin before the source lookup; real-profile rejection tests pin this gate", - "mcp/evidence_artifacts.py:_handle_alice_vnext_review_items": "legacy review list has no policy check", "vnext_projects.py:VNextProjectService.review_project_update": "write path; the route authorizes before this mutation", "vnext_projects.py:VNextProjectService.review_open_loop": "write path; the route and the open-loop tool settle the loop first", "vnext_memory_commit.py:VNextMemoryCommitService.confirm": "write path; _write_policy_decision settles the row", @@ -143,7 +144,6 @@ "vnext_queue.py:VNextQueueService._promote_artifact": "the HTTP review route authorizes through _vnext_authorized_artifact first", "vnext_queue.py:VNextQueueService.export_artifact_markdown": "the HTTP export route authorizes through _vnext_authorized_artifact first", "mcp/evidence_artifacts.py:_authorize_memory_audit_provenance": "original source pointers use SourceReadFence.admits before disclosure", - "routers/vnext_memories.py:get_vnext_source": "original source operator route; existing domain and project exemptions are preserved", "routers/vnext_memories.py:get_vnext_connector_status": "operator connector telemetry; original-source labels retain existing behavior", "routers/vnext_memories.py:review_vnext_source": "write path over an original source; existing exact policy applies", "routers/vnext_memories.py:delete_vnext_source": "owner mutation of an original source", diff --git a/tests/unit/test_source_scrub_handoff_passes.py b/tests/unit/test_source_scrub_handoff_passes.py index d3e8b9f34..fc6f5bc06 100644 --- a/tests/unit/test_source_scrub_handoff_passes.py +++ b/tests/unit/test_source_scrub_handoff_passes.py @@ -55,16 +55,18 @@ def test_reverse_lookup_limit_order_and_canonical_source_id(tmp_path): store = SQLiteVNextStore(conn, USER_ID) old = _create_loop(store, "old", metadata={"source_id": sid}) new = _create_loop(store, "new", metadata={"source_id": sid}) - conn.execute("UPDATE open_loops SET opened_at='2020-01-01T00:00:00Z', created_at='2020-01-01T00:00:00Z' WHERE id=?", (old,)) - conn.execute("UPDATE open_loops SET opened_at='2021-01-01T00:00:00Z', created_at='2021-01-01T00:00:00Z' WHERE id=?", (new,)) - assert [row["id"] for row in store.list_open_loops_referencing_source(source_id=UUID(sid).hex.upper(), limit=1)] == [new] + conn.execute("UPDATE open_loops SET opened_at='2020-01-01T00:00:00Z', created_at='2020-01-01T00:00:00Z', updated_at='2022-01-01T00:00:00Z' WHERE id=?", (old,)) + conn.execute("UPDATE open_loops SET opened_at='2021-01-01T00:00:00Z', created_at='2021-01-01T00:00:00Z', updated_at='2021-01-01T00:00:00Z' WHERE id=?", (new,)) + assert [row["id"] for row in store.list_open_loops_referencing_source(source_id=UUID(sid).hex.upper(), limit=1)] == [old] with pytest.raises(ValueError): store.list_open_loops_referencing_source(source_id=sid, limit=0) def test_plain_import_does_not_scan_loops(tmp_path, monkeypatch): db = _vault(tmp_path) + folder = _folder(tmp_path, note="Synthetic source.") + assert run_import(db, folder).imported_count == 1 def refuse(_store): raise AssertionError("plain import scanned open loops") monkeypatch.setattr(source_retirement, "_user_open_loops", refuse) - assert run_import(db, _folder(tmp_path, note="Synthetic source.")).imported_count == 1 + assert run_import(db, folder).imported_count == 0 From 63be8664ec7b8ba88627489af00c6fcec264709f Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 23:29:48 +0200 Subject: [PATCH 239/270] Close SQLite replay and legacy quote paths and verify regressions --- CHANGELOG.md | 2 +- .../alicebot_api/mcp/evidence_artifacts.py | 4 + apps/api/src/alicebot_api/sqlite_store.py | 13 ++- apps/api/src/alicebot_api/vnext_label_sql.py | 2 +- .../vnext_stores/sqlite/graph_open_loops.py | 4 +- docs/alpha/known-limitations.md | 2 +- ...est_label_round2_reader_fences_postgres.py | 31 +++++++- .../test_label_round2_read_budget.py | 27 +++++++ tests/unit/test_derived_label_input_filter.py | 17 +++- .../unit/test_label_round2_review_findings.py | 79 +++++++++++++++++++ ...mory_id_pointers_respect_the_read_fence.py | 10 +++ tests/unit/test_providers_router_split.py | 4 +- tests/unit/test_stage_a_vnext_auth_surface.py | 3 + .../unit/test_store_events_revisions_split.py | 5 +- .../unit/test_store_graph_open_loops_split.py | 20 +++-- tests/unit/test_store_memory_access_split.py | 6 +- .../unit/test_store_memory_lifecycle_split.py | 6 +- tests/unit/test_vnext_release_polish.py | 5 +- tests/unit/test_workspaces_router_split.py | 3 +- 19 files changed, 215 insertions(+), 28 deletions(-) create mode 100644 tests/unit/test_label_round2_review_findings.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c60e2031..0035b07a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Unreleased (on main, not in v0.20.0): October 6 round-two corrections recover a per-user open-loop digest after a project reassignment or stale duplicate extraction, and weekly candidates record the report inputs before insert so their stored floor matches the weekly artifact in the same transaction. Read-time labels reuse verified dependency signatures within one request; original rows are counted in SQL, and the derived partition still receives the kernel guard. PostgreSQL event filtering uses an indexed guarded UUID comparison. SQLite filters definitely hidden direct parents before its event limit, then applies the full effective-label guard. CI compares unique-metadata, varied-parent and repaired fixtures against pinned main 48873b03. No new migration is required. +- Unreleased (on main, not in v0.20.0): October 6 round-two corrections recover a per-user open-loop digest on both stores after a project reassignment or stale duplicate extraction, and weekly candidates record the report inputs before insert so their stored floor matches the weekly artifact in the same transaction. Read-time labels reuse verified dependency signatures within one request; original rows are counted in SQL, and the derived partition still receives the kernel guard. PostgreSQL event filtering uses an indexed guarded UUID comparison. SQLite filters definitely hidden direct parents before its event limit, then applies the full effective-label guard. The legacy review list also rechecks saved quotes on original or imported candidates against the current source fence. CI compares unique-metadata, varied-parent and repaired fixtures against pinned main 48873b03. No migration is required. - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings, including JSON escapes. Repeated keys and quote subtrees follow the limits in the draft security note. Lookup and removal both decode JSON text under source and memory references and inside trace fields. Admitted references and surrounding values stay, and the owner receives live references unchanged. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. diff --git a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py index feab58e69..82a824332 100644 --- a/apps/api/src/alicebot_api/mcp/evidence_artifacts.py +++ b/apps/api/src/alicebot_api/mcp/evidence_artifacts.py @@ -33,6 +33,7 @@ from alicebot_api.vnext_retrieval import MEMORY_ENTITY_EDGE_TYPES from alicebot_api.vnext_source_fence import ( EXPLAIN_DISCLOSURE_ACTION, + SavedProvenanceReader, SourceReadFence, cited_source_ids_in_memory_audit, source_rows_including_archived, @@ -755,6 +756,9 @@ def _handle_alice_vnext_review_items(context: MCPRuntimeContext, arguments: Mapp if len(items) >= limit or len(rows) < prefix: break prefix *= 2 + # Original/imported candidates may carry saved quotes without a + # derived marker. Judge their provenance against today's fence too. + items = SavedProvenanceReader(store, fence=fence).memories(items) return _json_object({"items": items, "count": len(items)}) diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index 6fb3fada9..c8b6e82b8 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -414,6 +414,17 @@ def _direct_source_hint(raw: object) -> str | None: return source_id +def _ensure_direct_source_hint(conn: sqlite3.Connection) -> None: + """Register once: SQLite refuses to replace a UDF with live statements.""" + cursor = conn.execute("SELECT 1 FROM pragma_function_list WHERE name='alice_direct_source_hint' AND narg=1 LIMIT 1") + try: + registered = cursor.fetchone() is not None + finally: + cursor.close() + if not registered: + conn.create_function("alice_direct_source_hint", 1, _direct_source_hint, deterministic=True) + + class SQLiteVNextStore: """SQLite-backed vNext repository facade for the second-brain kernel.""" @@ -427,7 +438,7 @@ def __init__(self, conn: sqlite3.Connection, user_id: UUID | str): self.user_id = str(user_id) _ensure_embedding_content_sha256_sqlite(self.conn) _ensure_project_scope_identity_sqlite(self.conn) - self.conn.create_function("alice_direct_source_hint", 1, _direct_source_hint, deterministic=True) + _ensure_direct_source_hint(self.conn) def lock_label_writes(self, *, exclusive: bool = False) -> None: """The SQLite writer lock is the label lock. Begin it when none is open.""" diff --git a/apps/api/src/alicebot_api/vnext_label_sql.py b/apps/api/src/alicebot_api/vnext_label_sql.py index bbb671e86..242f2b07e 100644 --- a/apps/api/src/alicebot_api/vnext_label_sql.py +++ b/apps/api/src/alicebot_api/vnext_label_sql.py @@ -13,7 +13,7 @@ def original_label_sql(kind: str, *, sqlite: bool = False) -> str: markers = ",".join("'" + key + "'" for key in sorted(MARKER_KEYS)) if sqlite: safe = "CASE WHEN json_valid(metadata_json) THEN metadata_json ELSE 'null' END" - predicate = f"(json_type({safe}) = 'object' AND NOT EXISTS (SELECT 1 FROM json_each({safe}) WHERE key IN ({markers})))" + predicate = f"(json_type({safe}) = 'object' AND NOT EXISTS (SELECT 1 FROM json_each({safe}) WHERE key IN ({markers})))" # nosec B608 - closed module constants, no caller input else: predicate = f"(jsonb_typeof(metadata_json) = 'object' AND NOT (metadata_json ?| ARRAY[{markers}]::text[]))" if kind == "artifact": diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py index 9ed0b0dfd..cc5733c06 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/graph_open_loops.py @@ -615,10 +615,10 @@ def upsert_open_loop_by_automation_digest( try: return self.create_open_loop(record, actor_type=actor_type) except sqlite3.IntegrityError: + # The unique digest belongs to the user; propagation may have cleared + # the mutable project/person columns since the first extraction. existing = self.find_open_loop_by_automation_digest( digest=normalized_digest, - project_id=str(loop["project_id"]) if loop.get("project_id") is not None else None, - person_id=str(loop["person_id"]) if loop.get("person_id") is not None else None, ) if existing is None: raise diff --git a/docs/alpha/known-limitations.md b/docs/alpha/known-limitations.md index a516c6694..fe23f1c89 100644 --- a/docs/alpha/known-limitations.md +++ b/docs/alpha/known-limitations.md @@ -75,7 +75,7 @@ Open in v0.20.0, with the detail in the [v0.20.0 release notes](../release/v0.20 - in v0.20.0 a key bound to one project can attach a source it cannot read through `alice_memory_commit` `source_refs`, the `provenance` of `alice_memory_correct` and, over HTTP on Postgres, `POST /v0/vnext/open-loops`, and a saved quote stays readable after its source is reclassified. Unreleased (on main, not in v0.20.0): those doors answer `not_found` (404 over HTTP) for a source or memory the caller may not read and the readers of a saved quote and of an open loop ask the reader's own fence again, but a link that passes the fence of an `admin_agent` writer still makes `alice_explain` of that memory fail for the keys of a project with a lower ceiling (see [Cited sources](mcp-tools.md#cited-sources) and [Saved quotes](mcp-tools.md#saved-quotes)) - the cited-source fence does not cover everything yet. Unreleased (on main, not in v0.20.0): memory proposals and the agent-output ingest store their `source_refs` as given, the owner dependency trace lists a memory by the first id of a multi-id ref only, `provenance_count` still counts a withheld link, and a memory or open loop saved before the fix keeps its link or id, so `alice_explain` still fails for such a memory; the rest is under [Cited sources](mcp-tools.md#cited-sources), [Saved quotes](mcp-tools.md#saved-quotes) and in the [CHANGELOG](../../CHANGELOG.md) - Unreleased (on main, not in v0.20.0): on SQLite, `import-markdown --supersede` matches resolved paths (not moved folders), and `sources delete` and `prune` keep source events, hashes, memory text and backups, and blank an open loop that names the source in any spelling the saved-quote reader names, in one pass. v0.20.0 kept loops using other spellings, and freed space keeps text until the [VACUUM step](../integrations/importers.md#list-delete-and-prune-sqlite-sources). -- in v0.20.0 a derived summary, report or copy keeps the label its inputs had when it was made, so relabelling an input leaves it readable by keys the input now refuses, and a report can show a key bound to one project the titles, ids or text of rows from other projects or with no project. Unreleased (on main, not in v0.20.0): a derived row follows its inputs when they are relabelled and stays as strict as they have been, a report with an input that has no project is global and keeps every input's project, and a row whose recorded inputs cannot be found is read only by the owner and an unbound admin key until it is regenerated, and the five screens apply the sensitivity ceiling, counting rows the caller may read. See [Derived row domains](mcp-tools.md#derived-row-domains) Graph edge explanations and the new doctor label counts remain outside that ceiling. Bounded retrieval, consolidation and staleness reads can under-fill; imported irregular memory references can retain ids. See the [draft security note](../release/derived-labels-security-note-draft.md) for these limits and reverse-lookup cost. +- in v0.20.0 a derived summary, report or copy keeps the label its inputs had when it was made. Unreleased (on main, not in v0.20.0): it follows its inputs when they are relabelled and keeps every input's project; unverified rows are read only by the owner and an unbound admin key, and the five screens apply the sensitivity ceiling, counting rows the caller may read, while graph edge explanations and the new doctor label counts remain outside that ceiling (see the [draft security note](../release/derived-labels-security-note-draft.md)). See [Derived row domains](mcp-tools.md#derived-row-domains) What v0.19.0 and v0.19.2 limited and v0.20.0 fixed or narrowed (request body size, the `Host` check, provider redirects, local-folder reads, the memory id fence, deep backup JSON, the lone surrogate turn, the query size bounds and the data directory variable) is recorded in the [v0.19.2 release notes](../release/v0.19.2-release-notes.md), the [v0.20.0 release notes](../release/v0.20.0-release-notes.md) and the [CHANGELOG](../../CHANGELOG.md). diff --git a/tests/integration/test_label_round2_reader_fences_postgres.py b/tests/integration/test_label_round2_reader_fences_postgres.py index 1383cf3ab..64a636db4 100644 --- a/tests/integration/test_label_round2_reader_fences_postgres.py +++ b/tests/integration/test_label_round2_reader_fences_postgres.py @@ -8,7 +8,7 @@ from tests.integration.derived_labels_postgres_support import label_harness -@pytest.mark.parametrize("profile,bound", [("trusted_local_agent", False), ("admin_agent", False), ("trusted_local_agent", True), ("admin_agent", True)]) +@pytest.mark.parametrize("profile,bound", [("trusted_local_agent", False), ("admin_agent", False), ("trusted_local_agent", True), ("admin_agent", True), ("read_only_agent", False), ("read_only_agent", True)]) def test_source_get_uses_the_callers_entire_fence(label_harness, profile, bound): h = label_harness alpha, beta = str(uuid4()), str(uuid4()) @@ -18,18 +18,43 @@ def test_source_get_uses_the_callers_entire_fence(label_harness, profile, bound) visible = h.source(scope=(alpha,)) private = h.source(scope=(alpha,), sensitivity="confidential") other = h.source(scope=(beta,)) + restricted_domain = h.source(scope=(alpha,)) + assert h.relabel("source", restricted_domain["id"], domain="health")[0] == 200 # The owner's control precedes key provisioning, as the real key gate requires. - for source in (visible, private, other): + for source in (visible, private, other, restricted_domain): status, body, _ = h.request("GET", "/v0/vnext/sources/" + str(source["id"])) assert status == 200 and str(body["id"]) == str(source["id"]) key = h.key(profile, project=alpha if bound else None) status, body, _ = h.request("GET", "/v0/vnext/sources/" + str(visible["id"]), key=key) assert status == 200 and str(body["id"]) == str(visible["id"]) - for source, permitted in ((private, profile == "admin_agent"), (other, not bound)): + for source, permitted in ((private, profile == "admin_agent"), (other, not bound), (restricted_domain, profile in {"trusted_local_agent", "admin_agent"})): status, body, _ = h.request("GET", "/v0/vnext/sources/" + str(source["id"]), key=key) assert status == (200 if permitted else 404), (profile, bound, body) if not permitted: assert body == {"detail": f"vNext source {source['id']} was not found"} + missing = str(uuid4()) + status, body, _ = h.request("GET", "/v0/vnext/sources/" + missing, key=key) + assert status == 404 and body == {"detail": f"vNext source {missing} was not found"} + + +@pytest.mark.parametrize("profile", ["trusted_local_agent", "admin_agent"]) +def test_legacy_review_list_rechecks_saved_quotes_postgres(label_harness, profile): + import json + h = label_harness + quote = "Synthetic PostgreSQL saved-quote sentinel" + source = h.source() + with h.store() as store: + memory = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Original synthetic candidate", "status": "candidate", + "domain": "project", "sensitivity": "public", + "metadata_json": {"provenance": {"source_id": str(source["id"]), "quote": quote}}}) + context = MCPRuntimeContext(database_url=h.urls["app"], user_id=h.user_id) + arguments = {"limit": 20, "agent_id": "reader", "permission_profile": profile} + assert quote in json.dumps(call_mcp_tool(context, name="alice_vnext_review_items", arguments=arguments)) + assert h.relabel("source", source["id"], sensitivity="confidential")[0] == 200 + result = call_mcp_tool(context, name="alice_vnext_review_items", arguments=arguments) + assert [str(row["id"]) for row in result["items"]] == [str(memory["id"])] + assert (quote in json.dumps(result)) is (profile == "admin_agent") + assert quote in json.dumps(call_mcp_tool(context, name="alice_vnext_review_items", arguments={"limit": 20})) @pytest.mark.parametrize("profile,nested", [("trusted_local_agent", False), ("trusted_local_agent", True), ("admin_agent", False), ("read_only_agent", True)]) diff --git a/tests/performance/test_label_round2_read_budget.py b/tests/performance/test_label_round2_read_budget.py index 273052bd2..470d3e182 100644 --- a/tests/performance/test_label_round2_read_budget.py +++ b/tests/performance/test_label_round2_read_budget.py @@ -4,6 +4,7 @@ from pathlib import Path import subprocess import sys +import time from uuid import uuid4 import pytest @@ -12,6 +13,9 @@ from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection from alicebot_api.vnext_agent_keys import create_agent_key from alicebot_api.vnext_derived_labels import with_derived_from +from alicebot_api.vnext_label_repair import label_gap_counts +from alicebot_api.vnext_label_guard import LabelGuard, label_read_request +from alicebot_api.vnext_agent_control import ALL_SENSITIVITY, DEFAULT_AGENT_SENSITIVITY USER = "11111111-1111-4111-8111-111111111111" @@ -48,6 +52,8 @@ def seed_varied(store, *, postgres=False, count=5000, source_count=300, mixed=Tr else: store.conn.executemany("INSERT INTO memories(id,user_id,memory_key,canonical_text,metadata_json,sensitivity,value,source_event_ids,status,domain) VALUES(?,?,?,?,?,?,'{}','[]','active','project')", values) store.conn.executemany("INSERT INTO event_log(id,user_id,target_id,event_type,actor_type,target_type,payload_json) VALUES(?,?,?,'memory.created','system','memory','{}')", [(str(uuid4()), values[0][1], row_id) for row_id in ids]) + if repaired: + assert label_gap_counts(store) == (0, 0), "post-repair budget must measure rows whose stored labels equal effective labels" keys = {} for profile in ("trusted_local_agent", "admin_agent"): _, keys[profile] = create_agent_key(store, user_id=store.user_id if not postgres else sources[0]["user_id"], @@ -78,6 +84,14 @@ def assert_budgets(backend, location, user, keys): assert head[action]["minimum_wall"] <= 1, (action, head) +@label_read_request +def complete_count(store, ceiling): + guard = LabelGuard.for_filters(store, (), ceiling) + first = guard.readable_status_counts("memory") + assert guard.readable_status_counts("memory") == first + return first + + @pytest.mark.parametrize("case,repaired", [("mixed", False), ("mixed", True), ("many-hidden", False), ("many-hidden", True), ("one-hidden", False)]) def test_sqlite_varied_read_budget(tmp_path, case, repaired): path = tmp_path / "round2.db" @@ -86,4 +100,17 @@ def test_sqlite_varied_read_budget(tmp_path, case, repaired): keys = seed_varied(SQLiteVNextStore(conn, USER), repaired=repaired, source_count=1 if case == "one-hidden" else 3000 if case == "many-hidden" else 300, all_hidden=case != "mixed", mixed=case == "mixed") + store = SQLiteVNextStore(conn, USER) + for profile in keys: + ceiling = ALL_SENSITIVITY if profile == "admin_agent" else DEFAULT_AGENT_SENSITIVITY + expected = 5000 if profile == "admin_agent" else 2500 if case == "mixed" else 0 + walls, cpus = [], [] + for _ in range(3): + wall_start, cpu_start = time.perf_counter(), time.process_time() + assert complete_count(store, ceiling) == ({"active": expected} if expected else {}) + walls.append(time.perf_counter() - wall_start) + cpus.append(time.process_time() - cpu_start) + print(json.dumps({"count_case": case, "repaired": repaired, "profile": profile, + "minimum_wall": min(walls), "minimum_cpu": min(cpus), "admitted": expected})) + assert min(walls) <= 1 and min(cpus) <= 1 assert_budgets("sqlite", "sqlite:///" + str(path), USER, keys) diff --git a/tests/unit/test_derived_label_input_filter.py b/tests/unit/test_derived_label_input_filter.py index 1eab25c73..ac734193c 100644 --- a/tests/unit/test_derived_label_input_filter.py +++ b/tests/unit/test_derived_label_input_filter.py @@ -3,6 +3,7 @@ from __future__ import annotations from datetime import UTC, datetime, timedelta +import pytest from alicebot_api.vnext_brain import _matches_report_scope from alicebot_api.vnext_derived_labels import input_admitted, locked_projects, stamp_derived_from @@ -45,6 +46,21 @@ def test_a_locked_brief_scope_rejects_a_shared_row() -> None: ) is False +@pytest.mark.parametrize("producer", ["brain", "connections", "contradictions"]) +def test_empty_projects_still_apply_the_locked_input_gate(producer): + from alicebot_api import vnext_connections, vnext_contradictions + row = {"id": "beta", "metadata_json": {"project_scope": [BETA]}, "captured_at": "2026-10-06T09:00:00Z"} + identity = {"project_scope_locked": True, "project_scope": [ALPHA]} + all_of = locked_projects(identity, ()) + if producer == "brain": + start = datetime(2026, 10, 6, tzinfo=UTC) + assert not _matches_report_scope(row, kind="source", projects=(), all_of=all_of, + window_start=start, window_end=start + timedelta(days=1)) + else: + module = vnext_connections if producer == "connections" else vnext_contradictions + assert not module._matches_projects(row, (), source_row=True, all_of=all_of) + + def test_stamp_derived_from_counts_match_the_lists() -> None: payload: dict[str, object] = {"metadata_json": {"workflow": "daily_brief"}} stamp_derived_from( @@ -63,7 +79,6 @@ def test_stamp_derived_from_counts_match_the_lists() -> None: assert record["counts"]["memories"] == 1 -import pytest from uuid import UUID from alicebot_api.vnext_agent_control import AgentIdentity, AgentPolicyBlockedError from alicebot_api.routers._vnext_shared import _vnext_authorized_artifact diff --git a/tests/unit/test_label_round2_review_findings.py b/tests/unit/test_label_round2_review_findings.py new file mode 100644 index 000000000..396a6a399 --- /dev/null +++ b/tests/unit/test_label_round2_review_findings.py @@ -0,0 +1,79 @@ +"""SQLite lifecycle and saved-quote regressions from the independent review.""" +from copy import deepcopy +import json +from uuid import uuid4 + +import pytest + +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext +from alicebot_api.onramp import bootstrap_database, sqlite_url_for_path +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_label_repair import label_gap_counts +from alicebot_api.vnext_projects import ProjectAutomationRequest, VNextProjectService + +USER = "11111111-1111-4111-8111-111111111111" + + +@pytest.mark.parametrize("stale", [False, True]) +def test_sqlite_digest_replays_after_project_moves(tmp_path, monkeypatch, stale): + path = tmp_path / "moves.db" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.invalid") + alpha, beta = str(uuid4()), str(uuid4()) + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source({"source_type": "note", "title": "Synthetic task", "content_hash": "move", + "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": [alpha], "raw_text": "TODO: Round trip task"}}) + request = ProjectAutomationRequest(agent_identity=None, project_id=alpha) + initial = VNextProjectService(store).extract_open_loops(request) + assert len(initial) == 1 + store.update_source(source_id=str(source["id"]), patch={"metadata_json": {**source["metadata_json"], "project_scope": [beta]}}) + if not stale: + store.update_source(source_id=str(source["id"]), patch={"metadata_json": {**source["metadata_json"], "project_scope": [alpha]}}) + assert store.get_open_loop(str(initial[0]["id"]))["project_id"] is None + if stale: + monkeypatch.setattr(store, "search_sources", lambda **_kwargs: [deepcopy(source)]) + for _ in range(2): + replay = VNextProjectService(store).extract_open_loops(request) + assert [row["id"] for row in replay] == [initial[0]["id"]] + rows = store.list_open_loops(status=None, limit=20) + assert len(rows) == 1 + assert set(rows[0]["metadata_json"]["project_floor"]) == {alpha, beta} + assert label_gap_counts(store) == (0, 0) + + +@pytest.mark.parametrize("profile", ["trusted_local_agent", "read_only_agent", "admin_agent"]) +@pytest.mark.parametrize("writer", ["imported", "legacy_commit"]) +def test_legacy_review_list_rechecks_saved_quotes(tmp_path, monkeypatch, profile, writer): + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + monkeypatch.setenv("ALICE_MCP_LEGACY_TOOLS", "1") + path = tmp_path / "quotes.db" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.invalid") + quote = "Synthetic saved-quote sentinel" + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + source = store.create_source({"source_type": "note", "title": "Synthetic source", "content_hash": "quote", + "domain": "project", "sensitivity": "public"}) + if writer == "imported": + memory = store.create_memory({"memory_key": "original", "canonical_text": "Original synthetic candidate", "status": "candidate", + "domain": "project", "sensitivity": "public", + "metadata_json": {"provenance": {"source_id": str(source["id"]), "quote": quote}}}) + context = MCPRuntimeContext(database_url=sqlite_url_for_path(path), user_id=USER) + if writer == "legacy_commit": + committed = call_mcp_tool(context, name="alice_vnext_commit_memory", arguments={ + "agent_id": "writer", "permission_profile": "trusted_local_agent", "title": "Synthetic observation", + "canonical_text": "Original synthetic candidate", "intent": "inferred_observation", "domain": "project", + "sensitivity": "public", "source_refs": [str(source["id"])], "conversation_excerpt": quote}) + memory = committed["memory"] + assert memory["status"] == "candidate" + arguments = {"limit": 20, "agent_id": "reader", "permission_profile": profile} + before = call_mcp_tool(context, name="alice_vnext_review_items", arguments=arguments) + assert quote in json.dumps(before) + with sqlite_user_connection(path, USER) as conn: + SQLiteVNextStore(conn, USER).update_source(source_id=str(source["id"]), patch={"sensitivity": "confidential"}) + after = call_mcp_tool(context, name="alice_vnext_review_items", arguments=arguments) + assert [str(row["id"]) for row in after["items"]] == [str(memory["id"])] + assert (quote in json.dumps(after)) is (profile == "admin_agent") + owner = call_mcp_tool(context, name="alice_vnext_review_items", arguments={"limit": 20}) + assert quote in json.dumps(owner) diff --git a/tests/unit/test_memory_id_pointers_respect_the_read_fence.py b/tests/unit/test_memory_id_pointers_respect_the_read_fence.py index 8cb5d9530..ef3aa67d8 100644 --- a/tests/unit/test_memory_id_pointers_respect_the_read_fence.py +++ b/tests/unit/test_memory_id_pointers_respect_the_read_fence.py @@ -827,6 +827,16 @@ def test_a_pack_keeps_the_visible_changes_found_before_the_scan_ceiling(tmp_path scans to the 16384 row ceiling and lists the older readable event as well. """ monkeypatch.setattr(vnext_retrieval_module, "RECENT_CHANGES_SCAN_MAX_ROWS", 64) + # An adapter without SQL sensitivity pushdown still needs the bounded + # fallback. Native SQLite now excludes definitely hidden targets first. + from alicebot_api.sqlite_store import SQLiteVNextStore + original_events = SQLiteVNextStore.list_memory_events + def legacy_events(self, *, event_type_prefix=None, scope_projects=(), scope_people=(), + scope_person_memory_ids=(), scope_window_start=None, scope_window_end=None, limit=500): + return original_events(self, event_type_prefix=event_type_prefix, scope_projects=scope_projects, + scope_people=scope_people, scope_person_memory_ids=scope_person_memory_ids, + scope_window_start=scope_window_start, scope_window_end=scope_window_end, limit=limit) + monkeypatch.setattr(SQLiteVNextStore, "list_memory_events", legacy_events) context = _context(tmp_path, monkeypatch) (old_readable,) = _seed_memories(context, count=1, sensitivity="private", label="Old readable") hidden = _seed_memories(context, count=100, sensitivity="confidential", label="Hidden") diff --git a/tests/unit/test_providers_router_split.py b/tests/unit/test_providers_router_split.py index 04a176b8d..6718496c7 100644 --- a/tests/unit/test_providers_router_split.py +++ b/tests/unit/test_providers_router_split.py @@ -120,7 +120,9 @@ # Re-pin 2026-10-06: source regeneration is a new protected write route in the # central vNext route policy; the app carrier keeps the same definitions. -EXPECTED_CARRIER_AST_SHA256 = "fcd6d722e2d6c3be28b138449555e703f6930168f52b8e94b41dacbcb937b605" +# Round-two re-pin: _VNEXT_ROUTE_LOCAL_POLICY and +# _VNEXT_CENTRAL_OPERATOR_ROUTES move source GET to its route-local full fence. +EXPECTED_CARRIER_AST_SHA256 = "b634b5cab5c2821bad5cc4a3eb718c5a825eee246fa4397c2c8add6336e9b718" EXPECTED_ROUTE_NAME_MANIFEST_SHA256 = "1a438538e16120361f92d30375cc94679d598fe4b78ba5a58a7d8a4dda6af83c" EXPECTED_OPERATION_MANIFEST_SHA256 = "8b79ceaf996b8c51b5bb2f3f38a8c19a4e33796955d8b8f7a66e7ac01ea1732d" EXPECTED_IMPORT_MANIFEST_SHA256 = "17484ccdd460e42e2ad5c82a8ca867664694feaf871118c410a134996a532358" diff --git a/tests/unit/test_stage_a_vnext_auth_surface.py b/tests/unit/test_stage_a_vnext_auth_surface.py index d8be03189..35270e558 100644 --- a/tests/unit/test_stage_a_vnext_auth_surface.py +++ b/tests/unit/test_stage_a_vnext_auth_surface.py @@ -52,6 +52,9 @@ ("POST", "/v0/vnext/memories/accept-consolidation"), ("POST", "/v0/vnext/memories/redact"), }, + "source_read_fence": { + ("GET", "/v0/vnext/sources/{source_id}"), + }, "persisted_artifact_scope": { ("POST", "/v0/vnext/artifacts/{artifact_id}/insight-feedback"), ("GET", "/v0/vnext/artifacts/{artifact_id}"), diff --git a/tests/unit/test_store_events_revisions_split.py b/tests/unit/test_store_events_revisions_split.py index c718a97b6..f905d02da 100644 --- a/tests/unit/test_store_events_revisions_split.py +++ b/tests/unit/test_store_events_revisions_split.py @@ -138,6 +138,7 @@ "_json_list_text": ("(value: 'object | None') -> 'str'", None), }, } +# Round two adds the conservative original-row SQL count on both facades. EXPECTED_CLASS_KEY_SHA256 = { # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. # Existing facade members retain their relative order. @@ -150,7 +151,7 @@ # Re-minted for the per-file importer savepoint (2026-10-02): ``savepoint`` is appended last on both façades. # Previous receipt: 650e2e0ff088d67c... Proof: the class key list equals the list at origin/main 040a2a10 # plus ``savepoint`` before ``__dict__``, with every other key in the same order. - "postgres": "3751e3fd145562485233765a14ae4e7bc7f396e6e72356a079e266fd98025839", + "postgres": "2a91ba3efd288f45d88d8eebd590ec76659e29b3d3223e818cc7828661807249", # Re-minted for the merge of #500 and #502: ``check_literal_match_query``. # Re-minted again for per-project memory S2 (2026-10-02): the two single-scan partition reads # ``list_memories_view_partitions`` and ``list_open_loops_view_partitions``, SQLite only on purpose @@ -158,7 +159,7 @@ # Re-minted again for the per-file importer savepoint (2026-10-02), the same appended method. # Previous receipt: 365b7a01acf7a8b5... Proof: as for postgres, one added key and no other change. # Re-minted for the derived-label lock: ``lock_label_writes`` and ``read_label_rows`` on both facades. - "sqlite": "265db9bfe184e712eb3bbb64356c7a9601b130c93ca847962697c35f68b84ecd", + "sqlite": "50d5aad840a245b6d01b72527ed559c97faad3b8cbbfbcb06982e1e9dce1f270", } EXPECTED_SUPPORT_AST_SHA256 = { "postgres_columns": { diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index 44c28de60..cea2b1d3d 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -106,6 +106,8 @@ # Reviewed label hooks: both open-loop updaters preserve protected metadata, # clamp derived labels and propagate a stricter label to later rows. +# Round two changes only the digest-conflict retry on each carrier: use the +# per-user digest after propagation clears mutable project/person columns. SOURCE_RECEIPTS = { # Re-minted for the filter-before-cut fix (2026-10-03): ``list_open_loop_events`` takes ``domains`` and # ``sensitivity_allowed`` as required arguments, and the SQLite one applies them in the join before ``LIMIT`` @@ -113,7 +115,7 @@ # the shared unscoped call site can state ``None`` for both, and it refuses anything else, since the Postgres # runtime resolves no project view (reviewed change, not drift). # The file hash now matches the carrier after the label lock. Previous receipt e4724ba1... - POSTGRES_CARRIER_PATH: "a9fecb0462324a46610f01134146fa73a2ab63692c7a6daa44ff56760840a812", + POSTGRES_CARRIER_PATH: "dda88f24e77df21886653e337ff8afdf325cd66dd62cee311eeb3bab31d0b86f", # The SQLite carrier is re-minted, with its method AST manifest below, for # ``list_open_loops`` and ``list_open_loop_events``: they bind a query through # ``literal_match_operand`` and so refuse one past the LIKE operand limit. @@ -132,26 +134,28 @@ # change (reviewed change, not drift). # Re-minted for the combined floor-aware partition read and canonical source-reference batch lookup. # Previous receipt 9a2634be... - SQLITE_CARRIER_PATH: "d80d53bc64e2395f9480eb6b06338f337c44b811b0eb86974b4dbc22308a88d0", + SQLITE_CARRIER_PATH: "cb4e420e905feb7416df16d3cd66365799e816e953ff9d4d440a5e8930f572ab", POSTGRES_COLUMNS_PATH: "1a782bf3eb87f68f67434508baab0f82d49cb40a0c547cde49bbc972e2f1d182", SQLITE_COLUMNS_PATH: "be81b8628d0831d3d02b280b5455fb02333db5740ebef8d85d58024384ae6556", } EXPECTED_METHOD_AST_MANIFESTS = { # Postgres manifest matches the carrier after the label lock. Previous 25580884... - POSTGRES_CARRIER_PATH: "69d4776f91829f5dc96f751c7d513c13149f851d878476c1c2c17c245d8cf0a3", + POSTGRES_CARRIER_PATH: "4de0ce4dc69301b45a8438db993f45178bf30f013f456223aa01011395905fbd", # SQLite manifest includes the floor identity on the partition read. Previous 2850ba60... - SQLITE_CARRIER_PATH: "a4da5f218817ad3c17ad887a2b4e643307e2f6f1c5b6bb16b0e15ccf25cb017f", + SQLITE_CARRIER_PATH: "e66e08ea2a973a5dcbb61cdaa493485d1b16291ded5e460343100484940acffe", } EXPECTED_METADATA_MANIFESTS = { POSTGRES_CARRIER_PATH: "6edb6a10e7a37dbbbbde97e5550422718a0112257666de8e23d49c60490fa13f", SQLITE_CARRIER_PATH: "121ed7dcea3f8565c181e844e49e0bc229098f6dc6c9124d03bb12e6062ab562", } EXPECTED_COMMENT_MANIFESTS = { - POSTGRES_CARRIER_PATH: (9, "bb34d175e716f5a929fa1ee5e7e30ba0e0b25be285cda3556a0c709719316c4e"), + POSTGRES_CARRIER_PATH: (11, "cd3d7d16060fbcac458f694d4201a6b176d7e8f353cd6684597ea34a173c59a9"), # The SQLite carrier gains a three-line comment above the two new required arguments of # ``list_open_loop_events`` (3 comments before, 6 now); the Postgres carrier gains none. - SQLITE_CARRIER_PATH: (6, "970028b5c929f0e749d8b40bdee571c872600de7a713e58d60e0da86f022af8a"), + SQLITE_CARRIER_PATH: (8, "66dd879bf3b6236c2253729ce03dc78830b552c710f01d1c42f6db9c381cdcfe"), } +# Round two adds count_original_label_statuses after read_label_rows on both +# facades; all existing members retain their relative order. EXPECTED_CLASS_ORDERS = { # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. # Existing facade members retain their relative order. @@ -161,7 +165,7 @@ # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # lock_label_writes and read_label_rows follow __init__. Previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (179, "71998141cd00c0639209285accbc29faa775b2e0250422c96101ee2f144df832"), + "PostgresVNextStore": (180, "98a0c0e5668366e07420aae305e3dd49f042e62a8cde9f0a27abb0ed9caf03e7"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -177,7 +181,7 @@ # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. # lock_label_writes and read_label_rows follow __init__. Previous receipt (134, 13012720...). - "SQLiteVNextStore": (138, "783c07140c4015bcf8e43a3c68f3c2837ad14ce3525f8bd56a6f9c319d38eada"), + "SQLiteVNextStore": (139, "d90f82579ad45d50bd22e9270d46e0b1ce50529e5ebfaf7047fb2f54799a6c54"), } EXPECTED_COLUMN_AST = { POSTGRES_COLUMNS_PATH: { diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index 1dc1d3837..0eb8fdb86 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -208,6 +208,8 @@ "_fts_match_any_expression", ) +# Round two adds count_original_label_statuses after read_label_rows on both +# facades; all existing members retain their relative order. EXPECTED_CLASS_ORDERS = { # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. # Existing facade members retain their relative order. @@ -220,7 +222,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (179, "71998141cd00c0639209285accbc29faa775b2e0250422c96101ee2f144df832"), + "PostgresVNextStore": (180, "98a0c0e5668366e07420aae305e3dd49f042e62a8cde9f0a27abb0ed9caf03e7"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -237,7 +239,7 @@ # prunable_sources here; every pre-existing class member keeps its order. # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (134, 13012720...). - "SQLiteVNextStore": (138, "783c07140c4015bcf8e43a3c68f3c2837ad14ce3525f8bd56a6f9c319d38eada"), + "SQLiteVNextStore": (139, "d90f82579ad45d50bd22e9270d46e0b1ce50529e5ebfaf7047fb2f54799a6c54"), } diff --git a/tests/unit/test_store_memory_lifecycle_split.py b/tests/unit/test_store_memory_lifecycle_split.py index 05a69e847..d693d363f 100644 --- a/tests/unit/test_store_memory_lifecycle_split.py +++ b/tests/unit/test_store_memory_lifecycle_split.py @@ -107,6 +107,8 @@ "postgres": "07a567e26d0f7c4f51ae2a1910d059397b513a575d85f06c2f8c0396ac1bb2ef", "sqlite": "1270ef0115988552418349aa9e94a7442ba04be41443f278f68a1fa81857903a", } +# Round two adds count_original_label_statuses after read_label_rows on both +# facades; all existing members retain their relative order. EXPECTED_CLASS_ORDERS = { # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. # Existing facade members retain their relative order. @@ -115,7 +117,7 @@ # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (179, "71998141cd00c0639209285accbc29faa775b2e0250422c96101ee2f144df832"), + "PostgresVNextStore": (180, "98a0c0e5668366e07420aae305e3dd49f042e62a8cde9f0a27abb0ed9caf03e7"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -130,7 +132,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. - "SQLiteVNextStore": (138, "783c07140c4015bcf8e43a3c68f3c2837ad14ce3525f8bd56a6f9c319d38eada"), + "SQLiteVNextStore": (139, "d90f82579ad45d50bd22e9270d46e0b1ce50529e5ebfaf7047fb2f54799a6c54"), } EXPECTED_FACADE_COMMENT_DIGESTS = { POSTGRES_FACADE_PATH: "d8599a46ee26dc35a3ae52c1a98a416509add9ae4a42ece780c5c5ed7e132b93", diff --git a/tests/unit/test_vnext_release_polish.py b/tests/unit/test_vnext_release_polish.py index 8e71bdce6..ad0baec55 100644 --- a/tests/unit/test_vnext_release_polish.py +++ b/tests/unit/test_vnext_release_polish.py @@ -525,7 +525,8 @@ def test_ci_action_dependency_carrier_uses_exact_atomic_pins() -> None: workflows, ) - # 28 since real-host-ci.yml checks out once for the pinned job, once + # 29 since round-two read budgets add one pinned baseline checkout. + # Previously 28 since real-host-ci.yml checks out once for the pinned job, once # for the weekly canary, once for the dispatch-only hook trial, once # for the dispatch-only plugin hook trial, once for the # dispatch-only marketplace check, and once for the dispatch-only @@ -539,7 +540,7 @@ def test_ci_action_dependency_carrier_uses_exact_atomic_pins() -> None: # rather than absorbed. assert checkout_refs == [ "3d3c42e5aac5ba805825da76410c181273ba90b1" - ] * 28 + ] * 29 assert codeql_refs == [ "ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd" ] * 3 diff --git a/tests/unit/test_workspaces_router_split.py b/tests/unit/test_workspaces_router_split.py index 583136566..b610c6b7a 100644 --- a/tests/unit/test_workspaces_router_split.py +++ b/tests/unit/test_workspaces_router_split.py @@ -132,7 +132,8 @@ # Re-pin 2026-10-06: source regeneration is a new protected write route in the # central vNext route policy; the app carrier keeps the same definitions. -EXPECTED_CARRIER_AST_SHA256 = "fcd6d722e2d6c3be28b138449555e703f6930168f52b8e94b41dacbcb937b605" +# Round two moves source GET to the route-local full fence; definitions are unchanged. +EXPECTED_CARRIER_AST_SHA256 = "b634b5cab5c2821bad5cc4a3eb718c5a825eee246fa4397c2c8add6336e9b718" EXPECTED_ROUTE_NODE_SHA256 = { "get_vnext_workspace": "52c12b20d7bb33759f8dafa2249b2d775b54666130402c0c75045e9ad57ed587", "bootstrap_v1_workspace": "07b1fe2a4cd03a5ba69abe76e258a457e85e92b0bfba592520ee02d01d759c4b", From 6817145f14d00ac66c12ed91a98b4433808d49db Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 23:36:34 +0200 Subject: [PATCH 240/270] Strengthen dependency-cache boundary fixtures --- tests/unit/test_label_dependency_cache.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_label_dependency_cache.py b/tests/unit/test_label_dependency_cache.py index 24592687f..11d3111c8 100644 --- a/tests/unit/test_label_dependency_cache.py +++ b/tests/unit/test_label_dependency_cache.py @@ -66,7 +66,7 @@ def counted(*args, **kwargs): @pytest.mark.parametrize("variant", ["self", "ancestor", "alias", "belief", "malformed", "missing", "floor", "class"]) def test_reused_signature_agrees_with_fresh_kernel_for_boundary_variants(variant): - source = row(SOURCE) + source = row(SOURCE, sensitivity="confidential") parent = row(PARENT, {"source_id": SOURCE}) first = row(ROOT, {"source_id": SOURCE, "observation": 1}) second = row("44444444-4444-4444-8444-444444444444", {"source_id": SOURCE, "observation": 2}) @@ -75,11 +75,11 @@ def test_reused_signature_agrees_with_fresh_kernel_for_boundary_variants(variant first["metadata_json"] = second["metadata_json"] = {"derived_from": stamp("memories", second["id"])} elif variant in {"ancestor", "alias"}: first["metadata_json"] = second["metadata_json"] = {"derived_from": stamp("memories", PARENT)} - second["id"] = PARENT.upper() if variant == "alias" else PARENT + second["id"] = "urn:uuid:" + PARENT if variant == "alias" else PARENT elif variant == "belief": belief = row("55555555-5555-4555-8555-555555555555", memory_id=second["id"]) first["metadata_json"] = second["metadata_json"] = {"derived_from": stamp("beliefs", belief["id"])} - rows.extend([("belief", belief), ("memory", row(second["id"]))]) + rows.extend([("belief", belief), ("memory", row(second["id"], sensitivity="confidential"))]) elif variant == "malformed": second["metadata_json"]["derived_from"] = {"sources": [SOURCE], "counts": {"sources": 99}} elif variant == "missing": @@ -89,7 +89,7 @@ def test_reused_signature_agrees_with_fresh_kernel_for_boundary_variants(variant elif variant == "class": second["metadata_json"]["workflow"] = "project_auto_update" if variant == "self": - rows.append(("memory", row(second["id"]))) + rows.append(("memory", row(second["id"], sensitivity="confidential"))) store = Store(rows) guard = LabelGuard(store, active=True) with label_read_scope(store): @@ -101,7 +101,10 @@ def test_reused_signature_agrees_with_fresh_kernel_for_boundary_variants(variant def test_implicit_weekly_candidate_parent_is_not_shared_between_candidate_ids(): weekly = row("66666666-6666-4666-8666-666666666666", - {"candidate_memory_ids": [ROOT], "derived_from": stamp("sources", SOURCE)}, artifact_type="weekly_synthesis") + {"candidate_memory_ids": [ROOT], "derived_from": stamp("sources", SOURCE), + "input_summary": {"source_ids": [SOURCE], "memory_ids": [], "open_loop_ids": [], "artifact_ids": [], + "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0}}}, + artifact_type="weekly_synthesis") first = row(ROOT, {"discovered_by": "vnext_weekly_synthesis", "source_artifact_id": weekly["id"]}) second = row(PARENT, deepcopy(first["metadata_json"])) store = Store([("artifact", weekly), ("source", row(SOURCE, sensitivity="confidential"))]) From 45d630047bf328d3dd19c5caac020dbf946f781a Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 23:40:31 +0200 Subject: [PATCH 241/270] Measure identical and varied parent budgets on both stores --- .../test_label_round2_read_budget_postgres.py | 11 +++++++---- .../test_label_round2_read_budget.py | 17 ++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_label_round2_read_budget_postgres.py b/tests/integration/test_label_round2_read_budget_postgres.py index 4bbe7d6d7..9e83c84c9 100644 --- a/tests/integration/test_label_round2_read_budget_postgres.py +++ b/tests/integration/test_label_round2_read_budget_postgres.py @@ -6,12 +6,15 @@ from tests.performance.test_label_round2_read_budget import seed_varied, assert_budgets -@pytest.mark.parametrize("case", ["varied", "repaired", "plain"]) -def test_postgres_round2_read_budgets(label_harness, case): +@pytest.mark.parametrize("case", ["varied", "repaired", "plain", "many-hidden", "many-hidden-repaired", "one-hidden", "identical"]) +def test_postgres_round2_read_budgets(label_harness, monkeypatch, case): + monkeypatch.setenv("ALICE_READ_BUDGET_CASE", case) h = label_harness with h.store() as store: - keys = seed_varied(store, postgres=True, repaired=case == "repaired", plain=case == "plain", - source_count=1000 if case == "plain" else 300) + keys = seed_varied(store, postgres=True, repaired="repaired" in case, plain=case == "plain", + source_count=1000 if case == "plain" else 3000 if case.startswith("many-hidden") else 1 if case in {"one-hidden", "identical"} else 300, + mixed=case in {"varied", "repaired"}, all_hidden=case in {"many-hidden", "many-hidden-repaired", "one-hidden", "identical"}, + unique_metadata=case != "identical") with psycopg.connect(h.urls["admin"], autocommit=True) as conn: for table in ("sources", "memories", "event_log"): conn.execute("ANALYZE " + table) diff --git a/tests/performance/test_label_round2_read_budget.py b/tests/performance/test_label_round2_read_budget.py index 470d3e182..99cc32bf3 100644 --- a/tests/performance/test_label_round2_read_budget.py +++ b/tests/performance/test_label_round2_read_budget.py @@ -20,7 +20,7 @@ USER = "11111111-1111-4111-8111-111111111111" -def seed_varied(store, *, postgres=False, count=5000, source_count=300, mixed=True, repaired=False, plain=False, all_hidden=False): +def seed_varied(store, *, postgres=False, count=5000, source_count=300, mixed=True, repaired=False, plain=False, all_hidden=False, unique_metadata=True): sources = [store.create_source({"source_type": "note", "title": "Synthetic budget input", "content_hash": str(uuid4()), "domain": "project", "sensitivity": "confidential" if all_hidden or i % 2 else "public"}) for i in range(source_count)] @@ -29,7 +29,9 @@ def seed_varied(store, *, postgres=False, count=5000, source_count=300, mixed=Tr for i, row_id in enumerate(ids): parent_index = i % source_count source_id = str(sources[parent_index]["id"]) - metadata = {"observation_index": i, "project_scope": [], "project_floor": []} + metadata = {"project_scope": [], "project_floor": []} + if unique_metadata: + metadata["observation_index"] = i if not plain: metadata["source_id"] = source_id if mixed and i >= source_count: @@ -76,7 +78,7 @@ def assert_budgets(backend, location, user, keys): for profile, key in keys.items(): baseline = probe(main, backend, location, user, profile, key) head = probe(repo, backend, location, user, profile, key) - print(json.dumps({"store": backend, "profile": profile, "main": baseline, "head": head})) + print(json.dumps({"store": backend, "case": os.environ.get("ALICE_READ_BUDGET_CASE", "unspecified"), "profile": profile, "main": baseline, "head": head})) for action in ("pack", "recall"): for clock in ("minimum_wall", "minimum_cpu"): assert head[action][clock] <= 2 * baseline[action][clock] + .1, (action, clock, head, baseline) @@ -92,14 +94,15 @@ def complete_count(store, ceiling): return first -@pytest.mark.parametrize("case,repaired", [("mixed", False), ("mixed", True), ("many-hidden", False), ("many-hidden", True), ("one-hidden", False)]) -def test_sqlite_varied_read_budget(tmp_path, case, repaired): +@pytest.mark.parametrize("case,repaired", [("mixed", False), ("mixed", True), ("many-hidden", False), ("many-hidden", True), ("one-hidden", False), ("identical", False)]) +def test_sqlite_varied_read_budget(tmp_path, monkeypatch, case, repaired): + monkeypatch.setenv("ALICE_READ_BUDGET_CASE", case + ("-repaired" if repaired else "")) path = tmp_path / "round2.db" bootstrap_database(path, user_id=USER, user_email="budget@example.invalid") with sqlite_user_connection(path, USER) as conn: keys = seed_varied(SQLiteVNextStore(conn, USER), repaired=repaired, - source_count=1 if case == "one-hidden" else 3000 if case == "many-hidden" else 300, - all_hidden=case != "mixed", mixed=case == "mixed") + source_count=1 if case in {"one-hidden", "identical"} else 3000 if case == "many-hidden" else 300, + all_hidden=case != "mixed", mixed=case == "mixed", unique_metadata=case != "identical") store = SQLiteVNextStore(conn, USER) for profile in keys: ceiling = ALL_SENSITIVITY if profile == "admin_agent" else DEFAULT_AGENT_SENSITIVITY From 38f674662edcf6b8f7344ff9645664fe16648ffc Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 23:53:47 +0200 Subject: [PATCH 242/270] Reuse pure label parsing within requests and patch Sharp audit failure --- CHANGELOG.md | 2 + .../src/alicebot_api/vnext_derived_labels.py | 36 ++- apps/web/package.json | 2 +- apps/web/pnpm-lock.yaml | 240 +++++++++--------- tests/unit/test_label_dependency_cache.py | 26 ++ 5 files changed, 182 insertions(+), 124 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0035b07a8..7fdf31b8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- The web package pins `sharp` to 0.35.5, including the patched librsvg binaries, for GHSA-wq5f-xc86-pv6w. Earlier pins below 0.35.5 fail the production dependency audit. + - Unreleased (on main, not in v0.20.0): October 6 round-two corrections recover a per-user open-loop digest on both stores after a project reassignment or stale duplicate extraction, and weekly candidates record the report inputs before insert so their stored floor matches the weekly artifact in the same transaction. Read-time labels reuse verified dependency signatures within one request; original rows are counted in SQL, and the derived partition still receives the kernel guard. PostgreSQL event filtering uses an indexed guarded UUID comparison. SQLite filters definitely hidden direct parents before its event limit, then applies the full effective-label guard. The legacy review list also rechecks saved quotes on original or imported candidates against the current source fence. CI compares unique-metadata, varied-parent and repaired fixtures against pinned main 48873b03. No migration is required. diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 861b431c7..8261bfdfc 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -289,12 +289,25 @@ def is_derived(kind: object, row: Mapping[str, object]) -> bool: does not count, so a forged ``value.kind`` does not make the row derived. """ - meta = _metadata(row) - if meta.get("redacted") is True: - return False name = canon_kind(kind) if name in {"source", "belief"}: return False + cache = _READ_METADATA.get() + raw = row.get("metadata_json") + key = ("derived", name, id(raw), repr(row.get("source_id")), repr(row.get("artifact_type"))) + cached = cache.get(key) if cache is not None else None + if cached is not None and cached[0] is raw: + return cached[1] + result = _is_derived(name, row) + if cache is not None: + cache[key] = (raw, result) + return result + + +def _is_derived(name: str, row: Mapping[str, object]) -> bool: + meta = _metadata(row) + if meta.get("redacted") is True: + return False if name == "project": return "derived_from" in meta if name == "open_loop": @@ -729,6 +742,23 @@ def dependency_record(kind: object, row: Mapping[str, object]) -> tuple[frozense An empty record is sound. A missing record on a derived row is ``no_record``. """ + # Closure walks and settlement revisit the same raw metadata within one + # guarded snapshot. Cache parsing, never labels or admission. The top-level + # fields used by this parser remain part of the key, including presence. + cache = _READ_METADATA.get() + raw = row.get("metadata_json") + key = ("dependencies", canon_kind(kind), id(raw), *((field in row, repr(row.get(field))) + for field in ("value", "source_id", "artifact_type", "project_floor"))) + cached = cache.get(key) if cache is not None else None + if cached is not None and cached[0] is raw: + return cached[1] + result = _dependency_record(kind, row) + if cache is not None: + cache[key] = (raw, result) + return result + + +def _dependency_record(kind: object, row: Mapping[str, object]) -> tuple[frozenset[tuple[str, str]], str]: if not is_derived(kind, row): return frozenset(), "" meta = _metadata(row) diff --git a/apps/web/package.json b/apps/web/package.json index 3a21d9c14..bb004af41 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -54,7 +54,7 @@ "js-yaml@4": "4.3.2", "nanoid": "3.3.18", "postcss": "8.5.26", - "sharp": "0.35.4", + "sharp": "0.35.5", "source-map-js": "1.2.2", "tinypool": "2.1.2", "vite": "6.4.3", diff --git a/apps/web/pnpm-lock.yaml b/apps/web/pnpm-lock.yaml index 5681eb734..c132e4e54 100644 --- a/apps/web/pnpm-lock.yaml +++ b/apps/web/pnpm-lock.yaml @@ -11,7 +11,7 @@ overrides: js-yaml@4: 4.3.2 nanoid: 3.3.18 postcss: 8.5.26 - sharp: 0.35.4 + sharp: 0.35.5 source-map-js: 1.2.2 tinypool: 2.1.2 vite: 6.4.3 @@ -450,144 +450,144 @@ packages: resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} engines: {node: '>=18'} - '@img/sharp-darwin-arm64@0.35.4': - resolution: {integrity: sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==} + '@img/sharp-darwin-arm64@0.35.5': + resolution: {integrity: sha512-QRUlFQ0WxvdWyqqG/WtI3iupfD5rBzmCHXSdPsY91sAtVtTo7Q4cb6zOccZ3gqEqkr0f1As1ehLqmEpDsRf+lg==} engines: {node: '>=20.9.0'} cpu: [arm64] os: [darwin] - '@img/sharp-darwin-x64@0.35.4': - resolution: {integrity: sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==} + '@img/sharp-darwin-x64@0.35.5': + resolution: {integrity: sha512-+BR255RhDlpygUpOc/Jdt1nT6DQ3XG/ERo5wbcdOf5Q320dKtPCKPLR1LJs9VGXRaMa8l1uUa0tkCNOXiAxZUw==} engines: {node: '>=20.9.0'} cpu: [x64] os: [darwin] - '@img/sharp-freebsd-wasm32@0.35.4': - resolution: {integrity: sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==} + '@img/sharp-freebsd-wasm32@0.35.5': + resolution: {integrity: sha512-Y/z91nEZ4uIBX5X3nfTovjU9lHNKFYbL2lpHCLVNmXQK03VIZvXBBt0KxbPGp2SdGSF+2mQU4e+hQaWOt86iAw==} engines: {node: '>=20.9.0'} os: [freebsd] - '@img/sharp-libvips-darwin-arm64@1.3.3': - resolution: {integrity: sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==} + '@img/sharp-libvips-darwin-arm64@1.3.4': + resolution: {integrity: sha512-5R89nBYiRdUlSWJxPhO+GVtaXzXSxKnRu/xqMn3KTA3L9EB9Oy/P+Nn2f2vlhPuUdy/Zusb2DarbyTpGCfEDuw==} cpu: [arm64] os: [darwin] - '@img/sharp-libvips-darwin-x64@1.3.3': - resolution: {integrity: sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==} + '@img/sharp-libvips-darwin-x64@1.3.4': + resolution: {integrity: sha512-iR2OKH80yi0U+dUplyh3/xdpFvps6YkCwsXenIJxqxR1v9o+xtKTGbS9H7cps+2Vxjc8B1j96p75NmTGjIhtpQ==} cpu: [x64] os: [darwin] - '@img/sharp-libvips-linux-arm64@1.3.3': - resolution: {integrity: sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==} + '@img/sharp-libvips-linux-arm64@1.3.4': + resolution: {integrity: sha512-Y3dgX/6lE2QhQb+Gxy0WZxfg9MEm/JBjamZpS2IklP7xIQoKN4hzAm7KcMVGtaVDt3neE9OKBC7vAfonA/Lr1A==} cpu: [arm64] os: [linux] - '@img/sharp-libvips-linux-arm@1.3.3': - resolution: {integrity: sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==} + '@img/sharp-libvips-linux-arm@1.3.4': + resolution: {integrity: sha512-LmRtTsOHuvM2+wlO2Db37dx5MiZhB0FvSunciw48YjdOkZz9KAiRbm8ujeMOA1INqmei5NapFxYEK1D1ZSidmw==} cpu: [arm] os: [linux] - '@img/sharp-libvips-linux-ppc64@1.3.3': - resolution: {integrity: sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==} + '@img/sharp-libvips-linux-ppc64@1.3.4': + resolution: {integrity: sha512-Le6boB8Tai0Nis+gIxIpKx68UDVVIqdR8Tin5Yf1z2LJJQLDJvCDRqRu+jC2qCoD+eIomonmOwB4smBRxfVpYQ==} cpu: [ppc64] os: [linux] - '@img/sharp-libvips-linux-riscv64@1.3.3': - resolution: {integrity: sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==} + '@img/sharp-libvips-linux-riscv64@1.3.4': + resolution: {integrity: sha512-aHkkIEHPRdQEegJN20MLmGtxYD9R2wQr3Cwpddnu5+YKMt6Uzax7S9h5gpZTo8wyrGuZSlfQ63OevL5mTyOC7Q==} cpu: [riscv64] os: [linux] - '@img/sharp-libvips-linux-s390x@1.3.3': - resolution: {integrity: sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==} + '@img/sharp-libvips-linux-s390x@1.3.4': + resolution: {integrity: sha512-ra/mB6MikESDUO7Yg+Mi95bFBb9GsObURuhnOv3OqknjGe9sZrG8tCe9q0xSIGrtLgvgw0gKnFWcK4blSgQOuQ==} cpu: [s390x] os: [linux] - '@img/sharp-libvips-linux-x64@1.3.3': - resolution: {integrity: sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==} + '@img/sharp-libvips-linux-x64@1.3.4': + resolution: {integrity: sha512-GJ//SSXbnwSDes02umB3nDJLFcQzw8a18V8fyhqr6tV515tOEMdImjjxj1AoafMRz56F3PHgftnj1QEKSU1zkw==} cpu: [x64] os: [linux] - '@img/sharp-libvips-linuxmusl-arm64@1.3.3': - resolution: {integrity: sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==} + '@img/sharp-libvips-linuxmusl-arm64@1.3.4': + resolution: {integrity: sha512-hvulFwtjUcagsis6BBxHwGFwWoNZjgYmULGVrZcyfNbjA8hKILbRxGg15/7w5HDyXHXUos/j6baAWqnCyQ2DWA==} cpu: [arm64] os: [linux] - '@img/sharp-libvips-linuxmusl-x64@1.3.3': - resolution: {integrity: sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==} + '@img/sharp-libvips-linuxmusl-x64@1.3.4': + resolution: {integrity: sha512-6zXKeE/p39I1AmA3cJG35eyBGNqNddLnUXjhwBnsGjFPWqf5VKkDBEqaEkPDoTEtkxwi2vv8Tcr2mDyP4So7Fg==} cpu: [x64] os: [linux] - '@img/sharp-linux-arm64@0.35.4': - resolution: {integrity: sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==} + '@img/sharp-linux-arm64@0.35.5': + resolution: {integrity: sha512-LYVx5JTsOM2CBzmxreh+nl64/3H6Xb09iSLknqH47z2T2DFFxDeFLP5y4dJwe6H7uGQlHPyEEtIqyo3DYsRwdQ==} engines: {node: '>=20.9.0'} cpu: [arm64] os: [linux] - '@img/sharp-linux-arm@0.35.4': - resolution: {integrity: sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==} + '@img/sharp-linux-arm@0.35.5': + resolution: {integrity: sha512-LEaXK2WdXVK5ykcw0buWyPMsmLLL2vpHLD6yrNSW+JGEL3BZPA4tpKN6iaMc4AxTTAoaX/sU1rOL51lcIz48ZQ==} engines: {node: '>=20.9.0'} cpu: [arm] os: [linux] - '@img/sharp-linux-ppc64@0.35.4': - resolution: {integrity: sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==} + '@img/sharp-linux-ppc64@0.35.5': + resolution: {integrity: sha512-QVxAAq8evVRI9ia2vqgwrmWucn5Dfv+JdWzj75pD8omHLPSP7f8p20O8jxzjCcuCEQEOtYOZUmX1hkiZ0kdevA==} engines: {node: '>=20.9.0'} cpu: [ppc64] os: [linux] - '@img/sharp-linux-riscv64@0.35.4': - resolution: {integrity: sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==} + '@img/sharp-linux-riscv64@0.35.5': + resolution: {integrity: sha512-LtdreXguaavKODPIfzJ4kffx7UNt1omwtK0rch4EBbbSTXPnxWmYSayXdLJw0fJzQ97kHt1gL/yh4tvU+nCyRQ==} engines: {node: '>=20.9.0'} cpu: [riscv64] os: [linux] - '@img/sharp-linux-s390x@0.35.4': - resolution: {integrity: sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==} + '@img/sharp-linux-s390x@0.35.5': + resolution: {integrity: sha512-UZasTOFiYzotTsGOCu42BfUzP6Tu6Do/947iRm1RsLKvlllxwGcn4RN27LibGWceix4Y+Pmw3jsnTcCQIgWjqA==} engines: {node: '>=20.9.0'} cpu: [s390x] os: [linux] - '@img/sharp-linux-x64@0.35.4': - resolution: {integrity: sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==} + '@img/sharp-linux-x64@0.35.5': + resolution: {integrity: sha512-SxFtLTeJInhAA9Q836kux2vZNeOBQEx658qvbboZScr0wIARym3IcGmW7KpVD5sbVg0Ojy+udFQdayYIZyoNog==} engines: {node: '>=20.9.0'} cpu: [x64] os: [linux] - '@img/sharp-linuxmusl-arm64@0.35.4': - resolution: {integrity: sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==} + '@img/sharp-linuxmusl-arm64@0.35.5': + resolution: {integrity: sha512-9HbMclmI1zlNkFRs3z9/eBtDjfD0sGlrX1z6b1qwmiFY5ElDLh4BC0LPBdVp7z1DXFiKlIcznf+ZlsuZzLxQqg==} engines: {node: '>=20.9.0'} cpu: [arm64] os: [linux] - '@img/sharp-linuxmusl-x64@0.35.4': - resolution: {integrity: sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==} + '@img/sharp-linuxmusl-x64@0.35.5': + resolution: {integrity: sha512-4KOphqB035HrVdqLZfCgMzzERrQkkzOwRhl4OAkRO1YCldbaFjySXMaK534Mo0V+LndnlJk+sbUyLeU0ULyD1A==} engines: {node: '>=20.9.0'} cpu: [x64] os: [linux] - '@img/sharp-wasm32@0.35.4': - resolution: {integrity: sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==} + '@img/sharp-wasm32@0.35.5': + resolution: {integrity: sha512-Ptsga1su4tQx+LLF1ECS9U6nz5kmrXKo6XVbtR48Ke3ZRxxgaWBu7IDtEe1quo8hiupwm6WFqxVlXaSf7IINGQ==} engines: {node: '>=20.9.0'} - '@img/sharp-webcontainers-wasm32@0.35.4': - resolution: {integrity: sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==} + '@img/sharp-webcontainers-wasm32@0.35.5': + resolution: {integrity: sha512-hfhF/FmoQyTUkA0bIKFOtw536BQSeBMe6BF6QyWlrPxT754+TFLaZ7sKKTfvvM0yJgKgaYTwnFCIZ/GuDw5SUA==} engines: {node: '>=20.9.0'} cpu: [wasm32] - '@img/sharp-win32-arm64@0.35.4': - resolution: {integrity: sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==} + '@img/sharp-win32-arm64@0.35.5': + resolution: {integrity: sha512-X4t7g+7ZA5DKblCBEXGjUqqemj4vczING/5viFwAL8h4N3qYeyjwdCvRLHi4EdOUI+2Z7UFlp1VM+p/AuEtm6Q==} engines: {node: '>=20.9.0'} cpu: [arm64] os: [win32] - '@img/sharp-win32-ia32@0.35.4': - resolution: {integrity: sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==} + '@img/sharp-win32-ia32@0.35.5': + resolution: {integrity: sha512-5Zm82LoBc43nhwNybZlG7Y1KO//Zhsn306fQl29ZOuStHLGTo3BWL83q3cznX0poxSAMuYL1On/BHBxkBeKr6A==} engines: {node: ^20.9.0} cpu: [ia32] os: [win32] - '@img/sharp-win32-x64@0.35.4': - resolution: {integrity: sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==} + '@img/sharp-win32-x64@0.35.5': + resolution: {integrity: sha512-x76eH0vEiHlcMQu8Y8IenntaACtddpT6W0wmXtWrnKcnKI7ME5DdgqhAD6SEWOEl1v2zDvkZDhFA9KnURwpfqg==} engines: {node: '>=20.9.0'} cpu: [x64] os: [win32] @@ -2292,8 +2292,8 @@ packages: resolution: {integrity: sha512-RJRdvCo6IAnPdsvP/7m6bsQqNnn1FCBX5ZNtFL98MmFF/4xAIJTIg1YbHW5DC2W5SKZanrC6i4HsJqlajw/dZw==} engines: {node: '>= 0.4'} - sharp@0.35.4: - resolution: {integrity: sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==} + sharp@0.35.5: + resolution: {integrity: sha512-Ywn4OnzGukp7CDMrp08RQ50YKmuwG47brZgIVPTvBaaAfQlRlygrRqSrxdCiL9M+LlzLBiJ68IR1QqvzHyjC7g==} engines: {node: '>=20.9.0'} peerDependencies: '@types/node': '*' @@ -3017,108 +3017,108 @@ snapshots: '@img/colour@1.1.0': optional: true - '@img/sharp-darwin-arm64@0.35.4': + '@img/sharp-darwin-arm64@0.35.5': optionalDependencies: - '@img/sharp-libvips-darwin-arm64': 1.3.3 + '@img/sharp-libvips-darwin-arm64': 1.3.4 optional: true - '@img/sharp-darwin-x64@0.35.4': + '@img/sharp-darwin-x64@0.35.5': optionalDependencies: - '@img/sharp-libvips-darwin-x64': 1.3.3 + '@img/sharp-libvips-darwin-x64': 1.3.4 optional: true - '@img/sharp-freebsd-wasm32@0.35.4': + '@img/sharp-freebsd-wasm32@0.35.5': dependencies: - '@img/sharp-wasm32': 0.35.4 + '@img/sharp-wasm32': 0.35.5 optional: true - '@img/sharp-libvips-darwin-arm64@1.3.3': + '@img/sharp-libvips-darwin-arm64@1.3.4': optional: true - '@img/sharp-libvips-darwin-x64@1.3.3': + '@img/sharp-libvips-darwin-x64@1.3.4': optional: true - '@img/sharp-libvips-linux-arm64@1.3.3': + '@img/sharp-libvips-linux-arm64@1.3.4': optional: true - '@img/sharp-libvips-linux-arm@1.3.3': + '@img/sharp-libvips-linux-arm@1.3.4': optional: true - '@img/sharp-libvips-linux-ppc64@1.3.3': + '@img/sharp-libvips-linux-ppc64@1.3.4': optional: true - '@img/sharp-libvips-linux-riscv64@1.3.3': + '@img/sharp-libvips-linux-riscv64@1.3.4': optional: true - '@img/sharp-libvips-linux-s390x@1.3.3': + '@img/sharp-libvips-linux-s390x@1.3.4': optional: true - '@img/sharp-libvips-linux-x64@1.3.3': + '@img/sharp-libvips-linux-x64@1.3.4': optional: true - '@img/sharp-libvips-linuxmusl-arm64@1.3.3': + '@img/sharp-libvips-linuxmusl-arm64@1.3.4': optional: true - '@img/sharp-libvips-linuxmusl-x64@1.3.3': + '@img/sharp-libvips-linuxmusl-x64@1.3.4': optional: true - '@img/sharp-linux-arm64@0.35.4': + '@img/sharp-linux-arm64@0.35.5': optionalDependencies: - '@img/sharp-libvips-linux-arm64': 1.3.3 + '@img/sharp-libvips-linux-arm64': 1.3.4 optional: true - '@img/sharp-linux-arm@0.35.4': + '@img/sharp-linux-arm@0.35.5': optionalDependencies: - '@img/sharp-libvips-linux-arm': 1.3.3 + '@img/sharp-libvips-linux-arm': 1.3.4 optional: true - '@img/sharp-linux-ppc64@0.35.4': + '@img/sharp-linux-ppc64@0.35.5': optionalDependencies: - '@img/sharp-libvips-linux-ppc64': 1.3.3 + '@img/sharp-libvips-linux-ppc64': 1.3.4 optional: true - '@img/sharp-linux-riscv64@0.35.4': + '@img/sharp-linux-riscv64@0.35.5': optionalDependencies: - '@img/sharp-libvips-linux-riscv64': 1.3.3 + '@img/sharp-libvips-linux-riscv64': 1.3.4 optional: true - '@img/sharp-linux-s390x@0.35.4': + '@img/sharp-linux-s390x@0.35.5': optionalDependencies: - '@img/sharp-libvips-linux-s390x': 1.3.3 + '@img/sharp-libvips-linux-s390x': 1.3.4 optional: true - '@img/sharp-linux-x64@0.35.4': + '@img/sharp-linux-x64@0.35.5': optionalDependencies: - '@img/sharp-libvips-linux-x64': 1.3.3 + '@img/sharp-libvips-linux-x64': 1.3.4 optional: true - '@img/sharp-linuxmusl-arm64@0.35.4': + '@img/sharp-linuxmusl-arm64@0.35.5': optionalDependencies: - '@img/sharp-libvips-linuxmusl-arm64': 1.3.3 + '@img/sharp-libvips-linuxmusl-arm64': 1.3.4 optional: true - '@img/sharp-linuxmusl-x64@0.35.4': + '@img/sharp-linuxmusl-x64@0.35.5': optionalDependencies: - '@img/sharp-libvips-linuxmusl-x64': 1.3.3 + '@img/sharp-libvips-linuxmusl-x64': 1.3.4 optional: true - '@img/sharp-wasm32@0.35.4': + '@img/sharp-wasm32@0.35.5': dependencies: '@emnapi/runtime': 1.11.3 optional: true - '@img/sharp-webcontainers-wasm32@0.35.4': + '@img/sharp-webcontainers-wasm32@0.35.5': dependencies: - '@img/sharp-wasm32': 0.35.4 + '@img/sharp-wasm32': 0.35.5 optional: true - '@img/sharp-win32-arm64@0.35.4': + '@img/sharp-win32-arm64@0.35.5': optional: true - '@img/sharp-win32-ia32@0.35.4': + '@img/sharp-win32-ia32@0.35.5': optional: true - '@img/sharp-win32-x64@0.35.4': + '@img/sharp-win32-x64@0.35.5': optional: true '@isaacs/cliui@8.0.2': @@ -4726,7 +4726,7 @@ snapshots: '@next/swc-win32-arm64-msvc': 16.3.6 '@next/swc-win32-x64-msvc': 16.3.6 '@playwright/test': 1.63.0 - sharp: 0.35.4(@types/node@26.1.2) + sharp: 0.35.5(@types/node@26.1.2) transitivePeerDependencies: - '@babel/core' - '@types/node' @@ -5010,37 +5010,37 @@ snapshots: es-errors: 1.3.0 es-object-atoms: 1.1.1 - sharp@0.35.4(@types/node@26.1.2): + sharp@0.35.5(@types/node@26.1.2): dependencies: '@img/colour': 1.1.0 detect-libc: 2.1.2 semver: 7.8.5 optionalDependencies: - '@img/sharp-darwin-arm64': 0.35.4 - '@img/sharp-darwin-x64': 0.35.4 - '@img/sharp-freebsd-wasm32': 0.35.4 - '@img/sharp-libvips-darwin-arm64': 1.3.3 - '@img/sharp-libvips-darwin-x64': 1.3.3 - '@img/sharp-libvips-linux-arm': 1.3.3 - '@img/sharp-libvips-linux-arm64': 1.3.3 - '@img/sharp-libvips-linux-ppc64': 1.3.3 - '@img/sharp-libvips-linux-riscv64': 1.3.3 - '@img/sharp-libvips-linux-s390x': 1.3.3 - '@img/sharp-libvips-linux-x64': 1.3.3 - '@img/sharp-libvips-linuxmusl-arm64': 1.3.3 - '@img/sharp-libvips-linuxmusl-x64': 1.3.3 - '@img/sharp-linux-arm': 0.35.4 - '@img/sharp-linux-arm64': 0.35.4 - '@img/sharp-linux-ppc64': 0.35.4 - '@img/sharp-linux-riscv64': 0.35.4 - '@img/sharp-linux-s390x': 0.35.4 - '@img/sharp-linux-x64': 0.35.4 - '@img/sharp-linuxmusl-arm64': 0.35.4 - '@img/sharp-linuxmusl-x64': 0.35.4 - '@img/sharp-webcontainers-wasm32': 0.35.4 - '@img/sharp-win32-arm64': 0.35.4 - '@img/sharp-win32-ia32': 0.35.4 - '@img/sharp-win32-x64': 0.35.4 + '@img/sharp-darwin-arm64': 0.35.5 + '@img/sharp-darwin-x64': 0.35.5 + '@img/sharp-freebsd-wasm32': 0.35.5 + '@img/sharp-libvips-darwin-arm64': 1.3.4 + '@img/sharp-libvips-darwin-x64': 1.3.4 + '@img/sharp-libvips-linux-arm': 1.3.4 + '@img/sharp-libvips-linux-arm64': 1.3.4 + '@img/sharp-libvips-linux-ppc64': 1.3.4 + '@img/sharp-libvips-linux-riscv64': 1.3.4 + '@img/sharp-libvips-linux-s390x': 1.3.4 + '@img/sharp-libvips-linux-x64': 1.3.4 + '@img/sharp-libvips-linuxmusl-arm64': 1.3.4 + '@img/sharp-libvips-linuxmusl-x64': 1.3.4 + '@img/sharp-linux-arm': 0.35.5 + '@img/sharp-linux-arm64': 0.35.5 + '@img/sharp-linux-ppc64': 0.35.5 + '@img/sharp-linux-riscv64': 0.35.5 + '@img/sharp-linux-s390x': 0.35.5 + '@img/sharp-linux-x64': 0.35.5 + '@img/sharp-linuxmusl-arm64': 0.35.5 + '@img/sharp-linuxmusl-x64': 0.35.5 + '@img/sharp-webcontainers-wasm32': 0.35.5 + '@img/sharp-win32-arm64': 0.35.5 + '@img/sharp-win32-ia32': 0.35.5 + '@img/sharp-win32-x64': 0.35.5 '@types/node': 26.1.2 optional: true diff --git a/tests/unit/test_label_dependency_cache.py b/tests/unit/test_label_dependency_cache.py index 11d3111c8..5a38b92e0 100644 --- a/tests/unit/test_label_dependency_cache.py +++ b/tests/unit/test_label_dependency_cache.py @@ -10,6 +10,32 @@ from alicebot_api.vnext_label_guard import LabelGuard, invalidate_read_labels, label_read_scope from alicebot_api.vnext_derived_labels import identifier, with_derived_from + +@pytest.mark.parametrize("field,initial,changed", [ + ("value", {"source_id": "first-input"}, {"source_id": "second-input"}), + ("source_id", "first-input", "second-input"), + ("project_floor", [], None), + ("artifact_type", "daily_brief", "note"), +]) +def test_pure_parsing_cache_keeps_top_level_variants_and_write_invalidation(field, initial, changed): + from alicebot_api.vnext_derived_labels import dependency_record, is_derived + + kind = "artifact" if field == "artifact_type" else "open_loop" if field == "source_id" else "memory" + metadata = {"discovered_by": "open_loop_extraction"} if kind == "open_loop" else {} if kind == "artifact" else {"source_id": "metadata-input"} + first = {"id": "first", "metadata_json": metadata, field: initial} + second = {**first, "id": "second", field: changed} + expected = [(is_derived(kind, item), dependency_record(kind, item)) for item in (first, second)] + assert expected[0] != expected[1] + store = Store([]) + with label_read_scope(store): + for _ in range(2): + assert [(is_derived(kind, item), dependency_record(kind, item)) for item in (first, second)] == expected + metadata["source_id"] = "new-input-after-write" + invalidate_read_labels(store) + after_write = (is_derived(kind, first), dependency_record(kind, first)) + assert after_write == (is_derived(kind, first), dependency_record(kind, first)) + assert ("source", "new-input-after-write") in after_write[1][0] + SOURCE = "11111111-1111-4111-8111-111111111111" PARENT = "22222222-2222-4222-8222-222222222222" ROOT = "33333333-3333-4333-8333-333333333333" From af992dc1c3eee1094751ba3edea2f711d8df4ef7 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Tue, 6 Oct 2026 23:54:52 +0200 Subject: [PATCH 243/270] Keep changelog and guard-mutation receipts aligned with parser reuse --- CHANGELOG.md | 3 +-- tests/unit/test_derived_labels_mutations.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fdf31b8c..faef4cdc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,9 @@ ## Unreleased -- The web package pins `sharp` to 0.35.5, including the patched librsvg binaries, for GHSA-wq5f-xc86-pv6w. Earlier pins below 0.35.5 fail the production dependency audit. - - Unreleased (on main, not in v0.20.0): October 6 round-two corrections recover a per-user open-loop digest on both stores after a project reassignment or stale duplicate extraction, and weekly candidates record the report inputs before insert so their stored floor matches the weekly artifact in the same transaction. Read-time labels reuse verified dependency signatures within one request; original rows are counted in SQL, and the derived partition still receives the kernel guard. PostgreSQL event filtering uses an indexed guarded UUID comparison. SQLite filters definitely hidden direct parents before its event limit, then applies the full effective-label guard. The legacy review list also rechecks saved quotes on original or imported candidates against the current source fence. CI compares unique-metadata, varied-parent and repaired fixtures against pinned main 48873b03. No migration is required. +- The web package pins `sharp` to 0.35.5, including the patched librsvg binaries, for GHSA-wq5f-xc86-pv6w. Earlier pins below 0.35.5 fail the production dependency audit. - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings, including JSON escapes. Repeated keys and quote subtrees follow the limits in the draft security note. Lookup and removal both decode JSON text under source and memory references and inside trace fields. Admitted references and surrounding values stay, and the owner receives live references unchanged. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. - Unreleased (on main, not in v0.20.0): the October 6 derived-label correction normalizes server UUIDs, carries bound scheduler identity, withholds encoded references and handles malformed marker shapes. Plain reviews share the label lock; label-changing reviews retain the bounded retryable refusal. Read labels and distinct parents are cached within one request. Context-pack rows dropped by effective admission are absent from the later stage's excluded counts and warnings. The search golden was regenerated through its recorder. Source GET and the legacy review list now apply the full caller fence. Graph edge explanations remain outside that ceiling, and the doctor label counts are new in this set, as disclosed in the draft security note. The sources_first golden token estimates each rise by 5 because derived inserts persist empty project_scope and project_floor. SQLite retirement deliberately follows the saved-quote reader for repeated keys and quote subtrees; an older raw scan blanked some such rows that it now retains. diff --git a/tests/unit/test_derived_labels_mutations.py b/tests/unit/test_derived_labels_mutations.py index 68cd1e888..9f56d7845 100644 --- a/tests/unit/test_derived_labels_mutations.py +++ b/tests/unit/test_derived_labels_mutations.py @@ -107,7 +107,7 @@ def test_derived_label_kernel_guard_mutations(): ) kill( labels, - "dependency_record", + "_dependency_record", "if not problem and not _record_present(meta, row) and not found:", "if not problem and not found:", checks.test_every_unverified_case_is_unverified, From 8976b8686da8107fc7662012351b454ba27d088e Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Wed, 7 Oct 2026 00:05:55 +0200 Subject: [PATCH 244/270] Reuse verified source-copy labels and batch native connector census --- apps/api/src/alicebot_api/vnext_connectors.py | 69 ++++++++++++++----- .../api/src/alicebot_api/vnext_label_guard.py | 21 +++++- tests/unit/test_label_dependency_cache.py | 28 ++++++++ tests/unit/test_vnext_connectors.py | 38 ++++++++++ 4 files changed, 136 insertions(+), 20 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_connectors.py b/apps/api/src/alicebot_api/vnext_connectors.py index 86911b8ad..c7097a428 100644 --- a/apps/api/src/alicebot_api/vnext_connectors.py +++ b/apps/api/src/alicebot_api/vnext_connectors.py @@ -1,5 +1,7 @@ from __future__ import annotations +from copy import deepcopy + from collections.abc import Iterator, Mapping, Sequence import csv from dataclasses import dataclass, field @@ -1179,15 +1181,14 @@ def _setting_row_to_config(self, row: Mapping[str, object]) -> JsonObject: def get_cursor(self, connector_name: str) -> str | None: definition = get_connector_definition(connector_name) - if hasattr(self.store, "get_connector_state"): - state = cast(Any, self.store).get_connector_state(definition.name) - if isinstance(state, dict): - cursor = _as_optional_text(state.get("cursor_value")) - if cursor is not None: - return cursor + state = self._connector_row("states", "get_connector_state", definition.name) + if isinstance(state, dict): + cursor = _as_optional_text(state.get("cursor_value")) + if cursor is not None: + return cursor events = [ event - for event in self.store.list_events(target_type="connector", target_id=definition.name) + for event in self._connector_events(definition.name) if event.get("event_type") == "connector.sync_completed" ] events.sort(key=lambda event: str(event.get("occurred_at") or ""), reverse=True) @@ -1327,13 +1328,12 @@ def update_config( def get_config(self, connector_name: str) -> JsonObject: definition = get_connector_definition(connector_name) - if hasattr(self.store, "get_connector_setting"): - row = cast(Any, self.store).get_connector_setting(definition.name) - if isinstance(row, dict): - return self._setting_row_to_config(row) + row = self._connector_row("settings", "get_connector_setting", definition.name) + if isinstance(row, dict): + return self._setting_row_to_config(row) events = [ event - for event in self.store.list_events(target_type="connector", target_id=definition.name) + for event in self._connector_events(definition.name) if event.get("event_type") == "connector.config_updated" ] events.sort(key=lambda event: str(event.get("occurred_at") or ""), reverse=True) @@ -1380,11 +1380,9 @@ def get_config(self, connector_name: str) -> JsonObject: def connector_health(self, connector_name: str) -> JsonObject: definition = get_connector_definition(connector_name) config = self.get_config(definition.name) - state = None - if hasattr(self.store, "get_connector_state"): - candidate_state = cast(Any, self.store).get_connector_state(definition.name) - state = candidate_state if isinstance(candidate_state, dict) else None - events = self.store.list_events(target_type="connector", target_id=definition.name) + candidate_state = self._connector_row("states", "get_connector_state", definition.name) + state = candidate_state if isinstance(candidate_state, dict) else None + events = self._connector_events(definition.name) events.sort(key=lambda event: str(event.get("occurred_at") or ""), reverse=True) sync_events = [ event @@ -1504,6 +1502,29 @@ def connector_health(self, connector_name: str) -> JsonObject: else None, } + def _connector_row(self, namespace: str, method: str, name: str): + from alicebot_api.vnext_label_guard import request_row_cache + + cache = request_row_cache(self.store, "connector_health_inputs") + if cache is not None and namespace in cache and name in cache[namespace]: + return deepcopy(cache[namespace][name]) + getter = getattr(self.store, method, None) + row = getter(name) if callable(getter) else None + if cache is not None: + cache.setdefault(namespace, {})[name] = deepcopy(row) + return row + + def _connector_events(self, name: str): + from alicebot_api.vnext_label_guard import request_row_cache + + cache = request_row_cache(self.store, "connector_health_inputs") + if cache is not None and name in cache.get("events", {}): + return deepcopy(cache["events"][name]) + events = self.store.list_events(target_type="connector", target_id=name) + if cache is not None: + cache.setdefault("events", {})[name] = deepcopy(events) + return events + def connector_health_all(self) -> JsonObject: from copy import deepcopy from alicebot_api.vnext_label_guard import request_row_cache @@ -1514,7 +1535,19 @@ def connector_health_all(self) -> JsonObject: cache = request_row_cache(self.store, "connector_health_all") if cache is not None and "result" in cache: return deepcopy(cache["result"]) - items = [self.connector_health(definition.name) for definition in list_connector_definitions()] + definitions = list_connector_definitions() + inputs = request_row_cache(self.store, "connector_health_inputs") + if inputs is not None: + # Native stores expose the same tenant-scoped settings and states + # in one read. Missing rows still use the historical event fallback. + for namespace, method in (("settings", "list_connector_settings"), ("states", "list_connector_states")): + if callable(getattr(type(self.store), method, None)): + rows = getattr(self.store, method)() + by_name = {str(row["connector_name"]): deepcopy(row) for row in rows + if isinstance(row, dict) and row.get("connector_name") + and (namespace != "states" or row.get("cursor_type") == "sync_cursor")} + inputs[namespace] = {definition.name: by_name.get(definition.name) for definition in definitions} + items = [self.connector_health(definition.name) for definition in definitions] result: JsonObject = {"items": items, "count": len(items), "order": [str(item["connector_name"]) for item in items]} if cache is not None: cache["result"] = deepcopy(result) diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index ee9c4f029..0da91919f 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -55,6 +55,7 @@ class _RequestLabels: counts: dict = field(default_factory=dict) dependency_labels: dict = field(default_factory=dict) dependency_ancestry: dict = field(default_factory=dict) + source_copies: dict = field(default_factory=dict) signatures: dict = field(default_factory=dict) parsed_signatures: dict = field(default_factory=dict) row_sets: dict = field(default_factory=dict) @@ -70,6 +71,7 @@ def clear(self): self.counts.clear() self.dependency_labels.clear() self.dependency_ancestry.clear() + self.source_copies.clear() self.signatures.clear() self.parsed_signatures.clear() self.row_sets.clear() @@ -212,8 +214,23 @@ def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[ label = state.dependency_labels.get(template) if label is None: nodes = self._collected(kind, row) - settled = settle_labels(nodes, on_cycle="unverified", max_hops=HOP_BOUND, max_nodes=NODE_BOUND) - label = settled.by_stored(kind, str(row.get("id") or ""), user_id=_GUARD_USER) + copy_template = None + # A copy with exactly one original source has no recursive input + # graph. First collect each root to check missing/ambiguous source + # spellings and the independent bounds, then reuse only a verified + # kernel result for identical root and parent label semantics. + if len(nodes) == 2 and template[3] == "copy" and not template[2] and len(template[1]) == 1: + parent = nodes[1] + ref = next(iter(template[1])) + if ref[0] == "source" and parent.get("kind") == "source" and identifier(parent.get("id")) == ref[1]: + parent_template = self._signature("source", parent, key=self._key("source", parent)) + copy_template = (template[:1] + template[2:], parent_template) + label = state.source_copies.get(copy_template) + if label is None: + settled = settle_labels(nodes, on_cycle="unverified", max_hops=HOP_BOUND, max_nodes=NODE_BOUND) + label = settled.by_stored(kind, str(row.get("id") or ""), user_id=_GUARD_USER) + if copy_template is not None and not label.unverified: + state.source_copies[copy_template] = label ancestry = frozenset( [*(ref for node in nodes for ref in dependencies_of(str(node["kind"]), node)), *((str(node["kind"]), identifier(node.get("id"))) for node in nodes[1:]), diff --git a/tests/unit/test_label_dependency_cache.py b/tests/unit/test_label_dependency_cache.py index 5a38b92e0..bd3797e92 100644 --- a/tests/unit/test_label_dependency_cache.py +++ b/tests/unit/test_label_dependency_cache.py @@ -90,6 +90,34 @@ def counted(*args, **kwargs): assert len(calls) == 3 +@pytest.mark.parametrize("variant", ["hidden", "scope", "floor", "scrubbed", "alias"]) +def test_distinct_source_copy_reuse_keeps_parent_labels_and_alias_refusal(variant): + first_parent = row(SOURCE, {"project_scope": ["alpha"]}, sensitivity="regulated" if variant == "alias" else "public") + second_parent = row(PARENT, {"project_scope": ["alpha"]}, sensitivity=first_parent["sensitivity"]) + if variant == "hidden": + second_parent["sensitivity"] = "confidential" + elif variant == "scope": + second_parent["metadata_json"]["project_scope"] = ["beta"] + elif variant == "floor": + second_parent["metadata_json"]["project_floor"] = ["beta"] + elif variant == "scrubbed": + second_parent["metadata_json"]["scrubbed"] = True + first = row(ROOT, {"source_id": SOURCE, "project_scope": ["alpha"]}) + second = row("44444444-4444-4444-8444-444444444444", {"source_id": PARENT, "project_scope": ["alpha"]}) + rows = [("source", first_parent), ("source", second_parent)] + if variant == "alias": + rows.append(("source", {**deepcopy(second_parent), "id": "urn:uuid:" + PARENT})) + store = Store(rows) + with label_read_scope(store): + guard = LabelGuard(store, active=True) + assert effective(guard, "memory", first)[-1] is False + reused = effective(guard, "memory", second) + fresh = effective(LabelGuard(store, active=True), "memory", deepcopy(second)) + assert reused == fresh + if variant == "alias": + assert reused[-1] is True + + @pytest.mark.parametrize("variant", ["self", "ancestor", "alias", "belief", "malformed", "missing", "floor", "class"]) def test_reused_signature_agrees_with_fresh_kernel_for_boundary_variants(variant): source = row(SOURCE, sensitivity="confidential") diff --git a/tests/unit/test_vnext_connectors.py b/tests/unit/test_vnext_connectors.py index 1e3023f67..129cba226 100644 --- a/tests/unit/test_vnext_connectors.py +++ b/tests/unit/test_vnext_connectors.py @@ -165,6 +165,44 @@ def _telegram_payload(update_id: int, text: str = "Fact: Telegram capture preser } +def test_health_census_preserves_settings_event_fallback_and_cursor_kind(monkeypatch): + from alicebot_api.vnext_label_guard import invalidate_read_labels, label_read_scope + + store = InMemoryConnectorSettingsStore() + service = VNextConnectorService(store, secret_provider=InMemorySecretProvider()) + store.upsert_connector_setting({"connector_name": "telegram", "enabled": True, "configured": True}) + store.upsert_connector_state({"connector_name": "telegram", "cursor_value": "native-cursor"}) + store.upsert_connector_state({"connector_name": "browser_clipper", "cursor_type": "unrelated", "cursor_value": "wrong-cursor"}) + store.append_event({"target_type": "connector", "target_id": "browser_clipper", + "event_type": "connector.config_updated", "occurred_at": "2026-10-06T00:00:00Z", + "payload_json": {"enabled": True, "configured": True, "validation_errors": []}}) + store.append_event({"target_type": "connector", "target_id": "browser_clipper", + "event_type": "connector.sync_completed", "occurred_at": "2026-10-06T00:01:00Z", + "payload_json": {"sync_cursor": "event-cursor", "item_count": 3, "imported_count": 2}}) + expected = service.connector_health_all() + by_name = {item["connector_name"]: item for item in expected["items"]} + assert by_name["telegram"]["cursor_state"] == "native-cursor" + assert by_name["browser_clipper"]["cursor_state"] == "event-cursor" + assert by_name["browser_clipper"]["items_captured"] == 2 + calls = [] + original = store.list_events + def counted(**kwargs): + calls.append(kwargs["target_id"]) + return original(**kwargs) + monkeypatch.setattr(store, "list_events", counted) + with label_read_scope(store): + result = service.connector_health_all() + assert result == expected + assert len(calls) == len(list_connector_definitions()) + result["items"][0]["validation_errors"].append("caller-mutation") + assert service.connector_health_all() == expected + store.upsert_connector_state({"connector_name": "telegram", "cursor_value": "after-write"}) + invalidate_read_labels(store) + updated = service.connector_health_all() + assert next(item for item in updated["items"] if item["connector_name"] == "telegram")["cursor_state"] == "after-write" + assert service.connector_health_all() == updated + + def test_connector_definitions_cover_sprint_11_sources_with_conservative_defaults() -> None: definitions = {definition.name: definition for definition in list_connector_definitions()} From 9aef641fa082de51268d5f6bd510bb4db4ce47eb Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Wed, 7 Oct 2026 00:11:34 +0200 Subject: [PATCH 245/270] Keep single-source walks and source event reads inside the request snapshot --- .../api/src/alicebot_api/vnext_label_guard.py | 20 ++++++++++++++++++- tests/unit/test_label_dependency_cache.py | 20 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index 0da91919f..148ad693e 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -213,7 +213,19 @@ def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[ if label is None and root not in state.dependency_ancestry.get(template, ()): label = state.dependency_labels.get(template) if label is None: - nodes = self._collected(kind, row) + refs = template[1] + direct_ref = next(iter(refs)) if len(refs) == 1 else None + parents = state.nodes.get(direct_ref, ()) if direct_ref is not None else () + if (template[3] == "copy" and not template[2] and direct_ref is not None + and direct_ref[0] == "source" and len(parents) == 1 + and identifier(parents[0].get("id")) == direct_ref[1] + and NODE_BOUND >= 2 and HOP_BOUND >= 1): + # A single original source terminates this walk. Ambiguous or + # absent parents still take the complete canonical collector. + nodes = [{**dict(row), "kind": canon_kind(kind), "user_id": _GUARD_USER}, + {**dict(parents[0]), "kind": "source", "user_id": _GUARD_USER}] + else: + nodes = self._collected(kind, row) copy_template = None # A copy with exactly one original source has no recursive input # graph. First collect each root to check missing/ambiguous source @@ -364,6 +376,12 @@ def admit_related_rows(self, rows: Sequence[_Row], *, kind: str, field: str) -> ids = list(dict.fromkeys(str(row.get(field)) for row in rows if row.get(field))) state = self._state() missing = [row_id for row_id in ids if (kind, row_id) not in state.targets] + if kind == "source": + for row_id in missing: + cached = state.nodes.get((kind, identifier(row_id)), ()) + if len(cached) == 1 and str(cached[0].get("id")) == row_id: + state.targets[(kind, row_id)] = cached[0] + missing = [row_id for row_id in missing if (kind, row_id) not in state.targets] for row in reader(kind, missing) if missing else []: state.targets[(kind, str(row.get("id")))] = row found = [state.targets[(kind, row_id)] for row_id in ids if (kind, row_id) in state.targets] diff --git a/tests/unit/test_label_dependency_cache.py b/tests/unit/test_label_dependency_cache.py index bd3797e92..99fbb39c9 100644 --- a/tests/unit/test_label_dependency_cache.py +++ b/tests/unit/test_label_dependency_cache.py @@ -118,6 +118,26 @@ def test_distinct_source_copy_reuse_keeps_parent_labels_and_alias_refusal(varian assert reused[-1] is True +def test_parent_source_event_reuse_applies_each_callers_filters_after_write(): + source = row(SOURCE, {"project_scope": ["beta"]}, sensitivity="confidential") + store = Store([("source", source)]) + copy = row(ROOT, {"source_id": SOURCE}) + event = {"target_type": "source", "target_id": SOURCE, "event_type": "source.created"} + with label_read_scope(store): + restricted = LabelGuard.for_filters(store, (), ("public",)) + assert restricted.admit_rows("memory", [copy]) == [] + assert restricted.admit_events([event]) == [] + admin = replace(restricted, sensitivity_allowed=ALL_SENSITIVITY) + assert admin.admit_events([event]) == [event] + bound = replace(admin, all_of=("alpha",)) + assert bound.admit_events([event]) == [] + source["sensitivity"] = "public" + source["metadata_json"]["project_scope"] = ["alpha"] + invalidate_read_labels(store) + assert restricted.admit_events([event]) == [event] + assert bound.admit_events([event]) == [event] + + @pytest.mark.parametrize("variant", ["self", "ancestor", "alias", "belief", "malformed", "missing", "floor", "class"]) def test_reused_signature_agrees_with_fresh_kernel_for_boundary_variants(variant): source = row(SOURCE, sensitivity="confidential") From 0a6f6ad5944e3874d17e51a1138b87659ac4d9a1 Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Wed, 7 Oct 2026 00:15:07 +0200 Subject: [PATCH 246/270] Declare connector cache reader and type the bounded source walk --- apps/api/src/alicebot_api/vnext_label_guard.py | 2 +- tests/unit/test_label_door_registry.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index 148ad693e..afa875724 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -215,7 +215,7 @@ def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[ if label is None: refs = template[1] direct_ref = next(iter(refs)) if len(refs) == 1 else None - parents = state.nodes.get(direct_ref, ()) if direct_ref is not None else () + parents: Sequence[Mapping[str, object]] = state.nodes.get(direct_ref, ()) if direct_ref is not None else () if (template[3] == "copy" and not template[2] and direct_ref is not None and direct_ref[0] == "source" and len(parents) == 1 and identifier(parents[0].get("id")) == direct_ref[1] diff --git a/tests/unit/test_label_door_registry.py b/tests/unit/test_label_door_registry.py index 98c03c502..56d8db28f 100644 --- a/tests/unit/test_label_door_registry.py +++ b/tests/unit/test_label_door_registry.py @@ -171,6 +171,7 @@ "vnext_connectors.py:VNextConnectorService.get_cursor": "connector cursor events only; no labelled targets", "vnext_connectors.py:VNextConnectorService.get_config": "connector configuration events only; no labelled targets", "vnext_connectors.py:VNextConnectorService.connector_health": "connector state telemetry only; no labels_raised events", + "vnext_connectors.py:VNextConnectorService._connector_events": "raw connector event cache; the query is constrained to connector targets and preserves the existing configuration/cursor telemetry readers", "vnext_dogfooding.py:VNextDogfoodingStore.list_artifact_quality_ratings": "store protocol declaration; no execution or response", "vnext_artifact_review.py:dispatch_vnext_artifact_review": "writer entry; calling route or MCP authorizes the artifact before dispatch", "vnext_memory_commit.py:VNextMemoryCommitService._guard_supersession_acyclic": "write validation traverses pointers without exposing their content", From 1754f8e0282f7f8d644757630d4217883b36b16c Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Wed, 7 Oct 2026 00:25:23 +0200 Subject: [PATCH 247/270] Exercise prefetched aliases in source-copy guard controls --- tests/unit/test_label_dependency_cache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_label_dependency_cache.py b/tests/unit/test_label_dependency_cache.py index 99fbb39c9..29f100d1b 100644 --- a/tests/unit/test_label_dependency_cache.py +++ b/tests/unit/test_label_dependency_cache.py @@ -110,6 +110,7 @@ def test_distinct_source_copy_reuse_keeps_parent_labels_and_alias_refusal(varian store = Store(rows) with label_read_scope(store): guard = LabelGuard(store, active=True) + guard.admit_rows("memory", [first, second]) assert effective(guard, "memory", first)[-1] is False reused = effective(guard, "memory", second) fresh = effective(LabelGuard(store, active=True), "memory", deepcopy(second)) From 89716150919997cb4828990ee7fe3e0c27313eab Mon Sep 17 00:00:00 2001 From: Sami Rusani Date: Wed, 7 Oct 2026 09:11:34 +0200 Subject: [PATCH 248/270] Restore derived-label reader compatibility and batch resolved inputs --- .github/workflows/tests.yml | 18 +- CHANGELOG.md | 8 +- .../src/alicebot_api/routers/_vnext_shared.py | 3 +- .../alicebot_api/routers/vnext_memories.py | 7 +- .../src/alicebot_api/routers/workspaces.py | 21 +- apps/api/src/alicebot_api/sqlite_schema.py | 12 + apps/api/src/alicebot_api/sqlite_store.py | 50 ++- .../src/alicebot_api/vnext_derived_labels.py | 27 ++ .../api/src/alicebot_api/vnext_label_guard.py | 135 ++++++- apps/api/src/alicebot_api/vnext_label_sql.py | 59 ++- .../src/alicebot_api/vnext_label_writes.py | 26 +- apps/api/src/alicebot_api/vnext_retrieval.py | 109 ++---- apps/api/src/alicebot_api/vnext_store.py | 21 +- .../vnext_stores/postgres/memory_access.py | 3 + .../vnext_stores/postgres/memory_lifecycle.py | 7 +- .../vnext_stores/sqlite/memory_access.py | 3 + .../vnext_stores/sqlite/memory_lifecycle.py | 7 +- docs/alpha/mcp-tools.md | 9 +- .../derived-labels-security-note-draft.md | 4 +- docs/release/v0.20.0-release-notes.md | 2 +- eval/scale/harness.py | 2 +- scripts/derived_label_mutations.json | 354 ++++++++++++++++++ scripts/verify_derived_label_mutations.py | 84 +++++ tests/integration/round3_parity_probe.py | 23 ++ ...est_label_round2_reader_fences_postgres.py | 8 +- .../test_label_round3_contracts_postgres.py | 104 +++++ .../test_label_round3_read_budget_postgres.py | 28 ++ tests/performance/round3_budget_probe.py | 67 ++++ .../test_label_round3_read_budget.py | 185 +++++++++ tests/unit/test_complete_readable_counts.py | 6 +- tests/unit/test_derived_labels_docs.py | 33 ++ .../unit/test_known_limitations_page_shape.py | 4 +- tests/unit/test_label_dependency_cache.py | 7 +- tests/unit/test_label_resolved_inputs.py | 131 +++++++ tests/unit/test_label_round3_sqlite_reads.py | 89 +++++ 35 files changed, 1481 insertions(+), 175 deletions(-) create mode 100644 scripts/derived_label_mutations.json create mode 100644 scripts/verify_derived_label_mutations.py create mode 100644 tests/integration/round3_parity_probe.py create mode 100644 tests/integration/test_label_round3_contracts_postgres.py create mode 100644 tests/integration/test_label_round3_read_budget_postgres.py create mode 100644 tests/performance/round3_budget_probe.py create mode 100644 tests/performance/test_label_round3_read_budget.py create mode 100644 tests/unit/test_label_resolved_inputs.py create mode 100644 tests/unit/test_label_round3_sqlite_reads.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a01911cf2..7b0e6097b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2283,7 +2283,7 @@ jobs: python-integration: name: ${{ matrix.integration_check }} runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -2304,7 +2304,7 @@ jobs: --health-retries 20 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: Paired read-budget baseline (main at the round-two handoff) + - name: Paired read-budget baseline (main at the round-three handoff) if: matrix.integration_check == 'Integration tests (Postgres + pgvector, role separation)' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -2337,17 +2337,17 @@ jobs: DATABASE_ADMIN_URL: postgresql://alicebot_admin:ci@localhost:5432/alicebot ALICE_READ_MAIN_CHECKOUT: ${{ github.workspace }}/.read-budget-baseline # --timeout=180 names one test that runs that long. The cancelled - # jobs were slow database setup, not a gap near 180s, so that flag - # would not have fired. --session-timeout=1500 is checked between - # tests, stops the run, and prints --durations before the 30 minute - # job cancel. - run: ALICE_LEGACY_SURFACES=1 ./.venv/bin/python -m pytest tests/integration -q -p no:cacheprovider --durations=20 --timeout=180 --session-timeout=1500 + # jobs were slow database setup, not a gap near 180s. The round-three + # grid includes 30 paired fixtures with ten interleaved samples per + # action and profile. Session bounds allow that grid to finish while + # each measured request retains its unchanged acceptance gate. + run: ALICE_LEGACY_SURFACES=1 ./.venv/bin/python -m pytest tests/integration -q -p no:cacheprovider --durations=20 --timeout=180 --session-timeout=3000 - - name: SQLite varied-parent read budgets against the same baseline + - name: SQLite random mixed-parent and keyless read budgets against the same baseline if: matrix.integration_check == 'Integration tests (Postgres + pgvector, role separation)' env: ALICE_READ_MAIN_CHECKOUT: ${{ github.workspace }}/.read-budget-baseline - run: ./.venv/bin/python -m pytest tests/performance/test_label_round2_read_budget.py -q --timeout=180 --session-timeout=240 + run: ./.venv/bin/python -m pytest tests/performance/test_label_round3_read_budget.py tests/performance/test_label_read_handoff_budget.py -q --timeout=180 --session-timeout=1800 - name: Default-surface core round-trip if: matrix.integration_check == 'Default surface integration smoke (Postgres)' diff --git a/CHANGELOG.md b/CHANGELOG.md index faef4cdc2..fb9889341 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,14 @@ ## Unreleased +- Unreleased (on main, not in v0.20.0): October 7 round-three corrections restore the source GET operator gate before its caller fence, preserve the owner's and unbound admin's complete source trace and workspace diagnostics and totals, and clamp an owner project assignment to the derived row's settled scope. A refused assignment sets `label_floor_applied`, records `floor_clamped` even when the existing label stays unchanged, and leaves no labels-check gap. Restricted reads batch ancestry lookups and reuse verified resolved parent labels within one request, preserving aliases, cycles and per-origin bounds. SQLite adds indexes for stored UUID aliases. Memory full-text reads refill after effective admission; exhausted legacy adapters retain the finite completeness error. CI measures the random mixed-parent grid before and after real repair, including the SQLite keyless agent path, against pinned main 48873b03. No migration is required. SQLite creates its alias indexes during the idempotent schema bootstrap. + - Unreleased (on main, not in v0.20.0): October 6 round-two corrections recover a per-user open-loop digest on both stores after a project reassignment or stale duplicate extraction, and weekly candidates record the report inputs before insert so their stored floor matches the weekly artifact in the same transaction. Read-time labels reuse verified dependency signatures within one request; original rows are counted in SQL, and the derived partition still receives the kernel guard. PostgreSQL event filtering uses an indexed guarded UUID comparison. SQLite filters definitely hidden direct parents before its event limit, then applies the full effective-label guard. The legacy review list also rechecks saved quotes on original or imported candidates against the current source fence. CI compares unique-metadata, varied-parent and repaired fixtures against pinned main 48873b03. No migration is required. -- The web package pins `sharp` to 0.35.5, including the patched librsvg binaries, for GHSA-wq5f-xc86-pv6w. Earlier pins below 0.35.5 fail the production dependency audit. +- Unreleased (on main, not in v0.20.0): the web package pins `sharp` to 0.35.5, including the patched librsvg binaries, for GHSA-wq5f-xc86-pv6w. Earlier pins below 0.35.5 fail the production dependency audit. - Unreleased (on main, not in v0.20.0): on SQLite, `sources delete`, `sources prune --superseded` and `import-markdown --supersede` blank an open loop that names the source in any spelling the saved-quote reader names, including capitals, no hyphens, braces, `urn:uuid:`, a list and JSON text. The lookup, the preview and the reader that withholds those ids use the same spellings, including JSON escapes. Repeated keys and quote subtrees follow the limits in the draft security note. Lookup and removal both decode JSON text under source and memory references and inside trace fields. Admitted references and surrounding values stay, and the owner receives live references unchanged. One pass over the user's loops answers for every source of the command. v0.20.0 blanked only the id as stored or `source:`, so any other spelling kept the loop's text. No migration is required. -- Unreleased (on main, not in v0.20.0): the October 6 derived-label correction normalizes server UUIDs, carries bound scheduler identity, withholds encoded references and handles malformed marker shapes. Plain reviews share the label lock; label-changing reviews retain the bounded retryable refusal. Read labels and distinct parents are cached within one request. Context-pack rows dropped by effective admission are absent from the later stage's excluded counts and warnings. The search golden was regenerated through its recorder. Source GET and the legacy review list now apply the full caller fence. Graph edge explanations remain outside that ceiling, and the doctor label counts are new in this set, as disclosed in the draft security note. The sources_first golden token estimates each rise by 5 because derived inserts persist empty project_scope and project_floor. SQLite retirement deliberately follows the saved-quote reader for repeated keys and quote subtrees; an older raw scan blanked some such rows that it now retains. +- Unreleased (on main, not in v0.20.0): the October 6 derived-label correction normalizes server UUIDs, carries bound scheduler identity, withholds encoded references and handles malformed marker shapes. Plain reviews share the label lock; label-changing reviews retain the bounded retryable refusal. Read labels and distinct parents are cached within one request. Context-pack rows dropped by effective admission are absent from the later stage's excluded counts and warnings. The search golden was regenerated through its recorder. Source GET retains its operator gate before applying the full caller fence; the owner and unbound trusted or admin keys are admitted, other profiles and project-bound keys receive 403, and admitted callers receive the same 404 for hidden and missing sources. The legacy review list applies the full caller fence. Graph edge explanations remain outside that ceiling, and the doctor label counts are new in this set and remain outside that ceiling, as disclosed in the draft security note. The sources_first golden token estimates each rise by 5 because derived inserts persist empty project_scope and project_floor. SQLite retirement deliberately follows the saved-quote reader for repeated keys and quote subtrees; an older raw scan blanked some such rows that it now retains. - Unreleased (on main, not in v0.20.0): the derived-row pages state the three-part label, who reads an unverified row, and that the five operator screens apply the caller's sensitivity ceiling. The v0.20.0 notes omitted both limits. No migration is required. - Unreleased (on main, not in v0.20.0): PostgreSQL migration `20261005_0096` raises a stored derived row to the labels of its inputs, and SQLite does the same on the next open and before a restore is published. The open pass can leave a vault open with its completion key unstamped after a failed repair; explicit repair and restore refuse the whole change on failure. `alicebot vnext labels` and `alice-memory labels` check and repair the same rows, including rows added after an earlier repair. The doctors print how many derived rows are below their inputs or unverified, as a warning, and report failed checks as unavailable. v0.20.0 left the stored label where it was written. Migration `20261005_0096` must run before serving requests as the NOSUPERUSER NOBYPASSRLS table owner. - Unreleased (on main, not in v0.20.0): recall, context packs, resume, recent decisions, the session brief, the review queue, open-loop lists, and the workspace, dashboard, context tree, and dogfooding lists drop a loaded row whose effective label is outside the request. The artifact list, the source trace, the project list, the dashboard project row, and the belief state route apply the caller's sensitivity ceiling, including titles, ids, and counts. Resume, recent decisions, the session brief, audit, and explain apply the target row's current effective label, so an excluded key does not learn of a label change. The producer input readers also drop an input whose effective label is outside the request before it appears in report text. v0.20.0 returned rows by their stored labels and could copy a public row with confidential inputs into a new report. No migration is required. @@ -48,7 +50,7 @@ - The `host-evidence` job (the dispatch-only job in `real-host-ci.yml` that records what the pinned hosts hand a hook and an MCP server) no longer passes a run whose `roots/list` probe never ended. Its final check read a server record marked `complete` as a finished probe, but a record is also marked complete when the client closes the connection, when no `notifications/initialized` arrives for 10 seconds, and when the server stops on an error, so a host that connected and left before it answered `roots/list` gave an empty problems list and a green job. Each server record now carries a separate `probe_finished` field, true only when the probe reached an end: an answer, an error answer, or the whole wait with no reply. The check reports a record without it as a problem that says why, for example `the roots/list probe did not finish: the client closed the connection first`. Three more things differ from main before this change. A client that declared the roots capability and answered `roots/list` with an error now fails the run; on main that run passed with no problem. That is a change of what passes, and it is meant: a capability the client declared and then could not use is the anomaly this evidence exists to show. A message that carries the stub's request id before the stub has sent the probe answers nothing the stub asked, so it no longer counts as the probe's answer: on main it was recorded as `answered`, finished the probe and released the host, and now it ends nothing, the real probe is still sent, and the check reports `the client answered roots/list before the stub asked` whatever follows. A server that stops on an error after the probe ended (a closed pipe on a late reply, say) is not a problem, on purpose, because the check is about the probe and not the process; the error name stays in the record as `error`, and a server that stops on an error before the probe ended is the problem described above. A client that declared nothing is still asked, as before, and whatever it answers is a finding, not a problem; a client that stays silent for the whole wait is still recorded as `no_reply` and passes. `complete` keeps its job of letting the stub API release the host. In v0.20.0 none of this exists: the `host-evidence` job and `scripts/real_host_evidence.py` are not in that release, so this changes only a dispatch-only CI script on main and no installed code. The S0 results already recorded for the two hosts stand, because a run in the bad state records `roots/list` as `not_sent` or `no_reply`, never `answered`. - A write that cites a source by id is now held to the caller's read fence. `alice_memory_commit` takes `source_refs` and `alice_memory_correct` (only an `admin_agent` key may call it) takes `provenance` and `replacement_provenance`, and each stores a link from the memory to the source that later readers follow. `POST /v0/vnext/open-loops` takes a `source_id` and a `memory_id` and keeps both on the loop, and is held to the same fence (the last paragraph of this entry). v0.20.0 checked only that the source existed for the acting user, so a key bound to one project could attach a source of another project, a global source, a source above its sensitivity ceiling, a source in a domain its profile may not read, or a deleted one, and the link was stored. A source id that did not exist failed (`tool_request_failed` on SQLite in v0.20.0, `precondition_failed` on main once the typed codes landed, both from the foreign key) while one that existed was stored, so a source id told a key whether it existed. The attached link did not show the text of a source of another project, a global source, a source above the key's ceiling or a deleted source to the key, because `alice_recall` and `alice_context_pack` already apply the read fence to the sources a memory cites, and that held when run against a real SQLite vault with real agent keys. (A `project_scoped_agent` key's recall returned a restricted-domain source of its own project when the call named no `domains`, link or no link. That is not changed here; the entry on a request that names no `domains` closes it.) It did show the foreign source's id to every key in the project that read the memory with `alice_memory_review` by id, and it made `alice_explain` of the memory fail for every key-bound caller, the key that wrote it included, because explain refuses a memory whose link names a source the caller may not read. An admin key could also attach, through `provenance` with `evidence_role: quoted_from` and a quote that covers a foreign source's excerpt, a link that marked that excerpt `derived_memory_corrected` in the other project's recall. Now a cited source must be one the caller could be shown, by the test `alice_explain` applies to each source it discloses, so a link that passes can never make that caller's own explain fail. The test is the project scope of a key bound to a project (a source of another project, a source shared with another project and a global source are outside it), the domains of the profile (a `project_scoped_agent` key may not cite a health, family, spiritual, legal or financial source), the profile's sensitivity ceiling, and deletion. A source that is missing, deleted or outside the fence answers `not_found` with the one fixed message over MCP and 404 `not_found` from `POST /v0/vnext/memories/commit`, from one raise site, so nothing in the answer tells the three apart. The check reads every id a ref names, not only the first one a link would use, runs in every write mode (a write that waits for confirmation or review stores the ref on its row too), runs after a policy refusal so a refused caller learns nothing about which ids exist, and runs before anything is written, so a refused call leaves no memory and no link. An idempotent replay returns the stored memory without reading the sources again. A ref that names no id, such as a URL or a label, is stored as before. The owner, a call with no agent identity, may still cite any live source of the vault. A keyless call that declares a profile is held to that profile's domains and ceiling, and its declared project is not enforced, as on every other keyless read and write. The fence is a required keyword-only argument of the functions that resolve ids (`resolve_attachable_sources`, `resolve_attachable_source_id`, `resolve_attachable_memory_id` and `_validated_review_provenance`), and tests list every call to `create_provenance_link` and to `create_open_loop`, every construction of the type that says an id was checked, and every field of an HTTP body or a tool schema that names a source, a memory or a provenance object, and fail for one they do not know. Memory proposals (`alice_vnext_propose_memory`, `POST /v0/vnext/memory-proposals`) and the agent-output ingest (`alice_vnext_ingest_agent_output`, `POST /v0/vnext/agents/ingest-output`) store the `source_refs` they are given and check none of them, as before, and links already stored are not touched. The new tests are in `tests/unit/test_source_refs_read_fence.py`. Two limits remain. The fence at write time is the writer's own read fence and not that of whoever reads later: a link that passes for a writer with a higher ceiling (an `admin_agent` key citing a confidential source of its own project) makes `alice_explain` of that memory fail for the keys of the project with a lower ceiling, which are shown no text of the source (measured on a real SQLite vault with real agent keys, and pinned by a test). The context pack's `supporting_evidence`, `alice_memory_review` by id and `alice_open_loops` showed that source's id to those keys when this entry was written; the entry on saved quotes and the entry on open loop references hold each of them to the reader's own fence. And links and open loops saved before the fix keep what they hold in storage: `alice_explain` still fails for such a memory, and the readers withhold a foreign id as those two entries say. The open-loop door, in full: `POST /v0/vnext/open-loops` kept the `source_id` and `memory_id` it was given and `alice_open_loops` returns both. Measured on a real SQLite vault with real agent keys, by running the route function over the SQLite store (the route runs on Postgres only, and Postgres was not run; the route's code is the same in v0.20.0): a key bound to one project got 201 and a stored loop for a source of another project, a global source, a source above its ceiling, a source in a domain its profile may not read and a deleted source, and for a memory of another project, a global memory, a memory above its ceiling, a memory in a restricted domain and a deleted memory, and `alice_open_loops` returned every one of those ids to the keys of the project. An id that did not exist raised the foreign key error out of the route, where one that existed was stored, so the id told the key whether it existed (on Postgres both foreign keys name the user and not the project, read from the migrations). Now both ids must be ones the caller could be shown, by the test `alice_explain` applies to a source and to a memory, and an id that is missing, deleted, malformed or outside the fence answers 404 with the public `not_found` body, the same for each. The check runs after a policy refusal, so a refused caller learns nothing about which ids exist, and before anything is written; a refused call is rolled back. The loop stores the id in canonical form, the one that was checked. The owner, a call with no agent key, may still name any live source and memory of the vault. -- An open loop no longer shows its reader the id of a source or memory the reader may not read. A loop keeps the `source_id` and `memory_id` it was made with in columns of its own, and its `metadata_json` can name sources too (the daily brief's candidate loops write their `source_id` there). In v0.20.0 every reader that returns a loop whole returned all of that as stored to any caller that could read the loop: `alice_open_loops` (the list, its legacy alias `alice_vnext_open_loops`, and the `close`, `snooze`, `edit` and `reopen` actions, which return the updated row), `POST /v0/vnext/open-loops/{id}/review`, the open loops of `POST /v0/vnext/context-packs`, and the report of the scheduler's `open_loop_review` workflow, which copies each loop's source id into its text and into its `source_refs` (PostgreSQL only). So a key that could read a loop was handed the id of a source or memory it could not read: a confidential source that an `admin_agent` key had linked, or, on a loop saved before the write fence of the entry on cited sources, the id of another project's source, a global source, a source in a domain the key's profile may not read, a deleted source, and the same kinds of memory. The id is not the text: `alice_memory_review` by id, `alice_explain` and `alice_recall` answer nothing for it, so it is an id and a sign that the row exists. Checked on v0.20.0 and on main with a real SQLite vault and real agent keys: a `project_scoped_agent` key bound to a project was returned the confidential source's id in `source_id` and in `metadata_json` by `alice_open_loops`. The two HTTP routes and the scheduler report run the same code in v0.20.0 as on main, and the routes were run over the SQLite store on main. Now each reader checks the loop's references against the reader's own read fence, the test `alice_explain` applies to each source and memory it discloses (the project scope of a key bound to a project, the domains of its profile, its sensitivity ceiling and deletion), and returns `null` for a reference the fence does not admit. The key stays in the row, so a protected, a deleted and a missing reference read alike. An id in `metadata_json` goes the same way. Under a key that names a reference (`source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references`, `selected_source_ids`, `memory_id`, `memory_ids`, `memory_ref`, `memory_refs` or `source_memory_ids`, at any depth) an id stays only if it names a row the reader may read. Anywhere else an id goes if it names a row the reader may not read, or if the same response withholds it from a reference position (the `source_id` and `memory_id` columns count as one). A deleted source or memory is a row the reader may not read: the lookup reads soft-deleted rows too, so its id is withheld wherever it stands, even when the loop names it nowhere else, and deleting a source does not bring back an id the loop had repeated under another key such as `evidence.quote_from`. The withheld ids are collected over every loop of one response (a list, a review, or the loops of one context pack), so one loop cannot show an id that another loop of the same response withholds. An id under another key that names no row and is linked nowhere in the response is kept, because it may be a trace id, and so is the id of a row that was removed outright and that no reference position of the response names. Ids are read in every spelling the link writer reads, whole or inside longer text: upper or lower case, hyphenated or as 32 hex digits in a row, in braces, and after `urn:uuid:`, `uuid:`, `source:` or `memory:`. An id without hyphens inside a URL or a sentence is withheld like the hyphenated one. Inside longer text the hyphenated layout is read wherever it stands. A run of 32 hex digits counts as an id only when no hex digit stands next to it, so a 40-digit git sha and a 64-digit digest are returned whole, and so is an id the reader may read that has a hyphen and more hex digits after it (`-20261003`). One layout of glued digits is cut under a reference key: a hyphen and groups of 4, 4, 4 and 12 digits right after an id, or groups of 8, 4, 4 and 4 digits and a hyphen right before a 32-digit id, read as a hyphenated id that names no row, so part of an id the reader may read is withheld there and wherever the same response repeats it; no id the reader may not read is shown by it. A value that is only an id is read the way the link writer reads it, which also takes hyphens in other places. Inside longer text an id with its hyphens in other places, a split id and an encoded id are not recognised. A column value that is no id is withheld as a missing one is, and whatever is shown is the value as stored. The two bulk reads of the stores, `get_sources_by_ids` and `get_memories_by_ids`, take an `include_deleted` argument for this, false by default, so every other caller reads what it read before. The loop and every other field of it are returned as before. The owner (a call with no agent identity) and a key that may read what the loop points at get every live reference back unchanged, and a reference to a source or memory that has been deleted is withheld from them too. A loop that an automation made over a global source, such as a daily brief's candidate loop, no longer shows its `source_id` to a key bound to a project, because a global source is outside the fence of such a key. The rule costs one batched read of the sources and one of the memories the loops name, and none when they name nothing. It lives in `alicebot_api/vnext_open_loop_references.py`, where the `fence` argument is required and keyword-only. A test lists every call that reads a loop row, a pack or a dashboard and fails for one it does not know, so a new reader has to be fenced or classified. The legacy MCP tools that return loops whole and take no identity argument (`alice_project_dashboard`, `alice_vnext_context_pack`, `alice_open_loop_extract`, `alice_open_loop_review`) are served only with no agent key, so every call to them is the owner's and nothing is withheld there, and a test fails if one gains an identity property. `alice_context_pack` and `alice_resume` return a fixed list of fields per loop that holds no id, and a test pins the list. Not changed here: the free-text columns of a loop (`title`, `description`, `resolution_note`), which are returned as stored and are not scanned for ids. One producer of such text is fixed: the extractor of candidate loops (`alicebot_api/vnext_projects.py`) wrote `Candidate task discovered from source .` into `description` and used the source's id when the source had no title, so that id was shown to a reader the loop's `source_id` column was withheld from. It now writes `a source with no title`. A loop saved before keeps the text it holds. The tests are in `tests/unit/test_open_loop_references_read_fence.py`, `tests/unit/test_open_loop_ids_every_spelling_and_after_delete.py` and `tests/unit/test_vnext_projects.py`, and in `tests/integration/test_open_loop_references_postgres.py`, which CI runs on PostgreSQL. +- An open loop no longer shows its reader the id of a source or memory the reader may not read. A loop keeps the `source_id` and `memory_id` it was made with in columns of its own, and its `metadata_json` can name sources too (the daily brief's candidate loops write their `source_id` there). In v0.20.0 every reader that returns a loop whole returned all of that as stored to any caller that could read the loop: `alice_open_loops` (the list, its legacy alias `alice_vnext_open_loops`, and the `close`, `snooze`, `edit` and `reopen` actions, which return the updated row), `POST /v0/vnext/open-loops/{id}/review`, the open loops of `POST /v0/vnext/context-packs`, and the report of the scheduler's `open_loop_review` workflow, which copies each loop's source id into its text and into its `source_refs` (PostgreSQL only). So a key that could read a loop was handed the id of a source or memory it could not read: a confidential source that an `admin_agent` key had linked, or, on a loop saved before the write fence of the entry on cited sources, the id of another project's source, a global source, a source in a domain the key's profile may not read, a deleted source, and the same kinds of memory. The id is not the text: `alice_memory_review` by id, `alice_explain` and `alice_recall` answer nothing for it, so it is an id and a sign that the row exists. Checked on v0.20.0 and on main with a real SQLite vault and real agent keys: a `project_scoped_agent` key bound to a project was returned the confidential source's id in `source_id` and in `metadata_json` by `alice_open_loops`. The two HTTP routes and the scheduler report run the same code in v0.20.0 as on main, and the routes were run over the SQLite store on main. Now each reader checks the loop's references against the reader's own read fence, the test `alice_explain` applies to each source and memory it discloses (the project scope of a key bound to a project, the domains of its profile, its sensitivity ceiling and deletion), and returns `null` for a reference the fence does not admit. The key stays in the row, so a protected, a deleted and a missing reference read alike. An id in `metadata_json` goes the same way. Under a key that names a reference (`source_id`, `source_ids`, `source_ref`, `source_refs`, `source_references`, `selected_source_ids`, `memory_id`, `memory_ids`, `memory_ref`, `memory_refs` or `source_memory_ids`, at any depth) an id stays only if it names a row the reader may read. Anywhere else an id goes if it names a row the reader may not read, or if the same response withholds it from a reference position (the `source_id` and `memory_id` columns count as one). A deleted source or memory is a row the reader may not read: the lookup reads soft-deleted rows too, so its id is withheld wherever it stands, even when the loop names it nowhere else, and deleting a source does not bring back an id the loop had repeated under another key such as `evidence.quote_from`. The withheld ids are collected over every loop of one response (a list, a review, or the loops of one context pack), so one loop cannot show an id that another loop of the same response withholds. An id under another key that names no row and is linked nowhere in the response is kept, because it may be a trace id, and so is the id of a row that was removed outright and that no reference position of the response names. Ids are read in every spelling the link writer reads, whole or inside longer text: upper or lower case, hyphenated or as 32 hex digits in a row, in braces, and after `urn:uuid:`, `uuid:`, `source:` or `memory:`. An id without hyphens inside a URL or a sentence is withheld like the hyphenated one. Inside longer text the hyphenated layout is read wherever it stands. A run of 32 hex digits counts as an id only when no hex digit stands next to it, so a 40-digit git sha and a 64-digit digest are returned whole, and so is an id the reader may read that has a hyphen and more hex digits after it (`<id>-20261003`). One layout of glued digits is cut under a reference key: a hyphen and groups of 4, 4, 4 and 12 digits right after an id, or groups of 8, 4, 4 and 4 digits and a hyphen right before a 32-digit id, read as a hyphenated id that names no row, so part of an id the reader may read is withheld there and wherever the same response repeats it; no id the reader may not read is shown by it. A value that is only an id is read the way the link writer reads it, which also takes hyphens in other places. Unreleased (on main, not in v0.20.0): a second source-only pass now withholds irregular, split and encoded source ids inside longer text. Irregular or non-ASCII MEMORY-prefixed ids and unnamed SOURCE whitespace forms can remain; the wider pass does not recognize those memory or whitespace spellings. A column value that is no id is withheld as a missing one is, and whatever is shown is the value as stored. The two bulk reads of the stores, `get_sources_by_ids` and `get_memories_by_ids`, take an `include_deleted` argument for this, false by default, so every other caller reads what it read before. The loop and every other field of it are returned as before. The owner (a call with no agent identity) and a key that may read what the loop points at get every live reference back unchanged, and a reference to a source or memory that has been deleted is withheld from them too. A loop that an automation made over a global source, such as a daily brief's candidate loop, no longer shows its `source_id` to a key bound to a project, because a global source is outside the fence of such a key. The rule costs one batched read of the sources and one of the memories the loops name, and none when they name nothing. It lives in `alicebot_api/vnext_open_loop_references.py`, where the `fence` argument is required and keyword-only. A test lists every call that reads a loop row, a pack or a dashboard and fails for one it does not know, so a new reader has to be fenced or classified. The legacy MCP tools that return loops whole and take no identity argument (`alice_project_dashboard`, `alice_vnext_context_pack`, `alice_open_loop_extract`, `alice_open_loop_review`) are served only with no agent key, so every call to them is the owner's and nothing is withheld there, and a test fails if one gains an identity property. `alice_context_pack` and `alice_resume` return a fixed list of fields per loop that holds no id, and a test pins the list. Not changed here: the free-text columns of a loop (`title`, `description`, `resolution_note`), which are returned as stored and are not scanned for ids. One producer of such text is fixed: the extractor of candidate loops (`alicebot_api/vnext_projects.py`) wrote `Candidate task discovered from source <title>.` into `description` and used the source's id when the source had no title, so that id was shown to a reader the loop's `source_id` column was withheld from. It now writes `a source with no title`. A loop saved before keeps the text it holds. The tests are in `tests/unit/test_open_loop_references_read_fence.py`, `tests/unit/test_open_loop_ids_every_spelling_and_after_delete.py` and `tests/unit/test_vnext_projects.py`, and in `tests/integration/test_open_loop_references_postgres.py`, which CI runs on PostgreSQL. - Internal change, no behaviour change: the CI job named "Unit tests + live eval battery (SQLite)" ran every unit test, the model-free eval battery and the coverage gate in one place, and its longest run was 17 minutes 50 seconds against a 20 minute limit. The unit tests now run as three parallel shard jobs, each a set of file patterns over `tests/unit`, and `tests/unit/test_ci_unit_job_split.py` fails if a test file matches no shard or two. The eval steps run once, in their own job. A coverage job combines the shard data with `scripts/combine_python_coverage.py`, which fails if a shard left no data, and holds the combined data to the same 50 percent threshold and per-file floor as before. The job that keeps the old name is now a summary: it runs whatever the others did and fails unless the shards, the eval job and the coverage job all succeeded, so the required status check of the main ruleset and the exact-SHA release check read the same name as before. `gates.json` records the new layout under `ci_time`, so its sha256 changes and no gate threshold does. - A quote that a memory saved from a source is now shown only to a caller who may read that source now. A link from a memory to a source keeps the quote it was made with, and the memory keeps copies of that quote in its own metadata: `metadata_json.provenance` (an edit-and-approve review), `metadata_json.replacement_provenance` (a supersede review) and `metadata_json.agentic_memory.conversation_excerpt` (the commit route, which also stores it as the quote of each link). The write fence checks the caller's read permission on the source once, when the link is made, so in v0.20.0, and on main before this change, a source that was reclassified afterwards (its sensitivity raised, its domain changed, its project moved, or the source archived) kept showing its saved quote to every caller below the new label. Measured on a real SQLite vault with real agent keys, by changing the source with the shipped `review_vnext_source` handler (`POST /v0/vnext/sources/{id}/review`) and searching each answer for a unique quote: `alice_memory_review` by id returned the quote twice (the link, and the memory's `metadata_json`) and `alice_context_pack` returned it in `supporting_evidence`, to a `trusted_local_agent` key and a `project_scoped_agent` key after the source became confidential, to a `read_only_agent` key after it became private, to a `project_scoped_agent` key after its domain became health, and, after the source was archived, to every key including the admin key (the pack's `supporting_evidence` to every key bound to no project, whatever its profile, because the pack's own scope test skips a source it cannot find only when the pack has a scope, and the full rows of `POST /v0/vnext/context-packs`, which carry `metadata_json`, to every key); after a move to another project the review leaked and the pack did not, because the pack's scope test catches a move. `alice_explain` refused throughout for a memory with a provenance link, and `alice_recall` left the source out. A memory with no provenance link keeps the quote and the id of its source only in its own metadata: the commit route stores a link only for a commit that is accepted at once, so a commit held for review (confidence under 0.5) and then approved, or one that waited for its author's confirmation (confidence under 0.85) and was confirmed, has none, and neither the approval nor the confirmation adds one. For such a memory, made through both doors on the same vault and read after the source became confidential, `alice_memory_review` by id and the full rows of `POST /v0/vnext/context-packs` returned the saved excerpt to every key that could not read the source, and `alice_explain` returned the memory row, its revisions and the payload of its events, with the quote and the source id, to every such key, because it authorized only the sources a link names. The full memory rows of `POST /v0/vnext/context-packs` and of the legacy context pack tool also carried the quote in `metadata_json`, and so did the row that `alice_memory_manage` (`expire`, `unexpire`, `undo`, `forget`), `alice_memory_correct`, the routes of the same verbs and `POST /v0/vnext/memories/{id}/review` hand back. Now every one of those readers asks the same fence again, with the source as it is when the call is made: the test `alice_explain` applies to each source it discloses (the key's project, its domains, its sensitivity ceiling, and deletion). The one object that asks is `SavedProvenanceReader` in `vnext_source_fence.py`. A link whose source is missing, archived or outside the fence is left out whole, as a link that was never stored would be, and a link that names no source at all is left out as well. When a memory has such a link, or its own copies name such a source, the three copies of the quote above are removed from the row, and the entries of its `source_refs` lists (in the metadata, in `agentic_memory`, in `value`, and the same list in a revision's `previous_value`, `new_value` and `metadata_json`) that name the source are dropped, so the source's id goes with its quote. A memory with no link is judged by the ids its own copies name. The pack asks before its own scope pass, which removes source refs from the rows of a scoped pack (those of a source the key may read too), because a row judged after that pass names no source and would keep its quote. `alice_explain` now authorizes every source that the memory row, its revisions and its event payloads name, as it does a source a link names, and fails closed for a key that may not read one (a source that is missing or deleted counts as one). When a link is left out, any other link of the same memory whose quote says the same text, ignoring the whitespace between words, is shown without its quote: the commit route saves one excerpt as the quote of every link it makes, so a memory committed with two sources showed the second source's link with the first source's excerpt. The same holds for every copy of a quote, found by an outside review: a commit with `{"source_ids": [A, B]}` links only A and keeps the excerpt as its own copy, which names B, and after B was made confidential or archived the copy was withheld while the link to A kept the same bytes in `alice_memory_review`, the MCP pack and the HTTP pack, so every text that is withheld (a link left out, or a copy removed from the row or a revision) is now withheld from the memory's other links too, and a memory is judged by every source id its refs name, in every spelling the link writer reads (an id that starts with `0` can be written with a space or a tab in the place of the zero, and the writer links it) and in the shapes it does not read (`selected_source_ids`, `source_references`, an upper case `SOURCE:`, several ids in one string, an id under another key). An id in a position that says it is a source must name a stored source the caller may read, while an id in a sentence, an outside URL or under another key is judged only when the store holds a row for it, an archived source's row included (`get_sources_by_ids(..., include_deleted=True)`, a keyword both stores take), so a ref such as `copied from source: <id>` whose id names nothing takes no quote from a caller who may read what the memory cites; the refs in the metadata of a revision are held to the fence like the memory's, and a ref is read in time that grows with its length. A ref that is JSON text is read as the value it decodes to, as the same ref stored as an object is, so an id inside its `quote` is not read as a citation (an outside review of this entry found that a literal `source:<an id that names no source>` in the quote of such a ref took the saved quote from `alice_memory_review` by id and from both context packs and made `alice_explain` refuse the memory, while the same ref stored as an object was read correctly); an id in any other field of it is judged as it is in an object, and a text that does not decode is scanned as before. A link whose quote says something else, such as the link from a captured candidate to the source it was captured from, keeps it. A source with no project (a source the owner captured has none) is outside the fence of every key bound to a project, the admin key included, so `alice_memory_review` by id no longer returns the link, the quote or the source id of a memory that cites such a source to those keys, which is what `alice_explain` and the pack already did, and v0.20.0 returned all three; a key bound to no project keeps them. The memory is still returned, with its text. A caller that may read every cited source gets the stored row, as the same object, and so a pack or a review for an authorized key is what it was. The owner, a call with no agent key, is shown what was stored, an archived source's quote included. The answers of callers who may read the source are unchanged: the review, the MCP pack and the HTTP pack at two depths each, and `alice_explain`, were read from copies of one vault file by the code of main and by this code, for six memories (one from each door that saves a quote, two of them with no link and one citing two sources), for every key and the owner before the source was reclassified, after it was made confidential and after it was archived. Every answer of a caller who may read the source (270 answers) was byte for byte the same apart from the ids, timestamps and hashes that the reads themselves create. For the callers who may no longer read the source, the only fields that differ are the withheld ones: the `provenance`, `replacement_provenance` and `conversation_excerpt` copies, the entries of `provenance_links`, `supporting_evidence` and the `source_refs` lists, the token estimates that counted them, and the `alice_explain` answer of a memory with no link, which is now refused. The pack takes the fence as a required keyword argument, `compile_context_pack(request, *, source_fence)`, with no default: the two doors an agent reaches pass the caller's own (`SourceReadFence.for_identity(identity)`), and the command line, the smoke runs and the evaluation harnesses pass `SourceReadFence.unfenced()` where a reviewer sees it. The eight verbs of `VNextMemoryCommitService` that return a row (`commit`, `confirm`, `undo`, `correct`, `forget`, `accept_consolidation_candidate`, `expire`, `unexpire`) hold the row to the fence of the `identity` they were called with, so the MCP tools, the legacy aliases and the HTTP routes get it by calling them. Tests list every call of `compile_context_pack`, every verb that returns a memory and every place that writes a quote, and fail for one they do not know. The new tests are `tests/unit/test_saved_quotes_follow_the_source_fence.py` and `tests/unit/test_saved_provenance_reader.py`, and the pinned limit in `tests/unit/test_source_refs_read_fence.py` changed to the new behaviour. Not changed: an id found where it could be a chunk id that names a source removed from the database (no door of the product removes one) is not judged, a caller that can store a memory can learn by reading it back whether an id it already holds names a stored source it may not read, and the check at write time still reads fewer ref shapes than the readers do; a memory that `alice_capture` derived from a source holds that text as its own text, and the ids that capture writes into such a candidate (`value`, `metadata_json`, `source_event_ids`) are not withheld; `provenance_count` still counts a withheld link; the legacy tool `alice_vnext_recent_memory_commits` lists commit rows with no row-level fence; the provenance links of artifacts are not held to this fence; the operator routes `GET /v0/vnext/memories/{id}/audit`, `GET /v0/vnext/memories/recent-commits` and `GET /v0/vnext/sources/{id}`, which only the owner and a `trusted_local_agent` or `admin_agent` key bound to no project reach, return what was stored (that key can read the whole source from the last of them); and on an install with no agent keys, a call that declares a restricted profile is held to it by `alice_memory_review` by id but not by `alice_explain`, which keeps its old tolerance for calls that are not key-bound (such a call can leave the declaration out and be the owner). The pack's `sources` section and `alice_recall` are a different reader, the source's own excerpt, and the entry on a request that names no `domains` holds them to the domains of a key that names none. The HTTP routes and the Postgres store run on Postgres only and Postgres was not run: the routes were run over the SQLite store, the reader uses only store methods both backends have (`list_provenance_links_for_targets`, `get_sources_by_ids`, with `include_deleted` on both, and `get_source`), and `tests/integration/test_saved_quotes_postgres.py` runs the same lifecycle, and one for a memory with no link, one for two links with the same quote, one for a nested ref and one for an archived source under a free key with a revision, in the Postgres CI job. diff --git a/apps/api/src/alicebot_api/routers/_vnext_shared.py b/apps/api/src/alicebot_api/routers/_vnext_shared.py index 0ba0ac414..75b45e1f2 100644 --- a/apps/api/src/alicebot_api/routers/_vnext_shared.py +++ b/apps/api/src/alicebot_api/routers/_vnext_shared.py @@ -335,7 +335,8 @@ def _vnext_load_source_trace( open_loop_ids=[str(open_loop["id"]) for open_loop in open_loops], limit=limit, ), caller, - admit=lambda rows: [event for event in rows if str(event.get("target_id") or "") in kept_ids], + admit=(lambda rows: [event for event in rows if str(event.get("target_id") or "") in kept_ids]) + if SourceReadFence.for_identity(caller).entity_read_fenced else None, ) events_complete = direct_events_complete and memories_complete and artifacts_complete and open_loops_complete return _vnext_source_trace( diff --git a/apps/api/src/alicebot_api/routers/vnext_memories.py b/apps/api/src/alicebot_api/routers/vnext_memories.py index 587ce8208..5df218dba 100644 --- a/apps/api/src/alicebot_api/routers/vnext_memories.py +++ b/apps/api/src/alicebot_api/routers/vnext_memories.py @@ -774,6 +774,9 @@ def get_vnext_source(source_id: UUID, user_id: UUID, authorization: str | None = identity = resolve_protected_agent_identity( store, user_id=user_id, raw_key=agent_key_from_authorization(authorization if isinstance(authorization, str) else None), payload={}) + operator = _vnext_policy_checked(store=store, identity=identity, action="http.operator.access") + if operator.decision == "blocked": + return _vnext_permission_response(operator) payload = store.get_source(str(source_id)) if payload is not None and not SourceReadFence.for_identity(identity).admits( effective_row_for_fence(store, identity, "source", payload) @@ -1337,7 +1340,9 @@ def review_vnext_memory( identity=identity, stage=f"http_review_{action}", ) - if action == "assign_project" and request.project_id is not None: + from alicebot_api.vnext_project_scope import resolve_project_scope + + if action == "assign_project" and request.project_id is not None and request.project_id in resolve_project_scope(updated).values: store.create_edge( { "from_type": "memory", diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index 499da32ac..08d0ba617 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -99,10 +99,13 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti requested_projects = identity.project_scope if identity is not None else () all_of = requested_projects if identity is not None and identity.project_scope_locked else None guard = LabelGuard.for_filters(store, (), sensitivity_allowed, requested_projects, all_of=all_of) + unfenced = not SourceReadFence.for_identity(identity).entity_read_fenced + if unfenced: + guard = LabelGuard(store=store, active=False) review_statuses = ["candidate", "needs_review", "private_only", "accepted", "rejected"] fetched_sources = store.list_sources(sensitivity_allowed=sensitivity_allowed, limit=20) sources = guard.admit_rows("source", fetched_sources) - source_count = sum(guard.readable_status_counts("source").values()) + source_count = store.count_sources() if unfenced else sum(guard.readable_status_counts("source").values()) list_memories_by_statuses = getattr(store, "list_memories_by_statuses", None) if callable(list_memories_by_statuses): fetched_memories = list_memories_by_statuses( @@ -115,24 +118,24 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti memory for memory in store.list_memories(status=None) if str(memory.get("status")) in set(review_statuses) ][:30] review_memories = guard.admit_rows("memory", fetched_memories) - memory_status_counts = guard.readable_status_counts("memory") + memory_status_counts = store.count_memories_by_status(sensitivity_allowed=sensitivity_allowed) if unfenced else guard.readable_status_counts("memory") review_memory_total = sum(memory_status_counts.get(status, 0) for status in review_statuses) fetched_artifacts = store.list_artifacts(sensitivity_allowed=sensitivity_allowed, limit=30) artifacts = guard.admit_rows("artifact", fetched_artifacts) - artifact_status_counts = guard.readable_status_counts("artifact") + artifact_status_counts = store.count_artifacts_by_status() if unfenced else guard.readable_status_counts("artifact") artifact_count = sum(artifact_status_counts.values()) quality_evals = guard.admit_related_rows(store.list_artifact_quality_ratings(limit=50), kind="artifact", field="artifact_id") - quality_eval_count = sum( + quality_eval_count = store.count_artifact_quality_ratings() if unfenced else sum( len(guard.admit_related_rows(batch, kind="artifact", field="artifact_id")) for batch in store.iter_label_ratings() ) fetched_projects = store.list_projects(status=None, sensitivity_allowed=sensitivity_allowed, limit=20) projects = guard.admit_rows("project", fetched_projects) - project_count = sum(guard.readable_status_counts("project").values()) + project_count = store.count_projects() if unfenced else sum(guard.readable_status_counts("project").values()) fetched_loops = store.list_open_loops(status=None, sensitivity_allowed=sensitivity_allowed, limit=30) open_loops = guard.admit_rows("open_loop", fetched_loops) open_loops = withhold_unreadable_references(store, open_loops, fence=SourceReadFence.for_identity(identity)) - open_loop_status_counts = guard.readable_status_counts("open_loop") + open_loop_status_counts = store.count_open_loops_by_status() if unfenced else guard.readable_status_counts("open_loop") open_loop_count = open_loop_status_counts.get("open", 0) people = store.list_people(sensitivity_allowed=sensitivity_allowed, limit=12) fetched_beliefs = store.list_beliefs(status=None, sensitivity_allowed=sensitivity_allowed, limit=12) @@ -140,7 +143,7 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti tasks = store.list_tasks(status=None, limit=12) fetched_events = store.list_events(limit=20) recent_events = guard.admit_events(fetched_events) - event_count = guard.readable_event_count() + event_count = store.count_events() if unfenced else guard.readable_event_count() agent_identities = store.list_agent_identities(limit=20) agent_count = store.count_agent_identities() agent_events = guard.admit_events(store.list_agent_events(limit=50)) @@ -162,8 +165,8 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti scheduler_status = VNextSchedulerService(store).status() scheduler_status = {**scheduler_status, "daemon": daemon_status()} connector_health = VNextConnectorService(store).connector_health_all() - dogfooding = VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(sensitivity_allowed), label_guard=guard) - doctor = VNextDoctorService(store).run(ci=True, include_content_diagnostics=False) + dogfooding = VNextDogfoodingService(store).dashboard() if unfenced else VNextDogfoodingService(store).dashboard(sensitivity_allowed=tuple(sensitivity_allowed), label_guard=guard) + doctor = VNextDoctorService(store).run(ci=True, include_content_diagnostics=unfenced) policy_telemetry = summarize_agent_policy_telemetry( agent_events=agent_events, artifacts=artifacts, diff --git a/apps/api/src/alicebot_api/sqlite_schema.py b/apps/api/src/alicebot_api/sqlite_schema.py index 3d8c688a2..569b967f3 100644 --- a/apps/api/src/alicebot_api/sqlite_schema.py +++ b/apps/api/src/alicebot_api/sqlite_schema.py @@ -823,6 +823,18 @@ def _sql_list(values: tuple[str, ...]) -> str: ON sources (user_id, captured_at DESC, id DESC) """, """ + CREATE INDEX IF NOT EXISTS sources_user_label_alias_idx + ON sources (user_id, replace(replace(replace(replace(lower(id),'urn:uuid:',''),'-',''),'{',''),'}','')) + """, + """ + CREATE INDEX IF NOT EXISTS memories_user_label_alias_idx + ON memories (user_id, replace(replace(replace(replace(lower(id),'urn:uuid:',''),'-',''),'{',''),'}','')) + """, + """ + CREATE INDEX IF NOT EXISTS open_loops_user_label_alias_idx + ON open_loops (user_id, replace(replace(replace(replace(lower(id),'urn:uuid:',''),'-',''),'{',''),'}','')) + """, + """ CREATE INDEX IF NOT EXISTS source_chunks_source_index_idx ON source_chunks (source_id, chunk_index) """, diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index c8b6e82b8..01ff1ed1b 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -407,7 +407,13 @@ def _direct_source_hint(raw: object) -> str | None: return None source_id = metadata.get("source_id") if not isinstance(source_id, str) or not source_id.strip(): - return None + record = metadata.get("derived_from") + sources = record.get("sources") if isinstance(record, Mapping) else None + if not isinstance(sources, list) or not sources or not isinstance(sources[0], str): + return None + # A named source in a canonical record is an input even when another + # input makes the row unverified. This is rejection only, never a grant. + source_id = sources[0] try: return str(UUID(source_id)) except (ValueError, TypeError): @@ -423,6 +429,35 @@ def _ensure_direct_source_hint(conn: sqlite3.Connection) -> None: cursor.close() if not registered: conn.create_function("alice_direct_source_hint", 1, _direct_source_hint, deterministic=True) + cursor = conn.execute("SELECT 1 FROM pragma_function_list WHERE name='alice_direct_memory_hint' AND narg=1 LIMIT 1") + try: + memory_registered = cursor.fetchone() is not None + finally: + cursor.close() + if not memory_registered: + conn.create_function("alice_direct_memory_hint", 1, _direct_memory_hint, deterministic=True) + + +def _direct_memory_hint(raw: object) -> str | None: + try: + metadata = json.loads(raw) if isinstance(raw, str) else raw + if isinstance(metadata, str): + metadata = json.loads(metadata) + except (ValueError, TypeError): + return None + if not isinstance(metadata, Mapping) or metadata.get("redacted") is True: + return None + consolidation = metadata.get("consolidation") + members = consolidation.get("cluster_member_ids") if isinstance(consolidation, Mapping) else None + if not isinstance(members, list) or not members: + record = metadata.get("derived_from") + members = record.get("memories") if isinstance(record, Mapping) else None + if not isinstance(members, list) or not members or not isinstance(members[0], str): + return None + try: + return str(UUID(members[0])) + except (ValueError, TypeError): + return members[0] class SQLiteVNextStore: @@ -764,17 +799,8 @@ def list_memory_events( if event_type_prefix is not None: prefix_sql = " AND e.event_type LIKE ?" params.append(f"{event_type_prefix}%") - from alicebot_api.vnext_derived_labels import SENSITIVITY_RANK - ceiling = max((SENSITIVITY_RANK.get(value, 0) for value in sensitivity_allowed or ()), default=0) - blocked = [value for value, rank in SENSITIVITY_RANK.items() if rank > ceiling] if sensitivity_allowed else [] - label_sql = "" - if blocked: - marks = self._placeholders(blocked) - label_sql = f""" AND m.sensitivity NOT IN ({marks}) AND NOT EXISTS ( - SELECT 1 FROM sources parent WHERE parent.user_id=m.user_id - AND parent.id=alice_direct_source_hint(m.metadata_json) - AND parent.sensitivity IN ({marks}))""" - params.extend((*blocked, *blocked)) + from alicebot_api.vnext_label_sql import hidden_memory_input_sql + label_sql = " AND " + hidden_memory_input_sql(sensitivity_allowed, sqlite=True) params.extend(project_params) people_sql = "" if people or person_ids: diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 8261bfdfc..1354b7579 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -978,6 +978,11 @@ def dependency_label_signature(kind: str, row: Mapping[str, object]) -> tuple: label.carries_scope, str(row.get("user_id") or "")) +def has_implicit_weekly_inputs(kind: str, row: Mapping[str, object]) -> bool: + """Recognize implicit candidate ancestry through canonical metadata decoding.""" + return canon_kind(kind) == "artifact" and bool(_metadata(row).get("candidate_memory_ids")) + + def _copy_scope(stored: tuple[str, ...], parents: Sequence[SettledLabel]) -> tuple[str, ...]: live = [item for item in parents if item.carries_scope] if not parents: @@ -1063,6 +1068,28 @@ def _changed(before: SettledLabel, after: SettledLabel) -> bool: ) +def settle_verified_inputs(kind: str, row: Mapping[str, object], parents: Sequence[SettledLabel]) -> SettledLabel: + """Apply the kernel rule to a complete, verified direct input set. + + The read guard separately proves unique stored identities, acyclic ancestry + and the per-origin bounds. Implicit weekly artifact inputs must use the + complete graph kernel. This helper does not accept partial input lists. + """ + + label = _node_label(canon_kind(kind), row) + if not label.derived: + if parents: + raise ValueError("original labels have no inputs") + return label + refs, problem = dependency_record(kind, row) + supplied = {(item.kind, item.normalized_id) for item in parents} + if problem or supplied != set(refs) or any(item.unverified or item.user_id != label.user_id for item in parents): + raise ValueError("verified labels require the complete input set") + return _apply_dependencies(label, parents, domain_fallback=label.stored_domain, + sensitivity_fallback=label.stored_sensitivity, + scope_fallback=label.stored_scope, floor_fallback=label.stored_floor) + + def _weekly_parent_deps( labels: Mapping[tuple[str, str, str], SettledLabel], own: dict[tuple[str, str, str], set[tuple[str, str, str]]], diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index afa875724..74ed2070d 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -23,15 +23,19 @@ from alicebot_api.vnext_derived_labels import ( HOP_BOUND, NODE_BOUND, + SENSITIVITY_RANK, + SettledLabel, canon_kind, dependencies_of, dependency_label_signature, dependency_syntax_key, label_metadata_cache, identifier, + has_implicit_weekly_inputs, is_derived, input_admitted, settle_labels, + settle_verified_inputs, ) from alicebot_api.vnext_label_closure import collect_label_rows from alicebot_api.vnext_project_scope import project_floor_shape, project_scope_identity, project_scopes_overlap, resolve_project_scope @@ -63,6 +67,8 @@ class _RequestLabels: source_admission: dict = field(default_factory=dict) normalized_metadata: dict = field(default_factory=dict) row_keys: dict = field(default_factory=dict) + resolved_inputs: dict = field(default_factory=dict) + parent_labels: dict = field(default_factory=dict) def clear(self): self.nodes.clear() @@ -79,6 +85,8 @@ def clear(self): self.source_admission.clear() self.normalized_metadata.clear() self.row_keys.clear() + self.resolved_inputs.clear() + self.parent_labels.clear() _REQUEST_LABELS: ContextVar[tuple[Any, _RequestLabels] | None] = ContextVar("request_labels", default=None) @@ -210,6 +218,10 @@ def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[ label = state.labels.get(key) template = self._signature(kind, row, key=key) root = (canon_kind(kind), identifier(row.get("id"))) + if label is None: + direct = self._settled_inputs(kind, row, frozenset()) + if direct is not None: + label = direct[0] if label is None and root not in state.dependency_ancestry.get(template, ()): label = state.dependency_labels.get(template) if label is None: @@ -251,8 +263,7 @@ def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[ ) implicit_parent = False for node in nodes: - metadata = node.get("metadata_json") - if node.get("kind") == "artifact" and isinstance(metadata, Mapping) and metadata.get("candidate_memory_ids"): + if has_implicit_weekly_inputs(str(node["kind"]), node): implicit_parent = True break # Complete verified ancestry has already passed the per-root hop, @@ -286,31 +297,78 @@ def effective_row(self, kind: str, row: Mapping[str, object] | None) -> Mapping[ copy["project_id"] = None return copy + def _settled_inputs(self, kind: str, row: Mapping[str, object], trail: frozenset) -> tuple[SettledLabel, frozenset, int] | None: + """Settle verified acyclic parents once, sharing only label semantics. + + Missing/ambiguous inputs, cycles, implicit weekly parents and exceeded + per-origin bounds fall back to the complete canonical graph walk. + Admission remains caller-specific and is never stored in this cache. + """ + + state = self._state() + key = self._key(kind, row) + cached = state.parent_labels.get(key) + root = (canon_kind(kind), identifier(row.get("id"))) + if root in trail or len(trail) > HOP_BOUND or kind == "belief": + return None + if cached is not None: + return None if cached[1] & trail else cached + template = self._signature(kind, row, key=key) + refs, problem = template[1:3] + if problem or has_implicit_weekly_inputs(kind, row): + return None + reader = getattr(self.store, "read_label_rows", None) + if refs and not callable(reader): + return None + parents = [] + ancestry = {root} + depth = 0 + for parent_kind, parent_id in sorted(refs): + if parent_kind == "belief": + return None + parent_key = (parent_kind, parent_id) + if parent_key not in state.nodes and callable(reader): + state.nodes[parent_key] = [dict(found) for found in reader(parent_kind, [parent_id]) + if identifier(found.get("id")) == parent_id] + raw = state.nodes[parent_key] + if len(raw) != 1 or identifier(raw[0].get("id")) != parent_id: + return None + parent = self._settled_inputs(parent_kind, raw[0], trail | {root}) + if parent is None or root in parent[1]: + return None + parents.append(parent[0]) + ancestry.update(parent[1]) + depth = max(depth, parent[2] + 1) + if depth > HOP_BOUND or len(ancestry) > NODE_BOUND: + return None + semantic_parents = tuple((parent.kind, parent.domain, parent.sensitivity, parent.project_scope, + parent.project_floor, parent.carries_scope) for parent in parents) + # Root rule and stored labels plus resolved input labels, excluding IDs. + semantic = (template[:1] + template[2:], semantic_parents) + label = state.resolved_inputs.get(semantic) + if label is None: + label = settle_verified_inputs(kind, {**dict(row), "user_id": _GUARD_USER}, parents) + state.resolved_inputs[semantic] = label + else: + label = replace(label, stored_id=str(row.get("id") or ""), normalized_id=root[1]) + result = (label, frozenset(ancestry), depth) + state.parent_labels[key] = result + return result + def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: """Rows whose effective labels pass this guard's filters. Originals of the rows, not copies.""" if not self.active: return [row for row in rows if isinstance(row, Mapping)] state = self._state() - reader = getattr(self.store, "read_label_rows", None) - wanted: dict[str, set[str]] = {} + if self.sensitivity_allowed: + highest = max(SENSITIVITY_RANK.get(value, SENSITIVITY_RANK["unknown"]) for value in self.sensitivity_allowed) + rows = [row for row in rows if isinstance(row, Mapping) + and SENSITIVITY_RANK.get(str(row.get("sensitivity") or "unknown"), SENSITIVITY_RANK["unknown"]) <= highest] for row in rows: if isinstance(row, Mapping): state.targets[(kind, str(row.get("id")))] = row - if not is_derived(kind, row): - continue - refs = self._signature(kind, row, key=self._key(kind, row))[1] - for ref_kind, ref_id in refs: - if (ref_kind, ref_id) not in state.nodes: - wanted.setdefault(ref_kind, set()).add(ref_id) - if callable(reader): - for ref_kind, ids in wanted.items(): - for ref_id in ids: - state.nodes[(ref_kind, ref_id)] = [] - for found in reader(ref_kind, sorted(ids)): - canonical = identifier(found.get("id")) - if canonical in ids: - state.nodes[(ref_kind, canonical)].append(dict(found)) + self._prefetch_inputs(kind, rows) kept: list[_Row] = [] for row in rows: if not isinstance(row, Mapping): @@ -336,6 +394,42 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: kept.append(row) return kept + def _prefetch_inputs(self, kind: str, rows: Sequence[Mapping[str, object]]) -> None: + """Load ancestry by frontier; each origin is still verified separately. + + This cache contains raw rows including every stored alias. A bounded + batch stops prefetching rather than changing the kernel's per-root + hop/node limits; unresolved inputs use the normal collector. + """ + reader = getattr(self.store, "read_label_rows", None) + if not callable(reader): + return + state = self._state() + frontier = [(kind, row) for row in rows if isinstance(row, Mapping)] + expanded: set[tuple[str, str]] = set() + for _ in range(HOP_BOUND + 1): + refs: set[tuple[str, str]] = set() + for row_kind, row in frontier: + if is_derived(row_kind, row): + refs.update(self._signature(row_kind, row, key=self._key(row_kind, row))[1]) + refs.difference_update(expanded) + if not refs or len(expanded | refs) > NODE_BOUND: + return + expanded.update(refs) + wanted: dict[str, set[str]] = {} + for ref_kind, ref_id in refs: + if (ref_kind, ref_id) not in state.nodes: + wanted.setdefault(ref_kind, set()).add(ref_id) + for ref_kind, ids in wanted.items(): + for ref_id in ids: + state.nodes[(ref_kind, ref_id)] = [] + for found in reader(ref_kind, sorted(ids)): + canonical = identifier(found.get("id")) + if canonical in ids: + state.nodes[(ref_kind, canonical)].append(dict(found)) + frontier = [(ref_kind, row) for ref_kind, ref_id in refs + for row in state.nodes[(ref_kind, ref_id)]] + def readable_status_counts(self, kind: str) -> dict[str, int]: """Count the complete population through the same effective admission. @@ -351,6 +445,11 @@ def readable_status_counts(self, kind: str) -> dict[str, int]: key = (kind, self.domains, self.sensitivity_allowed, self.projects, self.all_of) if key in state.counts: return dict(state.counts[key]) + native = getattr(self.store, "count_" + {"memory": "memories", "artifact": "artifacts", "open_loop": "open_loops"}.get(kind, kind) + "_by_status", None) + if not self.active and callable(native): + counts = native() + state.counts[key] = dict(counts) + return counts plain_counter = getattr(self.store, "count_original_label_statuses", None) if callable(getattr(type(self.store), "count_original_label_statuses", None)) and callable(plain_counter) and not self.projects and self.all_of is None: counts = plain_counter(kind, domains=self.domains, sensitivity_allowed=self.sensitivity_allowed) diff --git a/apps/api/src/alicebot_api/vnext_label_sql.py b/apps/api/src/alicebot_api/vnext_label_sql.py index 242f2b07e..e34220cf9 100644 --- a/apps/api/src/alicebot_api/vnext_label_sql.py +++ b/apps/api/src/alicebot_api/vnext_label_sql.py @@ -1,5 +1,62 @@ """Conservative SQL partition: anything uncertain still reaches the kernel.""" -from alicebot_api.vnext_derived_labels import MARKER_KEYS, DERIVED_ARTIFACT_TYPES +from alicebot_api.vnext_derived_labels import MARKER_KEYS, DERIVED_ARTIFACT_TYPES, SENSITIVITY_RANK + + +def hidden_memory_input_sql(sensitivity_allowed, *, sqlite: bool, alias: str = "m") -> str: + """Reject only a proved source floor above the highest requested rank. + + A direct canonical source, or a direct source of one recorded memory + input, suffices for rejection. Anything else reaches effective admission. + This predicate never grants a row and never filters by a derived domain. + """ + + if alias not in {"m", "memories"}: + raise ValueError("unsupported memory alias") + ceiling = max((SENSITIVITY_RANK.get(value, SENSITIVITY_RANK["unknown"]) for value in sensitivity_allowed or ()), default=0) + blocked = [value for value, rank in SENSITIVITY_RANK.items() if rank > ceiling] if sensitivity_allowed else [] + if not blocked: + return "TRUE" + names = ",".join("'" + value + "'" for value in blocked) # closed kernel constants + if sqlite: + source = f"alice_direct_source_hint({alias}.metadata_json)" + memory = f"alice_direct_memory_hint({alias}.metadata_json)" + parent_source = "alice_direct_source_hint(label_input.metadata_json)" + else: + def source_hint(meta): + return f"COALESCE({meta}->>'source_id', CASE WHEN jsonb_typeof({meta}->'derived_from'->'sources')='array' AND jsonb_typeof({meta}->'derived_from'->'sources'->0)='string' THEN {meta}->'derived_from'->'sources'->>0 END)" + def uuid_hint(value): + return f"CASE WHEN ({value}) ~* '^(?:[0-9a-f]{{32}}|[0-9a-f]{{8}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{12}})$' THEN ({value})::uuid END" + meta = f"{alias}.metadata_json" + source = uuid_hint(source_hint(meta)) + memory = uuid_hint(f"COALESCE(CASE WHEN jsonb_typeof({meta}->'consolidation'->'cluster_member_ids')='array' AND jsonb_typeof({meta}->'consolidation'->'cluster_member_ids'->0)='string' THEN {meta}->'consolidation'->'cluster_member_ids'->>0 END, CASE WHEN jsonb_typeof({meta}->'derived_from'->'memories')='array' AND jsonb_typeof({meta}->'derived_from'->'memories'->0)='string' THEN {meta}->'derived_from'->'memories'->>0 END)") + parent_source = uuid_hint(source_hint("label_input.metadata_json")) + not_redacted = "TRUE" if sqlite else f"{alias}.metadata_json->'redacted' IS DISTINCT FROM 'true'::jsonb" + input_not_redacted = "TRUE" if sqlite else "label_input.metadata_json->'redacted' IS DISTINCT FROM 'true'::jsonb" + if not sqlite: + # Uncorrelated row-valued sets are hashed once by PostgreSQL. Include + # tenant identity in each set rather than scanning parents per row. + return f"""({alias}.sensitivity NOT IN ({names}) AND NOT ( + {not_redacted} AND ( + COALESCE(({alias}.user_id, {source}) IN ( + SELECT user_id, id FROM sources WHERE sensitivity IN ({names}) + ), FALSE) + OR COALESCE(({alias}.user_id, {memory}) IN ( + SELECT label_input.user_id, label_input.id FROM memories label_input + JOIN sources label_source ON label_source.user_id=label_input.user_id + AND label_source.id={parent_source} + WHERE {input_not_redacted} AND label_source.sensitivity IN ({names}) + ), FALSE) + )))""" + return f"""({alias}.sensitivity NOT IN ({names}) AND NOT ( + {not_redacted} AND (EXISTS ( + SELECT 1 FROM sources label_source WHERE label_source.user_id={alias}.user_id + AND label_source.id={source} AND label_source.sensitivity IN ({names}) + ) OR EXISTS ( + SELECT 1 FROM memories label_input JOIN sources label_source + ON label_source.user_id=label_input.user_id AND label_source.id={parent_source} + WHERE label_input.user_id={alias}.user_id AND label_input.id={memory} + AND {input_not_redacted} AND label_source.sensitivity IN ({names}) + ))))""" def original_label_sql(kind: str, *, sqlite: bool = False) -> str: diff --git a/apps/api/src/alicebot_api/vnext_label_writes.py b/apps/api/src/alicebot_api/vnext_label_writes.py index 7dd394b8b..fc70ab8dd 100644 --- a/apps/api/src/alicebot_api/vnext_label_writes.py +++ b/apps/api/src/alicebot_api/vnext_label_writes.py @@ -257,6 +257,11 @@ def prepare_label_patch( invalidate_capture_label_inputs(store) proposed = dict(before or {}) proposed.update({key: value for key, value in patch.items() if value is not None}) + metadata = patch.get("metadata_json") + if isinstance(metadata, Mapping): + for field in ("project_scope", "project_floor"): + if field in metadata: + proposed[field] = metadata[field] proposed["kind"] = kind old = _label_fields({**before, "kind": kind}) if before else None new = _label_fields(proposed) @@ -598,7 +603,7 @@ def clamp_owner_patch( """Keep a derived row at or above its inputs when an edit would lower it. The store writes the higher label, records ``labels_raised`` with cause - ``floor_clamped`` when the stored label changes, and sets + ``floor_clamped`` when an assignment is refused, and sets ``store._label_floor_applied`` so the review answer can name it. """ @@ -627,6 +632,11 @@ def clamp_owner_patch( meta = dict(stored_meta) if isinstance(stored_meta, dict) else {} meta.update(patch_metadata) proposed["metadata_json"] = meta + # Store projections expose these aliases at the root, where the scope + # resolver reads them first. Judge the requested labels, not old ones. + for field in ("project_scope", "project_floor"): + if field in patch_metadata: + proposed[field] = meta[field] proposed["kind"] = kind nodes, exceeded = collect_label_rows(store, [proposed], max_nodes=PROPAGATION_BOUND) if exceeded: @@ -653,13 +663,15 @@ def clamp_owner_patch( metadata["project_scope"] = list(label.project_scope) metadata["project_floor"] = list(label.project_floor) proposed_patch["metadata_json"] = metadata + from alicebot_api.vnext_label_repair import label_project_id + + proposed_patch["project_id"] = label_project_id(label.project_scope) + if "project_id" in metadata: + metadata["project_id"] = proposed_patch["project_id"] stored = _label_fields(before) - if not ( - stored[0] == settled[0] - and stored[1] == settled[1] - and project_scope_identity(stored[2]) == project_scope_identity(settled[2]) - and project_scope_identity(stored[3]) == project_scope_identity(settled[3]) - ): + # A refused assignment is auditable even when the existing label was + # already settled and the clamp therefore preserves it exactly. + if requested != settled: require_exclusive_label_lock(store) event = build_event_log_record( event_type=f"{kind}.labels_raised", diff --git a/apps/api/src/alicebot_api/vnext_retrieval.py b/apps/api/src/alicebot_api/vnext_retrieval.py index 350186e61..f190eb17d 100644 --- a/apps/api/src/alicebot_api/vnext_retrieval.py +++ b/apps/api/src/alicebot_api/vnext_retrieval.py @@ -2710,6 +2710,7 @@ def _memories_by_ids( domains: Sequence[str] | None = None, sensitivity_allowed: Sequence[str] | None = None, projects: Sequence[str] | None = None, + effective: bool = True, ) -> dict[str, JsonObject]: normalized_ids = tuple(dict.fromkeys(str(memory_id) for memory_id in memory_ids if memory_id)) if not normalized_ids: @@ -2743,7 +2744,7 @@ def _memories_by_ids( domains=domains, sensitivity_allowed=sensitivity_allowed, projects=projects, - ) + ) if effective else rows return {str(row.get("id")): row for row in rows} def _sources_by_ids(self, source_ids: Sequence[str]) -> dict[str, JsonObject]: @@ -3079,7 +3080,7 @@ def _memory_fts_rows( filters = _optional_search_filters(memory_types, projects, created_by_agent_ids, run_id) scope_filters: dict[str, object] = {} search_memories_fts = getattr(self.store, "search_memories_fts", None) - active_search = search_memories_fts if callable(search_memories_fts) else self.store.search_memories + active_search = cast(Callable[..., list[JsonObject]], search_memories_fts if callable(search_memories_fts) else self.store.search_memories) effective_people = tuple(sorted(scope.people)) if scope is not None else scope_people effective_window_start = scope.window_start if scope is not None else scope_window_start effective_window_end = scope.window_end if scope is not None else scope_window_end @@ -3101,82 +3102,34 @@ def _memory_fts_rows( "scope_window_start": effective_window_start, "scope_window_end": effective_window_end, } - if callable(search_memories_fts): - rows = search_memories_fts( - query=query, - domains=domains or None, - sensitivity_allowed=sensitivity_allowed, - limit=limit, - **filters, - **scope_filters, - ) - # Display-only trace label; SQLite stores override it via - # ``fts_stage_source`` so traces do not claim a Postgres stage. - fts_source = str(getattr(self.store, "fts_stage_source", "postgres_fts")) - if not rows and len(fts_fallback_tokens(query)) >= 2: - # Strict AND semantics found nothing for a multi-word query. - # With no embeddings configured (the default first-hour - # setup) FTS is the whole recall path, so a natural-language - # question would return zero results against memories a - # keyword query finds instantly. Retry once with OR - # semantics; the source string keeps the trace honest about - # the relaxed pass, and fallback rows join RRF fusion - # exactly like strict FTS rows. Single-token queries skip - # the retry (OR and AND are identical there), and a strict - # hit above never reaches this branch. - try: - rows = search_memories_fts( - query=query, - domains=domains or None, - sensitivity_allowed=sensitivity_allowed, - limit=limit, - match_any=True, - **filters, - **scope_filters, - ) - except TypeError: - # Store predates the match_any kwarg; keep the strict - # (empty) result rather than guessing. - return [], fts_source - rows = admit_loaded( - self.store, - kind="memory", - rows=rows, - domains=domains, - sensitivity_allowed=sensitivity_allowed, - projects=projects, - ) - return _stabilize_scored_rows(rows), f"{fts_source}_or_fallback" - rows = admit_loaded( - self.store, - kind="memory", - rows=rows, - domains=domains, - sensitivity_allowed=sensitivity_allowed, - projects=projects, - ) - return _stabilize_scored_rows(rows), fts_source - legacy_search = cast( - Callable[..., list[JsonObject]], - getattr(self.store, "search_memories"), - ) - rows = legacy_search( - query=query, - domains=domains or None, - sensitivity_allowed=sensitivity_allowed, - limit=limit, - **filters, - **scope_filters, - ) - rows = admit_loaded( - self.store, - kind="memory", - rows=rows, - domains=domains, - sensitivity_allowed=sensitivity_allowed, - projects=projects, + options = {"query": query, "domains": domains or None, + "sensitivity_allowed": sensitivity_allowed, **filters, **scope_filters} + fts = callable(search_memories_fts) + source = str(getattr(self.store, "fts_stage_source", "postgres_fts")) if fts else "store_lexical" + rows = active_search(limit=limit, **options) + if fts and not rows and len(fts_fallback_tokens(query)) >= 2: + # Preserve the strict-AND to OR fallback and its disclosed stage. + try: + options["match_any"] = True + rows = active_search(limit=limit, **options) + except TypeError: + return [], source + source += "_or_fallback" + + def fetch(prefix_limit): + raw = rows if prefix_limit == limit else active_search(limit=prefix_limit, **options) + return raw, source + + selected, source = _fetch_filtered_prefix( + fetch, + select_rows=lambda raw: admit_loaded( + self.store, kind="memory", rows=raw, domains=domains, + sensitivity_allowed=sensitivity_allowed, projects=projects, + ), + target=limit, initial_limit=limit, ) - return list(rows), "store_lexical" + selected = selected[:limit] + return (_stabilize_scored_rows(selected) if fts else selected), source def _query_embedding(self, query: str) -> tuple[list[float] | None, str]: if self.embedding_provider is None: @@ -5389,7 +5342,7 @@ def _select_events(rows: Sequence[JsonObject]) -> list[JsonObject]: ) ] targets = self._memories_by_ids( - [str(event.get("target_id") or "") for event in eligible] + [str(event.get("target_id") or "") for event in eligible], effective=False ) admitted_targets = { str(row.get("id")) diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index ce302282e..04da14814 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -782,9 +782,8 @@ def list_memory_events( people_list = [str(value).strip().casefold() for value in scope_people if str(value).strip()] or None person_memory_ids = [str(value) for value in scope_person_memory_ids if str(value)] or None prefix_pattern = f"{event_type_prefix}%" if event_type_prefix is not None else None - from alicebot_api.vnext_derived_labels import SENSITIVITY_RANK - ceiling = max((SENSITIVITY_RANK.get(value, 0) for value in sensitivity_allowed or ()), default=0) - blocked = [value for value, rank in SENSITIVITY_RANK.items() if rank > ceiling] if sensitivity_allowed else None + from alicebot_api.vnext_label_sql import hidden_memory_input_sql + label_sql = hidden_memory_input_sql(sensitivity_allowed, sqlite=False) return self._fetch_all( f""" SELECT @@ -804,18 +803,7 @@ def list_memory_events( JOIN memories m ON e.target_type = 'memory' AND e.target_id = m.id::text - AND (%s::text[] IS NULL OR ( - NOT (m.sensitivity = ANY(%s::text[])) - AND NOT EXISTS ( - SELECT 1 FROM sources parent - WHERE parent.id = CASE - WHEN m.metadata_json->>'source_id' ~* '^(?:[0-9a-f]{{32}}|[0-9a-f]{{8}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{12}})$' - THEN (m.metadata_json->>'source_id')::uuid - END - AND m.metadata_json->>'redacted' IS DISTINCT FROM 'true' - AND parent.sensitivity = ANY(%s::text[]) - ) - )) + AND {label_sql} AND e.user_id = m.user_id WHERE m.deleted_at IS NULL AND (%s::text IS NULL OR e.event_type LIKE %s) @@ -831,9 +819,6 @@ def list_memory_events( LIMIT %s """, ( - blocked, - blocked, - blocked, prefix_pattern, prefix_pattern, project_list, diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py index 9d4690d72..e3e77609d 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py @@ -889,12 +889,15 @@ def search_memories_fts( else: tsquery_sql = "websearch_to_tsquery('english', %s)" tsquery_text = query + from alicebot_api.vnext_label_sql import hidden_memory_input_sql + label_sql = hidden_memory_input_sql(sensitivity_allowed, sqlite=False, alias="memories") return self._fetch_all( f""" SELECT {MEMORY_COLUMNS}, ts_rank(search_tsv, {tsquery_sql}) AS fts_score FROM memories WHERE deleted_at IS NULL + AND {label_sql} AND status IN {_MEMORY_SEARCHABLE_STATUSES_SQL} AND (%s::text[] IS NULL OR domain = ANY(%s::text[]) OR domain = 'unknown') AND (%s::text[] IS NULL OR sensitivity = ANY(%s::text[])) diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py index 0287fe060..f34673363 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py @@ -458,6 +458,10 @@ def update_memory( patch = prepare_label_patch(self, "memory", before_label, patch) patch = clamp_owner_patch(self, kind="memory", before=before_label, patch=patch) + project_metadata = patch.get("metadata_json") + project_patch_present = patch.get("project_id") is not None or ( + "project_id" in patch and isinstance(project_metadata, dict) and "project_scope" in project_metadata + ) row = self._fetch_one( "update_memory", f""" @@ -486,7 +490,7 @@ def update_memory( last_seen_at = COALESCE(%s, last_seen_at), last_reviewed_at = COALESCE(%s, last_reviewed_at), metadata_json = COALESCE(%s, metadata_json), - project_id = COALESCE(%s, project_id), + project_id = CASE WHEN %s THEN %s ELSE project_id END, superseded_by = COALESCE(%s::uuid, superseded_by), supersedes = COALESCE(%s::uuid, supersedes), updated_at = clock_timestamp(), @@ -523,6 +527,7 @@ def update_memory( patch.get("last_seen_at"), patch.get("last_reviewed_at"), _json_object(patch["metadata_json"]) if "metadata_json" in patch else None, + project_patch_present, patch.get("project_id"), patch.get("superseded_by"), patch.get("supersedes"), diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py index 2a019adae..6057f2ed7 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py @@ -1258,6 +1258,8 @@ def search_memories_fts( prefix="m.", ) prefixed_columns = ", ".join(f"m.{column}" for column in MEMORY_COLUMNS) + from alicebot_api.vnext_label_sql import hidden_memory_input_sql + label_sql = hidden_memory_input_sql(sensitivity_allowed, sqlite=True) params: list[object] = [match_expression, self.user_id] params.extend(domain_params) params.extend(sensitivity_params) @@ -1282,6 +1284,7 @@ def search_memories_fts( WHERE memories_fts MATCH ? AND m.user_id = ? AND m.deleted_at IS NULL + AND {label_sql} AND m.status IN {_MEMORY_SEARCHABLE_STATUSES_SQL}{domain_sql}{sensitivity_sql}{type_sql}{project_sql}{created_by_sql}{run_sql}{expiry_sql}{scope_sql} ORDER BY fts_score DESC, m.updated_at DESC, m.created_at DESC, m.id DESC LIMIT ? diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py index 728f8f359..980a28aca 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py @@ -308,6 +308,10 @@ def update_memory( from alicebot_api.vnext_label_writes import clamp_owner_patch patch = clamp_owner_patch(self, kind="memory", before=before_label, patch=patch) + project_metadata = patch.get("metadata_json") + project_patch_present = patch.get("project_id") is not None or ( + "project_id" in patch and isinstance(project_metadata, dict) and "project_scope" in project_metadata + ) # One clock reading for the write: an archive sets ``updated_at`` and ``deleted_at`` together. now = _utc_now_iso() cursor = self._execute( @@ -337,7 +341,7 @@ def update_memory( last_seen_at = COALESCE(?, last_seen_at), last_reviewed_at = COALESCE(?, last_reviewed_at), metadata_json = COALESCE(?, metadata_json), - project_id = COALESCE(?, project_id), + project_id = CASE WHEN ? THEN ? ELSE project_id END, superseded_by = COALESCE(?, superseded_by), supersedes = COALESCE(?, supersedes), updated_at = ?, @@ -374,6 +378,7 @@ def update_memory( _iso_or_none(patch.get("last_seen_at")), _iso_or_none(patch.get("last_reviewed_at")), _json_object_text(patch["metadata_json"]) if "metadata_json" in patch else None, + project_patch_present, patch.get("project_id"), _uuid_text(patch.get("superseded_by")), _uuid_text(patch.get("supersedes")), diff --git a/docs/alpha/mcp-tools.md b/docs/alpha/mcp-tools.md index 69a92bc92..16c83b1da 100644 --- a/docs/alpha/mcp-tools.md +++ b/docs/alpha/mcp-tools.md @@ -834,8 +834,9 @@ before a 32-digit id, read as a hyphenated id that names no row, so part of an i the reader may read is withheld there and wherever the same response repeats it. No id the reader may not read is shown by it. A value that is only an id is read the way the link writer reads it, which also takes hyphens -in other places. Inside longer text an id with its hyphens in other places, a -split id and an encoded id are not recognised. The free-text columns of a loop +in other places. Unreleased (on main, not in v0.20.0): a second source-only pass withholds +irregular, split and encoded source ids inside longer text. Irregular or +non-ASCII MEMORY-prefixed ids and unnamed SOURCE whitespace forms can remain. The free-text columns of a loop (`title`, `description`, `resolution_note`) are returned as stored and are not scanned. The extractor of candidate loops no longer writes the id of a source with no title into the `description` (it says the source has no title), and a @@ -945,10 +946,12 @@ the provenance links of artifacts are not held to this fence; the operator routes `GET /v0/vnext/memories/{id}/audit`, `GET /v0/vnext/memories/recent-commits` and `GET /v0/vnext/sources/{id}`, which only the owner and a `trusted_local_agent` or `admin_agent` key bound to no -project reach, return what was stored; and on an install with no agent keys, a +project reach, return what was stored subject to each route's current read fence; and on an install with no agent keys, a call that declares a restricted profile is held to it by `alice_memory_review` by id but not by `alice_explain`. +Unreleased (on main, not in v0.20.0): Source GET evaluates `http.operator.access` before its source fence. Only the owner and unbound trusted or admin keys reach the source lookup. Other profiles and project-bound keys receive HTTP 403, including for missing ids. Admitted callers receive the same HTTP 404 for a source outside their full domain, sensitivity and locked project fence as for a missing source. Owner and unbound admin source traces retain chunk and extraction events. Their workspace retains main's displayed rows, complete totals, unfiltered embedded dogfooding and full doctor diagnostics; fenced workspaces omit content diagnostics. + ## Domains a profile may read Every permission profile except `trusted_local_agent` and `admin_agent` is held diff --git a/docs/release/derived-labels-security-note-draft.md b/docs/release/derived-labels-security-note-draft.md index de1a0fe0c..473983008 100644 --- a/docs/release/derived-labels-security-note-draft.md +++ b/docs/release/derived-labels-security-note-draft.md @@ -2,12 +2,12 @@ Unreleased (on main, not in v0.20.0): the derived-label fixes tighten recorded-input inserts, restricted report inputs, exact reads, operator lists, and repair. This draft is for review before a release is tagged. -Unreleased (on main, not in v0.20.0): source GET and legacy `alice_vnext_review_items` now apply the caller's domain, sensitivity and locked project fence, including effective labels of recorded inputs. Graph neighborhood still returns edge explanations outside that ceiling; that behavior predates this set. Doctor's aggregate derived-label check counts are new in this set, absent from v0.20.0 and baseline main 48873b03, and remain visible to trusted keys. The owner should review that new count exposure separately before tagging. The five corrected screens and these two readers do not establish that every operator route observes the same ceiling. +Unreleased (on main, not in v0.20.0): source GET keeps the operator gate: only the owner and unbound trusted or admin keys reach the lookup; other profiles and project-bound keys receive 403. Behind that gate, source GET and legacy `alice_vnext_review_items` apply the caller's domain, sensitivity and locked project fence, including effective labels of recorded inputs. Graph neighborhood still returns edge explanations outside that ceiling; that behavior predates this set. Doctor's aggregate derived-label check counts are new in this set, absent from v0.20.0 and baseline main 48873b03, and remain outside the caller's sensitivity ceiling and visible to trusted keys. The owner should review that new count exposure separately before tagging. The five corrected screens and these two readers do not establish that every operator route observes the same ceiling. Unreleased (on main, not in v0.20.0): SQLite retirement uses the saved-quote reader's reference rules. Repeated JSON object keys are decoded with the last value retained, and quote subtrees are omitted by that reader. Retirement can consequently retain a loop that an older raw JSON scan would blank. The product does not write those two forms, but an imported or hand-edited row can contain them. Unreleased (on main, not in v0.20.0): capture overhead has only a few percentage points of headroom below the 15 percent budget. The earlier relabel measurements scale at roughly 2.4 ms per dependant over the measured sizes. Repeat those measurements on a quiet machine before tagging; the read budgets measure a separate path. -Unreleased (on main, not in v0.20.0): bounded retrieval, consolidation clustering and staleness sweeps can under-fill after restricted admission when newer hidden or shared rows consume their candidate window. Defaults are 2,000 embedded memories and 500 staleness rows. Complete effective-label counts use the full population, but retrieval completeness needs a separate refill and ranking change. +Unreleased (on main, not in v0.20.0): memory full-text retrieval refills after effective admission until its limit is met or candidates are exhausted. A legacy adapter that cannot prove exhaustion within 16,384 rows returns a completeness error. Consolidation clustering and staleness sweeps can still under-fill when hidden or shared rows consume their candidate window; their defaults are 2,000 embedded memories and 500 staleness rows. Complete effective-label counts use the full population. Unreleased (on main, not in v0.20.0): open-loop metadata withholding follows canonical saved-quote parsing for source ids. Its wider pass is source-only: irregular or non-ASCII memory-prefixed ids inside longer text, `alice://sources/` whitespace forms and equivalent imported spellings can retain the id. The product does not write these forms. Unknown trace ids and the loop's free-text columns remain as stored. At 20,000 loops the canonical reverse lookup costs about ten times the earlier scan in the external fixture; SQL text prefiltering is deferred because it must retain JSON escapes and every canonical parser spelling. diff --git a/docs/release/v0.20.0-release-notes.md b/docs/release/v0.20.0-release-notes.md index 279883aad..2935481d3 100644 --- a/docs/release/v0.20.0-release-notes.md +++ b/docs/release/v0.20.0-release-notes.md @@ -3,7 +3,7 @@ > **Correction (2026-10-05):** these notes do not say that a derived summary, report or copy kept the label its inputs had when it was made, or that a report could show a key bound to one project rows of other projects or with no project. Both are in v0.20.0 and are listed under known limitations. -> **Correction (2026-10-06):** three older operator or legacy readers expose withheld information in v0.20.0: source GET returns titles and raw text, graph neighborhood returns edge explanations, and legacy `alice_vnext_review_items` lists derived memories for a declared restricted identity. Unreleased corrections now fence source GET and the legacy review list. Doctor's derived-label check counts are new in this set, absent from v0.20.0 and baseline main 48873b03. See the [draft security note](derived-labels-security-note-draft.md) for the current boundary and SQLite retirement limits. +> **Correction (2026-10-06):** three older operator or legacy readers expose withheld information in v0.20.0: source GET returns titles and raw text, graph neighborhood returns edge explanations, and legacy `alice_vnext_review_items` lists derived memories for a declared restricted identity. Unreleased corrections now fence source GET and the legacy review list. Doctor's derived-label check counts are new in this set, absent from v0.20.0 and baseline main 48873b03, and remain outside the caller's sensitivity ceiling. See the [draft security note](derived-labels-security-note-draft.md) for the current boundary and SQLite retirement limits. **Take this release if you run the Postgres stack's HTTP API, use an embeddings endpoint (above all a hosted one), import ChatGPT or Markdown files, use the diff --git a/eval/scale/harness.py b/eval/scale/harness.py index 653ad1560..16c88d65d 100644 --- a/eval/scale/harness.py +++ b/eval/scale/harness.py @@ -505,7 +505,7 @@ def run_sweep() -> None: # materialization and clustering uses bounded float32 row blocks. cons_store = store if session.backend == "postgres" else ArtifactSinkStore(store) consolidation = VNextConsolidationService(cons_store, embedding_provider=provider) - cons_request = MemoryConsolidationRequest(agent_identity=None, ) + cons_request = MemoryConsolidationRequest(agent_identity=None) cons_notes: dict[str, object] = { "embedded_memory_hard_cap": MAX_EMBEDDED_MEMORIES_HARD_CAP, "artifact_persisted": session.backend == "postgres", diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json new file mode 100644 index 000000000..2fcaa0a41 --- /dev/null +++ b/scripts/derived_label_mutations.json @@ -0,0 +1,354 @@ +{ + "schema": 1, + "scope": "Round-three guard removals, each replayed from a green candidate in an isolated commit archive.", + "cases": [ + { + "name": "source-operator-gate", + "edits": [ + { + "file": "apps/api/src/alicebot_api/routers/vnext_memories.py", + "line": 778, + "before": "if operator.decision == \"blocked\":", + "after": "if False:", + "sha256": "860d8d74b5356ba0efe157186949f41bbedd7c56ffbd4335ca8740dbd1c150d9" + } + ], + "pytest": [ + "tests/integration/test_label_round2_reader_fences_postgres.py::test_source_get_uses_the_callers_entire_fence" + ], + "expected_failure": "test_source_get_uses_the_callers_entire_fence" + }, + { + "name": "source-caller-fence", + "edits": [ + { + "file": "apps/api/src/alicebot_api/routers/vnext_memories.py", + "line": 781, + "before": "if payload is not None and not SourceReadFence.for_identity(identity).admits(", + "after": "if False and payload is not None and not SourceReadFence.for_identity(identity).admits(", + "sha256": "860d8d74b5356ba0efe157186949f41bbedd7c56ffbd4335ca8740dbd1c150d9" + } + ], + "pytest": [ + "tests/integration/test_label_round2_reader_fences_postgres.py::test_source_get_uses_the_callers_entire_fence" + ], + "expected_failure": "test_source_get_uses_the_callers_entire_fence" + }, + { + "name": "unfenced-trace-events", + "edits": [ + { + "file": "apps/api/src/alicebot_api/routers/_vnext_shared.py", + "line": 339, + "before": "if SourceReadFence.for_identity(caller).entity_read_fenced else None,", + "after": "if True else None,", + "sha256": "17d1e4a84950e9646d7d41b044c0a691a0d077b589d46a44533978e944f35036" + } + ], + "pytest": [ + "tests/integration/test_label_round3_contracts_postgres.py::test_unfenced_source_trace_and_workspace_preserve_main", + "-k", + "trace" + ], + "expected_failure": "test_unfenced_source_trace_and_workspace_preserve_main" + }, + { + "name": "unfenced-workspace-parity", + "edits": [ + { + "file": "apps/api/src/alicebot_api/routers/workspaces.py", + "line": 102, + "before": "unfenced = not SourceReadFence.for_identity(identity).entity_read_fenced", + "after": "unfenced = False", + "sha256": "adf4c06a8434451f6de670ac2772ddd6d3f061e69b0d4ca56bafd36e4cd4ed13" + } + ], + "pytest": [ + "tests/integration/test_label_round3_contracts_postgres.py::test_unfenced_source_trace_and_workspace_preserve_main", + "-k", + "workspace" + ], + "expected_failure": "test_unfenced_source_trace_and_workspace_preserve_main" + }, + { + "name": "clamp-requested-scope-alias", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_writes.py", + "line": 639, + "before": "proposed[field] = meta[field]", + "after": "pass # removed requested scope alias", + "sha256": "a5ea46f1a0033b20b59851290352b052f5a92e46a70a21ba862cb015cda552ae" + } + ], + "pytest": [ + "tests/integration/test_label_round3_contracts_postgres.py::test_owner_scope_assignment_clamps_to_weekly_floor" + ], + "expected_failure": "test_owner_scope_assignment_clamps_to_weekly_floor" + }, + { + "name": "clamp-unchanged-attempt-event", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_writes.py", + "line": 674, + "before": "if requested != settled:", + "after": "if False:", + "sha256": "a5ea46f1a0033b20b59851290352b052f5a92e46a70a21ba862cb015cda552ae" + } + ], + "pytest": [ + "tests/integration/test_label_round3_contracts_postgres.py::test_owner_scope_assignment_clamps_to_weekly_floor" + ], + "expected_failure": "test_owner_scope_assignment_clamps_to_weekly_floor" + }, + { + "name": "postgres-clamped-project-null", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_stores/postgres/memory_lifecycle.py", + "line": 530, + "before": " project_patch_present,", + "after": " False,", + "sha256": "2b540dc4b52e74d564ef22ce9f9ec98a527d4e5a74f00497c33ab29d37bce109" + } + ], + "pytest": [ + "tests/integration/test_label_round3_contracts_postgres.py::test_owner_scope_assignment_clamps_to_weekly_floor", + "-k", + "True" + ], + "expected_failure": "test_owner_scope_assignment_clamps_to_weekly_floor" + }, + { + "name": "sqlite-clamped-project-null", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_stores/sqlite/memory_lifecycle.py", + "line": 381, + "before": " project_patch_present,", + "after": " False,", + "sha256": "0e8c2d50857d1064fc94b703c7cc05da234df63d4c278bc08a062fc90d450c3d" + } + ], + "pytest": [ + "tests/unit/test_label_round3_sqlite_reads.py::test_scope_clamp_clears_the_legacy_project_alias_and_records_the_attempt" + ], + "expected_failure": "test_scope_clamp_clears_the_legacy_project_alias_and_records_the_attempt" + }, + { + "name": "resolved-parent-label-key", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "line": 347, + "before": "semantic = (template[:1] + template[2:], semantic_parents)", + "after": "semantic = (template[:1] + template[2:], ())", + "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_random_mixed_deep_inputs_match_full_kernel_and_keep_distinct_domains" + ], + "expected_failure": "test_random_mixed_deep_inputs_match_full_kernel_and_keep_distinct_domains" + }, + { + "name": "resolved-parent-multiplicity", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "line": 347, + "before": "semantic = (template[:1] + template[2:], semantic_parents)", + "after": "semantic = (template[:1] + template[2:], tuple(sorted(set(semantic_parents))))", + "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_parent_label_multiplicity_changes_restricted_domain_selection" + ], + "expected_failure": "test_parent_label_multiplicity_changes_restricted_domain_selection" + }, + { + "name": "ambiguous-parent-refusal", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "line": 334, + "before": "if len(raw) != 1 or identifier(raw[0].get(\"id\")) != parent_id:", + "after": "if not raw or identifier(raw[0].get(\"id\")) != parent_id:", + "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_unproved_inputs_keep_the_complete_graph_fallback", + "-k", + "alias" + ], + "expected_failure": "test_unproved_inputs_keep_the_complete_graph_fallback" + }, + { + "name": "per-origin-ancestry-bounds", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "line": 342, + "before": "if depth > HOP_BOUND or len(ancestry) > NODE_BOUND:", + "after": "if False:", + "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_unproved_inputs_keep_the_complete_graph_fallback", + "-k", + "bound" + ], + "expected_failure": "test_unproved_inputs_keep_the_complete_graph_fallback" + }, + { + "name": "implicit-weekly-fallback", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "line": 318, + "before": "if problem or has_implicit_weekly_inputs(kind, row):", + "after": "if problem:", + "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_unproved_inputs_keep_the_complete_graph_fallback", + "-k", + "implicit" + ], + "expected_failure": "test_unproved_inputs_keep_the_complete_graph_fallback" + }, + { + "name": "unfenced-native-total", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "line": 449, + "before": "if not self.active and callable(native):", + "after": "if False:", + "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + } + ], + "pytest": [ + "tests/unit/test_label_round3_sqlite_reads.py::test_unfenced_status_counts_use_the_native_total" + ], + "expected_failure": "test_unfenced_status_counts_use_the_native_total" + }, + { + "name": "hidden-input-sql-prefilter", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_sql.py", + "line": 17, + "before": "if not blocked:", + "after": "if True:", + "sha256": "06c2349106e6bf5c61dfaaf41f3b285f388a501c7fe367a6394e71728fa49e9b" + } + ], + "pytest": [ + "tests/unit/test_label_round3_sqlite_reads.py::test_native_sql_prefilter_rejects_hidden_inputs_before_limit" + ], + "expected_failure": "test_native_sql_prefilter_rejects_hidden_inputs_before_limit" + }, + { + "name": "effective-ranked-refill", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_retrieval.py", + "line": 3129, + "before": "target=limit, initial_limit=limit,", + "after": "target=0, initial_limit=limit,", + "sha256": "b02277ef68d6f34a47d13de237d09e48e13c8f75f74a2c7e3cdae8cf7a2ccdb3" + } + ], + "pytest": [ + "tests/unit/test_label_round3_sqlite_reads.py::test_hidden_later_inputs_do_not_underfill_ranked_memory_reads" + ], + "expected_failure": "test_hidden_later_inputs_do_not_underfill_ranked_memory_reads" + }, + { + "name": "batched-parent-frontier", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "line": 371, + "before": " self._prefetch_inputs(kind, rows)", + "after": " pass # removed batched ancestry reads", + "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_dependency_frontier_batches_many_independent_origins" + ], + "expected_failure": "test_dependency_frontier_batches_many_independent_origins" + }, + { + "name": "october-six-new-doctor-disclosure", + "edits": [ + { + "file": "docs/release/v0.20.0-release-notes.md", + "line": 6, + "before": "Doctor's derived-label check counts are new in this set, absent from v0.20.0 and baseline main 48873b03, and remain outside the caller's sensitivity ceiling.", + "after": "", + "sha256": "1eef1e5173dd7603edbc40a40b6256ce4838c4628324bbe3ad6b92d7a9ef7f0b" + } + ], + "pytest": [ + "tests/unit/test_derived_labels_docs.py::test_october_six_correction_distinguishes_the_new_doctor_counts" + ], + "expected_failure": "test_october_six_correction_distinguishes_the_new_doctor_counts" + }, + { + "name": "known-limits-outside-ceiling", + "edits": [ + { + "file": "docs/alpha/known-limitations.md", + "line": 78, + "before": "while graph edge explanations and the new doctor label counts remain outside that ceiling (see the [draft security note](../release/derived-labels-security-note-draft.md))", + "after": "", + "sha256": "07044fd637c3c831865fab916de610b33f1c0151658719ca4d2f521f2de82512" + } + ], + "pytest": [ + "tests/unit/test_derived_labels_docs.py::test_known_limitations_discloses_graph_and_new_doctor_counts_outside_ceiling" + ], + "expected_failure": "test_known_limitations_discloses_graph_and_new_doctor_counts_outside_ceiling" + }, + { + "name": "draft-note-new-doctor-disclosure", + "edits": [ + { + "file": "docs/release/derived-labels-security-note-draft.md", + "line": 5, + "before": "Doctor's aggregate derived-label check counts are new in this set, absent from v0.20.0 and baseline main 48873b03, and remain outside the caller's sensitivity ceiling and visible to trusted keys.", + "after": "", + "sha256": "e1f5bd6c281d4e5c5c8a858a4f9cc5adbbd2e525b54f8151a5cf76cc14497bdb" + } + ], + "pytest": [ + "tests/unit/test_derived_labels_docs.py::test_draft_security_note_distinguishes_baseline_graph_from_new_doctor_counts" + ], + "expected_failure": "test_draft_security_note_distinguishes_baseline_graph_from_new_doctor_counts" + }, + { + "name": "changelog-new-doctor-ceiling", + "edits": [ + { + "file": "CHANGELOG.md", + "line": 12, + "before": "doctor label counts are new in this set and remain outside that ceiling", + "after": "doctor label counts are new in this set", + "sha256": "1bc64441c09f82aaa4f1147563121b97f33115fc0790f59e167116aa0bc2b74d" + } + ], + "pytest": [ + "tests/unit/test_derived_labels_docs.py::test_source_get_operator_gate_and_doctor_ceiling_are_explicit_in_changelog" + ], + "expected_failure": "test_source_get_operator_gate_and_doctor_ceiling_are_explicit_in_changelog" + } + ] +} diff --git a/scripts/verify_derived_label_mutations.py b/scripts/verify_derived_label_mutations.py new file mode 100644 index 000000000..26e096fce --- /dev/null +++ b/scripts/verify_derived_label_mutations.py @@ -0,0 +1,84 @@ +"""Replay exact guard removals in a disposable copy of an immutable commit. + +Run with the normal development Python and role-separated synthetic test DB +environment. The JSON manifest lists exact source spans, replacement text, +one-based lines, file digests and expected failing tests. No checkout is edited. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tarfile +import tempfile + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--only", action="append", default=[]) + args = parser.parse_args() + repository = Path(__file__).resolve().parents[1] + revision = subprocess.check_output(["git", "-C", str(repository), "rev-parse", "HEAD"], text=True).strip() + cases = json.loads((repository / "scripts/derived_label_mutations.json").read_text())["cases"] + selected = [case for case in cases if not args.only or case["name"] in args.only] + if not selected: + parser.error("no selected mutation cases") + args.output.mkdir(parents=True, exist_ok=True) + results = [] + with tempfile.TemporaryDirectory(prefix="alice-label-mutants-") as directory: + root = Path(directory) + archive = root / "source.tar" + with archive.open("wb") as destination: + subprocess.run(["git", "-C", str(repository), "archive", revision], stdout=destination, check=True) + checkout = root / "checkout" + checkout.mkdir() + with tarfile.open(archive) as source: + source.extractall(checkout, filter="data") + env = {**os.environ, "PYTHONPATH": str(checkout / "apps/api/src"), "PYTHONDONTWRITEBYTECODE": "1"} + for case in selected: + originals = {} + for edit in case["edits"]: + path = (checkout / edit["file"]).resolve() + if not path.is_relative_to(checkout): + raise ValueError("mutation path outside disposable checkout") + raw = path.read_bytes() + text = raw.decode() + if hashlib.sha256(raw).hexdigest() != edit["sha256"]: + raise ValueError("manifest digest mismatch: " + edit["file"]) + if text.count(edit["before"]) != 1 or text[:text.index(edit["before"])].count("\n") + 1 != edit["line"]: + raise ValueError("manifest source span mismatch: " + edit["file"]) + originals[path] = raw + command = [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", *case["pytest"]] + baseline = subprocess.run(command, cwd=checkout, env=env, capture_output=True, text=True, timeout=180) + (args.output / (case["name"] + "-baseline.log")).write_text(baseline.stdout + baseline.stderr) + mutant = None + try: + if baseline.returncode == 0: + for edit in case["edits"]: + path = checkout / edit["file"] + path.write_text(path.read_text().replace(edit["before"], edit["after"], 1)) + mutant = subprocess.run(command, cwd=checkout, env=env, capture_output=True, text=True, timeout=180) + (args.output / (case["name"] + "-mutant.log")).write_text(mutant.stdout + mutant.stderr) + finally: + for path, raw in originals.items(): + path.write_bytes(raw) + output = (mutant.stdout + mutant.stderr) if mutant else "" + killed = baseline.returncode == 0 and mutant is not None and mutant.returncode == 1 and any( + "FAILED " in line and case["expected_failure"] in line for line in output.splitlines() + ) + result = {"name": case["name"], "revision": revision, "baseline_exit": baseline.returncode, + "mutant_exit": mutant.returncode if mutant else None, "killed": killed, + "expected_failure": case["expected_failure"], "edits": case["edits"], "command": command} + results.append(result) + print(json.dumps({key: result[key] for key in ("name", "baseline_exit", "mutant_exit", "killed")}), flush=True) + (args.output / "summary.json").write_text(json.dumps({"revision": revision, "results": results}, indent=2) + "\n") + return 0 if all(result["killed"] for result in results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/round3_parity_probe.py b/tests/integration/round3_parity_probe.py new file mode 100644 index 000000000..ea0be887e --- /dev/null +++ b/tests/integration/round3_parity_probe.py @@ -0,0 +1,23 @@ +"""Read the same synthetic rows using the selected revision's implementation.""" +import inspect +import json +from pathlib import Path +import sys +from uuid import UUID + +repo, location, user, profile, source_id = sys.argv[1:6] +sys.path[:0] = [str(Path(repo) / "apps/api/src"), repo] +from alicebot_api.db import user_connection +from alicebot_api.vnext_agent_control import AgentIdentity +from alicebot_api.vnext_store import PostgresVNextStore +from alicebot_api.routers._vnext_shared import _vnext_load_source_trace +from alicebot_api.routers.workspaces import _vnext_workspace_payload + +identity = None if profile == "owner" else AgentIdentity(agent_id="parity", permission_profile=profile) +with user_connection(location, UUID(user)) as conn: + store = PostgresVNextStore(conn) + options = {"identity": identity} if "identity" in inspect.signature(_vnext_workspace_payload).parameters else {} + workspace = _vnext_workspace_payload(store, **options) + options = {"identity": identity} if "identity" in inspect.signature(_vnext_load_source_trace).parameters else {} + trace = _vnext_load_source_trace(store=store, source=store.get_source(source_id), **options) + print(json.dumps({"trace": trace, "workspace": workspace}, default=str)) diff --git a/tests/integration/test_label_round2_reader_fences_postgres.py b/tests/integration/test_label_round2_reader_fences_postgres.py index 64a636db4..9de434b2e 100644 --- a/tests/integration/test_label_round2_reader_fences_postgres.py +++ b/tests/integration/test_label_round2_reader_fences_postgres.py @@ -8,7 +8,7 @@ from tests.integration.derived_labels_postgres_support import label_harness -@pytest.mark.parametrize("profile,bound", [("trusted_local_agent", False), ("admin_agent", False), ("trusted_local_agent", True), ("admin_agent", True), ("read_only_agent", False), ("read_only_agent", True)]) +@pytest.mark.parametrize("profile,bound", [("trusted_local_agent", False), ("admin_agent", False), ("trusted_local_agent", True), ("admin_agent", True), ("read_only_agent", False), ("read_only_agent", True), ("project_scoped_agent", True), ("memory_proposal_agent", False)]) def test_source_get_uses_the_callers_entire_fence(label_harness, profile, bound): h = label_harness alpha, beta = str(uuid4()), str(uuid4()) @@ -25,6 +25,12 @@ def test_source_get_uses_the_callers_entire_fence(label_harness, profile, bound) status, body, _ = h.request("GET", "/v0/vnext/sources/" + str(source["id"])) assert status == 200 and str(body["id"]) == str(source["id"]) key = h.key(profile, project=alpha if bound else None) + if bound or profile not in {"trusted_local_agent", "admin_agent"}: + for source_id in [*(str(source["id"]) for source in (visible, private, other, restricted_domain)), str(uuid4())]: + status, body, _ = h.request("GET", "/v0/vnext/sources/" + source_id, key=key) + assert status == 403, (profile, bound, body) + assert "raw_text" not in body + return status, body, _ = h.request("GET", "/v0/vnext/sources/" + str(visible["id"]), key=key) assert status == 200 and str(body["id"]) == str(visible["id"]) for source, permitted in ((private, profile == "admin_agent"), (other, not bound), (restricted_domain, profile in {"trusted_local_agent", "admin_agent"})): diff --git a/tests/integration/test_label_round3_contracts_postgres.py b/tests/integration/test_label_round3_contracts_postgres.py new file mode 100644 index 000000000..4b367b8eb --- /dev/null +++ b/tests/integration/test_label_round3_contracts_postgres.py @@ -0,0 +1,104 @@ +"""Round-three owner compatibility and explicit scope-clamp contracts.""" +import json +import os +from pathlib import Path +import subprocess +import sys +from uuid import UUID, uuid4 + +import pytest + +from alicebot_api.vnext_agent_control import AgentIdentity +from alicebot_api.vnext_event_log import build_event_log_record +from alicebot_api.vnext_label_repair import label_gap_counts, plan_label_repairs, load_postgres_label_tables +from alicebot_api.routers._vnext_shared import _vnext_load_source_trace +from alicebot_api.routers.workspaces import _vnext_workspace_payload +from tests.integration.derived_labels_postgres_support import label_harness + + +@pytest.mark.parametrize("profile", ["owner", "admin_agent"]) +@pytest.mark.parametrize("component", ["trace", "workspace"]) +def test_unfenced_source_trace_and_workspace_preserve_main(label_harness, profile, component): + h = label_harness + sources = [h.source(sensitivity=value) for value in ("public", "internal", "private", "unknown", "confidential")] + source_id = str(sources[0]["id"]) + with h.store() as store: + store.create_source_chunk({"source_id": source_id, "chunk_index": 0, "text": "Synthetic chunk"}) + store.append_event(build_event_log_record(event_type="open_loop.extraction_completed", actor_type="system", + payload={"source_id": source_id})) + identity = None if profile == "owner" else AgentIdentity(agent_id="parity", permission_profile=profile) + # Default connector records are created on first load. Warm that + # legitimate side effect before comparing a stable, identical dataset. + _vnext_workspace_payload(store, identity=identity) + trace = _vnext_load_source_trace(store=store, source=store.get_source(source_id), identity=identity) + expected = store.list_events_for_source_trace(source_id=source_id) + if component == "trace": + assert {str(row["id"]) for row in trace["events"]} == {str(row["id"]) for row in expected} + assert {"source_chunk.created", "open_loop.extraction_completed"} <= {row["event_type"] for row in trace["events"]} + workspace = _vnext_workspace_payload(store, identity=identity) + checks = {row["name"]: row for row in workspace["doctor"]["checks"]} + if component == "workspace": + assert checks["flagged_sources"]["status"] == "pass" + assert checks["derived_labels"]["status"] == "pass" + assert workspace["dogfooding"]["captures_today"] == 5 + assert workspace["summary"]["source_count"] == 5 + # Main keeps its four-value display window, but complete totals and + # embedded diagnostics are unfiltered for these operators. + assert len(workspace["sources"]) == 4 + main = os.environ.get("ALICE_READ_MAIN_CHECKOUT") + if main: + probe = Path(__file__).with_name("round3_parity_probe.py") + completed = subprocess.run([sys.executable, str(probe), main, h.urls["app"], str(h.user_id), profile, source_id], + text=True, capture_output=True, timeout=120) + assert completed.returncode == 0, completed.stderr + baseline = json.loads(completed.stdout.strip().splitlines()[-1]) + assert [str(row["id"]) for row in trace["events"]] == [str(row["id"]) for row in baseline["trace"]["events"]] + assert workspace["summary"] == baseline["workspace"]["summary"] + assert workspace["dogfooding"] == baseline["workspace"]["dogfooding"] + baseline_checks = {row["name"]: row for row in baseline["workspace"]["doctor"]["checks"]} + assert checks["flagged_sources"] == baseline_checks["flagged_sources"] + + +@pytest.mark.parametrize("legacy_project", [False, True]) +def test_owner_scope_assignment_clamps_to_weekly_floor(label_harness, legacy_project): + from alicebot_api.routers import vnext_memories as router + from alicebot_api.vnext_derived_labels import with_derived_from + h = label_harness + projects = [str(uuid4()), str(uuid4())] + sources = [h.source(scope=(project,)) for project in projects] + with h.store() as store: + for project in projects: + store.create_project({"id": project, "name": project, "slug": project}) + metadata = with_derived_from({"discovered_by": "vnext_weekly_synthesis"}, {"sources": sources}) + memory = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Synthetic weekly summary", "status": "candidate", + "domain": "project", "sensitivity": "public", "metadata_json": metadata}) + assert memory["project_scope"] == [] + assert set(memory["metadata_json"]["project_floor"]) == set(projects) + assert label_gap_counts(store) == (0, 0) + if legacy_project: + store.conn.execute("UPDATE memories SET project_id=%s WHERE id=%s::uuid", (projects[0], str(memory["id"]))) + response = router.review_vnext_memory(UUID(str(memory["id"])), + router.VNextMemoryReviewRequest(user_id=h.user_id, action="assign_project", project_id=projects[0]), authorization=None) + assert response.status_code == 200, response.body + body = json.loads(response.body) + assert body.get("label_floor_applied") is True + with h.store() as store: + updated = store.get_memory(str(memory["id"])) + assert updated["project_scope"] == [] + assert updated["project_id"] is None + assert set(updated["metadata_json"]["project_floor"]) == set(projects) + assert label_gap_counts(store) == (0, 0) + assert plan_label_repairs(load_postgres_label_tables(store.conn)) == [] + raised = [event for event in store.list_events(limit=100) if event["event_type"] == "memory.labels_raised" + and event.get("target_id") == str(memory["id"]) and event["payload_json"].get("cause") == "floor_clamped"] + assert len(raised) == 1 + + +def test_fenced_workspace_keeps_diagnostics_omitted(label_harness): + h = label_harness + h.source() + with h.store() as store: + payload = _vnext_workspace_payload(store, identity=AgentIdentity(agent_id="fenced", permission_profile="trusted_local_agent")) + checks = {row["name"]: row for row in payload["doctor"]["checks"]} + assert checks["flagged_sources"]["status"] == "skipped" + assert checks["derived_labels"]["status"] == "skipped" diff --git a/tests/integration/test_label_round3_read_budget_postgres.py b/tests/integration/test_label_round3_read_budget_postgres.py new file mode 100644 index 000000000..4a60daaea --- /dev/null +++ b/tests/integration/test_label_round3_read_budget_postgres.py @@ -0,0 +1,28 @@ +"""Native reads meet the unchanged budget on the exact random grid.""" +import psycopg +import pytest + +from tests.integration.derived_labels_postgres_support import label_harness +from tests.performance.test_label_round3_read_budget import CASES, SOURCE_COUNTS, seed_grid, repair_fixture, paired_budgets + + +@pytest.mark.parametrize("case", CASES) +@pytest.mark.parametrize("source_count", SOURCE_COUNTS) +@pytest.mark.parametrize("repaired", (False, True)) +def test_postgres_round3_random_read_budgets(label_harness, case, source_count, repaired): + h = label_harness + with h.store() as store: + keys = seed_grid(store, postgres=True, case=case, source_count=source_count) + if repaired: + repair_fixture("postgres", h.urls["app"], h.user_id) + with psycopg.connect(h.urls["admin"], autocommit=True) as conn: + for table in ("sources", "memories", "event_log", "generated_artifacts", "open_loops"): + conn.execute("ANALYZE " + table) + paired_budgets("postgres", h.urls["app"], h.user_id, keys, case=case, source_count=source_count, repaired=repaired) + + +def test_postgres_round3_identical_copy_control(label_harness): + h = label_harness + with h.store() as store: + keys = seed_grid(store, postgres=True, source_count=1, identical=True) + paired_budgets("postgres", h.urls["app"], h.user_id, keys, case="identical-copies", source_count=1, repaired=False) diff --git a/tests/performance/round3_budget_probe.py b/tests/performance/round3_budget_probe.py new file mode 100644 index 000000000..2dbd2014e --- /dev/null +++ b/tests/performance/round3_budget_probe.py @@ -0,0 +1,67 @@ +"""Persistent revision-isolated probe, allowing interleaved same-data samples.""" +import importlib.util +import cProfile +import json +import os +from pathlib import Path +import sys +import time +from uuid import UUID + +repo, backend, location, user, profile, key = sys.argv[1:7] +sys.path[:0] = [str(Path(repo) / "apps/api/src"), repo] +os.environ["DATABASE_URL"] = location +if key: + os.environ["ALICE_AGENT_API_KEY"] = key +else: + os.environ.pop("ALICE_AGENT_API_KEY", None) +os.environ["ALICE_MCP_FULL_TOOLS"] = "1" +os.environ["ALICE_LEGACY_SURFACES"] = "1" +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext + +context = MCPRuntimeContext(database_url=location, user_id=UUID(user)) +arguments = {"query": "synthetic budget observation"} +if profile == "keyless_agent_id": + arguments["agent_id"] = "budget-keyless" +actions = {"pack": lambda: call_mcp_tool(context, name="alice_context_pack", arguments=arguments), + "recall": lambda: call_mcp_tool(context, name="alice_recall", arguments=arguments)} +if backend == "postgres": + from alicebot_api.config import Settings + from alicebot_api import main + from alicebot_api.routers import workspaces, vnext_memories + settings = Settings(database_url=location) + for module in (main, workspaces, vnext_memories): + module.get_settings = lambda: settings + support = Path(__file__).resolve().parents[1] / "integration/derived_labels_postgres_support.py" + spec = importlib.util.spec_from_file_location("round3_transport", support) + transport = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = transport + spec.loader.exec_module(transport) + + def read(path): + status, body, _headers = transport.invoke("GET", path, user_id=user, key=key) + assert status == 200, (status, body) + return body + + actions.update(workspace=lambda: read("/v0/vnext/workspace"), dogfooding=lambda: read("/v0/vnext/dogfooding")) + +profiled = set() +for line in sys.stdin: + name = line.strip() + if name == "stop": + break + wall_start, cpu_start = time.perf_counter(), time.process_time() + trace = os.environ.get("ALICE_READ_PROFILE") + profiler = cProfile.Profile() if trace and name not in profiled else None + if profiler: + profiler.enable() + result = actions[name]() + if profiler: + profiler.disable() + profiler.dump_stats(str(Path(trace) / (Path(repo).name + "-" + profile + "-" + name + ".prof"))) + profiled.add(name) + measurement = {"wall": time.perf_counter() - wall_start, "cpu": time.process_time() - cpu_start} + if name == "recall": + measurement["memories"] = len(result["results"]) + print(json.dumps(measurement), flush=True) diff --git a/tests/performance/test_label_round3_read_budget.py b/tests/performance/test_label_round3_read_budget.py new file mode 100644 index 000000000..0595c9fdf --- /dev/null +++ b/tests/performance/test_label_round3_read_budget.py @@ -0,0 +1,185 @@ +"""Exact random-parent grid requested by the third external handoff.""" +import json +import os +from pathlib import Path +import random +import subprocess +import sys +from types import SimpleNamespace +from uuid import uuid4 + +import pytest + +from alicebot_api.onramp import bootstrap_database +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_agent_keys import create_agent_key +from alicebot_api.vnext_derived_labels import with_derived_from +from alicebot_api.vnext_label_repair import label_gap_counts, relabel_labels_sqlite + +USER = "11111111-1111-4111-8111-111111111111" +CASES = ("half-hidden", "all-hidden", "all-visible", "deeper-ancestry", "extra-rows") +SOURCE_COUNTS = (300, 1000, 3000) + + +def seed_grid(store, *, postgres=False, count=5000, source_count=300, case="half-hidden", identical=False, provision_keys=True): + rng = random.Random(20261007) + sources = [store.create_source({"source_type": "note", "title": "Synthetic budget input", + "content_hash": str(uuid4()), "domain": "project", + "sensitivity": "confidential" if case == "all-hidden" or (case != "all-visible" and i % 2) else "public"}) + for i in range(source_count)] + user = str(sources[0]["user_id"]) + ids = [str(uuid4()) for _ in range(count)] + shapes = ["copy"] * (count * 30 // 100) + ["report"] * (count * 25 // 100) + ["consolidation"] * (count * 25 // 100) + shapes += ["weekly"] * (count - len(shapes)) + rng.shuffle(shapes) + first_copy = shapes.index("copy") + shapes[0], shapes[first_copy] = shapes[first_copy], shapes[0] + copies = [ids[i] for i, shape in enumerate(shapes) if shape == "copy"] + values = [] + for i, row_id in enumerate(ids): + metadata = {"project_scope": [], "project_floor": []} + if not identical: + metadata["observation_index"] = i + selected_sources = rng.sample(sources, min(source_count, 1 if shapes[i] == "copy" else rng.randint(1, 3))) + shape = "copy" if identical else shapes[i] + if shape == "copy": + metadata["source_id"] = str(sources[0]["id"] if identical else selected_sources[0]["id"]) + elif shape == "report": + metadata["workflow"] = "project_auto_update" + metadata = with_derived_from(metadata, {"sources": selected_sources}) + elif shape == "consolidation": + pool = ids[:i] if case == "deeper-ancestry" else copies + members = rng.sample(pool, min(len(pool), rng.randint(1, 3))) + metadata.update(candidate_kind="memory_consolidation", consolidation={"cluster_member_ids": members}) + else: + metadata["discovered_by"] = "vnext_weekly_synthesis" + metadata = with_derived_from(metadata, {"sources": selected_sources, + "memories": [{"id": item} for item in rng.sample(copies, rng.randint(1, 2))]}) + values.append((row_id, user, "budget." + row_id, "synthetic budget observation " + str(i), json.dumps(metadata))) + if postgres: + with store.conn.cursor() as cur: + cur.executemany("INSERT INTO memories(id,user_id,memory_key,canonical_text,metadata_json,value,source_event_ids,status,domain,sensitivity) VALUES(%s::uuid,%s::uuid,%s,%s,%s::jsonb,'{}','{}','active','project','public')", values) + cur.executemany("INSERT INTO event_log(id,user_id,target_id,event_type,actor_type,target_type,payload_json) VALUES(%s::uuid,%s::uuid,%s::uuid,'memory.created','system','memory','{}')", [(str(uuid4()), user, row_id) for row_id in ids]) + else: + store.conn.executemany("INSERT INTO memories(id,user_id,memory_key,canonical_text,metadata_json,value,source_event_ids,status,domain,sensitivity) VALUES(?,?,?,?,?,'{}','[]','active','project','public')", values) + store.conn.executemany("INSERT INTO event_log(id,user_id,target_id,event_type,actor_type,target_type,payload_json) VALUES(?,?,?,'memory.created','system','memory','{}')", [(str(uuid4()), user, row_id) for row_id in ids]) + if case == "extra-rows": + for i in range(400): + source = rng.choice(sources) + store.create_open_loop({"title": "Synthetic loop " + str(i), "status": "open", "source_id": str(source["id"]), + "domain": "project", "sensitivity": "public", "metadata_json": {"observation_index": i}}) + if postgres: + for i in range(500): + store.create_artifact({"artifact_type": "weekly_synthesis", "title": "Synthetic artifact " + str(i), + "content_markdown": "Synthetic report", "domain": "project", "sensitivity": "public", + "metadata_json": with_derived_from({"observation_index": i}, {"sources": rng.sample(sources, 2)})}) + return budget_keys(store, user_id=user) if provision_keys else {} + + +def budget_keys(store, *, user_id=None): + keys = {} + for profile in ("trusted_local_agent", "admin_agent"): + _, keys[profile] = create_agent_key(store, user_id=user_id or store.user_id, + agent_id="budget-" + profile, permission_profile=profile) + return keys + + +def repair_fixture(backend, location, user): + if backend == "sqlite": + with sqlite_user_connection(location.removeprefix("sqlite:///"), user) as conn: + applied = relabel_labels_sqlite(conn, explicit=True) + assert label_gap_counts(SQLiteVNextStore(conn, user)) == (0, 0) + else: + from alicebot_api.cli.labels import _run_vnext_labels_repair, _run_vnext_labels_check + ctx = SimpleNamespace(database_url=location, user_id=user) + applied = _run_vnext_labels_repair(ctx, None) + assert _run_vnext_labels_check(ctx, None) == "below_inputs 0" + return applied + + +def paired_budgets(backend, location, user, keys, *, case, source_count, repaired, samples=10, assert_budget=True): + assert samples >= 10 + repo = Path(__file__).resolve().parents[2] + main = os.environ.get("ALICE_READ_MAIN_CHECKOUT") + assert main, "round-three budgets require the exact paired main checkout" + script = Path(__file__).with_name("round3_budget_probe.py") + revisions = {name: subprocess.check_output(["git", "-C", str(checkout), "rev-parse", "HEAD"], text=True).strip() + for name, checkout in (("main", main), ("head", repo))} + failures = [] + for profile, key in keys.items(): + processes = [subprocess.Popen([sys.executable, str(script), str(checkout), backend, location, str(user), profile, key], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + for checkout in (main, repo)] + try: + measurements = {revision: {} for revision in ("main", "head")} + for action in ("pack", "recall", "workspace", "dogfooding") if backend == "postgres" else ("pack", "recall"): + for process in processes: + process.stdin.write(action + "\n") + process.stdin.flush() + line = process.stdout.readline() + assert line, process.stderr.read() + arrays = [{"wall": [], "cpu": [], "returned_memories": []} for _ in processes] + for sample in range(samples): + for index in (0, 1) if sample % 2 == 0 else (1, 0): + process = processes[index] + process.stdin.write(action + "\n") + process.stdin.flush() + line = process.stdout.readline() + assert line, process.stderr.read() + result = json.loads(line) + for clock in ("wall", "cpu"): + arrays[index][clock].append(result[clock]) + if "memories" in result: + arrays[index]["returned_memories"].append(result["memories"]) + for index, revision in enumerate(("main", "head")): + measurements[revision][action] = {**arrays[index], "minimum_wall": min(arrays[index]["wall"]), + "minimum_cpu": min(arrays[index]["cpu"])} + row = {"store": backend, "case": case, "source_count": source_count, "repaired": repaired, + "profile": profile, "revisions": revisions, "samples": samples, "seed": 20261007, + "memories": 5000, "mix": ({"source_copy": 5000} if case == "identical-copies" else {"source_copy": 1500, "stamped_report": 1250, "consolidation": 1250, "weekly": 1000}), + "artifacts": 500 if backend == "postgres" and case == "extra-rows" else 0, + "derived_loops": 400 if case == "extra-rows" else 0, "real_repair_zero_check": repaired, **measurements} + print(json.dumps(row), flush=True) + for action in ("pack", "recall"): + for clock in ("minimum_wall", "minimum_cpu"): + if measurements["head"][action][clock] > 2 * measurements["main"][action][clock] + .1: + failures.append((profile, action, clock, row)) + if backend == "postgres": + for action in ("workspace", "dogfooding"): + if measurements["head"][action]["minimum_wall"] > 1: + failures.append((profile, action, "minimum_wall", row)) + finally: + for process in processes: + if process.poll() is None: + process.stdin.write("stop\n") + process.stdin.flush() + process.communicate(timeout=30) + if assert_budget: + assert not failures, failures + return failures + + +@pytest.mark.parametrize("case", CASES) +@pytest.mark.parametrize("source_count", SOURCE_COUNTS) +@pytest.mark.parametrize("repaired", (False, True)) +def test_sqlite_round3_random_read_budgets(tmp_path, case, source_count, repaired): + path = tmp_path / "round3.db" + bootstrap_database(path, user_id=USER, user_email="budget@example.invalid") + with sqlite_user_connection(path, USER) as conn: + seed_grid(SQLiteVNextStore(conn, USER), case=case, source_count=source_count, provision_keys=False) + location = "sqlite:///" + str(path) + if repaired: + repair_fixture("sqlite", location, USER) + failures = paired_budgets("sqlite", location, USER, {"keyless_agent_id": ""}, case=case, source_count=source_count, repaired=repaired, assert_budget=False) + with sqlite_user_connection(path, USER) as conn: + keys = budget_keys(SQLiteVNextStore(conn, USER)) + failures.extend(paired_budgets("sqlite", location, USER, keys, case=case, source_count=source_count, repaired=repaired, assert_budget=False)) + assert not failures, failures + + +def test_sqlite_round3_identical_copy_control(tmp_path): + path = tmp_path / "identical.db" + bootstrap_database(path, user_id=USER, user_email="budget@example.invalid") + with sqlite_user_connection(path, USER) as conn: + keys = seed_grid(SQLiteVNextStore(conn, USER), source_count=1, identical=True) + paired_budgets("sqlite", "sqlite:///" + str(path), USER, keys, case="identical-copies", source_count=1, repaired=False) diff --git a/tests/unit/test_complete_readable_counts.py b/tests/unit/test_complete_readable_counts.py index 869c6071a..ca6d1c870 100644 --- a/tests/unit/test_complete_readable_counts.py +++ b/tests/unit/test_complete_readable_counts.py @@ -91,7 +91,7 @@ def test_workspace_counts_sql_hidden_and_beyond_display_page(monkeypatch): store.rows[kind] += [_row(f"{kind}-hidden-{index}", "confidential") for index in range(205)] store.rows["open_loop"] = [{**row, "status": "open"} for row in store.rows["open_loop"]] store.events = [{"id": str(index), "target_type": "memory", "target_id": row["id"], "event_type": "memory.labels_raised"} for index, row in enumerate(store.rows["memory"])] - body = workspaces._vnext_workspace_payload(store) + body = workspaces._vnext_workspace_payload(store, identity=AgentIdentity(agent_id="reader", permission_profile="trusted_local_agent")) summary = body["summary"] for field in ("source_count", "artifact_count", "project_count", "open_loop_count", "event_count", "candidate_memory_count"): assert summary[field] == 35, (field, summary[field]) @@ -107,7 +107,7 @@ def test_all_sql_prefiltered_rows_leave_zero_totals(monkeypatch): store = PopulationStore() for kind in store.rows: store.rows[kind] = [_row(f"{kind}-hidden", "confidential")] - body = workspaces._vnext_workspace_payload(store) + body = workspaces._vnext_workspace_payload(store, identity=AgentIdentity(agent_id="reader", permission_profile="trusted_local_agent")) for field in ("source_count", "artifact_count", "project_count", "open_loop_count", "candidate_memory_count"): assert body["summary"][field] == 0 assert all(not sample["has_more"] for sample in body["samples"].values()) @@ -124,7 +124,7 @@ def test_workspace_activity_and_nested_dashboard_share_the_guard(monkeypatch): store.list_agent_events = lambda **kwargs: store.events store.list_recent_agentic_commits = lambda **kwargs: [visible, hidden] store.list_pending_inline_confirmations = lambda **kwargs: [visible, hidden] - body = workspaces._vnext_workspace_payload(store) + body = workspaces._vnext_workspace_payload(store, identity=AgentIdentity(agent_id="reader", permission_profile="trusted_local_agent")) assert "hidden-memory" not in str(body) assert "hidden-event" not in str(body) activity = body["agent_activity"] diff --git a/tests/unit/test_derived_labels_docs.py b/tests/unit/test_derived_labels_docs.py index d25be896e..de75342ae 100644 --- a/tests/unit/test_derived_labels_docs.py +++ b/tests/unit/test_derived_labels_docs.py @@ -183,3 +183,36 @@ def test_source_move_docs_name_the_proved_recovery_and_failure_contract() -> Non assert "HTTP 409 with the cause" in tools for cause in ("propagation_bound", "row_changed", "dependency_cycle", "lock_order", "database_error"): assert f"`{cause}`" in tools + + +def test_october_six_correction_distinguishes_the_new_doctor_counts() -> None: + notes = _text("docs/release/v0.20.0-release-notes.md") + correction = notes.split("> **Correction (2026-10-06):**", 1)[1].split("\n", 1)[0] + assert "Doctor's derived-label check counts are new in this set, absent from v0.20.0 and baseline main 48873b03" in correction + assert "outside the caller's sensitivity ceiling" in correction + assert "These limits also occur in v0.20.0" not in correction + + +def test_known_limitations_discloses_graph_and_new_doctor_counts_outside_ceiling() -> None: + limits = _text("docs/alpha/known-limitations.md") + assert "graph edge explanations and the new doctor label counts remain outside that ceiling" in limits + assert "derived-labels-security-note-draft.md" in limits + + +def test_draft_security_note_distinguishes_baseline_graph_from_new_doctor_counts() -> None: + note = _text("docs/release/derived-labels-security-note-draft.md") + assert "Graph neighborhood still returns edge explanations outside that ceiling; that behavior predates this set" in note + assert "Doctor's aggregate derived-label check counts are new in this set, absent from v0.20.0 and baseline main 48873b03" in note + assert "remain outside the caller's sensitivity ceiling and visible to trusted keys" in note + assert "These behaviors also occur on the baseline" not in note + + +def test_source_get_operator_gate_and_doctor_ceiling_are_explicit_in_changelog() -> None: + changelog = _text("CHANGELOG.md") + assert "Source GET retains its operator gate before applying the full caller fence" in changelog + assert "doctor label counts are new in this set and remain outside that ceiling" in changelog + tools = _text("docs/alpha/mcp-tools.md") + assert "Source GET evaluates `http.operator.access` before its source fence" in tools + assert "Other profiles and project-bound keys receive HTTP 403, including for missing ids" in tools + note = _text("docs/release/derived-labels-security-note-draft.md") + assert "source GET keeps the operator gate" in note diff --git a/tests/unit/test_known_limitations_page_shape.py b/tests/unit/test_known_limitations_page_shape.py index 63e37b3f3..cd2d4455d 100644 --- a/tests/unit/test_known_limitations_page_shape.py +++ b/tests/unit/test_known_limitations_page_shape.py @@ -22,8 +22,8 @@ # eight sentences each. The caps sit just above the page as it is, so a limit can be reworded and a new one added, and # nothing grows back into a record without this test failing. A bullet is one or two sentences, and a closing # "See ..." pointer to the page that explains it does not count as one. -# Raised from 15,090 to 15,544 (2026-10-05) for the derived-row bullet (page length 15,524 plus 20). -MAX_PAGE_CHARS = 15_544 +# Recomputed 2026-10-07: current page length 15,336 plus 20 characters. +MAX_PAGE_CHARS = 15_356 MAX_BULLET_CHARS = 800 MAX_BULLET_SENTENCES = 2 diff --git a/tests/unit/test_label_dependency_cache.py b/tests/unit/test_label_dependency_cache.py index 29f100d1b..a77590314 100644 --- a/tests/unit/test_label_dependency_cache.py +++ b/tests/unit/test_label_dependency_cache.py @@ -67,11 +67,12 @@ def test_unique_metadata_and_text_share_one_settlement_but_never_admission(monke source = row(SOURCE, {"project_scope": ["alpha"]}, sensitivity="confidential") store = Store([("source", source)]) calls = [] - original = module.settle_labels + original = module.settle_verified_inputs def counted(*args, **kwargs): - calls.append(1) + if args[0] == "memory": + calls.append(1) return original(*args, **kwargs) - monkeypatch.setattr(module, "settle_labels", counted) + monkeypatch.setattr(module, "settle_verified_inputs", counted) with label_read_scope(store): restricted = LabelGuard.for_filters(store, (), ("public",)) copies = [row(str(UUID(int=i + 10)), {"source_id": SOURCE, "observation": i}, canonical_text=str(i)) for i in range(30)] diff --git a/tests/unit/test_label_resolved_inputs.py b/tests/unit/test_label_resolved_inputs.py new file mode 100644 index 000000000..b586b3df0 --- /dev/null +++ b/tests/unit/test_label_resolved_inputs.py @@ -0,0 +1,131 @@ +"""Dependency-order reuse agrees with the canonical kernel, including fallbacks.""" +from copy import deepcopy +from dataclasses import replace +import json +import random +from uuid import UUID + +import pytest + +from alicebot_api.vnext_derived_labels import identifier, settle_labels, with_derived_from +from alicebot_api.vnext_label_guard import LabelGuard, label_read_scope + + +class Rows: + def __init__(self, rows): + self.rows = rows + + def read_label_rows(self, kind, ids): + return [dict(row) for row in self.rows if row["kind"] == kind and identifier(row["id"]) in ids] + + +def labels(row): + meta = row["metadata_json"] + return row["domain"], row["sensitivity"], tuple(meta["project_scope"]), tuple(meta["project_floor"]), row["unverified"] + + +def expected(label): + return label.domain, "regulated" if label.unverified else label.sensitivity, (() if label.unverified else label.project_scope), (() if label.unverified else label.project_floor), label.unverified + + +def test_random_mixed_deep_inputs_match_full_kernel_and_keep_distinct_domains(): + rng = random.Random(20261007) + rows = [{"kind": "source", "id": str(UUID(int=i + 1)), "domain": ("health", "legal", "project")[i % 3], + "sensitivity": ("public", "confidential")[i % 2], "metadata_json": {"project_scope": ["P" + str(i % 3)]}} + for i in range(18)] + for i in range(180): + parents = rng.sample(rows, min(len(rows), rng.randint(1, 3))) + by_kind = {kind + "s": [row for row in parents if row["kind"] == kind] for kind in ("source", "memory")} + meta = with_derived_from({"project_scope": ["P0", "P1", "P2"], "observation_index": i}, by_kind) + if i % 4 == 1: + meta["workflow"] = "project_auto_update" + elif i % 4 == 2: + meta["candidate_kind"] = "memory_consolidation" + elif i % 4 == 3: + meta["discovered_by"] = "vnext_weekly_synthesis" + rows.append({"kind": "memory", "id": str(UUID(int=100 + i)), "domain": "project", "sensitivity": "public", "metadata_json": meta}) + canonical = settle_labels([{**row, "user_id": "label-guard"} for row in rows], on_cycle="unverified", max_nodes=5000, max_hops=32) + store = Rows(rows) + with label_read_scope(store): + guard = LabelGuard(store, active=True) + guard.admit_rows("memory", rows[18:]) + for row, label in zip(rows[18:], canonical.rows[18:], strict=True): + assert labels(guard.effective_row("memory", row)) == expected(label) + # Caller admission still uses its own ceiling/domain after settlement. + public = replace(guard, sensitivity_allowed=("public",), domains=("project",)) + visible = public.admit_rows("memory", rows[18:]) + assert all(labels(guard.effective_row("memory", row))[0:2] == ("project", "public") for row in visible) + + +def test_parent_label_multiplicity_changes_restricted_domain_selection(): + sources = [{"kind": "source", "id": str(UUID(int=i + 1)), "domain": domain, "sensitivity": "public", "metadata_json": {}} + for i, domain in enumerate(("health", "health", "legal", "legal"))] + roots = [{"kind": "memory", "id": str(UUID(int=100 + i)), "domain": "project", "sensitivity": "public", + "metadata_json": with_derived_from({}, {"sources": [sources[j] for j in indexes]})} + for i, indexes in enumerate(((0, 1, 2), (0, 2, 3)))] + store = Rows(sources) + with label_read_scope(store): + guard = LabelGuard(store, active=True) + assert [guard.effective_row("memory", root)["domain"] for root in roots] == ["health", "legal"] + + +def test_dependency_frontier_batches_many_independent_origins(): + sources = [{"kind": "source", "id": str(UUID(int=i + 1)), "domain": "project", "sensitivity": "confidential" if i % 2 else "public", "metadata_json": {}} + for i in range(30)] + parents = [{"kind": "memory", "id": str(UUID(int=i + 100)), "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": source["id"]}} + for i, source in enumerate(sources)] + roots = [{"kind": "memory", "id": str(UUID(int=i + 200)), "domain": "project", "sensitivity": "public", "metadata_json": {"consolidation": {"cluster_member_ids": [parents[i % 30]["id"]]}}} + for i in range(60)] + + class CountedRows(Rows): + calls = 0 + + def read_label_rows(self, kind, ids): + self.calls += 1 + return super().read_label_rows(kind, ids) + + store = CountedRows(sources + parents) + kept = LabelGuard(store, active=True, sensitivity_allowed=("public",)).admit_rows("memory", roots) + assert {row["id"] for row in kept} == {row["id"] for row in roots[::2]} + assert store.calls <= 2 + + +@pytest.mark.parametrize("variant", ["alias", "cycle", "missing", "malformed", "hop-bound", "node-bound", "implicit-weekly", "implicit-weekly-json"]) +def test_unproved_inputs_keep_the_complete_graph_fallback(monkeypatch, variant): + import alicebot_api.vnext_label_guard as module + source = {"kind": "source", "id": "source", "domain": "health", "sensitivity": "confidential", "metadata_json": {}} + parent = {"kind": "memory", "id": "parent", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": "source"}} + root = {"kind": "memory", "id": "root", "domain": "project", "sensitivity": "public", "metadata_json": {"consolidation": {"cluster_member_ids": ["parent"]}}} + rows = [source, parent] + if variant == "alias": + source["id"] = str(UUID(int=1)) + parent["metadata_json"]["source_id"] = source["id"] + rows.append({**deepcopy(source), "id": "urn:uuid:" + source["id"]}) + elif variant == "cycle": + parent["metadata_json"] = {"consolidation": {"cluster_member_ids": ["root"]}} + rows.append(root) + elif variant == "missing": + rows = [] + elif variant == "malformed": + root["metadata_json"]["candidate_kind"] = [] + elif variant == "hop-bound": + monkeypatch.setattr(module, "HOP_BOUND", 1) + elif variant == "node-bound": + monkeypatch.setattr(module, "NODE_BOUND", 2) + else: + artifact = {"kind": "artifact", "id": "artifact", "artifact_type": "weekly_synthesis", "domain": "project", "sensitivity": "public", + "metadata_json": {"candidate_memory_ids": ["root"], "input_summary": {"source_ids": ["source"]}}} + root["metadata_json"] = {"discovered_by": "vnext_weekly_synthesis", "source_artifact_id": "artifact"} + rows = [source, artifact] + if variant == "implicit-weekly-json": + artifact["metadata_json"] = json.dumps(artifact["metadata_json"]) + store = Rows(rows) + with label_read_scope(store): + guard = LabelGuard(store, active=True) + if variant in ("hop-bound", "node-bound"): + assert guard._settled_inputs("memory", parent, frozenset()) is not None + assert guard._settled_inputs("memory", root, frozenset()) is None + actual = guard.effective_row("memory", root) + # Same collector, same canonical kernel bounds, with optimization disabled. + monkeypatch.setattr(module.LabelGuard, "_settled_inputs", lambda *args: None) + assert labels(actual) == labels(LabelGuard(store, active=True).effective_row("memory", root)) diff --git a/tests/unit/test_label_round3_sqlite_reads.py b/tests/unit/test_label_round3_sqlite_reads.py new file mode 100644 index 000000000..3600d6755 --- /dev/null +++ b/tests/unit/test_label_round3_sqlite_reads.py @@ -0,0 +1,89 @@ +"""Native prelimit rejection and refill preserve ordinary restricted reads.""" +from uuid import uuid4 + +import pytest + +from alicebot_api.mcp.registry import call_mcp_tool +from alicebot_api.mcp.types import MCPRuntimeContext +from alicebot_api.onramp import bootstrap_database +from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection +from alicebot_api.vnext_derived_labels import with_derived_from +from alicebot_api.vnext_label_guard import LabelGuard +from alicebot_api.vnext_label_repair import label_gap_counts +from alicebot_api.vnext_label_writes import without_insert_floor + +USER = "11111111-1111-4111-8111-111111111111" + + +@pytest.mark.parametrize("tool", ["alice_recall", "alice_context_pack"]) +def test_hidden_later_inputs_do_not_underfill_ranked_memory_reads(tmp_path, monkeypatch, tool): + path = tmp_path / "refill.db" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.invalid") + with sqlite_user_connection(path, USER) as conn, without_insert_floor(): + store = SQLiteVNextStore(conn, USER) + public = store.create_source({"source_type": "note", "title": "Visible", "content_hash": "public", "sensitivity": "public", "domain": "project"}) + hidden = store.create_source({"source_type": "note", "title": "Hidden", "content_hash": "hidden", "sensitivity": "confidential", "domain": "project"}) + visible_ids = set() + for i in range(8): + row = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "refill observation", "status": "active", + "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": public["id"]}}) + visible_ids.add(str(row["id"])) + for i in range(100): + metadata = with_derived_from({"workflow": "project_auto_update"}, {"sources": [public, hidden]}) + # Keep the public parent first: the conservative SQL hint alone + # cannot reject these rows. Effective admission must refill. + metadata["derived_from"]["sources"] = [str(public["id"]), str(hidden["id"])] + store.create_memory({"memory_key": str(uuid4()), "canonical_text": "refill observation", "status": "active", + "domain": "project", "sensitivity": "public", "metadata_json": metadata}) + monkeypatch.setenv("ALICE_MCP_FULL_TOOLS", "1") + monkeypatch.delenv("ALICE_AGENT_API_KEY", raising=False) + result = call_mcp_tool(MCPRuntimeContext(database_url="sqlite:///" + str(path), user_id=USER), name=tool, + arguments={"query": "refill observation", "agent_id": "synthetic-reader"}) + rows = result["results"] if tool == "alice_recall" else result["memories"] + assert len(rows) == 8 + assert {str(row["id"]) for row in rows} == visible_ids + + +def test_native_sql_prefilter_rejects_hidden_inputs_before_limit(tmp_path): + path = tmp_path / "prefilter.db" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.invalid") + with sqlite_user_connection(path, USER) as conn, without_insert_floor(): + store = SQLiteVNextStore(conn, USER) + hidden = store.create_source({"source_type": "note", "title": "Hidden", "content_hash": "hidden", "sensitivity": "confidential", "domain": "project"}) + visible = store.create_memory({"memory_key": "visible", "canonical_text": "prefilter observation", "status": "active", "domain": "project", "sensitivity": "public"}) + for i in range(12): + parent = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "other", "status": "active", "domain": "project", + "sensitivity": "public", "metadata_json": {"source_id": hidden["id"]}}) + store.create_memory({"memory_key": str(uuid4()), "canonical_text": "prefilter observation", "status": "active", "domain": "project", + "sensitivity": "public", "metadata_json": {"consolidation": {"cluster_member_ids": [parent["id"]]}}}) + rows = store.search_memories_fts(query="prefilter observation", sensitivity_allowed=["public", "internal"], limit=1) + assert [row["id"] for row in rows] == [visible["id"]] + + +def test_scope_clamp_clears_the_legacy_project_alias_and_records_the_attempt(tmp_path): + path = tmp_path / "clamp.db" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.invalid") + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + sources = [store.create_source({"source_type": "note", "title": "Input", "content_hash": project, "sensitivity": "public", + "domain": "project", "metadata_json": {"project_scope": [project]}}) for project in ("P1", "P2")] + meta = with_derived_from({"discovered_by": "vnext_weekly_synthesis"}, {"sources": sources}) + memory = store.create_memory({"memory_key": "weekly", "canonical_text": "weekly observation", "status": "candidate", "domain": "project", + "sensitivity": "public", "metadata_json": meta}) + conn.execute("UPDATE memories SET project_id='P1' WHERE id=?", (str(memory["id"]),)) + updated = store.update_memory(memory_id=str(memory["id"]), patch={"project_id": "P1", "metadata_json": {**memory["metadata_json"], "project_scope": ["P1"]}}, label_write=True) + assert updated["project_scope"] == [] + assert updated["project_id"] is None + assert store._label_floor_applied is True + assert label_gap_counts(store) == (0, 0) + + +def test_unfenced_status_counts_use_the_native_total(): + class Native: + def count_memories_by_status(self): + return {"active": 5000} + + def iter_label_rows(self, kind): + raise AssertionError("unfenced counts must not enumerate the population") + + assert LabelGuard(Native(), active=False).readable_status_counts("memory") == {"active": 5000} From c0aaf4fde43d8a66c6cb8443b0c8847981ca2e81 Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:32:54 +0200 Subject: [PATCH 249/270] Page native effective reads to exhaustion and verify derived loop fixtures --- apps/api/src/alicebot_api/sqlite_store.py | 1 + apps/api/src/alicebot_api/vnext_retrieval.py | 32 +++++++++++++++---- apps/api/src/alicebot_api/vnext_store.py | 2 ++ .../vnext_stores/postgres/memory_access.py | 4 ++- .../vnext_stores/sqlite/memory_access.py | 5 +-- scripts/derived_label_mutations.json | 8 ++--- .../test_label_round3_contracts_postgres.py | 17 ++++++++++ .../test_label_round3_read_budget.py | 8 +++-- tests/unit/test_label_round3_sqlite_reads.py | 1 + 9 files changed, 62 insertions(+), 16 deletions(-) diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index 01ff1ed1b..c541de4c0 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -465,6 +465,7 @@ class SQLiteVNextStore: #: Retrieval-trace label for the full-text stage (FTS5, not Postgres tsvector). fts_stage_source = "sqlite_fts" + memory_fts_offset_paging = True def __init__(self, conn: sqlite3.Connection, user_id: UUID | str): if str(user_id).strip() == "": diff --git a/apps/api/src/alicebot_api/vnext_retrieval.py b/apps/api/src/alicebot_api/vnext_retrieval.py index f190eb17d..5a58d70fb 100644 --- a/apps/api/src/alicebot_api/vnext_retrieval.py +++ b/apps/api/src/alicebot_api/vnext_retrieval.py @@ -3120,14 +3120,34 @@ def fetch(prefix_limit): raw = rows if prefix_limit == limit else active_search(limit=prefix_limit, **options) return raw, source - selected, source = _fetch_filtered_prefix( - fetch, - select_rows=lambda raw: admit_loaded( + def select_rows(raw): + return admit_loaded( self.store, kind="memory", rows=raw, domains=domains, sensitivity_allowed=sensitivity_allowed, projects=projects, - ), - target=limit, initial_limit=limit, - ) + ) + + if fts and getattr(self.store, "memory_fts_offset_paging", False) is True: + # Native stores page their ranked SQL result to actual exhaustion. + # Only legacy prefix adapters have the 16,384-row compatibility + # ceiling. Do not turn that ceiling into a native false negative. + selected = [] + seen: set[str] = set() + offset, page_limit, raw = 0, limit, rows + while True: + fresh = [row for row in _dedupe_retrieval_rows(raw) if str(row.get("id")) not in seen] + if raw and not fresh: + raise VNextRetrievalCompletenessError("native memory search returned a non-progressing page") + seen.update(str(row.get("id")) for row in fresh) + selected.extend(select_rows(fresh)) + if len(selected) >= limit or len(raw) < page_limit: + break + offset += len(raw) + page_limit = min(max(limit, 128), 1024) + raw = active_search(limit=page_limit, offset=offset, **options) + else: + selected, source = _fetch_filtered_prefix( + fetch, select_rows=select_rows, target=limit, initial_limit=limit, + ) selected = selected[:limit] return (_stabilize_scored_rows(selected) if fts else selected), source diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 04da14814..34be30cf4 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -468,6 +468,8 @@ class PostgresVNextStore: """SQL-backed vNext repository facade for the second-brain kernel.""" + memory_fts_offset_paging = True + def __init__(self, conn: UserConnection): self.conn = conn self._label_floor_applied = False diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py index e3e77609d..ae8b317f4 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py @@ -859,6 +859,7 @@ def search_memories_fts( domains: list[str] | None = None, sensitivity_allowed: list[str] | None = None, limit: int = 50, + offset: int = 0, memory_types: tuple[str, ...] = (), projects: tuple[str, ...] = (), created_by_agent_ids: tuple[str, ...] = (), @@ -929,7 +930,7 @@ def search_memories_fts( ) AND search_tsv @@ {tsquery_sql} ORDER BY fts_score DESC, updated_at DESC, created_at DESC, id DESC - LIMIT %s + LIMIT %s OFFSET %s """, ( tsquery_text, @@ -959,6 +960,7 @@ def search_memories_fts( scope_window_end, tsquery_text, limit, + offset, ), ) diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py index 6057f2ed7..5094b82d6 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py @@ -1221,6 +1221,7 @@ def search_memories_fts( domains: list[str] | None = None, sensitivity_allowed: list[str] | None = None, limit: int = 50, + offset: int = 0, memory_types: tuple[str, ...] = (), projects: tuple[str, ...] = (), created_by_agent_ids: tuple[str, ...] = (), @@ -1269,7 +1270,7 @@ def search_memories_fts( params.extend(run_params) params.extend(expiry_params) params.extend(scope_params) - params.append(limit) + params.extend((limit, offset)) try: # Column weights follow the Postgres search_tsv setweights: # title 1.0 (A), canonical_text 0.4 (B), summary 0.2 (C), @@ -1287,7 +1288,7 @@ def search_memories_fts( AND {label_sql} AND m.status IN {_MEMORY_SEARCHABLE_STATUSES_SQL}{domain_sql}{sensitivity_sql}{type_sql}{project_sql}{created_by_sql}{run_sql}{expiry_sql}{scope_sql} ORDER BY fts_score DESC, m.updated_at DESC, m.created_at DESC, m.id DESC - LIMIT ? + LIMIT ? OFFSET ? """, tuple(params), ) diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index 2fcaa0a41..8eeee9f16 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -259,10 +259,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_retrieval.py", - "line": 3129, - "before": "target=limit, initial_limit=limit,", - "after": "target=0, initial_limit=limit,", - "sha256": "b02277ef68d6f34a47d13de237d09e48e13c8f75f74a2c7e3cdae8cf7a2ccdb3" + "line": 3142, + "before": "if len(selected) >= limit or len(raw) < page_limit:", + "after": "if True:", + "sha256": "15822c3a1613735c72782119a83403d44813b8425a8df3c12ed239d51a81b902" } ], "pytest": [ diff --git a/tests/integration/test_label_round3_contracts_postgres.py b/tests/integration/test_label_round3_contracts_postgres.py index 4b367b8eb..f3d9e0a01 100644 --- a/tests/integration/test_label_round3_contracts_postgres.py +++ b/tests/integration/test_label_round3_contracts_postgres.py @@ -102,3 +102,20 @@ def test_fenced_workspace_keeps_diagnostics_omitted(label_harness): checks = {row["name"]: row for row in payload["doctor"]["checks"]} assert checks["flagged_sources"]["status"] == "skipped" assert checks["derived_labels"]["status"] == "skipped" + + +def test_native_memory_refill_crosses_the_legacy_ceiling(label_harness, monkeypatch): + from alicebot_api.vnext_retrieval import VNextRetrievalService + from alicebot_api.vnext_derived_labels import with_derived_from + from alicebot_api.vnext_label_writes import without_insert_floor + monkeypatch.setattr("alicebot_api.vnext_retrieval.LEGACY_SCOPED_SCAN_MAX_ROWS", 16) + h = label_harness + public, hidden = h.source(), h.source(sensitivity="confidential") + with h.store() as store, without_insert_floor(): + visible = [store.create_memory({"memory_key": str(uuid4()), "canonical_text": "refill observation", "domain": "project", "sensitivity": "public", "status": "active"}) for _ in range(8)] + metadata = with_derived_from({"workflow": "project_auto_update"}, {"sources": [public, hidden]}) + metadata["derived_from"]["sources"] = [str(public["id"]), str(hidden["id"])] + for _ in range(100): + store.create_memory({"memory_key": str(uuid4()), "title": "refill observation", "canonical_text": "refill observation", "domain": "project", "sensitivity": "public", "status": "active", "metadata_json": metadata}) + rows, _ = VNextRetrievalService(store)._memory_fts_rows(query="refill observation", domains=[], sensitivity_allowed=["public", "internal"], limit=8) + assert {str(row["id"]) for row in rows} == {str(row["id"]) for row in visible} diff --git a/tests/performance/test_label_round3_read_budget.py b/tests/performance/test_label_round3_read_budget.py index 0595c9fdf..21379af56 100644 --- a/tests/performance/test_label_round3_read_budget.py +++ b/tests/performance/test_label_round3_read_budget.py @@ -13,7 +13,7 @@ from alicebot_api.onramp import bootstrap_database from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection from alicebot_api.vnext_agent_keys import create_agent_key -from alicebot_api.vnext_derived_labels import with_derived_from +from alicebot_api.vnext_derived_labels import with_derived_from, is_derived, dependencies_of from alicebot_api.vnext_label_repair import label_gap_counts, relabel_labels_sqlite USER = "11111111-1111-4111-8111-111111111111" @@ -66,8 +66,10 @@ def seed_grid(store, *, postgres=False, count=5000, source_count=300, case="half if case == "extra-rows": for i in range(400): source = rng.choice(sources) - store.create_open_loop({"title": "Synthetic loop " + str(i), "status": "open", "source_id": str(source["id"]), - "domain": "project", "sensitivity": "public", "metadata_json": {"observation_index": i}}) + loop = store.create_open_loop({"title": "Synthetic loop " + str(i), "status": "open", "source_id": str(source["id"]), + "domain": "project", "sensitivity": "public", "metadata_json": with_derived_from({"observation_index": i, "discovered_by": "vnext_project_open_loop_extraction"}, {"sources": [source]})}) + assert is_derived("open_loop", loop) + assert ("source", str(source["id"])) in dependencies_of("open_loop", loop) if postgres: for i in range(500): store.create_artifact({"artifact_type": "weekly_synthesis", "title": "Synthetic artifact " + str(i), diff --git a/tests/unit/test_label_round3_sqlite_reads.py b/tests/unit/test_label_round3_sqlite_reads.py index 3600d6755..652203bf2 100644 --- a/tests/unit/test_label_round3_sqlite_reads.py +++ b/tests/unit/test_label_round3_sqlite_reads.py @@ -17,6 +17,7 @@ @pytest.mark.parametrize("tool", ["alice_recall", "alice_context_pack"]) def test_hidden_later_inputs_do_not_underfill_ranked_memory_reads(tmp_path, monkeypatch, tool): + monkeypatch.setattr("alicebot_api.vnext_retrieval.LEGACY_SCOPED_SCAN_MAX_ROWS", 16) path = tmp_path / "refill.db" bootstrap_database(path, user_id=USER, user_email="synthetic@example.invalid") with sqlite_user_connection(path, USER) as conn, without_insert_floor(): From 804169d618b56f2196bad8de638eb8530d944f1b Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:03:11 +0200 Subject: [PATCH 250/270] Resolve disposable mutation checkout paths on macOS --- scripts/verify_derived_label_mutations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/verify_derived_label_mutations.py b/scripts/verify_derived_label_mutations.py index 26e096fce..2e6f09648 100644 --- a/scripts/verify_derived_label_mutations.py +++ b/scripts/verify_derived_label_mutations.py @@ -31,7 +31,7 @@ def main() -> int: args.output.mkdir(parents=True, exist_ok=True) results = [] with tempfile.TemporaryDirectory(prefix="alice-label-mutants-") as directory: - root = Path(directory) + root = Path(directory).resolve() archive = root / "source.tar" with archive.open("wb") as destination: subprocess.run(["git", "-C", str(repository), "archive", revision], stdout=destination, check=True) From dd2226879a06d583160d9b521599f5b3eebd5f36 Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:22:09 +0200 Subject: [PATCH 251/270] Preserve SQLite schema and pin round-three read contracts --- CHANGELOG.md | 2 +- apps/api/src/alicebot_api/sqlite_schema.py | 12 -------- apps/api/src/alicebot_api/sqlite_store.py | 19 ++++++++++++ scripts/derived_label_mutations.json | 2 +- ...derived_labels_read_acceptance_postgres.py | 23 +++++++++----- tests/performance/round3_budget_probe.py | 2 ++ tests/unit/test_label_reader_stage_matrix.py | 4 ++- tests/unit/test_label_round3_sqlite_reads.py | 30 +++++++++++++++++-- .../unit/test_legacy_surface_test_posture.py | 2 +- ...oop_ids_every_spelling_and_after_delete.py | 5 ++-- .../unit/test_store_events_revisions_split.py | 6 ++-- .../unit/test_store_graph_open_loops_split.py | 6 ++-- tests/unit/test_store_memory_access_split.py | 10 ++++--- .../unit/test_store_memory_lifecycle_split.py | 14 +++++---- tests/unit/test_vnext_store.py | 7 +++-- tests/unit/test_workspaces_router_split.py | 6 ++-- 16 files changed, 104 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb9889341..e93e4b6ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Unreleased (on main, not in v0.20.0): October 7 round-three corrections restore the source GET operator gate before its caller fence, preserve the owner's and unbound admin's complete source trace and workspace diagnostics and totals, and clamp an owner project assignment to the derived row's settled scope. A refused assignment sets `label_floor_applied`, records `floor_clamped` even when the existing label stays unchanged, and leaves no labels-check gap. Restricted reads batch ancestry lookups and reuse verified resolved parent labels within one request, preserving aliases, cycles and per-origin bounds. SQLite adds indexes for stored UUID aliases. Memory full-text reads refill after effective admission; exhausted legacy adapters retain the finite completeness error. CI measures the random mixed-parent grid before and after real repair, including the SQLite keyless agent path, against pinned main 48873b03. No migration is required. SQLite creates its alias indexes during the idempotent schema bootstrap. +- Unreleased (on main, not in v0.20.0): October 7 round-three corrections restore the source GET operator gate before its caller fence, preserve the owner's and unbound admin's complete source trace and workspace diagnostics and totals, and clamp an owner project assignment to the derived row's settled scope. A refused assignment sets `label_floor_applied`, records `floor_clamped` even when the existing label stays unchanged, and leaves no labels-check gap. Restricted reads batch ancestry lookups and reuse verified resolved parent labels within one request, preserving aliases, cycles and per-origin bounds. SQLite resolves stored UUID aliases once per request without changing the vault schema. Memory full-text reads refill after effective admission; exhausted legacy adapters retain the finite completeness error. CI measures the random mixed-parent grid before and after real repair, including the SQLite keyless agent path, against pinned main 48873b03. No migration is required. - Unreleased (on main, not in v0.20.0): October 6 round-two corrections recover a per-user open-loop digest on both stores after a project reassignment or stale duplicate extraction, and weekly candidates record the report inputs before insert so their stored floor matches the weekly artifact in the same transaction. Read-time labels reuse verified dependency signatures within one request; original rows are counted in SQL, and the derived partition still receives the kernel guard. PostgreSQL event filtering uses an indexed guarded UUID comparison. SQLite filters definitely hidden direct parents before its event limit, then applies the full effective-label guard. The legacy review list also rechecks saved quotes on original or imported candidates against the current source fence. CI compares unique-metadata, varied-parent and repaired fixtures against pinned main 48873b03. No migration is required. diff --git a/apps/api/src/alicebot_api/sqlite_schema.py b/apps/api/src/alicebot_api/sqlite_schema.py index 569b967f3..3d8c688a2 100644 --- a/apps/api/src/alicebot_api/sqlite_schema.py +++ b/apps/api/src/alicebot_api/sqlite_schema.py @@ -823,18 +823,6 @@ def _sql_list(values: tuple[str, ...]) -> str: ON sources (user_id, captured_at DESC, id DESC) """, """ - CREATE INDEX IF NOT EXISTS sources_user_label_alias_idx - ON sources (user_id, replace(replace(replace(replace(lower(id),'urn:uuid:',''),'-',''),'{',''),'}','')) - """, - """ - CREATE INDEX IF NOT EXISTS memories_user_label_alias_idx - ON memories (user_id, replace(replace(replace(replace(lower(id),'urn:uuid:',''),'-',''),'{',''),'}','')) - """, - """ - CREATE INDEX IF NOT EXISTS open_loops_user_label_alias_idx - ON open_loops (user_id, replace(replace(replace(replace(lower(id),'urn:uuid:',''),'-',''),'{',''),'}','')) - """, - """ CREATE INDEX IF NOT EXISTS source_chunks_source_index_idx ON source_chunks (source_id, chunk_index) """, diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index c541de4c0..d196d09dc 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -497,6 +497,25 @@ def read_label_rows(self, kind: str, ids: Sequence[str]) -> list[VNextRow]: extra = ", value, project_id, source_event_ids, deleted_at, status" elif table == "open_loops": extra = ", project_id, source_id, memory_id" + from alicebot_api.vnext_label_guard import request_row_cache + from alicebot_api.vnext_derived_labels import identifier + + cache = request_row_cache(self, "sqlite-label-id-map") + if cache is not None: + if kind not in cache: + aliases: dict[str, list[str]] = {} + for stored in self._fetch_all(f"SELECT id FROM {table} WHERE user_id = ?", (self.user_id,)): + stored_id = str(stored["id"]) + aliases.setdefault(identifier(stored_id), []).append(stored_id) + cache[kind] = aliases + resolved = set(wanted) + for item in wanted: + resolved.update(cache[kind].get(identifier(item), ())) + marks = ",".join("?" for _ in resolved) + return self._fetch_all( + f"SELECT id, user_id, domain, sensitivity, metadata_json{extra} FROM {table} WHERE user_id = ? AND id IN ({marks})", + (self.user_id, *sorted(resolved)), + ) from uuid import UUID canonical = [] for item in wanted: diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index 8eeee9f16..1a7646917 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -342,7 +342,7 @@ "line": 12, "before": "doctor label counts are new in this set and remain outside that ceiling", "after": "doctor label counts are new in this set", - "sha256": "1bc64441c09f82aaa4f1147563121b97f33115fc0790f59e167116aa0bc2b74d" + "sha256": "eaca6f2b2b12766a0551baf8bb70cc261f044a3861432145a2cb8ed7cff94193" } ], "pytest": [ diff --git a/tests/integration/test_derived_labels_read_acceptance_postgres.py b/tests/integration/test_derived_labels_read_acceptance_postgres.py index 8a9b0cb00..04c4178ba 100644 --- a/tests/integration/test_derived_labels_read_acceptance_postgres.py +++ b/tests/integration/test_derived_labels_read_acceptance_postgres.py @@ -14,7 +14,7 @@ from alicebot_api.mcp.types import MCPRuntimeContext, MCPToolError from alicebot_api.routers import vnext_review, vnext_retrieval, vnext_projects, vnext_memories, workspaces from alicebot_api.store import ContinuityStore -from alicebot_api.vnext_agent_control import ALL_SENSITIVITY +from alicebot_api.vnext_agent_control import ALL_SENSITIVITY, AgentIdentity from alicebot_api.vnext_agent_keys import resolve_agent_identity from alicebot_api.vnext_event_log import append_event from alicebot_api.vnext_label_guard import LabelGuard @@ -123,7 +123,9 @@ def test_workspace_counts_full_population_with_sql_hidden_and_stale_rows(migrate store.create_memory({"memory_key": f"hidden-{index}", "canonical_text": "Cedar hidden", "status": "candidate", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(secret["id"])}}) store.create_artifact({"artifact_type": "daily_brief", "title": "Cedar hidden", "content_markdown": "Cedar hidden", "domain": "project", "sensitivity": "confidential", "metadata_json": {"derived_from": {"v": 1, "sources": [], "memories": [], "open_loops": [], "artifacts": [], "beliefs": [], "counts": {"sources": 0, "memories": 0, "open_loops": 0, "artifacts": 0, "beliefs": 0}}}}) store.create_project({"name": "Cedar hidden", "slug": "hidden", "domain": "project", "sensitivity": "confidential"}) - body = workspaces._vnext_workspace_payload(store) + body = workspaces._vnext_workspace_payload( + store, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent") + ) assert body["summary"]["source_count"] == 205 assert body["summary"]["candidate_memory_count"] == 35 assert body["summary"]["artifact_count"] == 0 @@ -158,14 +160,19 @@ def test_workspace_activity_uses_actual_key_and_current_targets(migrated_databas response = workspaces.get_vnext_workspace(user_id, authorization=f"Bearer {key}" if key else None) assert response.status_code == 200 rendered = response.body.decode() - assert all(identifier not in rendered for identifier in hidden_ids) + if reader == "trusted": + assert all(identifier not in rendered for identifier in hidden_ids) + assert "Cedar hidden" not in rendered + else: + assert all(identifier in rendered for identifier in hidden_ids) + assert "Cedar hidden" in rendered assert all(identifier in rendered for identifier in visible_ids) - assert "Cedar hidden" not in rendered body = json.loads(rendered) - assert len(body["agent_activity"]["policy_blocks"]) == 1 - assert len(body["agent_activity"]["recent_commits"]) == 1 - assert len(body["agent_activity"]["inline_confirmations"]) == 1 - assert body["dogfooding"]["sample_scope"]["memories"]["total_count"] == 1 + expected_count = 1 if reader == "trusted" else 2 + assert len(body["agent_activity"]["policy_blocks"]) == expected_count + assert len(body["agent_activity"]["recent_commits"]) == expected_count + assert len(body["agent_activity"]["inline_confirmations"]) == expected_count + assert body["dogfooding"]["sample_scope"]["memories"]["total_count"] == expected_count @pytest.mark.parametrize("reader", ("owner", "admin", "trusted")) diff --git a/tests/performance/round3_budget_probe.py b/tests/performance/round3_budget_probe.py index 2dbd2014e..1e3f573d3 100644 --- a/tests/performance/round3_budget_probe.py +++ b/tests/performance/round3_budget_probe.py @@ -64,4 +64,6 @@ def read(path): measurement = {"wall": time.perf_counter() - wall_start, "cpu": time.process_time() - cpu_start} if name == "recall": measurement["memories"] = len(result["results"]) + elif name == "pack": + measurement["memories"] = len(result["memories"]) print(json.dumps(measurement), flush=True) diff --git a/tests/unit/test_label_reader_stage_matrix.py b/tests/unit/test_label_reader_stage_matrix.py index 59ee6ba55..07b15c645 100644 --- a/tests/unit/test_label_reader_stage_matrix.py +++ b/tests/unit/test_label_reader_stage_matrix.py @@ -112,7 +112,9 @@ def _stage(stage, store): if field == "memories": store.rows["memory"][0]["status"] = "candidate" field = "review_memories" - return workspaces._vnext_workspace_payload(store)[field] + return workspaces._vnext_workspace_payload( + store, identity=AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent") + )[field] trusted = AgentIdentity(agent_id="trusted", permission_profile="trusted_local_agent") if stage == "loop_memory_reference": shown = withhold_unreadable_references(store, [{"id": LOOP, "memory_id": MEMORY, "metadata_json": {}}], fence=SourceReadFence.for_identity(trusted)) diff --git a/tests/unit/test_label_round3_sqlite_reads.py b/tests/unit/test_label_round3_sqlite_reads.py index 652203bf2..5ffe76166 100644 --- a/tests/unit/test_label_round3_sqlite_reads.py +++ b/tests/unit/test_label_round3_sqlite_reads.py @@ -8,13 +8,39 @@ from alicebot_api.onramp import bootstrap_database from alicebot_api.sqlite_store import SQLiteVNextStore, sqlite_user_connection from alicebot_api.vnext_derived_labels import with_derived_from -from alicebot_api.vnext_label_guard import LabelGuard +from alicebot_api.vnext_label_guard import LabelGuard, label_read_scope from alicebot_api.vnext_label_repair import label_gap_counts from alicebot_api.vnext_label_writes import without_insert_floor USER = "11111111-1111-4111-8111-111111111111" +def test_request_id_map_keeps_ambiguous_aliases_and_refreshes_after_writes(tmp_path): + path = tmp_path / "aliases.db" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.invalid") + canonical = str(uuid4()) + with sqlite_user_connection(path, USER) as conn: + store = SQLiteVNextStore(conn, USER) + store.create_source({"id": canonical.upper(), "source_type": "note", "title": "First", + "content_hash": "first", "sensitivity": "public", "domain": "project"}) + # The facade also accepts ordinary tuple-returning SQLite connections. + conn.row_factory = None + with label_read_scope(store): + assert {row["id"] for row in store.read_label_rows("source", [canonical])} == {canonical.upper()} + store.create_source({"id": canonical, "source_type": "note", "title": "Alias", + "content_hash": "alias", "sensitivity": "confidential", "domain": "project"}) + aliases = store.read_label_rows("source", [canonical]) + assert {row["id"] for row in aliases} == {canonical, canonical.upper()} + # A canonical collision must remain visible to the guard, which refuses it. + effective = LabelGuard(store, active=True).effective_row("memory", { + "id": str(uuid4()), "user_id": USER, "domain": "project", "sensitivity": "public", + "metadata_json": {"source_id": canonical}, + }) + assert effective["unverified"] is True + with label_read_scope(store): + assert len(store.read_label_rows("source", [canonical])) == 2 + + @pytest.mark.parametrize("tool", ["alice_recall", "alice_context_pack"]) def test_hidden_later_inputs_do_not_underfill_ranked_memory_reads(tmp_path, monkeypatch, tool): monkeypatch.setattr("alicebot_api.vnext_retrieval.LEGACY_SCOPED_SCAN_MAX_ROWS", 16) @@ -55,7 +81,7 @@ def test_native_sql_prefilter_rejects_hidden_inputs_before_limit(tmp_path): for i in range(12): parent = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "other", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": hidden["id"]}}) - store.create_memory({"memory_key": str(uuid4()), "canonical_text": "prefilter observation", "status": "active", "domain": "project", + store.create_memory({"memory_key": str(uuid4()), "title": "prefilter observation", "canonical_text": "prefilter observation", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"consolidation": {"cluster_member_ids": [parent["id"]]}}}) rows = store.search_memories_fts(query="prefilter observation", sensitivity_allowed=["public", "internal"], limit=1) assert [row["id"] for row in rows] == [visible["id"]] diff --git a/tests/unit/test_legacy_surface_test_posture.py b/tests/unit/test_legacy_surface_test_posture.py index 832384d08..beda30df3 100644 --- a/tests/unit/test_legacy_surface_test_posture.py +++ b/tests/unit/test_legacy_surface_test_posture.py @@ -37,7 +37,7 @@ def test_integration_runners_enable_legacy_surfaces_without_changing_unit_postur assert run_lines == [ "run: ALICE_LEGACY_SURFACES=1 ./.venv/bin/python -m pytest " "tests/integration -q -p no:cacheprovider --durations=20 " - "--timeout=180 --session-timeout=1500" + "--timeout=180 --session-timeout=3000" ] assert ( "ALICE_LEGACY_SURFACES=1 $(PYTHON) -m pytest tests/integration -q" diff --git a/tests/unit/test_open_loop_ids_every_spelling_and_after_delete.py b/tests/unit/test_open_loop_ids_every_spelling_and_after_delete.py index 51be34da2..9a87065cc 100644 --- a/tests/unit/test_open_loop_ids_every_spelling_and_after_delete.py +++ b/tests/unit/test_open_loop_ids_every_spelling_and_after_delete.py @@ -1332,7 +1332,8 @@ def squashed(path: str) -> str: "and so is an id the reader may read that has a hyphen and more hex digits after it (`<id>-20261003`).", "One layout of glued digits is cut under a reference key:", "An id without hyphens inside a URL or a sentence is withheld like the hyphenated one.", - "Inside longer text an id with its hyphens in other places, a split id and an encoded id are not recognised.", + "a second source-only pass now withholds irregular, split and encoded source ids inside longer text.", + "Irregular or non-ASCII MEMORY-prefixed ids and unnamed SOURCE whitespace forms can remain;", "Not changed here: the free-text columns of a loop (`title`, `description`, `resolution_note`), which are returned as stored and are not scanned for ids.", "It now writes `a source with no title`.", "`tests/unit/test_open_loop_ids_every_spelling_and_after_delete.py`", @@ -1344,7 +1345,7 @@ def squashed(path: str) -> str: "A run of 32 hex digits is an id only when no hex digit stands next to it", "and so is an id the reader may read that has a hyphen and more hex digits after it.", "One layout of glued digits is cut under a reference key:", - "Inside longer text an id with its hyphens in other places, a split id and an encoded id are not recognised.", + "a second source-only pass withholds irregular, split and encoded source ids inside longer text.", "The free-text columns of a loop (`title`, `description`, `resolution_note`) are returned as stored and are not scanned.", "The extractor of candidate loops no longer writes the id of a source with no title into the `description`", # Moved here from the limitations page, which keeps one short statement of the rule. diff --git a/tests/unit/test_store_events_revisions_split.py b/tests/unit/test_store_events_revisions_split.py index f905d02da..38cdb6b91 100644 --- a/tests/unit/test_store_events_revisions_split.py +++ b/tests/unit/test_store_events_revisions_split.py @@ -1,3 +1,5 @@ +# Round three receipts: native FTS offset paging, explicit NULL project clamps, +# and owner/admin workspace parity. Existing graft identities and schema pins remain enforced. from __future__ import annotations import ast @@ -151,7 +153,7 @@ # Re-minted for the per-file importer savepoint (2026-10-02): ``savepoint`` is appended last on both façades. # Previous receipt: 650e2e0ff088d67c... Proof: the class key list equals the list at origin/main 040a2a10 # plus ``savepoint`` before ``__dict__``, with every other key in the same order. - "postgres": "2a91ba3efd288f45d88d8eebd590ec76659e29b3d3223e818cc7828661807249", + "postgres": "3e1113952b741f978c79325e4cd08067763bc1d2a770a8ace20667d9cbc71436", # Re-minted for the merge of #500 and #502: ``check_literal_match_query``. # Re-minted again for per-project memory S2 (2026-10-02): the two single-scan partition reads # ``list_memories_view_partitions`` and ``list_open_loops_view_partitions``, SQLite only on purpose @@ -159,7 +161,7 @@ # Re-minted again for the per-file importer savepoint (2026-10-02), the same appended method. # Previous receipt: 365b7a01acf7a8b5... Proof: as for postgres, one added key and no other change. # Re-minted for the derived-label lock: ``lock_label_writes`` and ``read_label_rows`` on both facades. - "sqlite": "50d5aad840a245b6d01b72527ed559c97faad3b8cbbfbcb06982e1e9dce1f270", + "sqlite": "a76b6dfeb7a77da3206245a6155f3b2fe2eeaebfe219111dcaa97ddf7b0f9c7a", } EXPECTED_SUPPORT_AST_SHA256 = { "postgres_columns": { diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index cea2b1d3d..d1a984a84 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -1,3 +1,5 @@ +# Round three receipts: native FTS offset paging, explicit NULL project clamps, +# and owner/admin workspace parity. Existing graft identities and schema pins remain enforced. from __future__ import annotations import ast @@ -165,7 +167,7 @@ # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # lock_label_writes and read_label_rows follow __init__. Previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (180, "98a0c0e5668366e07420aae305e3dd49f042e62a8cde9f0a27abb0ed9caf03e7"), + "PostgresVNextStore": (181, "5f3acdf8872b9bd81ef550ea96d645680ad35a1e4018451cae45778d0ef82449"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -181,7 +183,7 @@ # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. # lock_label_writes and read_label_rows follow __init__. Previous receipt (134, 13012720...). - "SQLiteVNextStore": (139, "d90f82579ad45d50bd22e9270d46e0b1ce50529e5ebfaf7047fb2f54799a6c54"), + "SQLiteVNextStore": (140, "2bd805be8a616fdc10839fa7abcf0c3e286dc3faef67757ba08810eca174cca7"), } EXPECTED_COLUMN_AST = { POSTGRES_COLUMNS_PATH: { diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index 0eb8fdb86..f9b3ef65b 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -1,3 +1,5 @@ +# Round three receipts: native FTS offset paging, explicit NULL project clamps, +# and owner/admin workspace parity. Existing graft identities and schema pins remain enforced. from __future__ import annotations import ast @@ -43,7 +45,7 @@ # Re-minted so the two roll-up lookups overlap scope united with floor. # Every other statement still uses the scope expression. Previous receipt 46946cc0... "apps/api/src/alicebot_api/vnext_stores/postgres/memory_access.py": ( - "ad6a1a81f077fbdaf13ec414558d43a8b6ad085350aec416e8a5e856a227d7c0" + "ea1e4a4ce9f58968e4e10ac66830822798aa26118207a39fb465b4f26da74aea" ), # Re-minted for per-project memory S2 (2026-10-02): the project fence builders read the reserved global # marker and take the domains to leave out, and the single-scan partition SQL and the materialized-CTE hint @@ -90,7 +92,7 @@ # Re-minted so the memory partition read passes the floor identity. # Previous receipt 1580dca3... "apps/api/src/alicebot_api/vnext_stores/sqlite/memory_access.py": ( - "58e22342c6aaf4ed73aa78e5f2ee85af1d8deeb7cbad6a3abe06f78fb8f9a0ac" + "345262de6a46660a7156dbdc4c8ff41effcf5861e513bb8717365d53d880bd4f" ), } @@ -222,7 +224,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (180, "98a0c0e5668366e07420aae305e3dd49f042e62a8cde9f0a27abb0ed9caf03e7"), + "PostgresVNextStore": (181, "5f3acdf8872b9bd81ef550ea96d645680ad35a1e4018451cae45778d0ef82449"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -239,7 +241,7 @@ # prunable_sources here; every pre-existing class member keeps its order. # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (134, 13012720...). - "SQLiteVNextStore": (139, "d90f82579ad45d50bd22e9270d46e0b1ce50529e5ebfaf7047fb2f54799a6c54"), + "SQLiteVNextStore": (140, "2bd805be8a616fdc10839fa7abcf0c3e286dc3faef67757ba08810eca174cca7"), } diff --git a/tests/unit/test_store_memory_lifecycle_split.py b/tests/unit/test_store_memory_lifecycle_split.py index d693d363f..eb536b02e 100644 --- a/tests/unit/test_store_memory_lifecycle_split.py +++ b/tests/unit/test_store_memory_lifecycle_split.py @@ -1,3 +1,5 @@ +# Round three receipts: native FTS offset paging, explicit NULL project clamps, +# and owner/admin workspace parity. Existing graft identities and schema pins remain enforced. from __future__ import annotations import ast @@ -86,7 +88,7 @@ # and the memory update checks exclusive L before changing labels. SOURCE_RECEIPTS = { COMMON_PATH: "8fc077dc71f0e631a2df81de2ebeec1fb6c768f341c2e7891309e4753eef7bb5", - POSTGRES_CARRIER_PATH: "23ab87cde159a285bf8c71dbc6d2eb4e0e15035ce0f509bef38ba799f3f92de3", + POSTGRES_CARRIER_PATH: "2b540dc4b52e74d564ef22ce9f9ec98a527d4e5a74f00497c33ab29d37bce109", # SQLite carrier re-minted for the Phase 4 Stage 2 resident vector cache # (reviewed change): redaction paths that NULL a live embedding now bump # the embedding_stamp token in the same transaction (prompt eviction). @@ -97,11 +99,11 @@ # back between two reads cannot fail memories_seen_range_check. Previous # sqlite receipt 67adaa61..., method AST 3f134ac9...; the metadata # manifests are unchanged. - SQLITE_CARRIER_PATH: "759cf44762c388e599b4e8c377fa3415ca2fdf9ba7884698259171ef3d2b8138", + SQLITE_CARRIER_PATH: "0e8c2d50857d1064fc94b703c7cc05da234df63d4c278bc08a062fc90d450c3d", } EXPECTED_METHOD_AST_MANIFESTS = { - "postgres": "827c4f391a1efe50dcb265b7bb50a5b191d2bae32c3f817dcc237d51bfb10d29", - "sqlite": "3577659fcf9e583bdb957bb1a959ac1d8cae07ce8b50321bc36697dec47acec4", + "postgres": "d9f5002319353a3ebe08cbdd8273faf2b2a11d79fcdf6d7cf779d157d8aae5e6", + "sqlite": "e5eecd8cc9b7922957adcf1a04b6238efa1fafb9e39eeefb310da86c4760ae24", } EXPECTED_METADATA_MANIFESTS = { "postgres": "07a567e26d0f7c4f51ae2a1910d059397b513a575d85f06c2f8c0396ac1bb2ef", @@ -117,7 +119,7 @@ # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (180, "98a0c0e5668366e07420aae305e3dd49f042e62a8cde9f0a27abb0ed9caf03e7"), + "PostgresVNextStore": (181, "5f3acdf8872b9bd81ef550ea96d645680ad35a1e4018451cae45778d0ef82449"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -132,7 +134,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. - "SQLiteVNextStore": (139, "d90f82579ad45d50bd22e9270d46e0b1ce50529e5ebfaf7047fb2f54799a6c54"), + "SQLiteVNextStore": (140, "2bd805be8a616fdc10839fa7abcf0c3e286dc3faef67757ba08810eca174cca7"), } EXPECTED_FACADE_COMMENT_DIGESTS = { POSTGRES_FACADE_PATH: "d8599a46ee26dc35a3ae52c1a98a416509add9ae4a42ece780c5c5ed7e132b93", diff --git a/tests/unit/test_vnext_store.py b/tests/unit/test_vnext_store.py index 3ff885712..c42c837c8 100644 --- a/tests/unit/test_vnext_store.py +++ b/tests/unit/test_vnext_store.py @@ -1918,6 +1918,7 @@ def test_fts_search_builds_websearch_tsquery_with_pushed_down_filters() -> None: None, "Alice provenance retrieval", 25, + 0, # first ranked page ) @@ -1969,6 +1970,7 @@ def test_fts_search_pushes_down_memory_type_project_agent_run_and_expiry_filters None, "Alice provenance retrieval", 25, + 0, # first ranked page ) @@ -1992,7 +1994,7 @@ def test_fts_search_pushes_people_and_time_scope_before_ranked_limit() -> None: assert "jsonb_path_query" in query assert "id::text = ANY" in query assert "COALESCE(valid_from, last_seen_at, updated_at, first_seen_at, created_at)" in query - assert params[-9:] == ( + assert params[-10:] == ( ["sam"], [linked_memory_id], ["sam"], @@ -2002,6 +2004,7 @@ def test_fts_search_pushes_people_and_time_scope_before_ranked_limit() -> None: window_end, "deployment", 1, + 0, ) @@ -2962,7 +2965,7 @@ def test_update_memory_reassigns_first_class_and_canonical_project_scope_togethe ) query, params = next((query, params) for query, params in cursor.statements if "UPDATE memories" in query) - assert "project_id = COALESCE(%s, project_id)" in query + assert "project_id = CASE WHEN %s THEN %s ELSE project_id END" in query assert params is not None metadata_param = next(param for param in params if isinstance(param, Jsonb)) assert metadata_param.obj["project_scope"] == ["project-new"] diff --git a/tests/unit/test_workspaces_router_split.py b/tests/unit/test_workspaces_router_split.py index b610c6b7a..5d5226fcf 100644 --- a/tests/unit/test_workspaces_router_split.py +++ b/tests/unit/test_workspaces_router_split.py @@ -1,3 +1,5 @@ +# Round three receipts: native FTS offset paging, explicit NULL project clamps, +# and owner/admin workspace parity. Existing graft identities and schema pins remain enforced. from __future__ import annotations import ast @@ -86,7 +88,7 @@ # Re-pin 2026-10-06: workspace reads authenticate the protected identity and # admit rows through effective labels before totals or dashboard disclosure. EXPECTED_ROUTE_AST_SHA256 = "b99f1435de67d9499819acb9ed7ed61b588a3fb0ff037e782eeca070b39af742" -EXPECTED_SUPPORT_AST_SHA256 = "000d3a03afeff74c5c7fbca8bfc5195553de4fec6dd962ea94aa9b7bfb58ad90" +EXPECTED_SUPPORT_AST_SHA256 = "997f71ac38b434aefb35a94414b03aabe52ffacbd3af2f2750b1de109d0a3dbb" EXPECTED_ROUTE_NAME_MANIFEST_SHA256 = "225c57c08bd8314156c56352dd1c53ffed3f556ce285c666dd6fca125115d0b4" EXPECTED_OPERATION_MANIFEST_SHA256 = "c320979b62d7ee8de244fe38bde5bf3761a4f9d76f76bf3cd8576c30fce9857e" EXPECTED_IMPORT_MANIFEST_SHA256 = "e8c18d6831ca012b55b22f46c9b2151d62575773a2452ef0d0f2e869cabc8abb" @@ -141,7 +143,7 @@ } EXPECTED_SUPPORT_NODE_SHA256 = { "_vnext_status_counts": "0bf0ed228a14bd648a9d18fcd5f99ebf8c585bd29f4b5e81e1df17fe0201fd15", - "_vnext_workspace_payload": "762ce1dc4c0d85bb48a56730d5c9ced74230c87a5b50ec1e1cd65cd25851ba96", + "_vnext_workspace_payload": "a5dd3efc718c0df7dac7d6cccc7ce749e28fab627024fd688eb2f316cbb22da9", "_workspace_rows": "070bdfbd1eae10608bd8208b08367e1c0ea10e2064f03ad5a84121a190ed4cf0", "_workspace_event_visible": "8343c060909326a5cb69fa6f671ac62f160630ecf989f04d78e792ea74c0ea90", } From 1e3fc41b1bae23d44c86eea1daa07455903749e7 Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:55:50 +0200 Subject: [PATCH 252/270] Reuse completed input prefetches and document closed SQL construction --- .../api/src/alicebot_api/vnext_label_guard.py | 17 ++++++++++++-- apps/api/src/alicebot_api/vnext_label_sql.py | 6 ++--- scripts/derived_label_mutations.json | 18 +++++++-------- tests/unit/test_label_resolved_inputs.py | 23 ++++++++++++++++++- tests/unit/test_label_sql_closed_literals.py | 22 ++++++++++++++++++ 5 files changed, 71 insertions(+), 15 deletions(-) create mode 100644 tests/unit/test_label_sql_closed_literals.py diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index 74ed2070d..e89049a1f 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -405,15 +405,24 @@ def _prefetch_inputs(self, kind: str, rows: Sequence[Mapping[str, object]]) -> N if not callable(reader): return state = self._state() + prefetched = state.row_sets.setdefault("prefetched-input-expansions", {}) frontier = [(kind, row) for row in rows if isinstance(row, Mapping)] expanded: set[tuple[str, str]] = set() for _ in range(HOP_BOUND + 1): refs: set[tuple[str, str]] = set() + pending: list[tuple] = [] for row_kind, row in frontier: if is_derived(row_kind, row): - refs.update(self._signature(row_kind, row, key=self._key(row_kind, row))[1]) + key = self._key(row_kind, row) + if key in prefetched: + continue + refs.update(self._signature(row_kind, row, key=key)[1]) + pending.append(key) refs.difference_update(expanded) - if not refs or len(expanded | refs) > NODE_BOUND: + if len(expanded | refs) > NODE_BOUND: + return + if not refs: + prefetched.update(dict.fromkeys(pending)) return expanded.update(refs) wanted: dict[str, set[str]] = {} @@ -427,6 +436,10 @@ def _prefetch_inputs(self, kind: str, rows: Sequence[Mapping[str, object]]) -> N canonical = identifier(found.get("id")) if canonical in ids: state.nodes[(ref_kind, canonical)].append(dict(found)) + # Mark only after these immediate inputs have actually been read. + # A bounded/partial frontier still uses per-origin settlement; + # this raw-row reuse never supplies a label or an admission grant. + prefetched.update(dict.fromkeys(pending)) frontier = [(ref_kind, row) for ref_kind, ref_id in refs for row in state.nodes[(ref_kind, ref_id)]] diff --git a/apps/api/src/alicebot_api/vnext_label_sql.py b/apps/api/src/alicebot_api/vnext_label_sql.py index e34220cf9..13840d28d 100644 --- a/apps/api/src/alicebot_api/vnext_label_sql.py +++ b/apps/api/src/alicebot_api/vnext_label_sql.py @@ -16,7 +16,7 @@ def hidden_memory_input_sql(sensitivity_allowed, *, sqlite: bool, alias: str = " blocked = [value for value, rank in SENSITIVITY_RANK.items() if rank > ceiling] if sensitivity_allowed else [] if not blocked: return "TRUE" - names = ",".join("'" + value + "'" for value in blocked) # closed kernel constants + names = ",".join("'" + value + "'" for value in blocked) # Closed kernel constants and whitelisted aliases are the only SQL inputs. if sqlite: source = f"alice_direct_source_hint({alias}.metadata_json)" memory = f"alice_direct_memory_hint({alias}.metadata_json)" @@ -46,7 +46,7 @@ def uuid_hint(value): AND label_source.id={parent_source} WHERE {input_not_redacted} AND label_source.sensitivity IN ({names}) ), FALSE) - )))""" + )))""" # nosec B608 return f"""({alias}.sensitivity NOT IN ({names}) AND NOT ( {not_redacted} AND (EXISTS ( SELECT 1 FROM sources label_source WHERE label_source.user_id={alias}.user_id @@ -56,7 +56,7 @@ def uuid_hint(value): ON label_source.user_id=label_input.user_id AND label_source.id={parent_source} WHERE label_input.user_id={alias}.user_id AND label_input.id={memory} AND {input_not_redacted} AND label_source.sensitivity IN ({names}) - ))))""" + ))))""" # nosec B608 def original_label_sql(kind: str, *, sqlite: bool = False) -> str: diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index 1a7646917..f707cbd79 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -144,7 +144,7 @@ "line": 347, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], ())", - "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" } ], "pytest": [ @@ -160,7 +160,7 @@ "line": 347, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], tuple(sorted(set(semantic_parents))))", - "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" } ], "pytest": [ @@ -176,7 +176,7 @@ "line": 334, "before": "if len(raw) != 1 or identifier(raw[0].get(\"id\")) != parent_id:", "after": "if not raw or identifier(raw[0].get(\"id\")) != parent_id:", - "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" } ], "pytest": [ @@ -194,7 +194,7 @@ "line": 342, "before": "if depth > HOP_BOUND or len(ancestry) > NODE_BOUND:", "after": "if False:", - "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" } ], "pytest": [ @@ -212,7 +212,7 @@ "line": 318, "before": "if problem or has_implicit_weekly_inputs(kind, row):", "after": "if problem:", - "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" } ], "pytest": [ @@ -227,10 +227,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 449, + "line": 462, "before": "if not self.active and callable(native):", "after": "if False:", - "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" } ], "pytest": [ @@ -246,7 +246,7 @@ "line": 17, "before": "if not blocked:", "after": "if True:", - "sha256": "06c2349106e6bf5c61dfaaf41f3b285f388a501c7fe367a6394e71728fa49e9b" + "sha256": "f733e792850cd192b5615d1a309af2ab1fe466fce0b95469d790828c6a820b92" } ], "pytest": [ @@ -278,7 +278,7 @@ "line": 371, "before": " self._prefetch_inputs(kind, rows)", "after": " pass # removed batched ancestry reads", - "sha256": "4f372e9a9d228d2e18d27b8c48374501501f61a282c71f185a62d89c4033f879" + "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" } ], "pytest": [ diff --git a/tests/unit/test_label_resolved_inputs.py b/tests/unit/test_label_resolved_inputs.py index b586b3df0..a11b7584f 100644 --- a/tests/unit/test_label_resolved_inputs.py +++ b/tests/unit/test_label_resolved_inputs.py @@ -8,7 +8,7 @@ import pytest from alicebot_api.vnext_derived_labels import identifier, settle_labels, with_derived_from -from alicebot_api.vnext_label_guard import LabelGuard, label_read_scope +from alicebot_api.vnext_label_guard import LabelGuard, invalidate_read_labels, label_read_scope class Rows: @@ -90,6 +90,27 @@ def read_label_rows(self, kind, ids): assert store.calls <= 2 +def test_partial_prefetch_keeps_per_origin_floors_and_refreshes_after_writes(monkeypatch): + # The combined frontier exceeds this bound; every individual graph fits. + monkeypatch.setattr("alicebot_api.vnext_label_guard.NODE_BOUND", 3) + sources = [{"kind": "source", "id": str(UUID(int=i + 1)), "domain": "project", + "sensitivity": "confidential" if i == 1 else "public", "metadata_json": {}} + for i in range(3)] + parents = [{"kind": "memory", "id": str(UUID(int=i + 100)), "domain": "project", "sensitivity": "public", + "metadata_json": {"source_id": source["id"]}} for i, source in enumerate(sources)] + roots = [{"kind": "memory", "id": str(UUID(int=i + 200)), "domain": "project", "sensitivity": "public", + "metadata_json": {"consolidation": {"cluster_member_ids": [parent["id"]]}}} + for i, parent in enumerate(parents)] + store = Rows(sources + parents) + with label_read_scope(store): + guard = LabelGuard(store, active=True, sensitivity_allowed=("public",)) + for _ in range(2): + assert [row["id"] for row in guard.admit_rows("memory", roots)] == [roots[0]["id"], roots[2]["id"]] + sources[0]["sensitivity"] = "confidential" + invalidate_read_labels(store) + assert [row["id"] for row in guard.admit_rows("memory", roots)] == [roots[2]["id"]] + + @pytest.mark.parametrize("variant", ["alias", "cycle", "missing", "malformed", "hop-bound", "node-bound", "implicit-weekly", "implicit-weekly-json"]) def test_unproved_inputs_keep_the_complete_graph_fallback(monkeypatch, variant): import alicebot_api.vnext_label_guard as module diff --git a/tests/unit/test_label_sql_closed_literals.py b/tests/unit/test_label_sql_closed_literals.py new file mode 100644 index 000000000..5777a4ea5 --- /dev/null +++ b/tests/unit/test_label_sql_closed_literals.py @@ -0,0 +1,22 @@ +"""Caller strings cannot enter the conservative native SQL predicate.""" +import pytest + +from alicebot_api.vnext_label_sql import hidden_memory_input_sql + + +@pytest.mark.parametrize("sqlite", (False, True)) +@pytest.mark.parametrize("alias", ("m", "memories")) +def test_prefilter_sensitivity_inputs_are_closed_kernel_literals(sqlite, alias): + hostile = "public'); DROP TABLE memories; --" + sql = hidden_memory_input_sql(["public", hostile], sqlite=sqlite, alias=alias) + assert hostile not in sql + assert "'confidential'" in sql + assert f"{alias}.sensitivity" in sql + + +@pytest.mark.parametrize("sqlite", (False, True)) +@pytest.mark.parametrize("values", ([], ["public"])) +def test_prefilter_refuses_a_caller_supplied_sql_alias(sqlite, values): + with pytest.raises(ValueError, match="unsupported memory alias"): + hidden_memory_input_sql(values, sqlite=sqlite, alias="m); DROP TABLE memories; --") + From 6a8079d812aa531992be02638dbd040c84dde5e5 Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:45:01 +0200 Subject: [PATCH 253/270] Reject proven hidden floors before count scans and reuse scoped admission --- .../src/alicebot_api/routers/workspaces.py | 5 +- apps/api/src/alicebot_api/sqlite_store.py | 30 ++++++--- apps/api/src/alicebot_api/vnext_dogfooding.py | 7 +- .../api/src/alicebot_api/vnext_label_guard.py | 23 +++++-- apps/api/src/alicebot_api/vnext_label_sql.py | 12 ++++ apps/api/src/alicebot_api/vnext_store.py | 30 ++++++--- .../vnext_stores/postgres/events_revisions.py | 6 +- .../vnext_stores/sqlite/events_revisions.py | 4 ++ scripts/derived_label_mutations.json | 64 ++++++++++++++----- .../test_label_round3_contracts_postgres.py | 6 ++ tests/unit/test_label_resolved_inputs.py | 21 ++++++ tests/unit/test_label_round3_sqlite_reads.py | 38 +++++++++++ tests/unit/test_label_sql_closed_literals.py | 10 ++- 13 files changed, 216 insertions(+), 40 deletions(-) diff --git a/apps/api/src/alicebot_api/routers/workspaces.py b/apps/api/src/alicebot_api/routers/workspaces.py index 08d0ba617..3a1fbf0cb 100644 --- a/apps/api/src/alicebot_api/routers/workspaces.py +++ b/apps/api/src/alicebot_api/routers/workspaces.py @@ -141,7 +141,10 @@ def _vnext_workspace_payload(store: PostgresVNextStore, *, identity: AgentIdenti fetched_beliefs = store.list_beliefs(status=None, sensitivity_allowed=sensitivity_allowed, limit=12) beliefs = guard.admit_beliefs(fetched_beliefs) tasks = store.list_tasks(status=None, limit=12) - fetched_events = store.list_events(limit=20) + if getattr(type(store), "label_count_input_prefilter", False): + fetched_events = store.list_events(limit=20, reject_sensitivity_allowed=guard.sensitivity_allowed) + else: + fetched_events = store.list_events(limit=20) recent_events = guard.admit_events(fetched_events) event_count = store.count_events() if unfenced else guard.readable_event_count() agent_identities = store.list_agent_identities(limit=20) diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index d196d09dc..9f3147741 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -466,6 +466,7 @@ class SQLiteVNextStore: #: Retrieval-trace label for the full-text stage (FTS5, not Postgres tsvector). fts_stage_source = "sqlite_fts" memory_fts_offset_paging = True + label_count_input_prefilter = True def __init__(self, conn: sqlite3.Connection, user_id: UUID | str): if str(user_id).strip() == "": @@ -555,7 +556,7 @@ def count_original_label_statuses(self, kind: str, *, domains=(), sensitivity_al rows = self._fetch_all(f"SELECT {status} AS status, COUNT(*) AS count FROM {table} WHERE {where} GROUP BY {status}", tuple(params)) return {str(row["status"]): int(cast(int, row["count"])) for row in rows} - def iter_label_rows(self, kind: str, *, batch_size: int = 500, derived_only: bool = False) -> Iterator[list[VNextRow]]: + def iter_label_rows(self, kind: str, *, batch_size: int = 500, derived_only: bool = False, reject_sensitivity_allowed: Sequence[str] = ()) -> Iterator[list[VNextRow]]: """Complete counted population, in narrow tenant-bound keyset batches.""" if batch_size < 1: @@ -572,35 +573,48 @@ def iter_label_rows(self, kind: str, *, batch_size: int = 500, derived_only: boo from alicebot_api.vnext_label_sql import original_label_sql if derived_only: live += " AND NOT COALESCE(" + original_label_sql(kind, sqlite=True) + ", FALSE)" + if kind == "memory" and reject_sensitivity_allowed: + from alicebot_api.vnext_label_sql import hidden_memory_input_sql + live += " AND " + hidden_memory_input_sql(reject_sensitivity_allowed, sqlite=True, alias="memories") + query_size = max(batch_size, 5000) if reject_sensitivity_allowed else batch_size after = "" while True: rows = self._fetch_all( f"""SELECT id, user_id, domain, sensitivity, metadata_json{extra} FROM {table} WHERE user_id = ? AND id > ?{live} ORDER BY id LIMIT ?""", - (self.user_id, after, batch_size), + (self.user_id, after, query_size), ) if not rows: return - yield rows + for start in range(0, len(rows), batch_size): + yield rows[start:start + batch_size] after = str(rows[-1]["id"]) + if len(rows) < query_size: + return - def iter_label_events(self, *, batch_size: int = 200) -> Iterator[list[VNextRow]]: + def iter_label_events(self, *, batch_size: int = 200, reject_sensitivity_allowed: Sequence[str] = ()) -> Iterator[list[VNextRow]]: """Complete event targets for readable counts, without event payloads.""" if batch_size < 1: raise ValueError("batch_size must be positive") + from alicebot_api.vnext_label_sql import hidden_memory_event_sql + label_sql = hidden_memory_event_sql(reject_sensitivity_allowed, sqlite=True) + query_size = max(batch_size, 5000) if reject_sensitivity_allowed else batch_size after = "" while True: rows = self._fetch_all( - """SELECT id, target_type, target_id, event_type FROM event_log - WHERE user_id = ? AND id > ? ORDER BY id LIMIT ?""", - (self.user_id, after, batch_size), + f"""SELECT id, target_type, target_id, event_type FROM event_log + WHERE user_id = ? AND id > ? AND {label_sql} ORDER BY id LIMIT ?""", + (self.user_id, after, query_size), ) if not rows: return - yield rows + for start in range(0, len(rows), batch_size): + yield rows[start:start + batch_size] after = str(rows[-1]["id"]) + if len(rows) < query_size: + return # -- fetch helpers (mirror PostgresVNextStore conventions) ------------ diff --git a/apps/api/src/alicebot_api/vnext_dogfooding.py b/apps/api/src/alicebot_api/vnext_dogfooding.py index 46cd085e3..631eacc7c 100644 --- a/apps/api/src/alicebot_api/vnext_dogfooding.py +++ b/apps/api/src/alicebot_api/vnext_dogfooding.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections import Counter +from collections.abc import Sequence from datetime import UTC, datetime, timedelta from statistics import mean from typing import TYPE_CHECKING, Protocol @@ -23,6 +24,7 @@ def list_events( target_type: str | None = None, target_id: str | None = None, limit: int | None = None, + reject_sensitivity_allowed: Sequence[str] = (), ) -> list[JsonObject]: ... def list_sources( @@ -196,7 +198,10 @@ def dashboard(self, *, sensitivity_allowed: tuple[str, ...] | None = None, label open_loops = guard.admit_rows("open_loop", open_loops) memory_status_counts = guard.readable_status_counts("memory") try: - events = self.store.list_events(limit=5_000) + if getattr(type(self.store), "label_count_input_prefilter", False): + events = self.store.list_events(limit=5_000, reject_sensitivity_allowed=guard.sensitivity_allowed) + else: + events = self.store.list_events(limit=5_000) except TypeError: # Compatibility for external/test stores on the old protocol. events = self.store.list_events()[:5_000] events = guard.admit_events(events) diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index e89049a1f..f7b742eb8 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -65,6 +65,7 @@ class _RequestLabels: row_sets: dict = field(default_factory=dict) dependencies: dict = field(default_factory=dict) source_admission: dict = field(default_factory=dict) + row_admission: dict = field(default_factory=dict) normalized_metadata: dict = field(default_factory=dict) row_keys: dict = field(default_factory=dict) resolved_inputs: dict = field(default_factory=dict) @@ -83,6 +84,7 @@ def clear(self): self.row_sets.clear() self.dependencies.clear() self.source_admission.clear() + self.row_admission.clear() self.normalized_metadata.clear() self.row_keys.clear() self.resolved_inputs.clear() @@ -378,16 +380,27 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: kept.append(row) continue key = self._key(kind, row) + row_admission_key = (key, self.domains, self.sensitivity_allowed, self.projects, self.all_of) + if row_admission_key in state.row_admission: + if state.row_admission[row_admission_key]: + kept.append(row) + continue template = self._signature(kind, row, key=key) admission_key = (template, self.domains, self.sensitivity_allowed, self.projects, self.all_of) root = (canon_kind(kind), identifier(row.get("id"))) reusable = template in state.dependency_labels and root not in state.dependency_ancestry.get(template, ()) if reusable and admission_key in state.source_admission: - if state.source_admission[admission_key]: + admitted = state.source_admission[admission_key] + state.row_admission[row_admission_key] = admitted + if admitted: kept.append(row) continue effective = self.effective_row(kind, row) admitted = isinstance(effective, Mapping) and self._admits_effective(effective, kind=kind) + # This is caller admission, separate from shared label settlement. + # Full label projections and every filter distinguish grants; all + # request caches are cleared together on writes and rollback. + state.row_admission[row_admission_key] = admitted if reusable: state.source_admission[admission_key] = admitted if admitted: @@ -414,7 +427,7 @@ def _prefetch_inputs(self, kind: str, rows: Sequence[Mapping[str, object]]) -> N for row_kind, row in frontier: if is_derived(row_kind, row): key = self._key(row_kind, row) - if key in prefetched: + if key in prefetched or key in state.labels or key in state.parent_labels: continue refs.update(self._signature(row_kind, row, key=key)[1]) pending.append(key) @@ -466,7 +479,8 @@ def readable_status_counts(self, kind: str) -> dict[str, int]: plain_counter = getattr(self.store, "count_original_label_statuses", None) if callable(getattr(type(self.store), "count_original_label_statuses", None)) and callable(plain_counter) and not self.projects and self.all_of is None: counts = plain_counter(kind, domains=self.domains, sensitivity_allowed=self.sensitivity_allowed) - batches = iterator(kind, derived_only=True) + prefilter = {"reject_sensitivity_allowed": self.sensitivity_allowed} if getattr(type(self.store), "label_count_input_prefilter", False) else {} + batches = iterator(kind, derived_only=True, **prefilter) else: counts = {} batches = iterator(kind) @@ -522,7 +536,8 @@ def readable_event_count(self) -> int: iterator = getattr(self.store, "iter_label_events", None) if not callable(iterator): raise TypeError("readable counts require complete event enumeration") - return sum(len(self.admit_events(batch)) for batch in iterator()) + prefilter = {"reject_sensitivity_allowed": self.sensitivity_allowed} if getattr(type(self.store), "label_count_input_prefilter", False) else {} + return sum(len(self.admit_events(batch)) for batch in iterator(**prefilter)) def admit_beliefs(self, beliefs: Sequence[_Row]) -> list[_Row]: """Beliefs whose backing memory the filters admit. One batched read.""" diff --git a/apps/api/src/alicebot_api/vnext_label_sql.py b/apps/api/src/alicebot_api/vnext_label_sql.py index 13840d28d..7ae25f2c2 100644 --- a/apps/api/src/alicebot_api/vnext_label_sql.py +++ b/apps/api/src/alicebot_api/vnext_label_sql.py @@ -59,6 +59,18 @@ def uuid_hint(value): ))))""" # nosec B608 +def hidden_memory_event_sql(sensitivity_allowed, *, sqlite: bool) -> str: + """Reject an event only when its same-tenant memory floor proves it hidden.""" + if not sensitivity_allowed: + return "TRUE" + label_sql = hidden_memory_input_sql(sensitivity_allowed, sqlite=sqlite) + memory_id = "m.id" if sqlite else "m.id::text" + # Both identifiers are closed literals; label_sql uses only kernel ranks. + return ("(target_type IS NULL OR target_type <> 'memory' OR " # nosec B608 + "(user_id, target_id) NOT IN (SELECT m.user_id, " + memory_id + + " FROM memories m WHERE NOT (" + label_sql + ")))") # nosec B608 + + def original_label_sql(kind: str, *, sqlite: bool = False) -> str: """Only rows definitely original by is_derived may be counted in SQL. diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 34be30cf4..16a180395 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -469,6 +469,7 @@ class PostgresVNextStore: """SQL-backed vNext repository facade for the second-brain kernel.""" memory_fts_offset_paging = True + label_count_input_prefilter = True def __init__(self, conn: UserConnection): self.conn = conn @@ -551,7 +552,7 @@ def count_original_label_statuses(self, kind: str, *, domains=(), sensitivity_al list(sensitivity_allowed) or None, list(sensitivity_allowed) or None)) return {str(row["status"]): int(cast(int, row["count"])) for row in rows} - def iter_label_rows(self, kind: str, *, batch_size: int = 1000, derived_only: bool = False) -> Iterator[list[VNextRow]]: + def iter_label_rows(self, kind: str, *, batch_size: int = 1000, derived_only: bool = False, reject_sensitivity_allowed: Sequence[str] = ()) -> Iterator[list[VNextRow]]: """Complete counted population, in narrow keyset batches under tenant RLS.""" if batch_size < 1: @@ -573,6 +574,10 @@ def iter_label_rows(self, kind: str, *, batch_size: int = 1000, derived_only: bo from alicebot_api.vnext_label_sql import original_label_sql if derived_only: live += " AND NOT COALESCE(" + original_label_sql(kind, sqlite=False) + ", FALSE)" + if kind == "memory" and reject_sensitivity_allowed: + from alicebot_api.vnext_label_sql import hidden_memory_input_sql + live += " AND " + hidden_memory_input_sql(reject_sensitivity_allowed, sqlite=False, alias="memories") + query_size = max(batch_size, 5000) if reject_sensitivity_allowed else batch_size after: str | None = None while True: rows = self._fetch_all( @@ -580,29 +585,38 @@ def iter_label_rows(self, kind: str, *, batch_size: int = 1000, derived_only: bo FROM {table} WHERE (%s::uuid IS NULL OR id > %s::uuid){live} ORDER BY id LIMIT %s""", - (after, after, batch_size), + (after, after, query_size), ) if not rows: return - yield rows + for start in range(0, len(rows), batch_size): + yield rows[start:start + batch_size] after = str(rows[-1]["id"]) + if len(rows) < query_size: + return - def iter_label_events(self, *, batch_size: int = 1000) -> Iterator[list[VNextRow]]: + def iter_label_events(self, *, batch_size: int = 1000, reject_sensitivity_allowed: Sequence[str] = ()) -> Iterator[list[VNextRow]]: """Complete event targets for readable counts, without event payloads.""" if batch_size < 1: raise ValueError("batch_size must be positive") + from alicebot_api.vnext_label_sql import hidden_memory_event_sql + label_sql = hidden_memory_event_sql(reject_sensitivity_allowed, sqlite=False) + query_size = max(batch_size, 5000) if reject_sensitivity_allowed else batch_size after: str | None = None while True: rows = self._fetch_all( - """SELECT id, target_type, target_id, event_type FROM event_log - WHERE (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s""", - (after, after, batch_size), + f"""SELECT id, target_type, target_id, event_type FROM event_log + WHERE (%s::uuid IS NULL OR id > %s::uuid) AND {label_sql} ORDER BY id LIMIT %s""", + (after, after, query_size), ) if not rows: return - yield rows + for start in range(0, len(rows), batch_size): + yield rows[start:start + batch_size] after = str(rows[-1]["id"]) + if len(rows) < query_size: + return def iter_label_ratings(self, *, batch_size: int = 1000) -> Iterator[list[VNextRow]]: """Complete rating targets for counts, without feedback text.""" diff --git a/apps/api/src/alicebot_api/vnext_stores/postgres/events_revisions.py b/apps/api/src/alicebot_api/vnext_stores/postgres/events_revisions.py index 6597b3560..cae3c39aa 100644 --- a/apps/api/src/alicebot_api/vnext_stores/postgres/events_revisions.py +++ b/apps/api/src/alicebot_api/vnext_stores/postgres/events_revisions.py @@ -131,10 +131,11 @@ def list_events( occurred_at_start: datetime | None = None, occurred_at_end: datetime | None = None, limit: int | None = None, + reject_sensitivity_allowed: Sequence[str] = (), ) -> list[VNextRow]: if limit is not None and limit < 1: raise ValueError("limit must be positive") - if target_type is None and target_id is None and occurred_at_start is None and occurred_at_end is None: + if not reject_sensitivity_allowed and target_type is None and target_id is None and occurred_at_start is None and occurred_at_end is None: limit_sql = "" params: list[object] = [] if limit is not None: @@ -153,6 +154,9 @@ def list_events( "(%s::text IS NULL OR target_id = %s)", ] params = [target_type, target_type, target_id, target_id] + if reject_sensitivity_allowed: + from alicebot_api.vnext_label_sql import hidden_memory_event_sql + clauses.append(hidden_memory_event_sql(reject_sensitivity_allowed, sqlite=False)) if occurred_at_start is not None: clauses.append("occurred_at >= %s::timestamptz") params.append(occurred_at_start) diff --git a/apps/api/src/alicebot_api/vnext_stores/sqlite/events_revisions.py b/apps/api/src/alicebot_api/vnext_stores/sqlite/events_revisions.py index 5ec75df3a..b2d64675a 100644 --- a/apps/api/src/alicebot_api/vnext_stores/sqlite/events_revisions.py +++ b/apps/api/src/alicebot_api/vnext_stores/sqlite/events_revisions.py @@ -130,11 +130,15 @@ def list_events( occurred_at_start: datetime | None = None, occurred_at_end: datetime | None = None, limit: int | None = None, + reject_sensitivity_allowed: Sequence[str] = (), ) -> list[VNextRow]: if limit is not None and limit < 1: raise ValueError("limit must be positive") clauses = ["user_id = ?"] params: list[object] = [self.user_id] + if reject_sensitivity_allowed: + from alicebot_api.vnext_label_sql import hidden_memory_event_sql + clauses.append(hidden_memory_event_sql(reject_sensitivity_allowed, sqlite=True)) if target_type is not None: clauses.append("target_type = ?") params.append(target_type) diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index f707cbd79..c5b461e4c 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -60,7 +60,7 @@ "line": 102, "before": "unfenced = not SourceReadFence.for_identity(identity).entity_read_fenced", "after": "unfenced = False", - "sha256": "adf4c06a8434451f6de670ac2772ddd6d3f061e69b0d4ca56bafd36e4cd4ed13" + "sha256": "3c6e6b909576c78fe78a5b2e2908a314bdf84db9d6d46ced22730263ea725b2c" } ], "pytest": [ @@ -141,10 +141,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 347, + "line": 349, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], ())", - "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" + "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" } ], "pytest": [ @@ -157,10 +157,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 347, + "line": 349, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], tuple(sorted(set(semantic_parents))))", - "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" + "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" } ], "pytest": [ @@ -173,10 +173,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 334, + "line": 336, "before": "if len(raw) != 1 or identifier(raw[0].get(\"id\")) != parent_id:", "after": "if not raw or identifier(raw[0].get(\"id\")) != parent_id:", - "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" + "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" } ], "pytest": [ @@ -191,10 +191,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 342, + "line": 344, "before": "if depth > HOP_BOUND or len(ancestry) > NODE_BOUND:", "after": "if False:", - "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" + "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" } ], "pytest": [ @@ -209,10 +209,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 318, + "line": 320, "before": "if problem or has_implicit_weekly_inputs(kind, row):", "after": "if problem:", - "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" + "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" } ], "pytest": [ @@ -227,10 +227,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 462, + "line": 475, "before": "if not self.active and callable(native):", "after": "if False:", - "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" + "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" } ], "pytest": [ @@ -246,7 +246,7 @@ "line": 17, "before": "if not blocked:", "after": "if True:", - "sha256": "f733e792850cd192b5615d1a309af2ab1fe466fce0b95469d790828c6a820b92" + "sha256": "bd1646e673033727148fbc112ceb6f10d079b1ab43dbed6595c185855e46a58f" } ], "pytest": [ @@ -275,10 +275,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 371, + "line": 373, "before": " self._prefetch_inputs(kind, rows)", "after": " pass # removed batched ancestry reads", - "sha256": "b6542485b899edf0e8e39aa7b923ec3364337a7e2d76edf6512a5bfa4ceac121" + "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" } ], "pytest": [ @@ -349,6 +349,38 @@ "tests/unit/test_derived_labels_docs.py::test_source_get_operator_gate_and_doctor_ceiling_are_explicit_in_changelog" ], "expected_failure": "test_source_get_operator_gate_and_doctor_ceiling_are_explicit_in_changelog" + }, + { + "name": "row-admission-caller-filters", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": "row_admission_key = (key, self.domains, self.sensitivity_allowed, self.projects, self.all_of)", + "after": "row_admission_key = (key,)", + "line": 383, + "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_repeated_admission_keeps_caller_filters_projections_and_write_refresh" + ], + "expected_failure": "test_repeated_admission_keeps_caller_filters_projections_and_write_refresh" + }, + { + "name": "row-admission-write-invalidation", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": " self.row_admission.clear()", + "after": " pass # removed admission invalidation", + "line": 87, + "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_repeated_admission_keeps_caller_filters_projections_and_write_refresh" + ], + "expected_failure": "test_repeated_admission_keeps_caller_filters_projections_and_write_refresh" } ] } diff --git a/tests/integration/test_label_round3_contracts_postgres.py b/tests/integration/test_label_round3_contracts_postgres.py index f3d9e0a01..3a4041dda 100644 --- a/tests/integration/test_label_round3_contracts_postgres.py +++ b/tests/integration/test_label_round3_contracts_postgres.py @@ -16,6 +16,12 @@ from tests.integration.derived_labels_postgres_support import label_harness +def test_native_count_prefilter_matches_complete_effective_admission(label_harness): + from tests.unit.test_label_round3_sqlite_reads import assert_native_count_prefilter_matches_complete_effective_admission + with label_harness.store() as store: + assert_native_count_prefilter_matches_complete_effective_admission(store) + + @pytest.mark.parametrize("profile", ["owner", "admin_agent"]) @pytest.mark.parametrize("component", ["trace", "workspace"]) def test_unfenced_source_trace_and_workspace_preserve_main(label_harness, profile, component): diff --git a/tests/unit/test_label_resolved_inputs.py b/tests/unit/test_label_resolved_inputs.py index a11b7584f..7135cbbcb 100644 --- a/tests/unit/test_label_resolved_inputs.py +++ b/tests/unit/test_label_resolved_inputs.py @@ -111,6 +111,27 @@ def test_partial_prefetch_keeps_per_origin_floors_and_refreshes_after_writes(mon assert [row["id"] for row in guard.admit_rows("memory", roots)] == [roots[2]["id"]] +def test_repeated_admission_keeps_caller_filters_projections_and_write_refresh(): + source = {"kind": "source", "id": "source", "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": ["P1"]}} + root = {"kind": "memory", "id": "root", "domain": "project", "sensitivity": "public", + "metadata_json": {"source_id": "source", "project_scope": ["P1"]}} + store = Rows([source]) + with label_read_scope(store): + public = LabelGuard(store, active=True, sensitivity_allowed=("public",)) + for _ in range(2): + assert public.admit_rows("memory", [root]) == [root] + assert replace(public, domains=("health",)).admit_rows("memory", [root]) == [] + assert replace(public, projects=("P2",), all_of=("P2",)).admit_rows("memory", [root]) == [] + assert replace(public, projects=("P1",), all_of=("P1",)).admit_rows("memory", [root]) == [root] + # The same ID with a different stored label is a different projection. + assert public.admit_rows("memory", [{**root, "sensitivity": "confidential"}]) == [] + source["sensitivity"] = "confidential" + invalidate_read_labels(store) + assert public.admit_rows("memory", [root]) == [] + assert replace(public, sensitivity_allowed=("public", "confidential")).admit_rows("memory", [root]) == [root] + + @pytest.mark.parametrize("variant", ["alias", "cycle", "missing", "malformed", "hop-bound", "node-bound", "implicit-weekly", "implicit-weekly-json"]) def test_unproved_inputs_keep_the_complete_graph_fallback(monkeypatch, variant): import alicebot_api.vnext_label_guard as module diff --git a/tests/unit/test_label_round3_sqlite_reads.py b/tests/unit/test_label_round3_sqlite_reads.py index 5ffe76166..70526f94c 100644 --- a/tests/unit/test_label_round3_sqlite_reads.py +++ b/tests/unit/test_label_round3_sqlite_reads.py @@ -114,3 +114,41 @@ def iter_label_rows(self, kind): raise AssertionError("unfenced counts must not enumerate the population") assert LabelGuard(Native(), active=False).readable_status_counts("memory") == {"active": 5000} + + +def assert_native_count_prefilter_matches_complete_effective_admission(store): + visible = store.create_source({"source_type": "note", "title": "Visible", "content_hash": str(uuid4()), "sensitivity": "public", "domain": "project"}) + hidden = store.create_source({"source_type": "note", "title": "Hidden", "content_hash": str(uuid4()), "sensitivity": "confidential", "domain": "project"}) + with without_insert_floor(): + public = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Visible observation", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(visible["id"])}}) + private = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Hidden observation", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(hidden["id"])}}) + metadata = with_derived_from({"workflow": "project_auto_update"}, {"sources": [visible, hidden]}) + metadata["derived_from"]["sources"] = [str(visible["id"]), str(hidden["id"])] + later = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Later hidden input", "status": "active", "domain": "project", "sensitivity": "public", "metadata_json": metadata}) + allowed = ("public", "internal") + raw = [row for batch in store.iter_label_rows("memory", reject_sensitivity_allowed=allowed) for row in batch] + assert str(private["id"]) not in {str(row["id"]) for row in raw} + assert {str(public["id"]), str(later["id"])} <= {str(row["id"]) for row in raw} + # A public first-parent hint is inconclusive. Admission must still discover + # the later hidden input rather than treating SQL as an admission grant. + all_rows = [row for batch in store.iter_label_rows("memory") for row in batch] + all_events = store.list_events() + with label_read_scope(store): + guard = LabelGuard(store, active=True, sensitivity_allowed=allowed) + expected_rows = guard.admit_rows("memory", all_rows) + expected_events = guard.admit_events(all_events) + with label_read_scope(store): + guard = LabelGuard(store, active=True, sensitivity_allowed=allowed) + assert guard.readable_status_counts("memory") == {"active": len(expected_rows)} + assert guard.readable_event_count() == len(expected_events) + events = guard.admit_events(store.list_events(reject_sensitivity_allowed=allowed)) + assert {str(row["id"]) for row in events} == {str(row["id"]) for row in expected_events} + assert {str(row["id"]) for row in guard.admit_rows("memory", raw)} == {str(public["id"])} + assert len(store.list_events()) == len(all_events) + + +def test_native_count_prefilter_matches_complete_effective_admission(tmp_path): + path = tmp_path / "count-prefilter.db" + bootstrap_database(path, user_id=USER, user_email="synthetic@example.invalid") + with sqlite_user_connection(path, USER) as conn: + assert_native_count_prefilter_matches_complete_effective_admission(SQLiteVNextStore(conn, USER)) diff --git a/tests/unit/test_label_sql_closed_literals.py b/tests/unit/test_label_sql_closed_literals.py index 5777a4ea5..555b161b5 100644 --- a/tests/unit/test_label_sql_closed_literals.py +++ b/tests/unit/test_label_sql_closed_literals.py @@ -1,7 +1,7 @@ """Caller strings cannot enter the conservative native SQL predicate.""" import pytest -from alicebot_api.vnext_label_sql import hidden_memory_input_sql +from alicebot_api.vnext_label_sql import hidden_memory_input_sql, hidden_memory_event_sql @pytest.mark.parametrize("sqlite", (False, True)) @@ -20,3 +20,11 @@ def test_prefilter_refuses_a_caller_supplied_sql_alias(sqlite, values): with pytest.raises(ValueError, match="unsupported memory alias"): hidden_memory_input_sql(values, sqlite=sqlite, alias="m); DROP TABLE memories; --") + +@pytest.mark.parametrize("sqlite", (False, True)) +def test_event_prefilter_sensitivity_inputs_are_closed_kernel_literals(sqlite): + hostile = "public'); DROP TABLE event_log; --" + sql = hidden_memory_event_sql(["public", hostile], sqlite=sqlite) + assert hostile not in sql + assert "'confidential'" in sql + assert "m.user_id" in sql From 70ae1fdc22b657d61a13f11b707e268fc5021186 Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:55:31 +0200 Subject: [PATCH 254/270] Keep the MCP fake aligned with the optional native rejection hint --- tests/unit/test_mcp.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/test_mcp.py b/tests/unit/test_mcp.py index a1db5d432..a7364ced0 100644 --- a/tests/unit/test_mcp.py +++ b/tests/unit/test_mcp.py @@ -2357,7 +2357,11 @@ def list_events( occurred_at_start: datetime | None = None, occurred_at_end: datetime | None = None, limit: int | None = None, + reject_sensitivity_allowed: Sequence[str] = (), ) -> list[dict[str, object]]: + # The SQL hint is an optional optimization. This fake returns the + # complete candidates; callers still perform effective admission. + del reject_sensitivity_allowed rows = [ event for event in self.events From fd03dac0c243ad8460db97589d46c9dce40d940a Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:00:36 +0200 Subject: [PATCH 255/270] Refresh strict carrier receipts for the count rejection capability --- tests/unit/test_store_events_revisions_split.py | 14 +++++++------- tests/unit/test_store_graph_open_loops_split.py | 4 ++-- tests/unit/test_store_memory_access_split.py | 4 ++-- tests/unit/test_store_memory_lifecycle_split.py | 4 ++-- tests/unit/test_workspaces_router_split.py | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/unit/test_store_events_revisions_split.py b/tests/unit/test_store_events_revisions_split.py index 38cdb6b91..6edb52cba 100644 --- a/tests/unit/test_store_events_revisions_split.py +++ b/tests/unit/test_store_events_revisions_split.py @@ -48,7 +48,7 @@ "postgres": { "_append_mutation_event": "294b2174082ac9f670e15cb6664c9968c2961d2c061a200c96c2bdecc55b87c9", "append_event": "e1624ec8cb156e52dfd8f821032689378ab49863ad0d4a2e68cf6176cfd8c89c", - "list_events": "7659f8bcbf050155eca1d61f575854e3c54120944e46d0d7aa24bce8af75e8fe", + "list_events": "6783e985fee09da2771113b215043ce5cd9725706a90a551a6250db831615a90", "list_events_for_source_trace": "20f22e3b75c3612c02c4242bc973295b2535b01f95e5451a0a0bff1196f187c3", "list_project_update_events": "2bd457ee19535f203da5c31e557387f7717fefc27cbef2c546a62bbad74962d3", "count_events": "e740e4b09ecfda973ef6b84acd0ea808b26d118faf6984e4057d7e104e595fb5", @@ -58,7 +58,7 @@ "sqlite": { "_append_mutation_event": "75922431a64c17ca369cd9b57e360a63b492e45d706af33d572b958ea287e27a", "append_event": "61927c2a8ed03ceff60c2b993ef1155cee01a5706e21ebee3e002a988820e052", - "list_events": "1fe66068556bfe8263a31b4da9e0e69aff91d54c16bb041c381ff55b720af0e6", + "list_events": "9da8d9c98e2245ad4e6425dc37bc23c732e1c757c68ff50d8d42f8e8cc472418", "list_events_for_source_trace": "64a122d098df8c8f62ef74654235f6bfd4d4322d908089c23e53564ef80f3eae", "list_project_update_events": "7e80021e9b6023c65c28f356ccda3242f202f446724628ec90aef87155c309ca", "count_events": "0387b6810557e231ab2dccacbf2f82da55984060ff986d43e1409b094f0ceb54", @@ -67,8 +67,8 @@ }, } EXPECTED_SOURCE_LINE_COUNTS = { - "postgres": (24, 48, 50, 58, 26, 18, 90, 10), - "sqlite": (24, 36, 38, 45, 26, 21, 72, 11), + "postgres": (24, 48, 54, 58, 26, 18, 90, 10), + "sqlite": (24, 36, 42, 45, 26, 21, 72, 11), } EXPECTED_SIGNATURES = { "_append_mutation_event": ( @@ -80,7 +80,7 @@ "list_events": ( "(self, *, target_type: 'str | None' = None, target_id: 'str | None' = None, " "occurred_at_start: 'datetime | None' = None, occurred_at_end: 'datetime | None' = None, " - "limit: 'int | None' = None) -> 'list[VNextRow]'" + "limit: 'int | None' = None, reject_sensitivity_allowed: 'Sequence[str]' = ()) -> 'list[VNextRow]'" ), "list_events_for_source_trace": ( "(self, *, source_id: 'str', memory_ids: 'Sequence[str]' = (), artifact_ids: 'Sequence[str]' = (), " @@ -153,7 +153,7 @@ # Re-minted for the per-file importer savepoint (2026-10-02): ``savepoint`` is appended last on both façades. # Previous receipt: 650e2e0ff088d67c... Proof: the class key list equals the list at origin/main 040a2a10 # plus ``savepoint`` before ``__dict__``, with every other key in the same order. - "postgres": "3e1113952b741f978c79325e4cd08067763bc1d2a770a8ace20667d9cbc71436", + "postgres": "0b5c4a0922ca22c14fc2965f06aeb460cc83b77f442ae12dadc504fa2b982b45", # Re-minted for the merge of #500 and #502: ``check_literal_match_query``. # Re-minted again for per-project memory S2 (2026-10-02): the two single-scan partition reads # ``list_memories_view_partitions`` and ``list_open_loops_view_partitions``, SQLite only on purpose @@ -161,7 +161,7 @@ # Re-minted again for the per-file importer savepoint (2026-10-02), the same appended method. # Previous receipt: 365b7a01acf7a8b5... Proof: as for postgres, one added key and no other change. # Re-minted for the derived-label lock: ``lock_label_writes`` and ``read_label_rows`` on both facades. - "sqlite": "a76b6dfeb7a77da3206245a6155f3b2fe2eeaebfe219111dcaa97ddf7b0f9c7a", + "sqlite": "82282bae5eeea904b50cb24ec8d2312d6173e1916f0d8db50228bcafd016bd43", } EXPECTED_SUPPORT_AST_SHA256 = { "postgres_columns": { diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index d1a984a84..647b42380 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -167,7 +167,7 @@ # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # lock_label_writes and read_label_rows follow __init__. Previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (181, "5f3acdf8872b9bd81ef550ea96d645680ad35a1e4018451cae45778d0ef82449"), + "PostgresVNextStore": (182, "b613fa15b028b26afe8ee90473288ba3d8e5926ea802b4aa70e39d75cc6a75e7"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -183,7 +183,7 @@ # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. # lock_label_writes and read_label_rows follow __init__. Previous receipt (134, 13012720...). - "SQLiteVNextStore": (140, "2bd805be8a616fdc10839fa7abcf0c3e286dc3faef67757ba08810eca174cca7"), + "SQLiteVNextStore": (141, "fc474bf2faa707e5e837846b66ae208e8795c0e65659c3c9cb54b84704a42b73"), } EXPECTED_COLUMN_AST = { POSTGRES_COLUMNS_PATH: { diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index f9b3ef65b..5f32493f3 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -224,7 +224,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (181, "5f3acdf8872b9bd81ef550ea96d645680ad35a1e4018451cae45778d0ef82449"), + "PostgresVNextStore": (182, "b613fa15b028b26afe8ee90473288ba3d8e5926ea802b4aa70e39d75cc6a75e7"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -241,7 +241,7 @@ # prunable_sources here; every pre-existing class member keeps its order. # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (134, 13012720...). - "SQLiteVNextStore": (140, "2bd805be8a616fdc10839fa7abcf0c3e286dc3faef67757ba08810eca174cca7"), + "SQLiteVNextStore": (141, "fc474bf2faa707e5e837846b66ae208e8795c0e65659c3c9cb54b84704a42b73"), } diff --git a/tests/unit/test_store_memory_lifecycle_split.py b/tests/unit/test_store_memory_lifecycle_split.py index eb536b02e..89458d2dc 100644 --- a/tests/unit/test_store_memory_lifecycle_split.py +++ b/tests/unit/test_store_memory_lifecycle_split.py @@ -119,7 +119,7 @@ # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (181, "5f3acdf8872b9bd81ef550ea96d645680ad35a1e4018451cae45778d0ef82449"), + "PostgresVNextStore": (182, "b613fa15b028b26afe8ee90473288ba3d8e5926ea802b4aa70e39d75cc6a75e7"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, @@ -134,7 +134,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Proof: the replacement branch gains only scrub_source, source_inventory and # prunable_sources here; every pre-existing class member keeps its order. - "SQLiteVNextStore": (140, "2bd805be8a616fdc10839fa7abcf0c3e286dc3faef67757ba08810eca174cca7"), + "SQLiteVNextStore": (141, "fc474bf2faa707e5e837846b66ae208e8795c0e65659c3c9cb54b84704a42b73"), } EXPECTED_FACADE_COMMENT_DIGESTS = { POSTGRES_FACADE_PATH: "d8599a46ee26dc35a3ae52c1a98a416509add9ae4a42ece780c5c5ed7e132b93", diff --git a/tests/unit/test_workspaces_router_split.py b/tests/unit/test_workspaces_router_split.py index 5d5226fcf..2b5a5017b 100644 --- a/tests/unit/test_workspaces_router_split.py +++ b/tests/unit/test_workspaces_router_split.py @@ -88,7 +88,7 @@ # Re-pin 2026-10-06: workspace reads authenticate the protected identity and # admit rows through effective labels before totals or dashboard disclosure. EXPECTED_ROUTE_AST_SHA256 = "b99f1435de67d9499819acb9ed7ed61b588a3fb0ff037e782eeca070b39af742" -EXPECTED_SUPPORT_AST_SHA256 = "997f71ac38b434aefb35a94414b03aabe52ffacbd3af2f2750b1de109d0a3dbb" +EXPECTED_SUPPORT_AST_SHA256 = "701353bcdfe16d5502eafbd3df6d0659acc2e8b85a911c33b7a594348cae5b9b" EXPECTED_ROUTE_NAME_MANIFEST_SHA256 = "225c57c08bd8314156c56352dd1c53ffed3f556ce285c666dd6fca125115d0b4" EXPECTED_OPERATION_MANIFEST_SHA256 = "c320979b62d7ee8de244fe38bde5bf3761a4f9d76f76bf3cd8576c30fce9857e" EXPECTED_IMPORT_MANIFEST_SHA256 = "e8c18d6831ca012b55b22f46c9b2151d62575773a2452ef0d0f2e869cabc8abb" @@ -143,7 +143,7 @@ } EXPECTED_SUPPORT_NODE_SHA256 = { "_vnext_status_counts": "0bf0ed228a14bd648a9d18fcd5f99ebf8c585bd29f4b5e81e1df17fe0201fd15", - "_vnext_workspace_payload": "a5dd3efc718c0df7dac7d6cccc7ce749e28fab627024fd688eb2f316cbb22da9", + "_vnext_workspace_payload": "d076d390f1667942ff6ef833eb8419cef7c4603f6ccba629675eedf384a336f8", "_workspace_rows": "070bdfbd1eae10608bd8208b08367e1c0ea10e2064f03ad5a84121a190ed4cf0", "_workspace_event_visible": "8343c060909326a5cb69fa6f671ac62f160630ecf989f04d78e792ea74c0ea90", } From 3e800a490005bd673a034554710b25f7034ed082 Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:06:19 +0200 Subject: [PATCH 256/270] Stabilize event target reads and reuse native count parents --- .../src/alicebot_api/vnext_derived_labels.py | 13 +++++-- .../api/src/alicebot_api/vnext_label_guard.py | 30 +++++++++++++--- .../src/alicebot_api/vnext_project_scope.py | 4 +++ apps/api/src/alicebot_api/vnext_store.py | 15 ++++---- scripts/derived_label_mutations.json | 36 +++++++++---------- .../test_label_round3_contracts_postgres.py | 18 ++++++++++ .../unit/test_store_events_revisions_split.py | 2 +- .../unit/test_store_graph_open_loops_split.py | 2 +- tests/unit/test_store_memory_access_split.py | 2 +- .../unit/test_store_memory_lifecycle_split.py | 2 +- 10 files changed, 89 insertions(+), 35 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 1354b7579..ccb440aae 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -197,10 +197,19 @@ def canon_kind(kind: object) -> str: def identifier(value: object) -> str: """Normalize any spelling ``uuid.UUID`` accepts. Anything else is kept as text.""" + if isinstance(value, UUID): + return str(value) + cache = _READ_METADATA.get() + key = ("identifier", value) if isinstance(value, str) else None + if cache is not None and key is not None and key in cache: + return cache[key] try: - return str(UUID(str(value))) + result = str(UUID(str(value))) except (ValueError, AttributeError, TypeError): - return str(value) + result = str(value) + if cache is not None and key is not None: + cache[key] = result + return result def _object(value: object) -> Mapping[str, object]: diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index f7b742eb8..a6f0ac746 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -13,6 +13,7 @@ from contextvars import ContextVar from functools import wraps from contextlib import contextmanager +from uuid import UUID from alicebot_api.vnext_agent_control import ( ALL_SENSITIVITY, @@ -46,9 +47,9 @@ def _row_label_key(kind: str, row: Mapping[str, object]) -> tuple: - return (kind, *((field in row, repr(row.get(field))) for field in ( + return (kind, *((field, repr(row[field])) for field in ( "id", "user_id", "domain", "sensitivity", "metadata_json", "value", "project_id", "project", "projects", "scope_json", "source_id", "artifact_type", "project_scope", "project_floor", - ))) + ) if field in row)) @dataclass @@ -395,8 +396,17 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: if admitted: kept.append(row) continue - effective = self.effective_row(kind, row) - admitted = isinstance(effective, Mapping) and self._admits_effective(effective, kind=kind) + direct = self._settled_inputs(kind, row, frozenset()) if not self.projects and self.all_of is None else None + if direct is not None: + # A count or unscoped list needs only the two settled labels. + # Scope-sensitive callers still use the complete effective + # projection, including its floor and malformed-input checks. + label = direct[0] + state.labels[key] = label + admitted = self._admits_effective({"domain": label.domain, "sensitivity": label.sensitivity}, kind=kind) + else: + effective = self.effective_row(kind, row) + admitted = isinstance(effective, Mapping) and self._admits_effective(effective, kind=kind) # This is caller admission, separate from shared label settlement. # Full label projections and every filter distinguish grants; all # request caches are cleared together on writes and rollback. @@ -480,11 +490,21 @@ def readable_status_counts(self, kind: str) -> dict[str, int]: if callable(getattr(type(self.store), "count_original_label_statuses", None)) and callable(plain_counter) and not self.projects and self.all_of is None: counts = plain_counter(kind, domains=self.domains, sensitivity_allowed=self.sensitivity_allowed) prefilter = {"reject_sensitivity_allowed": self.sensitivity_allowed} if getattr(type(self.store), "label_count_input_prefilter", False) else {} - batches = iterator(kind, derived_only=True, **prefilter) + unique_ids = getattr(type(self.store), "label_count_canonical_unique_ids", False) + batches = iterator(kind, derived_only=True, **prefilter, **({"batch_size": 5000} if unique_ids else {})) else: counts = {} + unique_ids = False batches = iterator(kind) for batch in batches: + if unique_ids: + # PostgreSQL stores canonical UUID primary keys. These complete + # native label rows already contain the parent projection, so + # resolving a counted parent need not fetch it again. Text-ID + # stores must still load all aliases through their reader. + for row in batch: + if isinstance(row.get("id"), UUID): + state.nodes.setdefault((canon_kind(kind), str(row["id"])), [row]) for row in self.admit_rows(kind, batch): status = str(row.get("status", "unknown")) counts[status] = counts.get(status, 0) + 1 diff --git a/apps/api/src/alicebot_api/vnext_project_scope.py b/apps/api/src/alicebot_api/vnext_project_scope.py index 4e6faca2c..704ca5e55 100644 --- a/apps/api/src/alicebot_api/vnext_project_scope.py +++ b/apps/api/src/alicebot_api/vnext_project_scope.py @@ -107,6 +107,8 @@ def project_identifier_identity(value: object) -> str: def normalize_project_scope(value: object) -> tuple[str, ...]: + if value is None or type(value) in (list, tuple) and not value: + return () values: list[str] = [] def add(item: object) -> None: @@ -396,6 +398,8 @@ def project_floor_shape(resource: Mapping[str, object] | None) -> tuple[str, tup break if not seen: return "absent", () + if type(raw) in (list, tuple) and not raw: + return "list", () if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes, bytearray)): return "malformed", () if any(not isinstance(item, str) for item in raw): diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 16a180395..3355cc4c4 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -470,6 +470,7 @@ class PostgresVNextStore: memory_fts_offset_paging = True label_count_input_prefilter = True + label_count_canonical_unique_ids = True def __init__(self, conn: UserConnection): self.conn = conn @@ -800,6 +801,9 @@ def list_memory_events( prefix_pattern = f"{event_type_prefix}%" if event_type_prefix is not None else None from alicebot_api.vnext_label_sql import hidden_memory_input_sql label_sql = hidden_memory_input_sql(sensitivity_allowed, sqlite=False) + # An uncorrelated target set is hashed once, including tenant identity. + # A join can rescan every memory for every event on fresh tenants whose + # planner statistics underestimate both tables. Keep exact text IDs. return self._fetch_all( f""" SELECT @@ -816,19 +820,18 @@ def list_memory_events( e.run_id, e.integrity_hash FROM event_log e - JOIN memories m - ON e.target_type = 'memory' - AND e.target_id = m.id::text - AND {label_sql} - AND e.user_id = m.user_id - WHERE m.deleted_at IS NULL + WHERE e.target_type = 'memory' AND (%s::text IS NULL OR e.event_type LIKE %s) + AND COALESCE((e.user_id, e.target_id) IN ( + SELECT m.user_id, m.id::text FROM memories m + WHERE m.deleted_at IS NULL AND {label_sql} AND (%s::text[] IS NULL OR ({_SCOPED_MEMORY_PROJECT_SQL}) ?| %s::text[]) AND ( %s::text[] IS NULL OR m.id::text = ANY(%s::text[]) OR {_SCOPED_MEMORY_DIRECT_PEOPLE_SQL} ) + ), FALSE) AND (%s::timestamptz IS NULL OR e.occurred_at >= %s::timestamptz) AND (%s::timestamptz IS NULL OR e.occurred_at <= %s::timestamptz) ORDER BY e.occurred_at DESC, e.id DESC diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index c5b461e4c..7afc0213b 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -141,10 +141,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 349, + "line": 350, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], ())", - "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" + "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" } ], "pytest": [ @@ -157,10 +157,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 349, + "line": 350, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], tuple(sorted(set(semantic_parents))))", - "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" + "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" } ], "pytest": [ @@ -173,10 +173,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 336, + "line": 337, "before": "if len(raw) != 1 or identifier(raw[0].get(\"id\")) != parent_id:", "after": "if not raw or identifier(raw[0].get(\"id\")) != parent_id:", - "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" + "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" } ], "pytest": [ @@ -191,10 +191,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 344, + "line": 345, "before": "if depth > HOP_BOUND or len(ancestry) > NODE_BOUND:", "after": "if False:", - "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" + "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" } ], "pytest": [ @@ -209,10 +209,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 320, + "line": 321, "before": "if problem or has_implicit_weekly_inputs(kind, row):", "after": "if problem:", - "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" + "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" } ], "pytest": [ @@ -227,10 +227,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 475, + "line": 485, "before": "if not self.active and callable(native):", "after": "if False:", - "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" + "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" } ], "pytest": [ @@ -275,10 +275,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 373, + "line": 374, "before": " self._prefetch_inputs(kind, rows)", "after": " pass # removed batched ancestry reads", - "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" + "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" } ], "pytest": [ @@ -357,8 +357,8 @@ "file": "apps/api/src/alicebot_api/vnext_label_guard.py", "before": "row_admission_key = (key, self.domains, self.sensitivity_allowed, self.projects, self.all_of)", "after": "row_admission_key = (key,)", - "line": 383, - "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" + "line": 384, + "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" } ], "pytest": [ @@ -373,8 +373,8 @@ "file": "apps/api/src/alicebot_api/vnext_label_guard.py", "before": " self.row_admission.clear()", "after": " pass # removed admission invalidation", - "line": 87, - "sha256": "5810b5f9ab22758ca734281a89b428a550e3ad14b5d34fbce168ead708269adc" + "line": 88, + "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" } ], "pytest": [ diff --git a/tests/integration/test_label_round3_contracts_postgres.py b/tests/integration/test_label_round3_contracts_postgres.py index 3a4041dda..f65d57320 100644 --- a/tests/integration/test_label_round3_contracts_postgres.py +++ b/tests/integration/test_label_round3_contracts_postgres.py @@ -22,6 +22,24 @@ def test_native_count_prefilter_matches_complete_effective_admission(label_harne assert_native_count_prefilter_matches_complete_effective_admission(store) +def test_memory_event_target_set_preserves_exact_target_matching(label_harness): + h = label_harness + visible = h.memory() + hidden = h.memory(source=h.source(sensitivity="confidential")) + with h.store() as store: + baseline = store.list_memory_events(sensitivity_allowed=["public", "internal"], limit=100) + assert any(event["target_id"] == str(visible["id"]) for event in baseline) + assert all(event["target_id"] != str(hidden["id"]) for event in baseline) + for target in ("legacy-not-a-uuid", str(visible["id"]).upper(), str(visible["id"]).replace("-", "")): + store.conn.execute( + "INSERT INTO event_log(id,user_id,event_type,actor_type,target_type,target_id,payload_json) " + "VALUES(%s,%s,'memory.updated','system','memory',%s,'{}')", + (uuid4(), h.user_id, target), + ) + actual = store.list_memory_events(sensitivity_allowed=["public", "internal"], limit=100) + assert [event["id"] for event in actual] == [event["id"] for event in baseline] + + @pytest.mark.parametrize("profile", ["owner", "admin_agent"]) @pytest.mark.parametrize("component", ["trace", "workspace"]) def test_unfenced_source_trace_and_workspace_preserve_main(label_harness, profile, component): diff --git a/tests/unit/test_store_events_revisions_split.py b/tests/unit/test_store_events_revisions_split.py index 6edb52cba..9f71008a1 100644 --- a/tests/unit/test_store_events_revisions_split.py +++ b/tests/unit/test_store_events_revisions_split.py @@ -153,7 +153,7 @@ # Re-minted for the per-file importer savepoint (2026-10-02): ``savepoint`` is appended last on both façades. # Previous receipt: 650e2e0ff088d67c... Proof: the class key list equals the list at origin/main 040a2a10 # plus ``savepoint`` before ``__dict__``, with every other key in the same order. - "postgres": "0b5c4a0922ca22c14fc2965f06aeb460cc83b77f442ae12dadc504fa2b982b45", + "postgres": "062e8006f55e3df7f369aac5173a32305a9850537e078c093bff71f5c5c5187c", # Re-minted for the merge of #500 and #502: ``check_literal_match_query``. # Re-minted again for per-project memory S2 (2026-10-02): the two single-scan partition reads # ``list_memories_view_partitions`` and ``list_open_loops_view_partitions``, SQLite only on purpose diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index 647b42380..cd2749541 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -167,7 +167,7 @@ # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # lock_label_writes and read_label_rows follow __init__. Previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (182, "b613fa15b028b26afe8ee90473288ba3d8e5926ea802b4aa70e39d75cc6a75e7"), + "PostgresVNextStore": (183, "b78e54887487bf23d24c250cec9499b6439b9164fa64c56fea29479d85550c94"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index 5f32493f3..303b70ad8 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -224,7 +224,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (182, "b613fa15b028b26afe8ee90473288ba3d8e5926ea802b4aa70e39d75cc6a75e7"), + "PostgresVNextStore": (183, "b78e54887487bf23d24c250cec9499b6439b9164fa64c56fea29479d85550c94"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, diff --git a/tests/unit/test_store_memory_lifecycle_split.py b/tests/unit/test_store_memory_lifecycle_split.py index 89458d2dc..db561c5cd 100644 --- a/tests/unit/test_store_memory_lifecycle_split.py +++ b/tests/unit/test_store_memory_lifecycle_split.py @@ -119,7 +119,7 @@ # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (182, "b613fa15b028b26afe8ee90473288ba3d8e5926ea802b4aa70e39d75cc6a75e7"), + "PostgresVNextStore": (183, "b78e54887487bf23d24c250cec9499b6439b9164fa64c56fea29479d85550c94"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, From d194b268d910dbc5099fb956c1c607a2cfd3cb55 Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:58:31 +0200 Subject: [PATCH 257/270] Reduce native label prefilter and signature work --- .../src/alicebot_api/vnext_derived_labels.py | 55 ++++++++++++------- .../api/src/alicebot_api/vnext_label_guard.py | 6 +- apps/api/src/alicebot_api/vnext_label_sql.py | 26 +++++---- scripts/derived_label_mutations.json | 36 ++++++------ .../test_label_round3_contracts_postgres.py | 40 ++++++++++++++ tests/unit/test_vnext_store.py | 8 ++- 6 files changed, 119 insertions(+), 52 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index ccb440aae..409a6c885 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -197,16 +197,17 @@ def canon_kind(kind: object) -> str: def identifier(value: object) -> str: """Normalize any spelling ``uuid.UUID`` accepts. Anything else is kept as text.""" - if isinstance(value, UUID): - return str(value) cache = _READ_METADATA.get() - key = ("identifier", value) if isinstance(value, str) else None + key = ("identifier", value) if isinstance(value, str) else ("uuid-identifier", value) if type(value) is UUID else None if cache is not None and key is not None and key in cache: return cache[key] - try: - result = str(UUID(str(value))) - except (ValueError, AttributeError, TypeError): + if isinstance(value, UUID): result = str(value) + else: + try: + result = str(UUID(str(value))) + except (ValueError, AttributeError, TypeError): + result = str(value) if cache is not None and key is not None: cache[key] = result return result @@ -510,6 +511,17 @@ def _plain_token(value: str) -> bool: def _source_token(value: str) -> str | None: + cache = _READ_METADATA.get() + key = ("source-token", value) + if cache is not None and key in cache: + return cache[key] + result = _parse_source_token(value) + if cache is not None: + cache[key] = result + return result + + +def _parse_source_token(value: str) -> str | None: text = value.strip() lowered = text.lower() for prefix in ("urn:uuid:", "source:", "uuid:"): @@ -936,6 +948,16 @@ def _node_label(kind: str, row: Mapping[str, object]) -> SettledLabel: ) +_DEPENDENCY_SYNTAX_KEYS = MARKER_KEYS | _ID_KIND.keys() | { + "redacted", "scrubbed", "project_scope", "project_floor", "project_id", "project", "projects", + "scope_json", "metadata_json", "agent_identity", "agentic_memory", "consolidation", +} +_DEPENDENCY_SYNTAX_FIELDS = ( + "user_id", "domain", "sensitivity", "value", "project_id", "project", "projects", "scope_json", + "source_id", "artifact_type", "project_scope", "project_floor", +) + + def dependency_syntax_key(kind: str, row: Mapping[str, object]) -> tuple: """Memoize parsing without allowing incidental scalar metadata to split it. @@ -943,18 +965,13 @@ def dependency_syntax_key(kind: str, row: Mapping[str, object]) -> tuple: unknown containers as the dependency parser does; their scalar text cannot name an input. This is only a parsing key, never a settled label or grant. """ - relevant = MARKER_KEYS | _ID_KIND.keys() | { - "redacted", "scrubbed", "project_scope", "project_floor", "project_id", "project", "projects", - "scope_json", "metadata_json", "agent_identity", "agentic_memory", "consolidation", - } - def metadata_key(value: object) -> tuple: if isinstance(value, Mapping): parts: list[tuple[str, object]] = [] for key, child in value.items(): if not isinstance(key, str): continue - if key in relevant: + if key in _DEPENDENCY_SYNTAX_KEYS: parts.append((key, repr(child))) elif isinstance(child, (Mapping, list)): nested = metadata_key(child) @@ -966,10 +983,8 @@ def metadata_key(value: object) -> tuple: and (nested := metadata_key(child))) return () - fields = ("user_id", "domain", "sensitivity", "value", "project_id", "project", "projects", "scope_json", - "source_id", "artifact_type", "project_scope", "project_floor") return (canon_kind(kind), isinstance(row.get("metadata_json"), Mapping), metadata_key(_metadata(row)), - *((field in row, repr(row.get(field))) for field in fields)) + *((field in row, repr(row.get(field))) for field in _DEPENDENCY_SYNTAX_FIELDS)) def dependency_label_signature(kind: str, row: Mapping[str, object]) -> tuple: @@ -981,10 +996,12 @@ def dependency_label_signature(kind: str, row: Mapping[str, object]) -> tuple: """ name = canon_kind(kind) deps, problem = dependency_record(name, row) - label = _node_label(name, row) - return (name, deps, problem, label.row_class, label.stored_domain, - label.stored_sensitivity, label.stored_scope, label.stored_floor, - label.carries_scope, str(row.get("user_id") or "")) + floor_shape, floor = _floor_of(row) + # A signature excludes row identity. Construct only its label semantics, + # rather than allocating a full node label and normalizing unused IDs. + return (name, deps, problem, row_class(name, row), str(row.get("domain") or "unknown"), + str(row.get("sensitivity") or "unknown"), stored_scope(name, row), + floor if floor_shape == "list" else (), carries_scope(name, row), str(row.get("user_id") or "")) def has_implicit_weekly_inputs(kind: str, row: Mapping[str, object]) -> bool: diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index a6f0ac746..de0e788f4 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -311,11 +311,13 @@ def _settled_inputs(self, kind: str, row: Mapping[str, object], trail: frozenset state = self._state() key = self._key(kind, row) cached = state.parent_labels.get(key) - root = (canon_kind(kind), identifier(row.get("id"))) - if root in trail or len(trail) > HOP_BOUND or kind == "belief": + if len(trail) > HOP_BOUND or kind == "belief": return None if cached is not None: return None if cached[1] & trail else cached + root = (canon_kind(kind), identifier(row.get("id"))) + if root in trail: + return None template = self._signature(kind, row, key=key) refs, problem = template[1:3] if problem or has_implicit_weekly_inputs(kind, row): diff --git a/apps/api/src/alicebot_api/vnext_label_sql.py b/apps/api/src/alicebot_api/vnext_label_sql.py index 7ae25f2c2..1cc7442b2 100644 --- a/apps/api/src/alicebot_api/vnext_label_sql.py +++ b/apps/api/src/alicebot_api/vnext_label_sql.py @@ -24,27 +24,29 @@ def hidden_memory_input_sql(sensitivity_allowed, *, sqlite: bool, alias: str = " else: def source_hint(meta): return f"COALESCE({meta}->>'source_id', CASE WHEN jsonb_typeof({meta}->'derived_from'->'sources')='array' AND jsonb_typeof({meta}->'derived_from'->'sources'->0)='string' THEN {meta}->'derived_from'->'sources'->>0 END)" - def uuid_hint(value): - return f"CASE WHEN ({value}) ~* '^(?:[0-9a-f]{{32}}|[0-9a-f]{{8}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{4}}-[0-9a-f]{{12}})$' THEN ({value})::uuid END" meta = f"{alias}.metadata_json" - source = uuid_hint(source_hint(meta)) - memory = uuid_hint(f"COALESCE(CASE WHEN jsonb_typeof({meta}->'consolidation'->'cluster_member_ids')='array' AND jsonb_typeof({meta}->'consolidation'->'cluster_member_ids'->0)='string' THEN {meta}->'consolidation'->'cluster_member_ids'->>0 END, CASE WHEN jsonb_typeof({meta}->'derived_from'->'memories')='array' AND jsonb_typeof({meta}->'derived_from'->'memories'->0)='string' THEN {meta}->'derived_from'->'memories'->>0 END)") - parent_source = uuid_hint(source_hint("label_input.metadata_json")) + source = "lower(" + source_hint(meta) + ")" + memory = f"lower(COALESCE(CASE WHEN jsonb_typeof({meta}->'consolidation'->'cluster_member_ids')='array' AND jsonb_typeof({meta}->'consolidation'->'cluster_member_ids'->0)='string' THEN {meta}->'consolidation'->'cluster_member_ids'->>0 END, CASE WHEN jsonb_typeof({meta}->'derived_from'->'memories')='array' AND jsonb_typeof({meta}->'derived_from'->'memories'->0)='string' THEN {meta}->'derived_from'->'memories'->>0 END))" + parent_source = "lower(" + source_hint("label_input.metadata_json") + ")" not_redacted = "TRUE" if sqlite else f"{alias}.metadata_json->'redacted' IS DISTINCT FROM 'true'::jsonb" input_not_redacted = "TRUE" if sqlite else "label_input.metadata_json->'redacted' IS DISTINCT FROM 'true'::jsonb" if not sqlite: - # Uncorrelated row-valued sets are hashed once by PostgreSQL. Include - # tenant identity in each set rather than scanning parents per row. + # Hash canonical and compact UUID spellings without casting untrusted + # JSON. Exact text membership accepts the same case-insensitive forms + # as the former guarded UUID cast. Tenant identity stays in every set; + # the parent partition also uses a hash instead of a per-row join. + source_set = f"SELECT user_id, unnest(ARRAY[id::text, replace(id::text, '-', '')]) FROM sources WHERE sensitivity IN ({names})" return f"""({alias}.sensitivity NOT IN ({names}) AND NOT ( {not_redacted} AND ( COALESCE(({alias}.user_id, {source}) IN ( - SELECT user_id, id FROM sources WHERE sensitivity IN ({names}) + {source_set} ), FALSE) OR COALESCE(({alias}.user_id, {memory}) IN ( - SELECT label_input.user_id, label_input.id FROM memories label_input - JOIN sources label_source ON label_source.user_id=label_input.user_id - AND label_source.id={parent_source} - WHERE {input_not_redacted} AND label_source.sensitivity IN ({names}) + SELECT label_input.user_id, unnest(ARRAY[label_input.id::text, replace(label_input.id::text, '-', '')]) + FROM memories label_input + WHERE {input_not_redacted} AND COALESCE((label_input.user_id, {parent_source}) IN ( + {source_set} + ), FALSE) ), FALSE) )))""" # nosec B608 return f"""({alias}.sensitivity NOT IN ({names}) AND NOT ( diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index 7afc0213b..1c42a1f40 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -141,10 +141,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 350, + "line": 352, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], ())", - "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" + "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" } ], "pytest": [ @@ -157,10 +157,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 350, + "line": 352, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], tuple(sorted(set(semantic_parents))))", - "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" + "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" } ], "pytest": [ @@ -173,10 +173,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 337, + "line": 339, "before": "if len(raw) != 1 or identifier(raw[0].get(\"id\")) != parent_id:", "after": "if not raw or identifier(raw[0].get(\"id\")) != parent_id:", - "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" + "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" } ], "pytest": [ @@ -191,10 +191,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 345, + "line": 347, "before": "if depth > HOP_BOUND or len(ancestry) > NODE_BOUND:", "after": "if False:", - "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" + "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" } ], "pytest": [ @@ -209,10 +209,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 321, + "line": 323, "before": "if problem or has_implicit_weekly_inputs(kind, row):", "after": "if problem:", - "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" + "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" } ], "pytest": [ @@ -227,10 +227,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 485, + "line": 487, "before": "if not self.active and callable(native):", "after": "if False:", - "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" + "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" } ], "pytest": [ @@ -246,7 +246,7 @@ "line": 17, "before": "if not blocked:", "after": "if True:", - "sha256": "bd1646e673033727148fbc112ceb6f10d079b1ab43dbed6595c185855e46a58f" + "sha256": "eca0734b71faf9d0a7b961b92ff31fedf6b262850ba81c47c53f399842e7973a" } ], "pytest": [ @@ -275,10 +275,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 374, + "line": 376, "before": " self._prefetch_inputs(kind, rows)", "after": " pass # removed batched ancestry reads", - "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" + "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" } ], "pytest": [ @@ -357,8 +357,8 @@ "file": "apps/api/src/alicebot_api/vnext_label_guard.py", "before": "row_admission_key = (key, self.domains, self.sensitivity_allowed, self.projects, self.all_of)", "after": "row_admission_key = (key,)", - "line": 384, - "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" + "line": 386, + "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" } ], "pytest": [ @@ -374,7 +374,7 @@ "before": " self.row_admission.clear()", "after": " pass # removed admission invalidation", "line": 88, - "sha256": "1e02fe9715c0e9d3b3b0ff22347250c3919ae319f92d991bccdff884ece1be6c" + "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" } ], "pytest": [ diff --git a/tests/integration/test_label_round3_contracts_postgres.py b/tests/integration/test_label_round3_contracts_postgres.py index f65d57320..48a1a0907 100644 --- a/tests/integration/test_label_round3_contracts_postgres.py +++ b/tests/integration/test_label_round3_contracts_postgres.py @@ -22,6 +22,46 @@ def test_native_count_prefilter_matches_complete_effective_admission(label_harne assert_native_count_prefilter_matches_complete_effective_admission(store) +def test_native_source_floor_sets_preserve_aliases_and_inconclusive_inputs(label_harness): + from alicebot_api.vnext_label_guard import LabelGuard, label_read_scope + from alicebot_api.vnext_label_sql import hidden_memory_input_sql + from alicebot_api.vnext_label_writes import without_insert_floor + + h = label_harness + hidden = h.source(sensitivity="confidential") + with h.store() as store, without_insert_floor(): + parent = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Hidden parent", "status": "active", + "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(hidden["id"])}}) + rejected, inconclusive = [], [] + for target_kind, target in (("source", hidden), ("memory", parent)): + canonical = str(target["id"]) + for spelling in (canonical, canonical.upper(), canonical.replace("-", ""), canonical.replace("-", "").upper(), "not-a-uuid", " " + canonical, "' OR TRUE --"): + metadata = {"source_id": spelling} if target_kind == "source" else {"consolidation": {"cluster_member_ids": [spelling]}} + row = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Floor-set observation", "status": "active", + "domain": "project", "sensitivity": "public", "metadata_json": metadata}) + (rejected if spelling in (canonical, canonical.upper(), canonical.replace("-", ""), canonical.replace("-", "").upper()) else inconclusive).append(row) + redacted = store.create_memory({"memory_key": str(uuid4()), "canonical_text": "Redacted original", "status": "active", + "domain": "project", "sensitivity": "public", "metadata_json": {"source_id": str(hidden["id"]), "redacted": True}}) + allowed = ("public", "internal") + predicate = hidden_memory_input_sql(allowed, sqlite=False) + prefetched = store._fetch_all("SELECT m.id FROM memories m WHERE " + predicate) + kept_ids = {str(row["id"]) for row in prefetched} + assert not ({str(row["id"]) for row in rejected} & kept_ids) + assert {str(row["id"]) for row in [*inconclusive, redacted]} <= kept_ids + # A SQL miss remains inconclusive and still goes through the canonical + # kernel. Preserve its behavior for unrecognised free text as well. + population = [row for batch in store.iter_label_rows("memory") for row in batch] + with label_read_scope(store): + full = LabelGuard(store, active=True, sensitivity_allowed=allowed).admit_rows("memory", population) + filtered = [row for batch in store.iter_label_rows("memory", reject_sensitivity_allowed=allowed) for row in batch] + with label_read_scope(store): + narrowed = LabelGuard(store, active=True, sensitivity_allowed=allowed).admit_rows("memory", filtered) + admitted = {str(row["id"]) for row in full} + assert {str(row["id"]) for row in narrowed} == admitted + assert not ({str(row["id"]) for row in rejected} & admitted) + assert str(redacted["id"]) in admitted + + def test_memory_event_target_set_preserves_exact_target_matching(label_harness): h = label_harness visible = h.memory() diff --git a/tests/unit/test_vnext_store.py b/tests/unit/test_vnext_store.py index c42c837c8..6a4562aad 100644 --- a/tests/unit/test_vnext_store.py +++ b/tests/unit/test_vnext_store.py @@ -1236,7 +1236,13 @@ def test_resume_store_queries_apply_admission_predicates_before_limit() -> None: assert "event.payload_json ->> 'text'" not in loop_event_query assert "event.payload_json::text" not in loop_event_query assert "event_type_prefix" not in shared_event_query - assert "JOIN memories m" in shared_event_query + assert "COALESCE((e.user_id, e.target_id) IN (" in shared_event_query + assert "SELECT m.user_id, m.id::text FROM memories m" in shared_event_query + assert shared_event_query.index("m.deleted_at IS NULL") < shared_event_query.index("LIMIT %s") + assert shared_event_query.index("SELECT m.user_id, m.id::text") < shared_event_query.index("ORDER BY e.occurred_at DESC") + assert cursor.statements[4][1] == ( + None, None, ["project a"], ["project a"], None, None, None, since, since, until, until, 2, + ) def test_project_update_event_lookup_is_one_bounded_target_and_payload_query() -> None: From e28d1560e9d74e0a09ad52e50c4ccd67c322cc7e Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:00:27 +0200 Subject: [PATCH 258/270] Document the closed-literal source lookup SQL --- apps/api/src/alicebot_api/vnext_label_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/alicebot_api/vnext_label_sql.py b/apps/api/src/alicebot_api/vnext_label_sql.py index 1cc7442b2..89c4dc558 100644 --- a/apps/api/src/alicebot_api/vnext_label_sql.py +++ b/apps/api/src/alicebot_api/vnext_label_sql.py @@ -35,7 +35,7 @@ def source_hint(meta): # JSON. Exact text membership accepts the same case-insensitive forms # as the former guarded UUID cast. Tenant identity stays in every set; # the parent partition also uses a hash instead of a per-row join. - source_set = f"SELECT user_id, unnest(ARRAY[id::text, replace(id::text, '-', '')]) FROM sources WHERE sensitivity IN ({names})" + source_set = f"SELECT user_id, unnest(ARRAY[id::text, replace(id::text, '-', '')]) FROM sources WHERE sensitivity IN ({names})" # nosec B608 - closed kernel sensitivity constants only return f"""({alias}.sensitivity NOT IN ({names}) AND NOT ( {not_redacted} AND ( COALESCE(({alias}.user_id, {source}) IN ( From 820798842ce10c1c06edcd598bbe042c910b7e1a Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:11:22 +0200 Subject: [PATCH 259/270] Reuse bulk label semantics and compact request cache keys --- .../src/alicebot_api/vnext_derived_labels.py | 79 +++++++++++++++---- .../api/src/alicebot_api/vnext_label_guard.py | 8 +- .../src/alicebot_api/vnext_project_scope.py | 20 +++-- scripts/derived_label_mutations.json | 20 ++--- tests/unit/test_derived_labels_kernel.py | 30 ++++++- 5 files changed, 123 insertions(+), 34 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 409a6c885..0e95c4126 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -304,7 +304,9 @@ def is_derived(kind: object, row: Mapping[str, object]) -> bool: return False cache = _READ_METADATA.get() raw = row.get("metadata_json") - key = ("derived", name, id(raw), repr(row.get("source_id")), repr(row.get("artifact_type"))) + key = ("derived", name, id(raw), + repr(row.get("source_id")) if name == "open_loop" else None, + repr(row.get("artifact_type")) if name == "artifact" else None) cached = cache.get(key) if cache is not None else None if cached is not None and cached[0] is raw: return cached[1] @@ -395,7 +397,13 @@ def _floor_of(row: Mapping[str, object]) -> tuple[str, tuple[str, ...]]: if "project_floor" in row: return project_floor_shape(row) - return project_floor_shape({"metadata_json": _metadata(row)}) + metadata = _metadata(row) + if "project_floor" not in metadata: + return "absent", () + floor = metadata.get("project_floor") + if type(floor) in (list, tuple) and not floor: + return "list", () + return project_floor_shape({"metadata_json": metadata}) def group_scope(row: Mapping[str, object], *, kind: str | None = None) -> tuple[str, ...]: @@ -424,6 +432,16 @@ def stored_scope(kind: object, row: Mapping[str, object]) -> tuple[str, ...]: return source_project_scope(row) if name == "project": return () + # Canonical presence is authoritative. The common native empty scope + # needs no alias/container resolution, but malformed and legacy shapes + # still use the canonical resolver. Metadata precedes scope_json. + if "project_scope" in row: + raw_scope = row.get("project_scope") + else: + raw_metadata = row.get("metadata_json") + raw_scope = raw_metadata.get("project_scope") if isinstance(raw_metadata, Mapping) else None + if type(raw_scope) in (list, tuple) and not raw_scope: + return () return resolve_project_scope(row).values @@ -468,6 +486,13 @@ def _source_ids_from(value: object) -> tuple[str, set[str]]: if value is None: return "", set() + # An exact canonical UUID contains one source and no surrounding text. + # Every other shape still runs the complete saved-quote parser, including + # encoded/split references and text naming more than one source. + if type(value) is str and len(value) == 36: + token = _source_token(value) + if token is not None and token == value.lower(): + return "", {token} named = {identifier(item) for item in cited_source_ids(value).named} problem = "" if isinstance(value, list): @@ -768,8 +793,8 @@ def dependency_record(kind: object, row: Mapping[str, object]) -> tuple[frozense # fields used by this parser remain part of the key, including presence. cache = _READ_METADATA.get() raw = row.get("metadata_json") - key = ("dependencies", canon_kind(kind), id(raw), *((field in row, repr(row.get(field))) - for field in ("value", "source_id", "artifact_type", "project_floor"))) + key = ("dependencies", canon_kind(kind), id(raw), *((field, repr(row[field])) + for field in ("value", "source_id", "artifact_type", "project_floor") if field in row)) cached = cache.get(key) if cache is not None else None if cached is not None and cached[0] is raw: return cached[1] @@ -984,7 +1009,8 @@ def metadata_key(value: object) -> tuple: return () return (canon_kind(kind), isinstance(row.get("metadata_json"), Mapping), metadata_key(_metadata(row)), - *((field in row, repr(row.get(field))) for field in _DEPENDENCY_SYNTAX_FIELDS)) + *((field, ("uuid", row[field]) if type(row[field]) is UUID else repr(row[field])) + for field in _DEPENDENCY_SYNTAX_FIELDS if field in row)) def dependency_label_signature(kind: str, row: Mapping[str, object]) -> tuple: @@ -1062,7 +1088,9 @@ def _apply_dependencies( fallback_domain = current.domain if domain_fallback is None else domain_fallback fallback_sensitivity = current.sensitivity if sensitivity_fallback is None else sensitivity_fallback base = current - if scope_fallback is not None or floor_fallback is not None: + if ((scope_fallback is not None and scope_fallback != current.stored_scope) + or (floor_fallback is not None and floor_fallback != current.stored_floor) + or fallback_domain != current.domain or fallback_sensitivity != current.sensitivity): base = replace( current, stored_scope=current.stored_scope if scope_fallback is None else scope_fallback, @@ -1082,6 +1110,9 @@ def _apply_dependencies( if base.row_class == "project_state": floor = () scope = _effective_scope(base, deps, floor) + if (base.domain == str(domain) and base.sensitivity == sensitivity + and base.project_scope == scope and base.project_floor == floor): + return base return replace(base, domain=str(domain), sensitivity=sensitivity, project_scope=scope, project_floor=floor) @@ -1285,7 +1316,8 @@ def resolve_belief(ref: tuple[str, str, str]) -> tuple[str, str, str]: and settled.stored_floor == label.stored_floor ) if same_stored_row: - published.append(replace(settled, unverified=False, reason=None)) + published.append(settled if not settled.unverified and settled.reason is None + else replace(settled, unverified=False, reason=None)) continue settled_deps = [labels[ref] for ref in sorted(resolved.get(label.key, set())) if ref in labels] recomputed = _apply_dependencies( @@ -1367,6 +1399,7 @@ def _iterate( problems: dict[tuple[str, str, str], str], ) -> None: dependants: dict[tuple[str, str, str], set[tuple[str, str, str]]] = {} + effective_parts: dict[tuple, tuple[str, str, tuple[str, ...], tuple[str, ...]]] = {} for key, refs in inputs.items(): for ref in refs: dependants.setdefault(ref, set()).add(key) @@ -1381,14 +1414,30 @@ def _iterate( queued.remove(key) current = labels[key] deps = [labels[ref] for ref in sorted(inputs[key]) if ref in labels] - updated = _apply_dependencies( - current, - deps, - domain_fallback=current.stored_domain, - sensitivity_fallback=current.stored_sensitivity, - scope_fallback=current.stored_scope, - floor_fallback=current.stored_floor, - ) + # The graph has already checked every identity, missing input and + # bound. Share only the pure rule's four outputs for equal stored + # root semantics and equal, ordered, current parent label sets. + # Cycles still revisit each member; a changed parent changes this + # key. No identity, row data or admission grant is shared. + semantic = (current.kind, current.row_class, current.derived, + current.stored_domain, current.stored_sensitivity, + current.stored_scope, current.stored_floor, + tuple((item.kind, item.domain, item.sensitivity, item.project_scope, + item.project_floor, item.carries_scope) for item in deps)) + parts = effective_parts.get(semantic) + if parts is None: + updated = _apply_dependencies( + current, deps, domain_fallback=current.stored_domain, + sensitivity_fallback=current.stored_sensitivity, + scope_fallback=current.stored_scope, floor_fallback=current.stored_floor, + ) + effective_parts[semantic] = (updated.domain, updated.sensitivity, + updated.project_scope, updated.project_floor) + else: + updated = current if (current.domain, current.sensitivity, current.project_scope, + current.project_floor) == parts else replace( + current, domain=parts[0], sensitivity=parts[1], + project_scope=parts[2], project_floor=parts[3]) if not _changed(current, updated): continue remaining_changes -= 1 diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index de0e788f4..1ebe4cb2f 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -620,7 +620,13 @@ def _key(self, kind: str, row: Mapping[str, object]) -> tuple: keys = current[1].row_keys raw_key = (kind, id(row)) if raw_key not in keys or keys[raw_key][0] is not row: - keys[raw_key] = (row, _row_label_key(kind, row)) + # This locked request already pins each raw projection and clears + # all entries on writes/rollback. A compact object key avoids + # repeatedly hashing full UUID/metadata representations. Strong + # references prevent id reuse, and distinct loaded projections + # never share an identity cache entry. Outside a snapshot the + # content key above still detects changes between guard calls. + keys[raw_key] = (row, raw_key) return keys[raw_key][1] def _signature(self, kind: str, row: Mapping[str, object], *, key: tuple) -> tuple: diff --git a/apps/api/src/alicebot_api/vnext_project_scope.py b/apps/api/src/alicebot_api/vnext_project_scope.py index 704ca5e55..d4f356528 100644 --- a/apps/api/src/alicebot_api/vnext_project_scope.py +++ b/apps/api/src/alicebot_api/vnext_project_scope.py @@ -6,7 +6,7 @@ from decimal import Decimal, InvalidOperation import math import re -from typing import Mapping, Sequence +from collections.abc import Mapping, Sequence from alicebot_api.vnext_repositories import JsonObject @@ -168,20 +168,22 @@ def resolve_project_scope(resource: Mapping[str, object] | None) -> ProjectScope return ProjectScopeResolution(present=False, values=()) def canonical_values(value: object) -> tuple[str, ...]: + if type(value) in (list, tuple) and not value: + return () if not isinstance(value, Sequence) or isinstance(value, (str, bytes, bytearray)): return () return normalize_project_scope(value) - containers = tuple( - container - for container_key in ("metadata_json", "scope_json") - if isinstance((container := resource.get(container_key)), Mapping) - ) if "project_scope" in resource: return ProjectScopeResolution( present=True, values=canonical_values(resource.get("project_scope")), ) + containers = tuple( + container + for container_key in ("metadata_json", "scope_json") + if isinstance((container := resource.get(container_key)), Mapping) + ) for container in containers: if "project_scope" in container: return ProjectScopeResolution( @@ -240,6 +242,8 @@ def resolve_source_metadata_project_scope( if not isinstance(metadata_json, Mapping): return ProjectScopeResolution(present=False, values=()) + if type(metadata_json) is dict and not metadata_json: + return ProjectScopeResolution(present=False, values=()) resource = dict(metadata_json) stored_container = resource.get("metadata_json") @@ -266,6 +270,10 @@ def source_project_scope(source: Mapping[str, object]) -> tuple[str, ...]: """ metadata_json = source.get("metadata_json") + if type(metadata_json) is dict and not metadata_json and not any( + key in source for key in ("project_scope", "scope_json", "project_id", "project", "projects") + ): + return () resolution = resolve_source_metadata_project_scope(metadata_json if isinstance(metadata_json, Mapping) else None) if resolution.present or resolution.values: return resolution.values diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index 1c42a1f40..d3c0121a0 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -144,7 +144,7 @@ "line": 352, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], ())", - "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" + "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" } ], "pytest": [ @@ -160,7 +160,7 @@ "line": 352, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], tuple(sorted(set(semantic_parents))))", - "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" + "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" } ], "pytest": [ @@ -176,7 +176,7 @@ "line": 339, "before": "if len(raw) != 1 or identifier(raw[0].get(\"id\")) != parent_id:", "after": "if not raw or identifier(raw[0].get(\"id\")) != parent_id:", - "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" + "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" } ], "pytest": [ @@ -194,7 +194,7 @@ "line": 347, "before": "if depth > HOP_BOUND or len(ancestry) > NODE_BOUND:", "after": "if False:", - "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" + "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" } ], "pytest": [ @@ -212,7 +212,7 @@ "line": 323, "before": "if problem or has_implicit_weekly_inputs(kind, row):", "after": "if problem:", - "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" + "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" } ], "pytest": [ @@ -230,7 +230,7 @@ "line": 487, "before": "if not self.active and callable(native):", "after": "if False:", - "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" + "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" } ], "pytest": [ @@ -246,7 +246,7 @@ "line": 17, "before": "if not blocked:", "after": "if True:", - "sha256": "eca0734b71faf9d0a7b961b92ff31fedf6b262850ba81c47c53f399842e7973a" + "sha256": "75729810b88316a6ff0611dfe01e1e214036226e19c57fa41bbe00f3c163870b" } ], "pytest": [ @@ -278,7 +278,7 @@ "line": 376, "before": " self._prefetch_inputs(kind, rows)", "after": " pass # removed batched ancestry reads", - "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" + "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" } ], "pytest": [ @@ -358,7 +358,7 @@ "before": "row_admission_key = (key, self.domains, self.sensitivity_allowed, self.projects, self.all_of)", "after": "row_admission_key = (key,)", "line": 386, - "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" + "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" } ], "pytest": [ @@ -374,7 +374,7 @@ "before": " self.row_admission.clear()", "after": " pass # removed admission invalidation", "line": 88, - "sha256": "b06fac8dbf66a9678bef2ef8b7d4adf724400bdf9288688f6cf70ac399be0d76" + "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" } ], "pytest": [ diff --git a/tests/unit/test_derived_labels_kernel.py b/tests/unit/test_derived_labels_kernel.py index 30ea192e6..43ad2dc74 100644 --- a/tests/unit/test_derived_labels_kernel.py +++ b/tests/unit/test_derived_labels_kernel.py @@ -22,6 +22,7 @@ labels_raised_payload, row_class, settle_labels, + with_derived_from, ) from alicebot_api.vnext_project_scope import GLOBAL_PROJECT_MARKER, project_floor_within, project_scopes_overlap @@ -102,7 +103,7 @@ def _memory(row_id: str, **fields: object) -> dict[str, object]: def test_a_chain_settles_in_one_pass(monkeypatch: pytest.MonkeyPatch) -> None: - """A chain of any length is labelled from the leaf inward, each row once.""" + """Visit the chain once and compute each equivalent parent-label set once.""" calls: list[str] = [] real = __import__("alicebot_api.vnext_derived_labels", fromlist=["_apply_dependencies"])._apply_dependencies @@ -125,10 +126,13 @@ def wrapped(*args: object, **kwargs: object) -> object: previous = f"n{index}" settled = settle_labels(rows) derived_ids = [row.stored_id for row in settled.derived_rows()] - assert calls == derived_ids + assert derived_ids == [f"n{index}" for index in range(size)] + assert calls == ["n0", "n1"] # source parent, then equivalent report parents for row in settled.derived_rows(): assert row.domain == "health" assert row.sensitivity == "confidential" + assert row.project_scope == (ALPHA,) + assert row.project_floor == (ALPHA,) assert row.unverified is False @@ -153,6 +157,28 @@ def wrapped(*args: object, **kwargs: object) -> object: assert settled.by_stored("artifact", "c").domain == "legal" +def test_shared_effective_parts_preserve_each_roots_labels_and_identity() -> None: + """Grouped rows agree with separately settled roots across label variants.""" + parents = [ + _source("public", domain="project", scope=[ALPHA]), + _source("private", domain="health", sensitivity="private", scope=[BETA]), + _source("scrubbed", domain="project", scope=[BETA], scrubbed=True), + _memory("plain", domain="project", sensitivity="public", metadata_json={"project_scope": [ALPHA]}), + ] + roots = [] + for index, parent in enumerate(parents): + for kind in ("memory", "artifact"): + for scope, floor in (([], []), ([ALPHA], []), ([ALPHA], [BETA])): + row_id = f"{kind}-{index}-{len(roots)}" + metadata = with_derived_from({"project_scope": scope, "project_floor": floor}, + {"sources" if parent["kind"] == "source" else "memories": [parent]}) + roots.append(_row(kind, row_id, domain="project", sensitivity="public", metadata_json=metadata)) + grouped = settle_labels([*parents, *roots]) + for root in roots: + separate = settle_labels([*parents, root]).by_stored(str(root["kind"]), str(root["id"])) + assert grouped.by_stored(str(root["kind"]), str(root["id"])) == separate + + def test_a_cycle_that_does_not_settle_is_refused() -> None: ring = [] names = ["r0", "r1", "r2"] From 8a5393ad936e8cf127eda51da6ba7edea82a3f4a Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:44:58 +0200 Subject: [PATCH 260/270] Retain paired budget samples and add maximum-workload controls --- .github/workflows/tests.yml | 29 +++++++++++++++++-- .../test_label_round3_read_budget.py | 9 +++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7b0e6097b..b9745564a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2287,7 +2287,7 @@ jobs: strategy: fail-fast: false matrix: - integration_check: ["Integration tests (Postgres + pgvector, role separation)", "Default surface integration smoke (Postgres)"] + integration_check: ["Integration tests (Postgres + pgvector, role separation)", "Default surface integration smoke (Postgres)", "Read-budget maximum workload smoke (Postgres)"] services: postgres: image: pgvector/pgvector:pg16@sha256:1d533553fefe4f12e5d80c7b80622ba0c382abb5758856f52983d8789179f0fb @@ -2305,7 +2305,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Paired read-budget baseline (main at the round-three handoff) - if: matrix.integration_check == 'Integration tests (Postgres + pgvector, role separation)' + if: matrix.integration_check != 'Default surface integration smoke (Postgres)' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: 48873b038013f4cf548099fcc4610a150972eedd @@ -2336,6 +2336,7 @@ jobs: DATABASE_URL: postgresql://alicebot_app:ci@localhost:5432/alicebot DATABASE_ADMIN_URL: postgresql://alicebot_admin:ci@localhost:5432/alicebot ALICE_READ_MAIN_CHECKOUT: ${{ github.workspace }}/.read-budget-baseline + ALICE_READ_BUDGET_EVIDENCE: ${{ runner.temp }}/round3-postgres-read-budgets.jsonl # --timeout=180 names one test that runs that long. The cancelled # jobs were slow database setup, not a gap near 180s. The round-three # grid includes 30 paired fixtures with ten interleaved samples per @@ -2347,8 +2348,32 @@ jobs: if: matrix.integration_check == 'Integration tests (Postgres + pgvector, role separation)' env: ALICE_READ_MAIN_CHECKOUT: ${{ github.workspace }}/.read-budget-baseline + ALICE_READ_BUDGET_EVIDENCE: ${{ runner.temp }}/round3-sqlite-read-budgets.jsonl run: ./.venv/bin/python -m pytest tests/performance/test_label_round3_read_budget.py tests/performance/test_label_read_handoff_budget.py -q --timeout=180 --session-timeout=1800 + - name: Maximum-workload read-budget controls + if: matrix.integration_check == 'Read-budget maximum workload smoke (Postgres)' + env: + DATABASE_URL: postgresql://alicebot_app:ci@localhost:5432/alicebot + DATABASE_ADMIN_URL: postgresql://alicebot_admin:ci@localhost:5432/alicebot + ALICE_READ_MAIN_CHECKOUT: ${{ github.workspace }}/.read-budget-baseline + ALICE_READ_BUDGET_EVIDENCE: ${{ runner.temp }}/round3-postgres-read-budgets.jsonl + run: >- + ALICE_LEGACY_SURFACES=1 ./.venv/bin/python -m pytest -q -s --timeout=180 + 'tests/integration/test_label_round3_read_budget_postgres.py::test_postgres_round3_random_read_budgets[False-3000-all-visible]' + 'tests/integration/test_label_round3_read_budget_postgres.py::test_postgres_round3_random_read_budgets[True-3000-extra-rows]' + + - name: Retain paired read-budget samples, including failed runs + if: always() && matrix.integration_check != 'Default surface integration smoke (Postgres)' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: round3-paired-read-budgets-${{ strategy.job-index }} + path: | + ${{ runner.temp }}/round3-postgres-read-budgets.jsonl + ${{ runner.temp }}/round3-sqlite-read-budgets.jsonl + if-no-files-found: error + retention-days: 7 + - name: Default-surface core round-trip if: matrix.integration_check == 'Default surface integration smoke (Postgres)' env: diff --git a/tests/performance/test_label_round3_read_budget.py b/tests/performance/test_label_round3_read_budget.py index 21379af56..c7e199d5c 100644 --- a/tests/performance/test_label_round3_read_budget.py +++ b/tests/performance/test_label_round3_read_budget.py @@ -141,7 +141,14 @@ def paired_budgets(backend, location, user, keys, *, case, source_count, repaire "memories": 5000, "mix": ({"source_copy": 5000} if case == "identical-copies" else {"source_copy": 1500, "stamped_report": 1250, "consolidation": 1250, "weekly": 1000}), "artifacts": 500 if backend == "postgres" and case == "extra-rows" else 0, "derived_loops": 400 if case == "extra-rows" else 0, "real_repair_zero_check": repaired, **measurements} - print(json.dumps(row), flush=True) + encoded_row = json.dumps(row) + print(encoded_row, flush=True) + evidence_path = os.environ.get("ALICE_READ_BUDGET_EVIDENCE") + if evidence_path: + evidence = Path(evidence_path) + evidence.parent.mkdir(parents=True, exist_ok=True) + with evidence.open("a") as output: + output.write(encoded_row + "\n") for action in ("pack", "recall"): for clock in ("minimum_wall", "minimum_cpu"): if measurements["head"][action][clock] > 2 * measurements["main"][action][clock] + .1: From 09cc6361366d447ef35cee6b761f915657d1bacc Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:14:07 +0200 Subject: [PATCH 261/270] Settle complete native count batches with the canonical bulk kernel --- .../src/alicebot_api/vnext_derived_labels.py | 2 +- .../api/src/alicebot_api/vnext_label_guard.py | 43 +++++++++- .../src/alicebot_api/vnext_label_repair.py | 6 ++ scripts/derived_label_mutations.json | 56 ++++++++++--- tests/unit/test_label_resolved_inputs.py | 79 +++++++++++++++++++ .../unit/test_legacy_surface_test_posture.py | 3 +- 6 files changed, 174 insertions(+), 15 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 0e95c4126..d577778f3 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -606,7 +606,7 @@ def _collect_metadata_ids(value: object, found: set[tuple[str, str]]) -> str: if isinstance(child, (str, list)): child_problem, source_ids = _source_ids_from(child) problem = problem or child_problem - source_ids.update(item for item in _strings(child) if _plain_token(item)) + source_ids.update(item for item in _strings(child) if item not in source_ids and _plain_token(item)) _add_ids(found, "source", source_ids) else: problem = problem or "malformed" diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index 1ebe4cb2f..e0ce6faf1 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -388,6 +388,15 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: if state.row_admission[row_admission_key]: kept.append(row) continue + settled = state.labels.get(key) + if settled is not None and not settled.unverified: + admitted = self._admits_effective({"domain": settled.domain, "sensitivity": settled.sensitivity, + "project_scope": settled.project_scope, "project_floor": settled.project_floor, + "unverified": settled.unverified}, kind=kind) + state.row_admission[row_admission_key] = admitted + if admitted: + kept.append(row) + continue template = self._signature(kind, row, key=key) admission_key = (template, self.domains, self.sensitivity_allowed, self.projects, self.all_of) root = (canon_kind(kind), identifier(row.get("id"))) @@ -441,7 +450,7 @@ def _prefetch_inputs(self, kind: str, rows: Sequence[Mapping[str, object]]) -> N key = self._key(row_kind, row) if key in prefetched or key in state.labels or key in state.parent_labels: continue - refs.update(self._signature(row_kind, row, key=key)[1]) + refs.update(dependencies_of(row_kind, row)) pending.append(key) refs.difference_update(expanded) if len(expanded | refs) > NODE_BOUND: @@ -507,12 +516,44 @@ def readable_status_counts(self, kind: str) -> dict[str, int]: for row in batch: if isinstance(row.get("id"), UUID): state.nodes.setdefault((canon_kind(kind), str(row["id"])), [row]) + self._settle_native_count_batch(kind, batch) for row in self.admit_rows(kind, batch): status = str(row.get("status", "unknown")) counts[status] = counts.get(status, 0) + 1 state.counts[key] = dict(counts) return counts + def _settle_native_count_batch(self, kind: str, rows: Sequence[Mapping[str, object]]) -> None: + """The complete UUID-native population uses the canonical bulk kernel. + + Raw inputs are still loaded through the same bounded frontier. Cache + only verified results for these exact count projections: incomplete + ancestry, implicit weekly parents and every unverified row retain the + ordinary per-origin collector. Text-ID stores never enter this path. + """ + if not rows or any(type(row.get("id")) is not UUID for row in rows): + return + self._prefetch_inputs(kind, rows) + state = self._state() + for row in rows: + stored = state.nodes.get((canon_kind(kind), identifier(row["id"])), ()) + if len(stored) != 1 or (stored[0] is not row and _row_label_key(kind, stored[0]) != _row_label_key(kind, row)): + return + nodes: list[dict[str, object]] = [] + for (node_kind, _node_id), found in state.nodes.items(): + if len(found) != 1: + return + row = found[0] + if has_implicit_weekly_inputs(node_kind, row) or node_kind == "belief": + return + nodes.append({**row, "kind": node_kind, "user_id": _GUARD_USER}) + result = settle_labels(nodes, on_cycle="unverified", max_hops=HOP_BOUND, max_nodes=NODE_BOUND) + settled = {label.key: label for label in result.rows} + for row in rows: + label = settled.get((canon_kind(kind), _GUARD_USER, identifier(row["id"]))) + if label is not None and not label.unverified: + state.labels[self._key(kind, row)] = label + def admit_related_rows(self, rows: Sequence[_Row], *, kind: str, field: str) -> list[_Row]: """Admit an event or rating by its target's current effective label.""" diff --git a/apps/api/src/alicebot_api/vnext_label_repair.py b/apps/api/src/alicebot_api/vnext_label_repair.py index e3163aaa4..13858d4d3 100644 --- a/apps/api/src/alicebot_api/vnext_label_repair.py +++ b/apps/api/src/alicebot_api/vnext_label_repair.py @@ -257,6 +257,12 @@ def classify_stored_labels( continue if table not in _WRITABLE: continue + if (label.stored_domain == label.domain and label.stored_sensitivity == label.sensitivity + and label.stored_scope == label.project_scope and label.stored_floor == label.project_floor): + # Exact equality already implies the canonical identity equality + # below. Keep the full comparison and repair evidence for every + # changed or merely equivalent spelling. + continue previous: LabelParts = { "domain": str(node.get("domain") or "unknown"), "sensitivity": str(node.get("sensitivity") or "unknown"), diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index d3c0121a0..59ce805d1 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -144,7 +144,7 @@ "line": 352, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], ())", - "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" + "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" } ], "pytest": [ @@ -160,7 +160,7 @@ "line": 352, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], tuple(sorted(set(semantic_parents))))", - "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" + "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" } ], "pytest": [ @@ -176,7 +176,7 @@ "line": 339, "before": "if len(raw) != 1 or identifier(raw[0].get(\"id\")) != parent_id:", "after": "if not raw or identifier(raw[0].get(\"id\")) != parent_id:", - "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" + "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" } ], "pytest": [ @@ -194,7 +194,7 @@ "line": 347, "before": "if depth > HOP_BOUND or len(ancestry) > NODE_BOUND:", "after": "if False:", - "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" + "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" } ], "pytest": [ @@ -212,7 +212,7 @@ "line": 323, "before": "if problem or has_implicit_weekly_inputs(kind, row):", "after": "if problem:", - "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" + "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" } ], "pytest": [ @@ -227,10 +227,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 487, + "line": 496, "before": "if not self.active and callable(native):", "after": "if False:", - "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" + "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" } ], "pytest": [ @@ -276,9 +276,9 @@ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", "line": 376, - "before": " self._prefetch_inputs(kind, rows)", - "after": " pass # removed batched ancestry reads", - "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" + "before": " self._prefetch_inputs(kind, rows)\n kept: list[_Row] = []", + "after": " pass # removed batched ancestry reads\n kept: list[_Row] = []", + "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" } ], "pytest": [ @@ -358,7 +358,7 @@ "before": "row_admission_key = (key, self.domains, self.sensitivity_allowed, self.projects, self.all_of)", "after": "row_admission_key = (key,)", "line": 386, - "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" + "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" } ], "pytest": [ @@ -374,13 +374,45 @@ "before": " self.row_admission.clear()", "after": " pass # removed admission invalidation", "line": 88, - "sha256": "12c1e16eef9573699ddf7a80948b175dabf23b3b818178c3caae83d75097c06b" + "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" } ], "pytest": [ "tests/unit/test_label_resolved_inputs.py::test_repeated_admission_keeps_caller_filters_projections_and_write_refresh" ], "expected_failure": "test_repeated_admission_keeps_caller_filters_projections_and_write_refresh" + }, + { + "name": "cached-unverified-admission", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "line": 392, + "before": " if settled is not None and not settled.unverified:", + "after": " if settled is not None:", + "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_cached_unverified_result_still_uses_regulated_admission" + ], + "expected_failure": "test_cached_unverified_result_still_uses_regulated_admission" + }, + { + "name": "native-bulk-projection", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "line": 540, + "before": " if len(stored) != 1 or (stored[0] is not row and _row_label_key(kind, stored[0]) != _row_label_key(kind, row)):", + "after": " if len(stored) != 1:", + "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_native_bulk_count_does_not_reuse_a_distinct_projection" + ], + "expected_failure": "test_native_bulk_count_does_not_reuse_a_distinct_projection" } ] } diff --git a/tests/unit/test_label_resolved_inputs.py b/tests/unit/test_label_resolved_inputs.py index 7135cbbcb..b15c63874 100644 --- a/tests/unit/test_label_resolved_inputs.py +++ b/tests/unit/test_label_resolved_inputs.py @@ -171,3 +171,82 @@ def test_unproved_inputs_keep_the_complete_graph_fallback(monkeypatch, variant): # Same collector, same canonical kernel bounds, with optimization disabled. monkeypatch.setattr(module.LabelGuard, "_settled_inputs", lambda *args: None) assert labels(actual) == labels(LabelGuard(store, active=True).effective_row("memory", root)) + + +@pytest.mark.parametrize("variant", ["mixed", "deep", "missing", "malformed", "cycle", "alias", "implicit-weekly"]) +def test_native_bulk_count_matches_independent_root_admission(variant): + sources = [{"kind": "source", "id": UUID(int=i + 1), "domain": "project", + "sensitivity": "confidential" if i % 2 else "public", "metadata_json": {"project_scope": ["P1"]}} + for i in range(8)] + roots = [] + for i in range(32): + parents = roots[max(0, i - 2):i] if variant == "deep" and i else [sources[i % 8]] + meta = with_derived_from({"observation_index": i}, {"sources": [p for p in parents if p["kind"] == "source"], + "memories": [p for p in parents if p["kind"] == "memory"]}) + if i % 4 == 1: + meta["workflow"] = "project_auto_update" + elif i % 4 == 2: + meta["candidate_kind"] = "memory_consolidation" + elif i % 4 == 3: + meta["discovered_by"] = "vnext_weekly_synthesis" + roots.append({"kind": "memory", "id": UUID(int=100 + i), "domain": "project", "sensitivity": "public", + "status": "active", "metadata_json": meta}) + rows = sources + roots + if variant == "missing": + rows = roots + elif variant == "malformed": + roots[0]["metadata_json"]["derived_from"]["counts"]["sources"] += 1 + elif variant == "cycle": + roots[0]["metadata_json"] = with_derived_from({}, {"memories": [roots[1]]}) + roots[1]["metadata_json"] = with_derived_from({}, {"memories": [roots[0]], "sources": [sources[1]]}) + elif variant == "alias": + rows.append({**deepcopy(sources[0]), "id": "urn:uuid:" + str(sources[0]["id"])}) + elif variant == "implicit-weekly": + artifact = {"kind": "artifact", "id": UUID(int=300), "domain": "project", "sensitivity": "public", + "artifact_type": "weekly_synthesis", "metadata_json": {"candidate_memory_ids": [str(roots[0]["id"])], + "input_summary": {"source_ids": [str(sources[1]["id"])]}}} + rows.append(artifact) + roots[0]["metadata_json"] = {"discovered_by": "vnext_weekly_synthesis", "source_artifact_id": str(artifact["id"])} + + class NativeRows(Rows): + label_count_canonical_unique_ids = True + + def count_original_label_statuses(self, *args, **kwargs): + return {} + + def iter_label_rows(self, kind, **kwargs): + yield roots + + independent = LabelGuard(Rows(rows), active=True, sensitivity_allowed=("public",)) + expected_ids = {row["id"] for row in independent.admit_rows("memory", roots)} + store = NativeRows(rows) + with label_read_scope(store): + guard = LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard.readable_status_counts("memory") == ({"active": len(expected_ids)} if expected_ids else {}) + assert {row["id"] for row in guard.admit_rows("memory", roots)} == expected_ids + for row in roots: + assert labels(guard.effective_row("memory", row)) == labels(independent.effective_row("memory", row)) + + +def test_cached_unverified_result_still_uses_regulated_admission(): + root = {"kind": "memory", "id": "root", "domain": "project", "sensitivity": "public", + "metadata_json": {"source_id": "missing"}} + store = Rows([]) + with label_read_scope(store): + guard = LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard.effective_row("memory", root)["unverified"] is True + assert guard.admit_rows("memory", [root]) == [] + + +def test_native_bulk_count_does_not_reuse_a_distinct_projection(): + source = {"kind": "source", "id": UUID(int=1), "domain": "project", "sensitivity": "public", "metadata_json": {}} + root = {"kind": "memory", "id": UUID(int=2), "domain": "project", "sensitivity": "public", + "metadata_json": {"source_id": str(source["id"])}} + store = Rows([source]) + with label_read_scope(store): + guard = LabelGuard(store, active=True) + state = guard._state() + state.nodes[("memory", str(root["id"]))] = [{**root, "sensitivity": "confidential"}] + guard._settle_native_count_batch("memory", [root]) + assert guard._key("memory", root) not in state.labels + assert labels(guard.effective_row("memory", root))[1] == "public" diff --git a/tests/unit/test_legacy_surface_test_posture.py b/tests/unit/test_legacy_surface_test_posture.py index beda30df3..8dd890fc5 100644 --- a/tests/unit/test_legacy_surface_test_posture.py +++ b/tests/unit/test_legacy_surface_test_posture.py @@ -61,7 +61,8 @@ def test_postgres_matrix_has_a_required_flag_off_default_surface_row() -> None: assert "name: ${{ matrix.integration_check }}" in integration_job assert ( 'integration_check: ["Integration tests (Postgres + pgvector, role separation)", ' - '"Default surface integration smoke (Postgres)"]' + '"Default surface integration smoke (Postgres)", ' + '"Read-budget maximum workload smoke (Postgres)"]' ) in integration_job assert ( "if: matrix.integration_check == " From 984691b51847fa9c49cf058ff5627d56aea2fb97 Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:31:50 +0200 Subject: [PATCH 262/270] Reuse native JSON decoding and identical doctor scan inputs --- .../src/alicebot_api/vnext_derived_labels.py | 36 ++++++++++++++++--- apps/api/src/alicebot_api/vnext_doctor.py | 15 +++++++- .../src/alicebot_api/vnext_label_repair.py | 7 +++- apps/api/src/alicebot_api/vnext_store.py | 3 ++ scripts/derived_label_mutations.json | 32 +++++++++++++++++ tests/unit/test_label_resolved_inputs.py | 25 +++++++++++++ tests/unit/test_vnext_doctor.py | 30 ++++++++++++++++ 7 files changed, 142 insertions(+), 6 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index d577778f3..c5ad15564 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -11,6 +11,7 @@ from __future__ import annotations import json +import re from collections import deque from collections.abc import Iterable, Mapping, Sequence from contextlib import contextmanager @@ -239,6 +240,30 @@ def label_metadata_cache(cache: dict): _READ_METADATA.reset(token) +def _cache_native_json_metadata(rows: Iterable[Mapping[str, object]]) -> None: + """Reuse standard PostgreSQL JSON-decoder output in a locked request. + + Only native database read sites call this: their JSON objects cannot + contain Python UUID instances. JSON strings and all other input forms + still take the canonical decoding/UUID-normalization path. + """ + cache = _READ_METADATA.get() + if cache is None: + return + for row in rows: + raw = row.get("metadata_json") + if type(raw) is dict: + cache[id(raw)] = (raw, raw) + + +def _share_metadata_decode(raw: object, projection: object) -> None: + """A shallow metadata copy has exactly the same pure decoding result.""" + cache = _READ_METADATA.get() + cached = cache.get(id(raw)) if cache is not None else None + if cache is not None and type(raw) is dict and type(projection) is dict and cached is not None and cached[0] is raw: + cache[id(projection)] = (projection, cached[1]) + + def _metadata(row: Mapping[str, object]) -> Mapping[str, object]: raw = row.get("metadata_json") cache = _READ_METADATA.get() @@ -255,6 +280,8 @@ def _uuid_strings(value: object) -> object: """Database UUID objects and JSON strings name the same recorded input.""" if value is None or type(value) in (str, int, float, bool): return value + if type(value) is dict and not value: + return value if isinstance(value, UUID): return str(value) if isinstance(value, Mapping): @@ -481,6 +508,9 @@ def _as_string_list(value: object) -> tuple[str, list[str]] | None: return ("", _strings(value)) +_CANONICAL_SOURCE_UUID = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") + + def _source_ids_from(value: object) -> tuple[str, set[str]]: """Named source ids, including every spelling the saved-quote reader names.""" @@ -489,10 +519,8 @@ def _source_ids_from(value: object) -> tuple[str, set[str]]: # An exact canonical UUID contains one source and no surrounding text. # Every other shape still runs the complete saved-quote parser, including # encoded/split references and text naming more than one source. - if type(value) is str and len(value) == 36: - token = _source_token(value) - if token is not None and token == value.lower(): - return "", {token} + if type(value) is str and _CANONICAL_SOURCE_UUID.fullmatch(value): + return "", {value.lower()} named = {identifier(item) for item in cited_source_ids(value).named} problem = "" if isinstance(value, list): diff --git a/apps/api/src/alicebot_api/vnext_doctor.py b/apps/api/src/alicebot_api/vnext_doctor.py index 5eda25f00..f66bce668 100644 --- a/apps/api/src/alicebot_api/vnext_doctor.py +++ b/apps/api/src/alicebot_api/vnext_doctor.py @@ -356,8 +356,21 @@ def _flagged_source_scan(store: object) -> tuple[list[str], bool]: rows = list(lister()) stopped_early = False ids: list[str] = [] + empty_metadata_verdicts: dict[tuple[object, ...], bool] = {} for row in rows: - if not source_row_is_flagged(row): + verdict = None + if isinstance(row, Mapping) and type(row.get("metadata_json")) is dict and not row["metadata_json"]: + fields = tuple(row.get(key) for key in ("title", "author", "uri", "raw_path", "external_id")) + if all(value is None or type(value) is str for value in fields): + if fields not in empty_metadata_verdicts: + empty_metadata_verdicts[fields] = source_row_is_flagged(row) + verdict = empty_metadata_verdicts[fields] + # The classifier reads these five fields and the complete metadata. + # Only exact empty metadata and equal immutable scalar inputs share a + # verdict in this scan. Every other row uses the full commit-door scan. + if verdict is None: + verdict = source_row_is_flagged(row) + if not verdict: continue source_id = row.get("id") if isinstance(row, Mapping) else None if source_id is not None: diff --git a/apps/api/src/alicebot_api/vnext_label_repair.py b/apps/api/src/alicebot_api/vnext_label_repair.py index 13858d4d3..a44439e09 100644 --- a/apps/api/src/alicebot_api/vnext_label_repair.py +++ b/apps/api/src/alicebot_api/vnext_label_repair.py @@ -7,11 +7,12 @@ from __future__ import annotations import json +import psycopg from collections.abc import Mapping, Sequence from typing import TypedDict from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError, require_changed -from alicebot_api.vnext_derived_labels import labels_raised_payload, settle_labels +from alicebot_api.vnext_derived_labels import _cache_native_json_metadata, _share_metadata_decode, labels_raised_payload, settle_labels from alicebot_api.vnext_event_log import build_event_log_record from alicebot_api.vnext_project_scope import project_scope_identity @@ -94,6 +95,7 @@ def plan_label_repairs( node["kind"] = kind node["_stored_metadata"] = row.get("metadata_json") node["metadata_json"] = _json_object(row.get("metadata_json")) + _share_metadata_decode(row.get("metadata_json"), node["metadata_json"]) if isinstance(row.get("value"), str): node["value"] = _json_object(row.get("value")) nodes.append(node) @@ -241,6 +243,7 @@ def classify_stored_labels( node["kind"] = kind node["_stored_metadata"] = row.get("metadata_json") node["metadata_json"] = _json_object(row.get("metadata_json")) + _share_metadata_decode(row.get("metadata_json"), node["metadata_json"]) if isinstance(row.get("value"), str): node["value"] = _json_object(row.get("value")) nodes.append(node) @@ -311,6 +314,8 @@ def load_postgres_label_tables(conn) -> dict[str, list[dict[str, object]]]: cursor = conn.execute(statement) names = [column[0] for column in cursor.description] tables[table] = [row if isinstance(row, dict) else dict(zip(names, row)) for row in cursor.fetchall()] + if isinstance(conn, psycopg.Connection): + _cache_native_json_metadata(tables[table]) return tables diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 3355cc4c4..9a6c57e53 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -692,6 +692,9 @@ def _fetch_all( with self.conn.cursor() as cur: cur.execute(query, params) rows = cur.fetchall() + if isinstance(self.conn, psycopg.Connection): + from alicebot_api.vnext_derived_labels import _cache_native_json_metadata + _cache_native_json_metadata(rows) return [expose_memory_project_scope(cast(VNextRow, row)) for row in rows] _append_mutation_event = _events_append_mutation_event diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index 59ce805d1..98d0f962c 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -413,6 +413,38 @@ "tests/unit/test_label_resolved_inputs.py::test_native_bulk_count_does_not_reuse_a_distinct_projection" ], "expected_failure": "test_native_bulk_count_does_not_reuse_a_distinct_projection" + }, + { + "name": "doctor-verdict-fields", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_doctor.py", + "before": " fields = tuple(row.get(key) for key in (\"title\", \"author\", \"uri\", \"raw_path\", \"external_id\"))", + "after": " fields = tuple(row.get(key) for key in (\"title\",))", + "line": 363, + "sha256": "7bcb4f4fe4c67d9ea58d73bc4affbe28c62c64a8de0191b3d4b0532ed24281d5" + } + ], + "pytest": [ + "tests/unit/test_vnext_doctor.py::test_shared_empty_metadata_scan_preserves_every_flagged_id_and_field" + ], + "expected_failure": "test_shared_empty_metadata_scan_preserves_every_flagged_id_and_field" + }, + { + "name": "doctor-verdict-metadata", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_doctor.py", + "before": " if isinstance(row, Mapping) and type(row.get(\"metadata_json\")) is dict and not row[\"metadata_json\"]:", + "after": " if isinstance(row, Mapping) and type(row.get(\"metadata_json\")) is dict:", + "line": 362, + "sha256": "7bcb4f4fe4c67d9ea58d73bc4affbe28c62c64a8de0191b3d4b0532ed24281d5" + } + ], + "pytest": [ + "tests/unit/test_vnext_doctor.py::test_shared_empty_metadata_scan_preserves_every_flagged_id_and_field" + ], + "expected_failure": "test_shared_empty_metadata_scan_preserves_every_flagged_id_and_field" } ] } diff --git a/tests/unit/test_label_resolved_inputs.py b/tests/unit/test_label_resolved_inputs.py index b15c63874..d35a8a986 100644 --- a/tests/unit/test_label_resolved_inputs.py +++ b/tests/unit/test_label_resolved_inputs.py @@ -250,3 +250,28 @@ def test_native_bulk_count_does_not_reuse_a_distinct_projection(): guard._settle_native_count_batch("memory", [root]) assert guard._key("memory", root) not in state.labels assert labels(guard.effective_row("memory", root))[1] == "public" + + +def test_native_json_decode_cache_preserves_recorded_reference_forms_and_refresh(): + from alicebot_api.vnext_derived_labels import _cache_native_json_metadata + source = {"kind": "source", "id": UUID(int=1), "domain": "health", "sensitivity": "confidential", "metadata_json": {}} + source_id = str(source["id"]) + encoded = source_id.replace("-", "%2d") + roots = [{"kind": "memory", "id": UUID(int=100 + i), "domain": "project", "sensitivity": "public", + "metadata_json": {"source_refs": [ref], "derived_from": {"v": 1, "sources": [], "counts": {}}}} + for i, ref in enumerate((source_id, "SOURCE:" + source_id.upper(), "quoted source:" + encoded))] + # A JSON string containing metadata is not a decoded JSON object and must + # retain the canonical parser; source-reference text must never disappear. + roots.append({**roots[0], "id": UUID(int=200), "metadata_json": json.dumps(roots[0]["metadata_json"])}) + rows = [source, *roots] + expected_rows = settle_labels([{**row, "user_id": "label-guard"} for row in rows], on_cycle="unverified").rows + store = Rows(rows) + with label_read_scope(store): + _cache_native_json_metadata(rows) + guard = LabelGuard(store, active=True) + for root, expected_label in zip(roots, expected_rows[1:], strict=True): + assert labels(guard.effective_row("memory", root)) == expected(expected_label) + source["sensitivity"] = "regulated" + invalidate_read_labels(store) + _cache_native_json_metadata(rows) + assert guard.effective_row("memory", roots[0])["sensitivity"] == "regulated" diff --git a/tests/unit/test_vnext_doctor.py b/tests/unit/test_vnext_doctor.py index ecfc77103..7c2bacc42 100644 --- a/tests/unit/test_vnext_doctor.py +++ b/tests/unit/test_vnext_doctor.py @@ -265,6 +265,36 @@ def list_sources(self, **kwargs) -> list[dict[str, object]]: assert "stopped after 10000 sources" in wide["message"] +def test_shared_empty_metadata_scan_preserves_every_flagged_id_and_field(monkeypatch) -> None: + import alicebot_api.vnext_doctor as doctor + token = "ghp_" + "0123456789abcdefghijklmnopqrstuvwxyz" + rows = [{"id": "clean-1", "title": "clean", "metadata_json": {}}, + {"id": "clean-2", "title": "clean", "metadata_json": {}}, + {"id": "title-1", "title": token, "metadata_json": {}}, + {"id": "title-2", "title": token, "metadata_json": {}}] + for field in ("author", "uri", "raw_path", "external_id"): + rows.append({"id": field, "title": "clean", field: token, "metadata_json": {}}) + rows.extend([{"id": "metadata", "title": "clean", "metadata_json": {"raw_text": token}}, + {"id": "nested", "title": "clean", "metadata_json": {"provenance": {"token": token}}}, + {"id": "json", "title": "clean", "metadata_json": '{"raw_text":"' + token + '"}'}]) + + class Store: + def list_sources(self, **kwargs): + return rows + + original = doctor.source_row_is_flagged + expected = [row["id"] for row in rows if original(row)] + calls = [] + def counted(row): + calls.append(row["id"]) + return original(row) + monkeypatch.setattr(doctor, "source_row_is_flagged", counted) + assert doctor._flagged_source_scan(Store()) == (expected, False) + assert "clean-2" not in calls and "title-2" not in calls + rows[1]["metadata_json"] = {"raw_text": token} + assert doctor._flagged_source_scan(Store()) == ([row["id"] for row in rows if original(row)], False) + + def test_doctor_passes_when_local_live_cors_is_explicit(tmp_path, monkeypatch) -> None: web_dir = tmp_path / "apps" / "web" web_dir.mkdir(parents=True) From c4532df6e11457cddd6ea12b0fc68d002534edd5 Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:49:48 +0200 Subject: [PATCH 263/270] Reuse verified native count labels for equivalent display projections --- .../api/src/alicebot_api/vnext_label_guard.py | 23 ++++++ scripts/derived_label_mutations.json | 76 +++++++++++++------ tests/unit/test_label_resolved_inputs.py | 32 ++++++++ 3 files changed, 109 insertions(+), 22 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index e0ce6faf1..79d69bd0a 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -71,6 +71,7 @@ class _RequestLabels: row_keys: dict = field(default_factory=dict) resolved_inputs: dict = field(default_factory=dict) parent_labels: dict = field(default_factory=dict) + native_labels: dict = field(default_factory=dict) def clear(self): self.nodes.clear() @@ -90,6 +91,7 @@ def clear(self): self.row_keys.clear() self.resolved_inputs.clear() self.parent_labels.clear() + self.native_labels.clear() _REQUEST_LABELS: ContextVar[tuple[Any, _RequestLabels] | None] = ContextVar("request_labels", default=None) @@ -450,6 +452,10 @@ def _prefetch_inputs(self, kind: str, rows: Sequence[Mapping[str, object]]) -> N key = self._key(row_kind, row) if key in prefetched or key in state.labels or key in state.parent_labels: continue + native = self._native_label_for_projection(row_kind, row, key=key) + if native is not None: + state.labels[key] = native + continue refs.update(dependencies_of(row_kind, row)) pending.append(key) refs.difference_update(expanded) @@ -553,6 +559,23 @@ def _settle_native_count_batch(self, kind: str, rows: Sequence[Mapping[str, obje label = settled.get((canon_kind(kind), _GUARD_USER, identifier(row["id"]))) if label is not None and not label.unverified: state.labels[self._key(kind, row)] = label + state.native_labels[(canon_kind(kind), identifier(row["id"]))] = (row, label) + + def _native_label_for_projection(self, kind: str, row: Mapping[str, object], *, key: tuple) -> SettledLabel | None: + """Reuse one verified UUID origin only for equivalent label semantics.""" + current = _REQUEST_LABELS.get() + if current is None or current[0] is not self.store or type(row.get("id")) is not UUID: + return None + entry = current[1].native_labels.get((canon_kind(kind), identifier(row["id"]))) + if entry is None: + return None + raw, label = entry + if self._signature(kind, row, key=key) != self._signature(kind, raw, key=self._key(kind, raw)): + return None + # Same origin, same direct inputs and stored semantics in this locked + # snapshot: its alias/cycle/bound proof remains the same. Caller filters + # are checked separately; writes and rollback clear this map together. + return label def admit_related_rows(self, rows: Sequence[_Row], *, kind: str, field: str) -> list[_Row]: """Admit an event or rating by its target's current effective label.""" diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index 98d0f962c..4707c52a8 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -141,10 +141,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 352, + "line": 354, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], ())", - "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" } ], "pytest": [ @@ -157,10 +157,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 352, + "line": 354, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], tuple(sorted(set(semantic_parents))))", - "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" } ], "pytest": [ @@ -173,10 +173,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 339, + "line": 341, "before": "if len(raw) != 1 or identifier(raw[0].get(\"id\")) != parent_id:", "after": "if not raw or identifier(raw[0].get(\"id\")) != parent_id:", - "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" } ], "pytest": [ @@ -191,10 +191,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 347, + "line": 349, "before": "if depth > HOP_BOUND or len(ancestry) > NODE_BOUND:", "after": "if False:", - "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" } ], "pytest": [ @@ -209,10 +209,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 323, + "line": 325, "before": "if problem or has_implicit_weekly_inputs(kind, row):", "after": "if problem:", - "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" } ], "pytest": [ @@ -227,10 +227,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 496, + "line": 502, "before": "if not self.active and callable(native):", "after": "if False:", - "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" } ], "pytest": [ @@ -275,10 +275,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 376, + "line": 378, "before": " self._prefetch_inputs(kind, rows)\n kept: list[_Row] = []", "after": " pass # removed batched ancestry reads\n kept: list[_Row] = []", - "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" } ], "pytest": [ @@ -357,8 +357,8 @@ "file": "apps/api/src/alicebot_api/vnext_label_guard.py", "before": "row_admission_key = (key, self.domains, self.sensitivity_allowed, self.projects, self.all_of)", "after": "row_admission_key = (key,)", - "line": 386, - "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + "line": 388, + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" } ], "pytest": [ @@ -373,8 +373,8 @@ "file": "apps/api/src/alicebot_api/vnext_label_guard.py", "before": " self.row_admission.clear()", "after": " pass # removed admission invalidation", - "line": 88, - "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + "line": 89, + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" } ], "pytest": [ @@ -387,10 +387,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 392, + "line": 394, "before": " if settled is not None and not settled.unverified:", "after": " if settled is not None:", - "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" } ], "pytest": [ @@ -403,10 +403,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 540, + "line": 546, "before": " if len(stored) != 1 or (stored[0] is not row and _row_label_key(kind, stored[0]) != _row_label_key(kind, row)):", "after": " if len(stored) != 1:", - "sha256": "3376123a4ef4fc86a4f7544bff7ae5a152506d8aa67b3dd70d7befecbb863fc7" + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" } ], "pytest": [ @@ -445,6 +445,38 @@ "tests/unit/test_vnext_doctor.py::test_shared_empty_metadata_scan_preserves_every_flagged_id_and_field" ], "expected_failure": "test_shared_empty_metadata_scan_preserves_every_flagged_id_and_field" + }, + { + "name": "native-projection-write-invalidation", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": " self.native_labels.clear()", + "after": " pass # removed native projection invalidation", + "line": 94, + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_native_projection_reuse_keeps_filters_and_refreshes_after_writes" + ], + "expected_failure": "test_native_projection_reuse_keeps_filters_and_refreshes_after_writes" + }, + { + "name": "native-projection-signature", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": " if self._signature(kind, row, key=key) != self._signature(kind, raw, key=self._key(kind, raw)):\n return None", + "after": " pass # removed native projection signature equality", + "line": 573, + "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_native_projection_reuse_keeps_filters_and_refreshes_after_writes" + ], + "expected_failure": "test_native_projection_reuse_keeps_filters_and_refreshes_after_writes" } ] } diff --git a/tests/unit/test_label_resolved_inputs.py b/tests/unit/test_label_resolved_inputs.py index d35a8a986..1780ba95f 100644 --- a/tests/unit/test_label_resolved_inputs.py +++ b/tests/unit/test_label_resolved_inputs.py @@ -275,3 +275,35 @@ def test_native_json_decode_cache_preserves_recorded_reference_forms_and_refresh invalidate_read_labels(store) _cache_native_json_metadata(rows) assert guard.effective_row("memory", roots[0])["sensitivity"] == "regulated" + + +def test_native_projection_reuse_keeps_filters_and_refreshes_after_writes(): + source = {"kind": "source", "id": UUID(int=1), "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": ["P1"]}} + hidden = {**deepcopy(source), "id": UUID(int=3), "sensitivity": "confidential"} + root = {"kind": "memory", "id": UUID(int=2), "domain": "project", "sensitivity": "public", "status": "active", + "metadata_json": {"source_id": str(source["id"]), "project_scope": ["P1"], "project_floor": ["P1"]}} + + class NativeRows(Rows): + label_count_canonical_unique_ids = True + + def count_original_label_statuses(self, *args, **kwargs): + return {} + + def iter_label_rows(self, kind, **kwargs): + yield [root] + + store = NativeRows([source, hidden, root]) + with label_read_scope(store): + guard = LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard.readable_status_counts("memory") == {"active": 1} + reloaded = {**deepcopy(root), "canonical_text": "Reloaded display row", "memory_key": "display"} + assert guard.admit_rows("memory", [reloaded]) == [reloaded] + assert replace(guard, domains=("health",)).admit_rows("memory", [reloaded]) == [] + assert replace(guard, projects=("P2",), all_of=("P2",)).admit_rows("memory", [reloaded]) == [] + assert replace(guard, projects=("P1",), all_of=("P1",)).admit_rows("memory", [reloaded]) == [reloaded] + changed = {**deepcopy(root), "metadata_json": {**root["metadata_json"], "source_id": str(hidden["id"])}} + assert guard.admit_rows("memory", [changed]) == [] + source["sensitivity"] = "confidential" + invalidate_read_labels(store) + assert guard.admit_rows("memory", [deepcopy(reloaded)]) == [] From 0929651fd5a8835b977594ed47d36af0421afe6e Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:10:58 +0200 Subject: [PATCH 264/270] Register the maximum-workload read check for release validation --- scripts/check_github_release_checks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/check_github_release_checks.py b/scripts/check_github_release_checks.py index c2f831d07..370463793 100644 --- a/scripts/check_github_release_checks.py +++ b/scripts/check_github_release_checks.py @@ -24,6 +24,7 @@ "Python 3.13 install smoke", "Python 3.14 install smoke", "Integration tests (Postgres + pgvector, role separation)", + "Read-budget maximum workload smoke (Postgres)", "Default surface integration smoke (Postgres)", "Web tests, types, accessibility, and budgets", "Semantic eval attestation (exact SHA)", From 1cc6ca0e547ffb32923f516c55a74656452d9bc8 Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:53:15 +0200 Subject: [PATCH 265/270] Reuse pure SQLite parent hints within the label snapshot --- apps/api/src/alicebot_api/sqlite_store.py | 43 ++++++++++++++----- .../src/alicebot_api/vnext_derived_labels.py | 6 +++ scripts/derived_label_mutations.json | 32 ++++++++++++++ tests/unit/test_label_count_partition.py | 20 ++++++++- tests/unit/test_label_resolved_inputs.py | 18 ++++++++ 5 files changed, 107 insertions(+), 12 deletions(-) diff --git a/apps/api/src/alicebot_api/sqlite_store.py b/apps/api/src/alicebot_api/sqlite_store.py index 9f3147741..405614c85 100644 --- a/apps/api/src/alicebot_api/sqlite_store.py +++ b/apps/api/src/alicebot_api/sqlite_store.py @@ -46,6 +46,7 @@ capture_dedupe_key_for_source, source_capture_raw_text, ) +from alicebot_api.vnext_derived_labels import _read_metadata_cache, identifier from alicebot_api.vnext_embeddings import ( EMBEDDING_SIGNATURE_METADATA_KEY, EMBEDDING_VECTOR_DIMENSIONS, @@ -390,7 +391,7 @@ def sqlite_user_connection(path: str | Path, user_id: UUID | str, *, repair_labe conn.close() -def _direct_source_hint(raw: object) -> str | None: +def _direct_source_hint_uncached(raw: object) -> str | None: """A direct parent for a conservative prefilter, never an admission grant. Decode like the store and kernel, including JSON strings and duplicate or @@ -414,10 +415,28 @@ def _direct_source_hint(raw: object) -> str | None: # A named source in a canonical record is an input even when another # input makes the row unverified. This is rejection only, never a grant. source_id = sources[0] - try: - return str(UUID(source_id)) - except (ValueError, TypeError): - return source_id + return identifier(source_id) + + +def _cached_direct_parent_hints(raw: str, cache: dict) -> tuple[str | None, str | None]: + """Share pure JSON/ID decoding, never a parent's current label or grant.""" + key = ("sqlite-parent-hints", raw) + if key not in cache: + try: + metadata = json.loads(raw) + if isinstance(metadata, str): + metadata = json.loads(metadata) + except (ValueError, TypeError): + metadata = None + cache[key] = (_direct_source_hint_uncached(metadata), _direct_memory_hint_uncached(metadata)) if isinstance(metadata, Mapping) else (None, None) + return cast(tuple[str | None, str | None], cache[key]) + + +def _direct_source_hint(raw: object) -> str | None: + cache = _read_metadata_cache() + if cache is not None and type(raw) is str: + return _cached_direct_parent_hints(raw, cache)[0] + return _direct_source_hint_uncached(raw) def _ensure_direct_source_hint(conn: sqlite3.Connection) -> None: @@ -438,7 +457,7 @@ def _ensure_direct_source_hint(conn: sqlite3.Connection) -> None: conn.create_function("alice_direct_memory_hint", 1, _direct_memory_hint, deterministic=True) -def _direct_memory_hint(raw: object) -> str | None: +def _direct_memory_hint_uncached(raw: object) -> str | None: try: metadata = json.loads(raw) if isinstance(raw, str) else raw if isinstance(metadata, str): @@ -454,10 +473,14 @@ def _direct_memory_hint(raw: object) -> str | None: members = record.get("memories") if isinstance(record, Mapping) else None if not isinstance(members, list) or not members or not isinstance(members[0], str): return None - try: - return str(UUID(members[0])) - except (ValueError, TypeError): - return members[0] + return identifier(members[0]) + + +def _direct_memory_hint(raw: object) -> str | None: + cache = _read_metadata_cache() + if cache is not None and type(raw) is str: + return _cached_direct_parent_hints(raw, cache)[1] + return _direct_memory_hint_uncached(raw) class SQLiteVNextStore: diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index c5ad15564..1c3346352 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -204,6 +204,8 @@ def identifier(value: object) -> str: return cache[key] if isinstance(value, UUID): result = str(value) + elif type(value) is str and _CANONICAL_SOURCE_UUID.fullmatch(value): + result = value.lower() else: try: result = str(UUID(str(value))) @@ -226,6 +228,10 @@ def _object(value: object) -> Mapping[str, object]: _READ_METADATA: ContextVar[dict | None] = ContextVar("label_read_metadata", default=None) +def _read_metadata_cache() -> dict | None: + return _READ_METADATA.get() + + @contextmanager def label_metadata_cache(cache: dict): """Reuse pure decoding only inside a guarded store snapshot. diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index 4707c52a8..fd299ef63 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -477,6 +477,38 @@ "tests/unit/test_label_resolved_inputs.py::test_native_projection_reuse_keeps_filters_and_refreshes_after_writes" ], "expected_failure": "test_native_projection_reuse_keeps_filters_and_refreshes_after_writes" + }, + { + "name": "sqlite-hint-full-input", + "edits": [ + { + "file": "apps/api/src/alicebot_api/sqlite_store.py", + "before": " key = (\"sqlite-parent-hints\", raw)", + "after": " key = (\"sqlite-parent-hints\", \"\")", + "sha256": "461f288fd66bb5c80a3408fc450f7169d9206a6609a8d6891d742819a01f8329", + "line": 423 + } + ], + "pytest": [ + "tests/unit/test_label_count_partition.py::test_cached_parent_hints_keep_the_complete_json_and_original_decoding_depth" + ], + "expected_failure": "test_cached_parent_hints_keep_the_complete_json_and_original_decoding_depth" + }, + { + "name": "canonical-identifier-grammar", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_derived_labels.py", + "before": " elif type(value) is str and _CANONICAL_SOURCE_UUID.fullmatch(value):", + "after": " elif type(value) is str:", + "sha256": "398581751be87665c48c8ed33ba9fe5b1254125022b69f00d2879751443a17f6", + "line": 207 + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_text_identifiers_keep_case_while_uuid_aliases_share_an_origin" + ], + "expected_failure": "test_text_identifiers_keep_case_while_uuid_aliases_share_an_origin" } ] } diff --git a/tests/unit/test_label_count_partition.py b/tests/unit/test_label_count_partition.py index 8ca321a50..6f0fc56a1 100644 --- a/tests/unit/test_label_count_partition.py +++ b/tests/unit/test_label_count_partition.py @@ -4,8 +4,8 @@ import pytest -from alicebot_api.sqlite_store import _direct_source_hint -from alicebot_api.vnext_derived_labels import MARKER_KEYS, DERIVED_ARTIFACT_TYPES, is_derived +from alicebot_api.sqlite_store import _direct_source_hint, _direct_memory_hint +from alicebot_api.vnext_derived_labels import MARKER_KEYS, DERIVED_ARTIFACT_TYPES, is_derived, label_metadata_cache from alicebot_api.vnext_label_sql import original_label_sql @@ -41,3 +41,19 @@ def test_sqlite_original_partition_is_conservative_for_legacy_and_marker_shapes( ]) def test_sqlite_parent_hint_matches_store_and_kernel_decoding(metadata, expected): assert _direct_source_hint(metadata) == expected + + +def test_cached_parent_hints_keep_the_complete_json_and_original_decoding_depth(): + rows = [ + ('{"source_id":"Public-A","consolidation":{"cluster_member_ids":["Memory-A"]}}', ("Public-A", "Memory-A")), + ('{"source_id":"Hidden-B","consolidation":{"cluster_member_ids":["Memory-B"]}}', ("Hidden-B", "Memory-B")), + ('{"source_id":"Hidden-B","redacted":true}', (None, None)), + ('{"source_id":"old","source_\\u0069d":"new","derived_from":{"memories":["Parent"]}}', ("new", "Parent")), + (json.dumps(json.dumps({"source_id": "Source-C", "derived_from": {"memories": ["Memory-C"]}})), ("Source-C", "Memory-C")), + (json.dumps(json.dumps(json.dumps({"source_id": "Source-C"}))), (None, None)), + ('not JSON', (None, None)), + ] + for _ in range(2): + with label_metadata_cache({}): + for raw, expected in rows * 2: + assert (_direct_source_hint(raw), _direct_memory_hint(raw)) == expected diff --git a/tests/unit/test_label_resolved_inputs.py b/tests/unit/test_label_resolved_inputs.py index 1780ba95f..b380e2895 100644 --- a/tests/unit/test_label_resolved_inputs.py +++ b/tests/unit/test_label_resolved_inputs.py @@ -307,3 +307,21 @@ def iter_label_rows(self, kind, **kwargs): source["sensitivity"] = "confidential" invalidate_read_labels(store) assert guard.admit_rows("memory", [deepcopy(reloaded)]) == [] + + +def test_text_identifiers_keep_case_while_uuid_aliases_share_an_origin(): + canonical = str(UUID(int=0xABCDEF)) + sources = [ + {"kind": "source", "id": "Source-A", "domain": "project", "sensitivity": "public", "metadata_json": {}}, + {"kind": "source", "id": "source-a", "domain": "project", "sensitivity": "confidential", "metadata_json": {}}, + {"kind": "source", "id": canonical, "domain": "project", "sensitivity": "public", "metadata_json": {}}, + {"kind": "source", "id": "urn:uuid:" + canonical, "domain": "project", "sensitivity": "confidential", "metadata_json": {}}, + ] + roots = [{"kind": "memory", "id": "root-" + str(i), "domain": "project", "sensitivity": "public", + "metadata_json": {"source_id": value}} for i, value in enumerate(("Source-A", "source-a", canonical.upper()))] + store = Rows(sources) + with label_read_scope(store): + guard = LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard.admit_rows("memory", roots) == [roots[0]] + assert guard.effective_row("memory", roots[1])["sensitivity"] == "confidential" + assert guard.effective_row("memory", roots[2])["unverified"] is True From c003c760cf24cf35fa37e0d4bac63a1bdac869ba Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:13:52 +0200 Subject: [PATCH 266/270] Reuse verified native count inputs for memory event admission --- .../api/src/alicebot_api/vnext_label_guard.py | 15 +++++++ scripts/derived_label_mutations.json | 42 ++++++++++++------ tests/unit/test_label_resolved_inputs.py | 43 +++++++++++++++++++ 3 files changed, 87 insertions(+), 13 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index 79d69bd0a..c9a1d42f3 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -588,6 +588,21 @@ def admit_related_rows(self, rows: Sequence[_Row], *, kind: str, field: str) -> ids = list(dict.fromkeys(str(row.get(field)) for row in rows if row.get(field))) state = self._state() missing = [row_id for row_id in ids if (kind, row_id) not in state.targets] + current = _REQUEST_LABELS.get() + if kind == "memory" and current is not None and current[0] is self.store: + for row_id in missing: + native = state.native_labels.get((kind, identifier(row_id))) + if native is None: + continue + raw, _label = native + if type(raw.get("id")) is UUID and str(raw.get("id")) == row_id: + # A verified native count row has the same complete label + # projection as read_label_rows. Reuse the raw input only; + # this caller still applies its own admission below. Exact + # stored spelling preserves target lookup behavior, and + # writes/rollback clear both maps in this locked request. + state.targets[(kind, row_id)] = raw + missing = [row_id for row_id in missing if (kind, row_id) not in state.targets] if kind == "source": for row_id in missing: cached = state.nodes.get((kind, identifier(row_id)), ()) diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index fd299ef63..55685cd09 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -144,7 +144,7 @@ "line": 354, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], ())", - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -160,7 +160,7 @@ "line": 354, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], tuple(sorted(set(semantic_parents))))", - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -176,7 +176,7 @@ "line": 341, "before": "if len(raw) != 1 or identifier(raw[0].get(\"id\")) != parent_id:", "after": "if not raw or identifier(raw[0].get(\"id\")) != parent_id:", - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -194,7 +194,7 @@ "line": 349, "before": "if depth > HOP_BOUND or len(ancestry) > NODE_BOUND:", "after": "if False:", - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -212,7 +212,7 @@ "line": 325, "before": "if problem or has_implicit_weekly_inputs(kind, row):", "after": "if problem:", - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -230,7 +230,7 @@ "line": 502, "before": "if not self.active and callable(native):", "after": "if False:", - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -278,7 +278,7 @@ "line": 378, "before": " self._prefetch_inputs(kind, rows)\n kept: list[_Row] = []", "after": " pass # removed batched ancestry reads\n kept: list[_Row] = []", - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -358,7 +358,7 @@ "before": "row_admission_key = (key, self.domains, self.sensitivity_allowed, self.projects, self.all_of)", "after": "row_admission_key = (key,)", "line": 388, - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -374,7 +374,7 @@ "before": " self.row_admission.clear()", "after": " pass # removed admission invalidation", "line": 89, - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -390,7 +390,7 @@ "line": 394, "before": " if settled is not None and not settled.unverified:", "after": " if settled is not None:", - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -406,7 +406,7 @@ "line": 546, "before": " if len(stored) != 1 or (stored[0] is not row and _row_label_key(kind, stored[0]) != _row_label_key(kind, row)):", "after": " if len(stored) != 1:", - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -454,7 +454,7 @@ "before": " self.native_labels.clear()", "after": " pass # removed native projection invalidation", "line": 94, - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -470,7 +470,7 @@ "before": " if self._signature(kind, row, key=key) != self._signature(kind, raw, key=self._key(kind, raw)):\n return None", "after": " pass # removed native projection signature equality", "line": 573, - "sha256": "41c479f7a30158065c2ad25f878d40337d58686246dd3a346f05292a94b5e6cf" + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" } ], "pytest": [ @@ -509,6 +509,22 @@ "tests/unit/test_label_resolved_inputs.py::test_text_identifiers_keep_case_while_uuid_aliases_share_an_origin" ], "expected_failure": "test_text_identifiers_keep_case_while_uuid_aliases_share_an_origin" + }, + { + "name": "native-target-stored-spelling", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": "if type(raw.get(\"id\")) is UUID and str(raw.get(\"id\")) == row_id:", + "after": "if type(raw.get(\"id\")) is UUID:", + "line": 598, + "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + } + ], + "pytest": [ + "tests/unit/test_label_resolved_inputs.py::test_native_count_targets_keep_spelling_filters_and_refresh_after_writes" + ], + "expected_failure": "test_native_count_targets_keep_spelling_filters_and_refresh_after_writes" } ] } diff --git a/tests/unit/test_label_resolved_inputs.py b/tests/unit/test_label_resolved_inputs.py index b380e2895..9621ad935 100644 --- a/tests/unit/test_label_resolved_inputs.py +++ b/tests/unit/test_label_resolved_inputs.py @@ -309,6 +309,49 @@ def iter_label_rows(self, kind, **kwargs): assert guard.admit_rows("memory", [deepcopy(reloaded)]) == [] +def test_native_count_targets_keep_spelling_filters_and_refresh_after_writes(): + source = {"kind": "source", "id": UUID(int=1), "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": ["P1"]}} + root = {"kind": "memory", "id": UUID(int=0xABCDEF), "domain": "project", "sensitivity": "public", "status": "active", + "metadata_json": {"source_id": str(source["id"]), "project_scope": ["P1"], "project_floor": ["P1"]}} + + class NativeRows(Rows): + label_count_canonical_unique_ids = True + + def __init__(self, rows): + super().__init__(rows) + self.reads = [] + + def read_label_rows(self, kind, ids): + self.reads.append((kind, tuple(ids))) + return super().read_label_rows(kind, ids) + + def count_original_label_statuses(self, *args, **kwargs): + return {} + + def iter_label_rows(self, kind, **kwargs): + yield [root] + + store = NativeRows([source, root]) + event = {"target_id": str(root["id"])} + with label_read_scope(store): + guard = LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard.readable_status_counts("memory") == {"active": 1} + before = len(store.reads) + assert guard.admit_related_rows([event], kind="memory", field="target_id") == [event] + assert len(store.reads) == before + assert replace(guard, domains=("health",)).admit_related_rows([event], kind="memory", field="target_id") == [] + assert replace(guard, projects=("P2",), all_of=("P2",)).admit_related_rows([event], kind="memory", field="target_id") == [] + assert replace(guard, projects=("P1",), all_of=("P1",)).admit_related_rows([event], kind="memory", field="target_id") == [event] + alias = {"target_id": str(root["id"]).upper()} + assert guard.admit_related_rows([alias], kind="memory", field="target_id") == [] + assert len(store.reads) == before + 1 + source["sensitivity"] = "confidential" + invalidate_read_labels(store) + assert guard.admit_related_rows([event], kind="memory", field="target_id") == [] + assert len(store.reads) > before + 1 + + def test_text_identifiers_keep_case_while_uuid_aliases_share_an_origin(): canonical = str(UUID(int=0xABCDEF)) sources = [ From 5770c26e52956036b9ea0e94b03ab748dbdc239e Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:30:34 +0200 Subject: [PATCH 267/270] Count unscoped original source events without repeated target reads --- .../api/src/alicebot_api/vnext_label_guard.py | 35 +++--- apps/api/src/alicebot_api/vnext_store.py | 28 ++++- scripts/derived_label_mutations.json | 104 ++++++++++++++---- .../test_label_round3_contracts_postgres.py | 55 +++++++++ tests/unit/test_label_resolved_inputs.py | 41 +++++++ 5 files changed, 223 insertions(+), 40 deletions(-) diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index c9a1d42f3..0e1f5e9a1 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -588,21 +588,6 @@ def admit_related_rows(self, rows: Sequence[_Row], *, kind: str, field: str) -> ids = list(dict.fromkeys(str(row.get(field)) for row in rows if row.get(field))) state = self._state() missing = [row_id for row_id in ids if (kind, row_id) not in state.targets] - current = _REQUEST_LABELS.get() - if kind == "memory" and current is not None and current[0] is self.store: - for row_id in missing: - native = state.native_labels.get((kind, identifier(row_id))) - if native is None: - continue - raw, _label = native - if type(raw.get("id")) is UUID and str(raw.get("id")) == row_id: - # A verified native count row has the same complete label - # projection as read_label_rows. Reuse the raw input only; - # this caller still applies its own admission below. Exact - # stored spelling preserves target lookup behavior, and - # writes/rollback clear both maps in this locked request. - state.targets[(kind, row_id)] = raw - missing = [row_id for row_id in missing if (kind, row_id) not in state.targets] if kind == "source": for row_id in missing: cached = state.nodes.get((kind, identifier(row_id)), ()) @@ -637,8 +622,24 @@ def readable_event_count(self) -> int: iterator = getattr(self.store, "iter_label_events", None) if not callable(iterator): raise TypeError("readable counts require complete event enumeration") - prefilter = {"reject_sensitivity_allowed": self.sensitivity_allowed} if getattr(type(self.store), "label_count_input_prefilter", False) else {} - return sum(len(self.admit_events(batch)) for batch in iterator(**prefilter)) + prefilter: dict[str, Any] = {"reject_sensitivity_allowed": self.sensitivity_allowed} if self.active and getattr(type(self.store), "label_count_input_prefilter", False) else {} + canonical = getattr(type(self.store), "label_count_canonical_unique_ids", False) + counter = getattr(self.store, "count_source_label_events", None) + source_count = 0 + if canonical and callable(counter) and self.active and not self.projects and self.all_of is None: + # Sources are original inputs, so this unscoped count has no + # ancestry to settle. The native counter preserves exact target + # spelling, current tenant, domains and sensitivity. Scoped and + # inactive callers keep the complete per-target path below. + source_count = counter(domains=self.domains, sensitivity_allowed=self.sensitivity_allowed) + prefilter["exclude_source_targets"] = True + if canonical: + # Native UUID stores already read up to 5,000 event targets in + # each SQL page. Admit that complete page together instead of + # splitting its missing source reads into five database trips. + # Every target still goes through its current effective guard. + prefilter["batch_size"] = 5000 + return source_count + sum(len(self.admit_events(batch)) for batch in iterator(**prefilter)) def admit_beliefs(self, beliefs: Sequence[_Row]) -> list[_Row]: """Beliefs whose backing memory the filters admit. One batched read.""" diff --git a/apps/api/src/alicebot_api/vnext_store.py b/apps/api/src/alicebot_api/vnext_store.py index 9a6c57e53..1428d54f3 100644 --- a/apps/api/src/alicebot_api/vnext_store.py +++ b/apps/api/src/alicebot_api/vnext_store.py @@ -596,7 +596,27 @@ def iter_label_rows(self, kind: str, *, batch_size: int = 1000, derived_only: bo if len(rows) < query_size: return - def iter_label_events(self, *, batch_size: int = 1000, reject_sensitivity_allowed: Sequence[str] = ()) -> Iterator[list[VNextRow]]: + def count_source_label_events(self, *, domains=(), sensitivity_allowed=()) -> int: + """Original-source events for an active, unscoped label guard. + + Match the reader's canonical stored target text without casting legacy + event IDs. Deleted source rows remain readable by the label reader. + RLS and the explicit tenant join preserve the same input population. + """ + row = self._fetch_one( + "count_source_label_events", + """SELECT COUNT(*) AS count FROM event_log AS e + JOIN sources AS s ON s.user_id = e.user_id AND e.target_id = s.id::text + WHERE e.target_type = 'source' + AND (%s::text[] IS NULL OR COALESCE(NULLIF(s.domain, ''), 'unknown') = ANY(%s::text[]) + OR COALESCE(NULLIF(s.domain, ''), 'unknown') = 'unknown') + AND (%s::text[] IS NULL OR COALESCE(NULLIF(s.sensitivity, ''), 'unknown') = ANY(%s::text[]))""", + (list(domains) or None, list(domains) or None, + list(sensitivity_allowed) or None, list(sensitivity_allowed) or None), + ) + return int(cast(int, row["count"])) + + def iter_label_events(self, *, batch_size: int = 1000, reject_sensitivity_allowed: Sequence[str] = (), exclude_source_targets: bool = False) -> Iterator[list[VNextRow]]: """Complete event targets for readable counts, without event payloads.""" if batch_size < 1: @@ -608,8 +628,10 @@ def iter_label_events(self, *, batch_size: int = 1000, reject_sensitivity_allowe while True: rows = self._fetch_all( f"""SELECT id, target_type, target_id, event_type FROM event_log - WHERE (%s::uuid IS NULL OR id > %s::uuid) AND {label_sql} ORDER BY id LIMIT %s""", - (after, after, query_size), + WHERE (%s::uuid IS NULL OR id > %s::uuid) AND {label_sql} + AND (NOT %s::boolean OR COALESCE(target_type, '') <> 'source') + ORDER BY id LIMIT %s""", + (after, after, exclude_source_targets, query_size), ) if not rows: return diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index 55685cd09..92513e891 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -144,7 +144,7 @@ "line": 354, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], ())", - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -160,7 +160,7 @@ "line": 354, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], tuple(sorted(set(semantic_parents))))", - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -176,7 +176,7 @@ "line": 341, "before": "if len(raw) != 1 or identifier(raw[0].get(\"id\")) != parent_id:", "after": "if not raw or identifier(raw[0].get(\"id\")) != parent_id:", - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -194,7 +194,7 @@ "line": 349, "before": "if depth > HOP_BOUND or len(ancestry) > NODE_BOUND:", "after": "if False:", - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -212,7 +212,7 @@ "line": 325, "before": "if problem or has_implicit_weekly_inputs(kind, row):", "after": "if problem:", - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -230,7 +230,7 @@ "line": 502, "before": "if not self.active and callable(native):", "after": "if False:", - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -278,7 +278,7 @@ "line": 378, "before": " self._prefetch_inputs(kind, rows)\n kept: list[_Row] = []", "after": " pass # removed batched ancestry reads\n kept: list[_Row] = []", - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -358,7 +358,7 @@ "before": "row_admission_key = (key, self.domains, self.sensitivity_allowed, self.projects, self.all_of)", "after": "row_admission_key = (key,)", "line": 388, - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -374,7 +374,7 @@ "before": " self.row_admission.clear()", "after": " pass # removed admission invalidation", "line": 89, - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -390,7 +390,7 @@ "line": 394, "before": " if settled is not None and not settled.unverified:", "after": " if settled is not None:", - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -406,7 +406,7 @@ "line": 546, "before": " if len(stored) != 1 or (stored[0] is not row and _row_label_key(kind, stored[0]) != _row_label_key(kind, row)):", "after": " if len(stored) != 1:", - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -454,7 +454,7 @@ "before": " self.native_labels.clear()", "after": " pass # removed native projection invalidation", "line": 94, - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -470,7 +470,7 @@ "before": " if self._signature(kind, row, key=key) != self._signature(kind, raw, key=self._key(kind, raw)):\n return None", "after": " pass # removed native projection signature equality", "line": 573, - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ @@ -511,20 +511,84 @@ "expected_failure": "test_text_identifiers_keep_case_while_uuid_aliases_share_an_origin" }, { - "name": "native-target-stored-spelling", + "name": "native-event-source-batching", "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "before": "if type(raw.get(\"id\")) is UUID and str(raw.get(\"id\")) == row_id:", - "after": "if type(raw.get(\"id\")) is UUID:", - "line": 598, - "sha256": "1cce5c12319a8c7e18b52d0a19a62e35898b4ae67fe2d53750e274c1f84a0326" + "before": "prefilter[\"batch_size\"] = 5000", + "after": "prefilter[\"batch_size\"] = 1000", + "line": 641, + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" } ], "pytest": [ - "tests/unit/test_label_resolved_inputs.py::test_native_count_targets_keep_spelling_filters_and_refresh_after_writes" + "tests/unit/test_label_resolved_inputs.py::test_native_event_counts_batch_sources_without_changing_admission" ], - "expected_failure": "test_native_count_targets_keep_spelling_filters_and_refresh_after_writes" + "expected_failure": "test_native_event_counts_batch_sources_without_changing_admission" + }, + { + "name": "source-event-domain", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_store.py", + "before": "OR COALESCE(NULLIF(s.domain, ''), 'unknown') = ANY(%s::text[])", + "after": "OR %s::text[] IS NOT NULL", + "line": 611, + "sha256": "301c170d486a8dfc7b6a3bacc43c293501e8361af29dab92a8b74cbb91dda1ef" + } + ], + "pytest": [ + "tests/integration/test_label_round3_contracts_postgres.py::test_native_source_event_counts_match_the_complete_per_target_guard" + ], + "expected_failure": "test_native_source_event_counts_match_the_complete_per_target_guard" + }, + { + "name": "source-event-sensitivity", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_store.py", + "before": "COALESCE(NULLIF(s.sensitivity, ''), 'unknown') = ANY(%s::text[])", + "after": "%s::text[] IS NOT NULL", + "line": 613, + "sha256": "301c170d486a8dfc7b6a3bacc43c293501e8361af29dab92a8b74cbb91dda1ef" + } + ], + "pytest": [ + "tests/integration/test_label_round3_contracts_postgres.py::test_native_source_event_counts_match_the_complete_per_target_guard" + ], + "expected_failure": "test_native_source_event_counts_match_the_complete_per_target_guard" + }, + { + "name": "source-event-scope-fallback", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": "if canonical and callable(counter) and self.active and not self.projects and self.all_of is None:", + "after": "if canonical and callable(counter) and self.active:", + "line": 629, + "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + } + ], + "pytest": [ + "tests/integration/test_label_round3_contracts_postgres.py::test_native_source_event_counts_match_the_complete_per_target_guard" + ], + "expected_failure": "test_native_source_event_counts_match_the_complete_per_target_guard" + }, + { + "name": "source-event-exact-target", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_store.py", + "before": "e.target_id = s.id::text", + "after": "lower(e.target_id) = s.id::text", + "line": 609, + "sha256": "301c170d486a8dfc7b6a3bacc43c293501e8361af29dab92a8b74cbb91dda1ef" + } + ], + "pytest": [ + "tests/integration/test_label_round3_contracts_postgres.py::test_native_source_event_counts_match_the_complete_per_target_guard" + ], + "expected_failure": "test_native_source_event_counts_match_the_complete_per_target_guard" } ] } diff --git a/tests/integration/test_label_round3_contracts_postgres.py b/tests/integration/test_label_round3_contracts_postgres.py index 48a1a0907..59b016e6d 100644 --- a/tests/integration/test_label_round3_contracts_postgres.py +++ b/tests/integration/test_label_round3_contracts_postgres.py @@ -80,6 +80,61 @@ def test_memory_event_target_set_preserves_exact_target_matching(label_harness): assert [event["id"] for event in actual] == [event["id"] for event in baseline] +def test_native_source_event_counts_match_the_complete_per_target_guard(label_harness): + from alicebot_api.vnext_label_guard import LabelGuard, label_read_scope + from alicebot_api.store import ContinuityStore + from tests.integration.derived_labels_postgres_support import LabelHarness + + h = label_harness + sources = [h.source(scope=("P1" if i % 2 == 0 else "P2",), sensitivity=value) + for i, value in enumerate(("public", "public", "internal", "confidential", "private", "regulated", "unknown", "public"))] + foreign = LabelHarness(h.urls, uuid4()) + with foreign.store() as store: + ContinuityStore(store.conn).create_user(foreign.user_id, "foreign@example.invalid", "Synthetic foreign tenant") + foreign_source = foreign.source() + h.memory(source=sources[3]) + with h.store() as store: + for source, domain in zip(sources, ("project", "health", "unknown", "project", "legal", "project", "project", "project"), strict=True): + store.conn.execute("UPDATE sources SET domain=%s WHERE id=%s", (domain, source["id"])) + store.conn.execute("UPDATE sources SET deleted_at=now() WHERE id=%s", (sources[-1]["id"],)) + fixed = store.create_source({"id": str(UUID(int=0xABCDEF)), "source_type": "note", "title": "Synthetic canonical target", + "content_hash": str(uuid4()), "domain": "project", "sensitivity": "public", + "metadata_json": {"project_scope": ["P1"]}}) + canonical = str(fixed["id"]) + for target in (canonical.upper(), canonical.replace("-", ""), " " + canonical, "urn:uuid:" + canonical, + "source:" + canonical, "legacy-not-a-uuid", str(uuid4()), str(foreign_source["id"])): + store.conn.execute( + "INSERT INTO event_log(id,user_id,event_type,actor_type,target_type,target_id,payload_json) " + "VALUES(%s,%s,'source.updated','system','source',%s,'{}')", (uuid4(), h.user_id, target)) + store.append_event(build_event_log_record(event_type="source_chunk.created", actor_type="system", payload={})) + store.append_event(build_event_log_record(event_type="unknown.labels_raised", actor_type="system", payload={})) + + class PerTarget: + def read_label_rows(self, kind, ids): + return store.read_label_rows(kind, ids) + + def iter_label_events(self): + return store.iter_label_events() + + oracle = PerTarget() + filters = [ + {"sensitivity_allowed": ("public",)}, + {"domains": ("health",), "sensitivity_allowed": ("public", "internal")}, + {"domains": ("legal",), "sensitivity_allowed": ("private", "unknown")}, + {"sensitivity_allowed": ("public", "internal", "private", "unknown")}, + {"sensitivity_allowed": ("public",), "projects": ("P1",)}, + {"sensitivity_allowed": ("public",), "all_of": ("P1",)}, + {}, + ] + for active in (True, False): + for options in filters: + with label_read_scope(oracle): + expected = LabelGuard(oracle, active=active, **options).readable_event_count() + with label_read_scope(store): + actual = LabelGuard(store, active=active, **options).readable_event_count() + assert actual == expected, (active, options, actual, expected) + + @pytest.mark.parametrize("profile", ["owner", "admin_agent"]) @pytest.mark.parametrize("component", ["trace", "workspace"]) def test_unfenced_source_trace_and_workspace_preserve_main(label_harness, profile, component): diff --git a/tests/unit/test_label_resolved_inputs.py b/tests/unit/test_label_resolved_inputs.py index 9621ad935..d3a8a1a74 100644 --- a/tests/unit/test_label_resolved_inputs.py +++ b/tests/unit/test_label_resolved_inputs.py @@ -352,6 +352,47 @@ def iter_label_rows(self, kind, **kwargs): assert len(store.reads) > before + 1 +def test_native_event_counts_batch_sources_without_changing_admission(): + sources = [{"kind": "source", "id": UUID(int=i + 1), "domain": "project", + "sensitivity": "confidential" if i % 2 else "public", "metadata_json": {}} + for i in range(3000)] + events = [{"target_type": "source", "target_id": str(source["id"]), "event_type": "source.created"} + for source in sources for _ in range(3)] + events.extend([ + {"target_type": "source", "target_id": str(UUID(int=10000)), "event_type": "source.created"}, + {"target_type": "source", "target_id": str(sources[0]["id"]).replace("-", ""), "event_type": "source.created"}, + {"target_type": "source_chunk", "event_type": "source_chunk.labels_raised"}, + {"target_type": "source_chunk", "event_type": "source_chunk.created"}, + ]) + + class NativeEvents(Rows): + label_count_canonical_unique_ids = True + + def __init__(self, rows): + super().__init__(rows) + self.reads = 0 + + def read_label_rows(self, kind, ids): + self.reads += 1 + return super().read_label_rows(kind, ids) + + def iter_label_events(self, *, batch_size=1000): + for start in range(0, len(events), batch_size): + yield events[start:start + batch_size] + + store = NativeEvents(sources) + with label_read_scope(store): + guard = LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard.readable_event_count() == 4501 + assert store.reads <= 2 + assert replace(guard, domains=("health",)).readable_event_count() == 1 + before_write = store.reads + sources[0]["sensitivity"] = "confidential" + invalidate_read_labels(store) + assert guard.readable_event_count() == 4498 + assert store.reads - before_write <= 2 + + def test_text_identifiers_keep_case_while_uuid_aliases_share_an_origin(): canonical = str(UUID(int=0xABCDEF)) sources = [ From d393908e17332b21edd76daf658740938b027cbb Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:53:51 +0200 Subject: [PATCH 268/270] Pin the added native source-event counter without relaxing store graft checks --- tests/unit/test_store_events_revisions_split.py | 4 +++- tests/unit/test_store_graph_open_loops_split.py | 4 +++- tests/unit/test_store_memory_access_split.py | 4 +++- tests/unit/test_store_memory_lifecycle_split.py | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_store_events_revisions_split.py b/tests/unit/test_store_events_revisions_split.py index 9f71008a1..d52a8dd82 100644 --- a/tests/unit/test_store_events_revisions_split.py +++ b/tests/unit/test_store_events_revisions_split.py @@ -141,6 +141,8 @@ }, } # Round two adds the conservative original-row SQL count on both facades. +# Round three adds only the unscoped original-source event counter to Postgres. +# Runtime metadata and every grafted method remain pinned below. EXPECTED_CLASS_KEY_SHA256 = { # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. # Existing facade members retain their relative order. @@ -153,7 +155,7 @@ # Re-minted for the per-file importer savepoint (2026-10-02): ``savepoint`` is appended last on both façades. # Previous receipt: 650e2e0ff088d67c... Proof: the class key list equals the list at origin/main 040a2a10 # plus ``savepoint`` before ``__dict__``, with every other key in the same order. - "postgres": "062e8006f55e3df7f369aac5173a32305a9850537e078c093bff71f5c5c5187c", + "postgres": "cc43f69fcf43c87332e24786f58cbddb6b64bce2af9d1f559d02eb5e4f18f288", # Re-minted for the merge of #500 and #502: ``check_literal_match_query``. # Re-minted again for per-project memory S2 (2026-10-02): the two single-scan partition reads # ``list_memories_view_partitions`` and ``list_open_loops_view_partitions``, SQLite only on purpose diff --git a/tests/unit/test_store_graph_open_loops_split.py b/tests/unit/test_store_graph_open_loops_split.py index cd2749541..011159ff0 100644 --- a/tests/unit/test_store_graph_open_loops_split.py +++ b/tests/unit/test_store_graph_open_loops_split.py @@ -158,6 +158,8 @@ } # Round two adds count_original_label_statuses after read_label_rows on both # facades; all existing members retain their relative order. +# Round three adds only count_source_label_events before iter_label_events. +# Removing that new member reproduces the previous exact native member order. EXPECTED_CLASS_ORDERS = { # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. # Existing facade members retain their relative order. @@ -167,7 +169,7 @@ # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # lock_label_writes and read_label_rows follow __init__. Previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (183, "b78e54887487bf23d24c250cec9499b6439b9164fa64c56fea29479d85550c94"), + "PostgresVNextStore": (184, "e5bb6fbb9a63e4027d028928a9fafb6f4a65e64e85c15bbb48f0a605b9326511"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, diff --git a/tests/unit/test_store_memory_access_split.py b/tests/unit/test_store_memory_access_split.py index 303b70ad8..729093afa 100644 --- a/tests/unit/test_store_memory_access_split.py +++ b/tests/unit/test_store_memory_access_split.py @@ -212,6 +212,8 @@ # Round two adds count_original_label_statuses after read_label_rows on both # facades; all existing members retain their relative order. +# Round three adds only count_source_label_events before iter_label_events. +# Removing that new member reproduces the previous exact native member order. EXPECTED_CLASS_ORDERS = { # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. # Existing facade members retain their relative order. @@ -224,7 +226,7 @@ # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). # Label lock: lock_label_writes and read_label_rows follow __init__. Dropping # those two names restores the previous receipt (172, 6f1a459f...). - "PostgresVNextStore": (183, "b78e54887487bf23d24c250cec9499b6439b9164fa64c56fea29479d85550c94"), + "PostgresVNextStore": (184, "e5bb6fbb9a63e4027d028928a9fafb6f4a65e64e85c15bbb48f0a605b9326511"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, diff --git a/tests/unit/test_store_memory_lifecycle_split.py b/tests/unit/test_store_memory_lifecycle_split.py index db561c5cd..2fae9c1a0 100644 --- a/tests/unit/test_store_memory_lifecycle_split.py +++ b/tests/unit/test_store_memory_lifecycle_split.py @@ -111,6 +111,8 @@ } # Round two adds count_original_label_statuses after read_label_rows on both # facades; all existing members retain their relative order. +# Round three adds only count_source_label_events before iter_label_events. +# Removing that new member reproduces the previous exact native member order. EXPECTED_CLASS_ORDERS = { # Reviewed additions: label/event enumeration, PG ratings, belief aliases and source recovery. # Existing facade members retain their relative order. @@ -119,7 +121,7 @@ # Per-file importer savepoint (2026-10-02): one paired method more, ``savepoint``, appended last. # Previous receipt: (171, 526374782104a2a1...). Proof: the member list equals the list at origin/main # 040a2a10 with ``savepoint`` added at the end and nothing else moved (reviewed change, not drift). - "PostgresVNextStore": (183, "b78e54887487bf23d24c250cec9499b6439b9164fa64c56fea29479d85550c94"), + "PostgresVNextStore": (184, "e5bb6fbb9a63e4027d028928a9fafb6f4a65e64e85c15bbb48f0a605b9326511"), # One SQLite-only method more, ``check_source_search_query``: the Postgres # source search has no expression-depth or LIKE-length limit to check. # Merge of #500 and #502 (2026-10-01): one more SQLite-only method, From cf818b695b5aef82dadd3eb5d0037cf66c542f74 Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:52:01 +0200 Subject: [PATCH 269/270] Reduce native read admission cost and retain separate profiles --- .github/workflows/tests.yml | 4 +- .../src/alicebot_api/vnext_derived_labels.py | 57 ++- .../api/src/alicebot_api/vnext_label_guard.py | 143 ++++++++ .../src/alicebot_api/vnext_label_repair.py | 2 +- scripts/derived_label_mutations.json | 174 +++++++-- tests/performance/round3_budget_probe.py | 172 ++++++--- .../test_label_round3_read_budget.py | 43 +++ tests/unit/test_derived_labels_mutations.py | 2 +- tests/unit/test_label_doctor_preparation.py | 163 +++++++++ .../unit/test_label_native_rank_admission.py | 340 ++++++++++++++++++ .../test_round3_budget_profile_protocol.py | 252 +++++++++++++ 11 files changed, 1251 insertions(+), 101 deletions(-) create mode 100644 tests/unit/test_label_doctor_preparation.py create mode 100644 tests/unit/test_label_native_rank_admission.py create mode 100644 tests/unit/test_round3_budget_profile_protocol.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b9745564a..9a14e6a12 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2358,6 +2358,7 @@ jobs: DATABASE_ADMIN_URL: postgresql://alicebot_admin:ci@localhost:5432/alicebot ALICE_READ_MAIN_CHECKOUT: ${{ github.workspace }}/.read-budget-baseline ALICE_READ_BUDGET_EVIDENCE: ${{ runner.temp }}/round3-postgres-read-budgets.jsonl + ALICE_READ_PROFILE: ${{ runner.temp }}/round3-read-profiles run: >- ALICE_LEGACY_SURFACES=1 ./.venv/bin/python -m pytest -q -s --timeout=180 'tests/integration/test_label_round3_read_budget_postgres.py::test_postgres_round3_random_read_budgets[False-3000-all-visible]' @@ -2367,10 +2368,11 @@ jobs: if: always() && matrix.integration_check != 'Default surface integration smoke (Postgres)' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: round3-paired-read-budgets-${{ strategy.job-index }} + name: round3-paired-read-budgets-${{ strategy.job-index }}-attempt-${{ github.run_attempt }} path: | ${{ runner.temp }}/round3-postgres-read-budgets.jsonl ${{ runner.temp }}/round3-sqlite-read-budgets.jsonl + ${{ runner.temp }}/round3-read-profiles/ if-no-files-found: error retention-days: 7 diff --git a/apps/api/src/alicebot_api/vnext_derived_labels.py b/apps/api/src/alicebot_api/vnext_derived_labels.py index 1c3346352..0c78b0313 100644 --- a/apps/api/src/alicebot_api/vnext_derived_labels.py +++ b/apps/api/src/alicebot_api/vnext_derived_labels.py @@ -13,7 +13,7 @@ import json import re from collections import deque -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence, Set from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass, replace @@ -390,7 +390,12 @@ def row_class(kind: object, row: Mapping[str, object]) -> str: if not is_derived(kind, row): return "original" - name = canon_kind(kind) + return _derived_row_class(canon_kind(kind), row) + + +def _derived_row_class(name: str, row: Mapping[str, object]) -> str: + """Classify a row whose canonical kind and derived marker are already known.""" + if name == "project": return "project_state" if name == "artifact": @@ -841,23 +846,28 @@ def dependency_record(kind: object, row: Mapping[str, object]) -> tuple[frozense def _dependency_record(kind: object, row: Mapping[str, object]) -> tuple[frozenset[tuple[str, str]], str]: if not is_derived(kind, row): return frozenset(), "" + return _derived_dependency_record(canon_kind(kind), row) + + +def _derived_dependency_record(name: str, row: Mapping[str, object]) -> tuple[frozenset[tuple[str, str]], str]: + """The canonical parser after the caller has checked this row is derived.""" + meta = _metadata(row) found: set[tuple[str, str]] = set() - problem = "malformed_marker" if _malformed_marker(canon_kind(kind), meta) else "" + problem = "malformed_marker" if _malformed_marker(name, meta) else "" problem = problem or _collect_metadata_ids(meta, found) if "derived_from" in meta: derived_problem, derived_deps = _derived_from_deps(meta.get("derived_from")) problem = problem or derived_problem found |= derived_deps - if is_derived(kind, row): - found |= _value_dependencies(row) - if canon_kind(kind) == "open_loop" and _nonempty_str(row.get("source_id")): + found |= _value_dependencies(row) + if name == "open_loop" and _nonempty_str(row.get("source_id")): found.add(("source", identifier(row.get("source_id")))) floor_shape, _floor = _floor_of(row) if floor_shape == "malformed": problem = problem or "malformed_floor" if not problem: - problem = _legacy_count_problem(canon_kind(kind), row, meta) + problem = _legacy_count_problem(name, row, meta) if not problem and not _record_present(meta, row) and not found: # A weekly candidate may still be completed from its parent artifact # inside settle_labels. The structural pass says no_record until then. @@ -986,6 +996,7 @@ def _node_label(kind: str, row: Mapping[str, object]) -> SettledLabel: scope = stored_scope(kind, row) domain = str(row.get("domain") or "unknown") sensitivity = str(row.get("sensitivity") or "unknown") + derived = is_derived(kind, row) return SettledLabel( kind=kind, user_id=str(row.get("user_id") or ""), @@ -1002,8 +1013,8 @@ def _node_label(kind: str, row: Mapping[str, object]) -> SettledLabel: unverified=False, reason=None, carries_scope=carries_scope(kind, row), - derived=is_derived(kind, row), - row_class=row_class(kind, row), + derived=derived, + row_class=_derived_row_class(kind, row) if derived else "original", ) @@ -1221,6 +1232,7 @@ def settle_labels( on_cycle: str = "raise", max_hops: int | None = None, max_nodes: int | None = None, + one_pass: bool = False, ) -> SettleResult: """Settle every derived row in ``nodes``. @@ -1228,6 +1240,8 @@ def settle_labels( Originals are fixed inputs. A cycle that does not settle raises ``DerivedDomainRepairError`` unless ``on_cycle`` is ``unverified``. ``max_hops`` and ``max_nodes`` bound a read. The repair leaves them unset. + A fresh full classification may set ``one_pass`` to avoid dependency-cache + setup. Ordinary closure reads keep their request parsing cache by default. """ if on_cycle not in {"raise", "unverified"}: @@ -1252,7 +1266,10 @@ def settle_labels( for label, row in prepared: if not label.derived: continue - deps, problem = dependency_record(label.kind, row) + if one_pass: + deps, problem = _derived_dependency_record(label.kind, row) + else: + deps, problem = dependency_record(label.kind, row) if problem == "no_record" and label.row_class == "aggregate": # Filled from the parent artifact below when one names this row. problem = "" @@ -1404,8 +1421,24 @@ def _mark_bounds( ) -> None: """Mark a derived row unverified when the walk from that row passes a bound.""" - for origin, label in labels.items(): - if not label.derived or origin in problems: + _mark_dependency_bounds( + (origin for origin, label in labels.items() if label.derived), + resolved, problems, max_hops=max_hops, max_nodes=max_nodes, + ) + + +def _mark_dependency_bounds( + origins: Iterable[tuple[str, str, str]], + resolved: Mapping[tuple[str, str, str], Set[tuple[str, str, str]]], + problems: dict[tuple[str, str, str], str], + *, + max_hops: int | None, + max_nodes: int | None, +) -> None: + """Apply the canonical per-origin BFS bounds to an already-parsed graph.""" + + for origin in origins: + if origin in problems: continue pending: deque[tuple[tuple[str, str, str], int]] = deque([(origin, 0)]) seen: set[tuple[str, str, str]] = set() diff --git a/apps/api/src/alicebot_api/vnext_label_guard.py b/apps/api/src/alicebot_api/vnext_label_guard.py index 0e1f5e9a1..125cae790 100644 --- a/apps/api/src/alicebot_api/vnext_label_guard.py +++ b/apps/api/src/alicebot_api/vnext_label_guard.py @@ -7,6 +7,7 @@ from __future__ import annotations +from collections import deque from collections.abc import Mapping, Sequence from dataclasses import dataclass, replace, field from typing import Any, TypeVar @@ -29,6 +30,7 @@ canon_kind, dependencies_of, dependency_label_signature, + dependency_record, dependency_syntax_key, label_metadata_cache, identifier, @@ -37,6 +39,7 @@ input_admitted, settle_labels, settle_verified_inputs, + _mark_dependency_bounds, ) from alicebot_api.vnext_label_closure import collect_label_rows from alicebot_api.vnext_project_scope import project_floor_shape, project_scope_identity, project_scopes_overlap, resolve_project_scope @@ -52,6 +55,21 @@ def _row_label_key(kind: str, row: Mapping[str, object]) -> tuple: ) if field in row)) +def _rank_projection_supported(row: Mapping[str, object]) -> bool: + """Only canonical empty scope/floor shapes use a reduced admission proof.""" + metadata = row.get("metadata_json") + if type(metadata) is not dict: + return False + for container in (row, metadata): + for name in ("project_scope", "project_floor"): + if name in container and (type(container[name]) not in (list, tuple) or container[name]): + return False + for name in ("project_id", "project", "projects", "scope_json", "agent_identity", "agentic_memory"): + if name in container and container[name] is not None: + return False + return True + + @dataclass class _RequestLabels: nodes: dict = field(default_factory=dict) @@ -72,6 +90,9 @@ class _RequestLabels: resolved_inputs: dict = field(default_factory=dict) parent_labels: dict = field(default_factory=dict) native_labels: dict = field(default_factory=dict) + rank_rows: dict = field(default_factory=dict) + rank_origins: dict = field(default_factory=dict) + rank_admission: dict = field(default_factory=dict) def clear(self): self.nodes.clear() @@ -92,6 +113,9 @@ def clear(self): self.resolved_inputs.clear() self.parent_labels.clear() self.native_labels.clear() + self.rank_rows.clear() + self.rank_origins.clear() + self.rank_admission.clear() _REQUEST_LABELS: ContextVar[tuple[Any, _RequestLabels] | None] = ContextVar("request_labels", default=None) @@ -368,6 +392,7 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: if not self.active: return [row for row in rows if isinstance(row, Mapping)] state = self._state() + rank_ceiling = self._rank_ceiling() if self.sensitivity_allowed: highest = max(SENSITIVITY_RANK.get(value, SENSITIVITY_RANK["unknown"]) for value in self.sensitivity_allowed) rows = [row for row in rows if isinstance(row, Mapping) @@ -380,6 +405,14 @@ def admit_rows(self, kind: str, rows: Sequence[_Row]) -> list[_Row]: for row in rows: if not isinstance(row, Mapping): continue + rank = self._native_rank_for_projection(kind, row) if rank_ceiling is not None else None + if rank_ceiling is not None and rank is not None: + grant_key = (self._key(kind, row), self.domains, self.sensitivity_allowed, self.projects, self.all_of) + if grant_key not in state.rank_admission: + state.rank_admission[grant_key] = rank <= rank_ceiling + if state.rank_admission[grant_key]: + kept.append(row) + continue if not is_derived(kind, row): if self._admits_effective(row, kind=kind): kept.append(row) @@ -443,6 +476,9 @@ def _prefetch_inputs(self, kind: str, rows: Sequence[Mapping[str, object]]) -> N state = self._state() prefetched = state.row_sets.setdefault("prefetched-input-expansions", {}) frontier = [(kind, row) for row in rows if isinstance(row, Mapping)] + if self._rank_ceiling() is not None and state.rank_origins: + frontier = [(row_kind, row) for row_kind, row in frontier + if self._native_rank_for_projection(row_kind, row) is None] expanded: set[tuple[str, str]] = set() for _ in range(HOP_BOUND + 1): refs: set[tuple[str, str]] = set() @@ -541,6 +577,8 @@ def _settle_native_count_batch(self, kind: str, rows: Sequence[Mapping[str, obje return self._prefetch_inputs(kind, rows) state = self._state() + if self._settle_native_rank_batch(kind, rows): + return for row in rows: stored = state.nodes.get((canon_kind(kind), identifier(row["id"])), ()) if len(stored) != 1 or (stored[0] is not row and _row_label_key(kind, stored[0]) != _row_label_key(kind, row)): @@ -561,6 +599,111 @@ def _settle_native_count_batch(self, kind: str, rows: Sequence[Mapping[str, obje state.labels[self._key(kind, row)] = label state.native_labels[(canon_kind(kind), identifier(row["id"]))] = (row, label) + def _rank_ceiling(self) -> int | None: + """A complete rank prefix may use a verified sensitivity-only proof.""" + current = _REQUEST_LABELS.get() + if (not self.active or self.domains or self.projects or self.all_of is not None + or current is None or current[0] is not self.store + or not getattr(type(self.store), "label_count_canonical_unique_ids", False)): + return None + allowed = frozenset(self.sensitivity_allowed) + if not allowed or not allowed.issubset(SENSITIVITY_RANK): + return None + ceiling = max(SENSITIVITY_RANK[value] for value in allowed) + if allowed != frozenset(value for value, rank in SENSITIVITY_RANK.items() if rank <= ceiling): + return None + return ceiling + + def _settle_native_rank_batch(self, kind: str, rows: Sequence[Mapping[str, object]]) -> bool: + """Prove a complete native DAG before publishing any reduced result. + + Canonical syntax, source-kind rules and independent origin bounds stay + authoritative. Any unsupported graph uses the full label kernel. + These ranks never enter a full-label or effective-row cache. + """ + if self._rank_ceiling() is None: + return False + state = self._state() + for row in rows: + found = state.nodes.get((canon_kind(kind), identifier(row["id"])), ()) + if len(found) != 1 or (found[0] is not row and _row_label_key(kind, found[0]) != _row_label_key(kind, row)): + return False + graph: dict[tuple[str, str, str], frozenset[tuple[str, str, str]]] = {} + ranks: dict[tuple[str, str, str], int] = {} + origins: list[tuple[str, str, str]] = [] + for (node_kind, node_id), found in state.nodes.items(): + node_kind = canon_kind(node_kind) + if len(found) != 1: + return False + row = found[0] + if type(row.get("id")) is not UUID or identifier(row["id"]) != node_id: + return False + if (node_kind == "belief" or has_implicit_weekly_inputs(node_kind, row) + or not _rank_projection_supported(row)): + return False + sensitivity = str(row.get("sensitivity") or "unknown") + if sensitivity not in SENSITIVITY_RANK: + return False + key = (node_kind, _GUARD_USER, node_id) + if key in graph: + return False + derived = is_derived(node_kind, row) + refs, problem = dependency_record(node_kind, row) if derived else (frozenset(), "") + if problem: + return False + graph[key] = frozenset((ref_kind, _GUARD_USER, ref_id) for ref_kind, ref_id in refs) + ranks[key] = SENSITIVITY_RANK[sensitivity] + if derived: + origins.append(key) + if any(ref not in graph for refs in graph.values() for ref in refs): + return False + pending_count = {key: len(refs) for key, refs in graph.items()} + dependants: dict[tuple[str, str, str], list[tuple[str, str, str]]] = {} + for node_key, input_keys in graph.items(): + for input_key in input_keys: + dependants.setdefault(input_key, []).append(node_key) + pending = deque(key for key, count in pending_count.items() if not count) + visited = 0 + while pending: + key = pending.popleft() + visited += 1 + for child in dependants.get(key, ()): + ranks[child] = max(ranks[child], ranks[key]) + pending_count[child] -= 1 + if not pending_count[child]: + pending.append(child) + if visited != len(graph): + return False + problems: dict[tuple[str, str, str], str] = {} + _mark_dependency_bounds(origins, graph, problems, max_hops=HOP_BOUND, max_nodes=NODE_BOUND) + if problems: + return False + for row in rows: + canonical = (canon_kind(kind), identifier(row["id"])) + rank = ranks[(canonical[0], _GUARD_USER, canonical[1])] + state.rank_rows[self._key(kind, row)] = rank + state.rank_origins[canonical] = (row, rank) + return True + + def _native_rank_for_projection(self, kind: str, row: Mapping[str, object]) -> int | None: + """A reduced proof belongs to one supported origin and its semantics.""" + current = _REQUEST_LABELS.get() + if current is None or current[0] is not self.store or type(row.get("id")) is not UUID: + return None + state = current[1] + entry = state.rank_origins.get((canon_kind(kind), identifier(row["id"]))) + if entry is None or not _rank_projection_supported(row): + return None + key = self._key(kind, row) + if key in state.rank_rows: + return state.rank_rows[key] + raw, rank = entry + projection_signature = self._signature(kind, row, key=key) + if projection_signature != self._signature(kind, raw, key=self._key(kind, raw)): + return None + state.rank_rows[key] = rank + return rank + def _native_label_for_projection(self, kind: str, row: Mapping[str, object], *, key: tuple) -> SettledLabel | None: """Reuse one verified UUID origin only for equivalent label semantics.""" current = _REQUEST_LABELS.get() diff --git a/apps/api/src/alicebot_api/vnext_label_repair.py b/apps/api/src/alicebot_api/vnext_label_repair.py index a44439e09..9bc429102 100644 --- a/apps/api/src/alicebot_api/vnext_label_repair.py +++ b/apps/api/src/alicebot_api/vnext_label_repair.py @@ -248,7 +248,7 @@ def classify_stored_labels( node["value"] = _json_object(row.get("value")) nodes.append(node) index.append((table, node)) - settled = settle_labels(nodes, on_cycle="unverified") + settled = settle_labels(nodes, on_cycle="unverified", one_pass=True) below: list[LabelRepair] = [] unverified: dict[str, list[str]] = {} for (table, node), label in zip(index, settled.rows, strict=True): diff --git a/scripts/derived_label_mutations.json b/scripts/derived_label_mutations.json index 92513e891..f3df5418e 100644 --- a/scripts/derived_label_mutations.json +++ b/scripts/derived_label_mutations.json @@ -141,10 +141,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 354, + "line": 378, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], ())", - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -157,10 +157,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 354, + "line": 378, "before": "semantic = (template[:1] + template[2:], semantic_parents)", "after": "semantic = (template[:1] + template[2:], tuple(sorted(set(semantic_parents))))", - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -173,10 +173,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 341, + "line": 365, "before": "if len(raw) != 1 or identifier(raw[0].get(\"id\")) != parent_id:", "after": "if not raw or identifier(raw[0].get(\"id\")) != parent_id:", - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -191,10 +191,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 349, + "line": 373, "before": "if depth > HOP_BOUND or len(ancestry) > NODE_BOUND:", "after": "if False:", - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -209,10 +209,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 325, + "line": 349, "before": "if problem or has_implicit_weekly_inputs(kind, row):", "after": "if problem:", - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -227,10 +227,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 502, + "line": 538, "before": "if not self.active and callable(native):", "after": "if False:", - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -275,10 +275,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 378, + "line": 403, "before": " self._prefetch_inputs(kind, rows)\n kept: list[_Row] = []", "after": " pass # removed batched ancestry reads\n kept: list[_Row] = []", - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -357,8 +357,8 @@ "file": "apps/api/src/alicebot_api/vnext_label_guard.py", "before": "row_admission_key = (key, self.domains, self.sensitivity_allowed, self.projects, self.all_of)", "after": "row_admission_key = (key,)", - "line": 388, - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "line": 421, + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -373,8 +373,8 @@ "file": "apps/api/src/alicebot_api/vnext_label_guard.py", "before": " self.row_admission.clear()", "after": " pass # removed admission invalidation", - "line": 89, - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "line": 110, + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -387,10 +387,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 394, + "line": 427, "before": " if settled is not None and not settled.unverified:", "after": " if settled is not None:", - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -403,10 +403,10 @@ "edits": [ { "file": "apps/api/src/alicebot_api/vnext_label_guard.py", - "line": 546, + "line": 584, "before": " if len(stored) != 1 or (stored[0] is not row and _row_label_key(kind, stored[0]) != _row_label_key(kind, row)):", "after": " if len(stored) != 1:", - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -453,8 +453,8 @@ "file": "apps/api/src/alicebot_api/vnext_label_guard.py", "before": " self.native_labels.clear()", "after": " pass # removed native projection invalidation", - "line": 94, - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "line": 115, + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -469,8 +469,8 @@ "file": "apps/api/src/alicebot_api/vnext_label_guard.py", "before": " if self._signature(kind, row, key=key) != self._signature(kind, raw, key=self._key(kind, raw)):\n return None", "after": " pass # removed native projection signature equality", - "line": 573, - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "line": 716, + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -501,7 +501,7 @@ "file": "apps/api/src/alicebot_api/vnext_derived_labels.py", "before": " elif type(value) is str and _CANONICAL_SOURCE_UUID.fullmatch(value):", "after": " elif type(value) is str:", - "sha256": "398581751be87665c48c8ed33ba9fe5b1254125022b69f00d2879751443a17f6", + "sha256": "529ef81c4c4fcf4d5a756535b98334dd03ee45da7c9e1b8308d23e8a11e65e71", "line": 207 } ], @@ -517,8 +517,8 @@ "file": "apps/api/src/alicebot_api/vnext_label_guard.py", "before": "prefilter[\"batch_size\"] = 5000", "after": "prefilter[\"batch_size\"] = 1000", - "line": 641, - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "line": 784, + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -565,8 +565,8 @@ "file": "apps/api/src/alicebot_api/vnext_label_guard.py", "before": "if canonical and callable(counter) and self.active and not self.projects and self.all_of is None:", "after": "if canonical and callable(counter) and self.active:", - "line": 629, - "sha256": "6b05eb5600db7e1da3d2578639de821dcd1f4959396734d1536cec087d36a28e" + "line": 772, + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e" } ], "pytest": [ @@ -589,6 +589,118 @@ "tests/integration/test_label_round3_contracts_postgres.py::test_native_source_event_counts_match_the_complete_per_target_guard" ], "expected_failure": "test_native_source_event_counts_match_the_complete_per_target_guard" + }, + { + "name": "native-rank-prefix-completeness", + "pytest": [ + "tests/unit/test_label_native_rank_admission.py::test_all_rank_boundaries_and_split_allowances_match_full_labels" + ], + "expected_failure": "test_all_rank_boundaries_and_split_allowances_match_full_labels", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": "if allowed != frozenset(value for value, rank in SENSITIVITY_RANK.items() if rank <= ceiling):", + "after": "if False:", + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e", + "line": 613 + } + ] + }, + { + "name": "native-rank-origin-bounds", + "pytest": [ + "tests/unit/test_label_native_rank_admission.py::test_unsupported_graphs_use_full_kernel_without_publishing_ranks" + ], + "expected_failure": "test_unsupported_graphs_use_full_kernel_without_publishing_ranks", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": "_mark_dependency_bounds(origins, graph, problems, max_hops=HOP_BOUND, max_nodes=NODE_BOUND)", + "after": "pass # removed independent origin bounds", + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e", + "line": 678 + } + ] + }, + { + "name": "native-rank-cycle-refusal", + "pytest": [ + "tests/unit/test_label_native_rank_admission.py::test_unsupported_graphs_use_full_kernel_without_publishing_ranks" + ], + "expected_failure": "test_unsupported_graphs_use_full_kernel_without_publishing_ranks", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": "if visited != len(graph):", + "after": "if False:", + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e", + "line": 675 + } + ] + }, + { + "name": "native-rank-reloaded-shape", + "pytest": [ + "tests/unit/test_label_native_rank_admission.py::test_reloaded_projections_require_supported_shape_and_canonical_signature" + ], + "expected_failure": "test_reloaded_projections_require_supported_shape_and_canonical_signature", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": "if entry is None or not _rank_projection_supported(row):", + "after": "if entry is None:", + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e", + "line": 695 + } + ] + }, + { + "name": "native-rank-reloaded-signature", + "pytest": [ + "tests/unit/test_label_native_rank_admission.py::test_reloaded_projections_require_supported_shape_and_canonical_signature" + ], + "expected_failure": "test_reloaded_projections_require_supported_shape_and_canonical_signature", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": "if projection_signature != self._signature(kind, raw, key=self._key(kind, raw)):", + "after": "if False:", + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e", + "line": 702 + } + ] + }, + { + "name": "native-rank-write-rollback-invalidation", + "pytest": [ + "tests/unit/test_label_native_rank_admission.py::test_event_spelling_filter_grants_and_writer_rollback_invalidation" + ], + "expected_failure": "test_event_spelling_filter_grants_and_writer_rollback_invalidation", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": " self.rank_rows.clear()\n self.rank_origins.clear()\n self.rank_admission.clear()", + "after": " pass # removed reduced-proof invalidation", + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e", + "line": 116 + } + ] + }, + { + "name": "native-rank-caller-filter-grants", + "pytest": [ + "tests/unit/test_label_native_rank_admission.py::test_same_origin_keeps_counts_and_event_grants_separate_across_rank_ceilings" + ], + "expected_failure": "test_same_origin_keeps_counts_and_event_grants_separate_across_rank_ceilings", + "edits": [ + { + "file": "apps/api/src/alicebot_api/vnext_label_guard.py", + "before": "grant_key = (self._key(kind, row), self.domains, self.sensitivity_allowed, self.projects, self.all_of)", + "after": "grant_key = (self._key(kind, row),)", + "sha256": "709fe6008d367346257112b118a7be18a5e5d3c6e49d36c041de710e9b6e5d5e", + "line": 410 + } + ] } ] } diff --git a/tests/performance/round3_budget_probe.py b/tests/performance/round3_budget_probe.py index 1e3f573d3..2e9750713 100644 --- a/tests/performance/round3_budget_probe.py +++ b/tests/performance/round3_budget_probe.py @@ -1,69 +1,131 @@ """Persistent revision-isolated probe, allowing interleaved same-data samples.""" import importlib.util import cProfile +from importlib.metadata import PackageNotFoundError, version +import io import json import os from pathlib import Path +import platform +import pstats +import subprocess import sys import time from uuid import UUID -repo, backend, location, user, profile, key = sys.argv[1:7] -sys.path[:0] = [str(Path(repo) / "apps/api/src"), repo] -os.environ["DATABASE_URL"] = location -if key: - os.environ["ALICE_AGENT_API_KEY"] = key -else: - os.environ.pop("ALICE_AGENT_API_KEY", None) -os.environ["ALICE_MCP_FULL_TOOLS"] = "1" -os.environ["ALICE_LEGACY_SURFACES"] = "1" -from alicebot_api.mcp.registry import call_mcp_tool -from alicebot_api.mcp.types import MCPRuntimeContext +EXPECTED_CAPTURE_FUNCTIONS = { + "workspace": ("get_vnext_workspace", "_vnext_workspace_payload"), + "dogfooding": ("get_vnext_dogfooding_dashboard", "dashboard"), +} -context = MCPRuntimeContext(database_url=location, user_id=UUID(user)) -arguments = {"query": "synthetic budget observation"} -if profile == "keyless_agent_id": - arguments["agent_id"] = "budget-keyless" -actions = {"pack": lambda: call_mcp_tool(context, name="alice_context_pack", arguments=arguments), - "recall": lambda: call_mcp_tool(context, name="alice_recall", arguments=arguments)} -if backend == "postgres": - from alicebot_api.config import Settings - from alicebot_api import main - from alicebot_api.routers import workspaces, vnext_memories - settings = Settings(database_url=location) - for module in (main, workspaces, vnext_memories): - module.get_settings = lambda: settings - support = Path(__file__).resolve().parents[1] / "integration/derived_labels_postgres_support.py" - spec = importlib.util.spec_from_file_location("round3_transport", support) - transport = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = transport - spec.loader.exec_module(transport) - def read(path): - status, body, _headers = transport.invoke("GET", path, user_id=user, key=key) - assert status == 200, (status, body) - return body - - actions.update(workspace=lambda: read("/v0/vnext/workspace"), dogfooding=lambda: read("/v0/vnext/dogfooding")) - -profiled = set() -for line in sys.stdin: - name = line.strip() - if name == "stop": - break - wall_start, cpu_start = time.perf_counter(), time.process_time() - trace = os.environ.get("ALICE_READ_PROFILE") - profiler = cProfile.Profile() if trace and name not in profiled else None - if profiler: +def profile_action(action, name, trace, context): + """Save diagnostics separately; neither timings nor host paths enter the protocol.""" + directory = Path(trace) + directory.mkdir(parents=True, exist_ok=True) + profiler = cProfile.Profile() + error = None + try: profiler.enable() - result = actions[name]() - if profiler: + action() + except Exception as exc: + # Exception text can contain request data or paths; retain only its type. + error = type(exc).__name__ + finally: profiler.disable() - profiler.dump_stats(str(Path(trace) / (Path(repo).name + "-" + profile + "-" + name + ".prof"))) - profiled.add(name) - measurement = {"wall": time.perf_counter() - wall_start, "cpu": time.process_time() - cpu_start} - if name == "recall": - measurement["memories"] = len(result["results"]) - elif name == "pack": - measurement["memories"] = len(result["memories"]) - print(json.dumps(measurement), flush=True) + stats = pstats.Stats(profiler).strip_dirs() + stats.dump_stats(str(directory / (name + ".prof"))) + captured = {key[2] for key in stats.stats} + expected = EXPECTED_CAPTURE_FUNCTIONS[name] + missing = sorted(set(expected) - captured) + report = io.StringIO() + for sort in ("cumulative", "tottime"): + pstats.Stats(profiler, stream=report).strip_dirs().sort_stats(sort).print_stats(40) + (directory / (name + ".txt")).write_text(report.getvalue(), encoding="utf-8") + dependencies = {} + for package in ("fastapi", "starlette", "anyio", "psycopg"): + try: + dependencies[package] = version(package) + except PackageNotFoundError: + dependencies[package] = "unavailable" + receipt = { + **context, "diagnostic": True, "action": name, + "status": "error" if error else "incomplete" if missing else "complete", + "error_type": error, "expected_functions": list(expected), "missing_functions": missing, + "runtime": {"python": platform.python_version(), "implementation": platform.python_implementation(), + "system": platform.system(), "machine": platform.machine(), "dependencies": dependencies}, + } + (directory / (name + ".json")).write_text(json.dumps(receipt) + "\n", encoding="utf-8") + return receipt + + +def serve(actions, commands, output, *, trace=None, profile_context=None): + for line in commands: + name = line.strip() + if name == "stop": + break + if name.startswith("profile:"): + action = name.removeprefix("profile:") + assert trace and action in EXPECTED_CAPTURE_FUNCTIONS, "invalid diagnostic command" + receipt = profile_action(actions[action], action, trace, profile_context() if profile_context else {}) + print(json.dumps(receipt), file=output, flush=True) + continue + wall_start, cpu_start = time.perf_counter(), time.process_time() + result = actions[name]() + measurement = {"wall": time.perf_counter() - wall_start, "cpu": time.process_time() - cpu_start} + if name == "recall": + measurement["memories"] = len(result["results"]) + elif name == "pack": + measurement["memories"] = len(result["memories"]) + print(json.dumps(measurement), file=output, flush=True) + + +def main(): + repo, backend, location, user, profile, key = sys.argv[1:7] + sys.path[:0] = [str(Path(repo) / "apps/api/src"), repo] + os.environ["DATABASE_URL"] = location + if key: + os.environ["ALICE_AGENT_API_KEY"] = key + else: + os.environ.pop("ALICE_AGENT_API_KEY", None) + os.environ["ALICE_MCP_FULL_TOOLS"] = "1" + os.environ["ALICE_LEGACY_SURFACES"] = "1" + from alicebot_api.mcp.registry import call_mcp_tool + from alicebot_api.mcp.types import MCPRuntimeContext + + context = MCPRuntimeContext(database_url=location, user_id=UUID(user)) + arguments = {"query": "synthetic budget observation"} + if profile == "keyless_agent_id": + arguments["agent_id"] = "budget-keyless" + actions = {"pack": lambda: call_mcp_tool(context, name="alice_context_pack", arguments=arguments), + "recall": lambda: call_mcp_tool(context, name="alice_recall", arguments=arguments)} + if backend == "postgres": + from alicebot_api.config import Settings + from alicebot_api import main as main_module + from alicebot_api.routers import workspaces, vnext_memories + settings = Settings(database_url=location) + for module in (main_module, workspaces, vnext_memories): + module.get_settings = lambda: settings + support = Path(__file__).resolve().parents[1] / "integration/derived_labels_postgres_support.py" + spec = importlib.util.spec_from_file_location("round3_transport", support) + transport = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = transport + spec.loader.exec_module(transport) + + def read(path): + status, body, _headers = transport.invoke("GET", path, user_id=user, key=key) + assert status == 200, (status, body) + return body + + actions.update(workspace=lambda: read("/v0/vnext/workspace"), dogfooding=lambda: read("/v0/vnext/dogfooding")) + + def profile_context(): + receipt = json.loads(os.environ.get("ALICE_READ_PROFILE_CONTEXT", "{}")) + receipt["revision"] = subprocess.check_output(["git", "-C", repo, "rev-parse", "HEAD"], text=True).strip() + return receipt + + serve(actions, sys.stdin, sys.stdout, trace=os.environ.get("ALICE_READ_PROFILE"), profile_context=profile_context) + + +if __name__ == "__main__": + main() diff --git a/tests/performance/test_label_round3_read_budget.py b/tests/performance/test_label_round3_read_budget.py index c7e199d5c..db71e37d5 100644 --- a/tests/performance/test_label_round3_read_budget.py +++ b/tests/performance/test_label_round3_read_budget.py @@ -99,6 +99,46 @@ def repair_fixture(backend, location, user): return applied +def capture_read_profiles(backend, location, user, keys, *, case, source_count, repaired, revisions, main, repo, script): + """Run optional diagnostics only after every profile's budget samples are retained.""" + trace = os.environ.get("ALICE_READ_PROFILE") + if not trace or backend != "postgres": + return [] + failures = [] + for profile, key in keys.items(): + for revision, checkout in (("main", main), ("head", repo)): + directory = Path(trace) / backend / f"{case}-{source_count}-{'repaired' if repaired else 'unrepaired'}" / profile / revision + directory.mkdir(parents=True, exist_ok=True) + context = {"store": backend, "case": case, "source_count": source_count, "repaired": repaired, + "profile": profile, "revision_label": revision, "revision": revisions[revision]} + env = {**os.environ, "ALICE_READ_PROFILE": str(directory), "ALICE_READ_PROFILE_CONTEXT": json.dumps(context)} + receipt = {**context, "status": "error", "captures": []} + try: + completed = subprocess.run( + [sys.executable, str(script), str(checkout), backend, location, str(user), profile, key], + input="workspace\nprofile:workspace\ndogfooding\nprofile:dogfooding\nstop\n", + capture_output=True, text=True, timeout=60, env=env, + ) + receipt["exit_code"] = completed.returncode + if completed.returncode == 0: + messages = [json.loads(line) for line in completed.stdout.splitlines()] + captures = [message for message in messages if message.get("diagnostic") is True] + receipt["captures"] = captures + if (len(captures) == 2 and {item.get("action") for item in captures} == {"workspace", "dogfooding"} + and all(item.get("status") == "complete" and item.get("revision") == revisions[revision] + for item in captures)): + receipt["status"] = "complete" + else: + receipt["status"] = "incomplete" + except (OSError, subprocess.SubprocessError, ValueError, AttributeError) as exc: + # Do not persist exception text, child stderr, keys, or database URLs. + receipt["error_type"] = type(exc).__name__ + (directory / "capture-summary.json").write_text(json.dumps(receipt) + "\n", encoding="utf-8") + if receipt["status"] != "complete": + failures.append(receipt) + return failures + + def paired_budgets(backend, location, user, keys, *, case, source_count, repaired, samples=10, assert_budget=True): assert samples >= 10 repo = Path(__file__).resolve().parents[2] @@ -163,8 +203,11 @@ def paired_budgets(backend, location, user, keys, *, case, source_count, repaire process.stdin.write("stop\n") process.stdin.flush() process.communicate(timeout=30) + profile_failures = capture_read_profiles(backend, location, user, keys, case=case, source_count=source_count, + repaired=repaired, revisions=revisions, main=main, repo=repo, script=script) if assert_budget: assert not failures, failures + assert not profile_failures, profile_failures return failures diff --git a/tests/unit/test_derived_labels_mutations.py b/tests/unit/test_derived_labels_mutations.py index 9f56d7845..751514fe8 100644 --- a/tests/unit/test_derived_labels_mutations.py +++ b/tests/unit/test_derived_labels_mutations.py @@ -107,7 +107,7 @@ def test_derived_label_kernel_guard_mutations(): ) kill( labels, - "_dependency_record", + "_derived_dependency_record", "if not problem and not _record_present(meta, row) and not found:", "if not problem and not found:", checks.test_every_unverified_case_is_unverified, diff --git a/tests/unit/test_label_doctor_preparation.py b/tests/unit/test_label_doctor_preparation.py new file mode 100644 index 000000000..6f3ce3792 --- /dev/null +++ b/tests/unit/test_label_doctor_preparation.py @@ -0,0 +1,163 @@ +"""A full doctor scan keeps the canonical labels and guarded-read cache contract.""" +from __future__ import annotations + +import json + +import pytest + +from alicebot_api import vnext_derived_labels as kernel, vnext_label_repair as repair +from alicebot_api.vnext_derived_domain_backfill import DerivedDomainRepairError + + +def row(kind, stored, **fields): + return {"kind": kind, "id": stored, "user_id": "u", "domain": "project", "sensitivity": "public", + "metadata_json": {}, **fields} + + +def source(): + return row("source", "s", domain="health", sensitivity="confidential", metadata_json={"project_scope": ["P1"]}) + + +def report(stored, parents, **fields): + return row("artifact", stored, artifact_type="daily_brief", + metadata_json={"workflow": "daily_brief", "derived_from": {"artifacts": parents, + "counts": {"artifacts": len(parents)}}}, **fields) + + +def oscillating_ring(): + return [report("r0", ["r1"], domain="health"), report("r1", ["r2"], domain="legal"), + report("r2", ["r0"], domain="health")] + + +def cases(): + copy = row("memory", "r", metadata_json={"source_id": "s", "project_scope": ["P1", "P2"], "project_floor": ["P2"]}) + yield pytest.param([source(), copy], {}, "memory", "r", True, None, "confidential", id="copy-floor") + for metadata, reason in (({"source_id": "missing"}, "missing_dependency"), + ({"source_id": "s", "candidate_kind": []}, "malformed_marker"), + ({"source_id": "s", "project_floor": "P1"}, "malformed_floor"), + ({"derived_from": {"sources": ["s"], "counts": {"sources": 2}}}, "counts_disagree"), + ({"derived_from": {"sources": ["s"], "counts": {"sources": True}}}, "malformed")): + yield pytest.param([source(), row("memory", "r", metadata_json=metadata)], {}, + "memory", "r", True, reason, "public", id=reason) + yield pytest.param([source(), copy], {"unavailable_kinds": ["source"]}, + "memory", "r", True, "missing_table", "public", id="missing-table") + alias = "550e8400-e29b-41d4-a716-446655440000" + yield pytest.param([source(), {**copy, "id": alias}, {**copy, "id": alias.upper()}], {}, + "memory", alias, True, "ambiguous_identity", "public", id="uuid-alias") + backing = row("memory", "backing", metadata_json={"source_id": "s"}) + belief_report = row("artifact", "r", artifact_type="daily_brief", + metadata_json={"derived_from": {"beliefs": ["b"], "counts": {"beliefs": 1}}}) + yield pytest.param([source(), backing, row("belief", "b", memory_id="backing"), belief_report], {}, + "artifact", "r", True, None, "confidential", id="belief") + weekly = row("memory", "r", metadata_json={"discovered_by": "vnext_weekly_synthesis"}) + parent = row("artifact", "weekly", artifact_type="weekly_synthesis", metadata_json={ + "workflow": "weekly_synthesis", "candidate_memory_ids": ["r"], + "input_summary": {"source_ids": ["s"], "memory_ids": [], "open_loop_ids": [], "artifact_ids": [], + "counts": {"sources": 1, "memories": 0, "open_loops": 0, "artifacts": 0}}}) + yield pytest.param([source(), weekly, parent], {}, "memory", "r", True, None, "confidential", id="weekly-parent") + yield pytest.param([source(), row("memory", "r", value={"kind": "memory_consolidation", "source_id": "s"})], {}, + "memory", "r", False, None, "public", id="value-only-marker") + yield pytest.param([source(), row("memory", "r", value=json.dumps({"source_id": "s"}), + metadata_json={"consolidation": {}})], {}, + "memory", "r", True, None, "confidential", id="encoded-value-input") + yield pytest.param([source(), row("memory", "r", metadata_json=json.dumps({"source_id": "s"}))], {}, + "memory", "r", True, None, "confidential", id="encoded-metadata") + yield pytest.param([source(), row("memory", "r", user_id="other", metadata_json={"source_id": "s"})], {}, + "memory", "r", True, "missing_dependency", "public", id="tenant-separation") + yield pytest.param([source(), copy], {"max_nodes": 1}, "memory", "r", True, + "bound_exceeded", "public", id="node-bound") + tail = row("artifact", "tail", artifact_type="daily_brief", metadata_json={"source_id": "s"}) + yield pytest.param([source(), tail, report("mid", ["tail"]), report("r", ["mid"])], {"max_hops": 1}, + "artifact", "r", True, "bound_exceeded", "public", id="hop-bound") + yield pytest.param(oscillating_ring(), {}, "artifact", "r0", True, "cycle_unsettled", "public", id="cycle") + + +@pytest.mark.parametrize("nodes,options,kind,stored,derived,reason,sensitivity", list(cases())) +def test_one_pass_preserves_every_label_field_and_classification(monkeypatch, nodes, options, kind, stored, + derived, reason, sensitivity): + with kernel.label_metadata_cache({}): + cached = kernel.settle_labels(nodes, on_cycle="unverified", **options) + with kernel.label_metadata_cache({}): + single = kernel.settle_labels(nodes, on_cycle="unverified", one_pass=True, **options) + assert single.rows == cached.rows + label = single.by_stored(kind, stored) + assert (label.derived, label.reason, label.sensitivity) == (derived, reason, sensitivity) + assert label.unverified is (reason is not None) + if label.stored_id == "r" and label.stored_floor == ("P2",) and reason is None: + assert (label.project_scope, label.project_floor, label.domain) == (("P1",), ("P1", "P2"), "health") + tables = {} + table_names = {"source": "sources", "memory": "memories", "artifact": "generated_artifacts", "belief": "beliefs"} + for node in nodes: + tables.setdefault(table_names[node["kind"]], []).append(node) + actual = repair.classify_stored_labels(tables) + + def cached_settle(inputs, **kwargs): + return kernel.settle_labels(inputs, **{**kwargs, "one_pass": False}) + + monkeypatch.setattr(repair, "settle_labels", cached_settle) + assert actual == repair.classify_stored_labels(tables) + + +@pytest.mark.parametrize("one_pass", (False, True)) +def test_both_modes_keep_the_cycle_repair_abort(one_pass): + with pytest.raises(DerivedDomainRepairError, match="did not settle"): + kernel.settle_labels(oscillating_ring(), one_pass=one_pass) + + +def test_default_settlement_reuses_parsing_and_rechecks_a_changed_value(monkeypatch): + root = row("memory", "r", metadata_json={"source_id": "s"}) + original = kernel._derived_dependency_record + parsed = [] + + def traced(name, item): + parsed.append(item["id"]) + return original(name, item) + + monkeypatch.setattr(kernel, "_derived_dependency_record", traced) + with kernel.label_metadata_cache({}): + assert kernel.dependency_record("memory", root) == (frozenset({("source", "s")}), "") + for _ in range(2): + assert kernel.settle_labels([source(), root]).by_stored("memory", "r").sensitivity == "confidential" + assert parsed == ["r"] + root["value"] = {"source_id": "missing"} + assert kernel.settle_labels([source(), root]).by_stored("memory", "r").reason == "missing_dependency" + assert parsed == ["r", "r"] + + +def test_only_full_classification_opts_into_one_pass(monkeypatch): + tables = {"sources": [source()], "memories": [row("memory", "r", metadata_json={"source_id": "s"})]} + original = repair.settle_labels + options = [] + + def traced(nodes, **kwargs): + options.append(kwargs.get("one_pass", False)) + return original(nodes, **kwargs) + + monkeypatch.setattr(repair, "settle_labels", traced) + below, unverified = repair.classify_stored_labels(tables) + planned = repair.plan_label_repairs(tables) + assert options == [True, False] + assert below == planned + assert len(below) == 1 and below[0][2] == "r" and unverified == {} + + +@pytest.mark.parametrize("max_hops,max_nodes,exceeded", ((0, None, {"r", "mid"}), (1, None, {"r"}), + (2, None, set()), (None, 0, {"r", "mid"}), + (None, 1, {"r", "mid"}), (None, 2, {"r"}), + (None, 3, set()))) +def test_shared_bounds_keep_original_selection_order_and_existing_problems(max_hops, max_nodes, exceeded): + nodes = [source(), row("memory", "mid", metadata_json={"source_id": "s"}), + row("memory", "r", metadata_json={"consolidation": {"cluster_member_ids": ["mid"]}}), + row("memory", "skip", metadata_json={"source_id": "missing"})] + labels = {label.key: label for node in nodes for label in [kernel._node_label(node["kind"], node)]} + key = lambda stored: ("source" if stored == "s" else "memory", "u", stored) + resolved = {key("mid"): {key("s")}, key("r"): {key("mid")}, key("skip"): {key("missing")}} + initial = {key("skip"): "missing_dependency"} + wrapped, shared = dict(initial), dict(initial) + kernel._mark_bounds(labels, resolved, wrapped, max_hops=max_hops, max_nodes=max_nodes) + kernel._mark_dependency_bounds((origin for origin, label in labels.items() if label.derived), resolved, shared, + max_hops=max_hops, max_nodes=max_nodes) + expected = {**initial, **{key(stored): "bound_exceeded" for stored in exceeded}} + assert wrapped == shared == expected + assert list(wrapped) == list(shared) + assert key("s") not in wrapped diff --git a/tests/unit/test_label_native_rank_admission.py b/tests/unit/test_label_native_rank_admission.py new file mode 100644 index 000000000..92f822b34 --- /dev/null +++ b/tests/unit/test_label_native_rank_admission.py @@ -0,0 +1,340 @@ +"""Reduced native admission has an independent complete-label oracle.""" +from copy import deepcopy +from dataclasses import replace +from itertools import product +import json +import random +from types import SimpleNamespace +from uuid import UUID + +import pytest + +from alicebot_api import vnext_label_guard as guards +from alicebot_api.vnext_derived_labels import SENSITIVITY_RANK, SettledLabel, identifier, settle_labels, with_derived_from +from alicebot_api.vnext_label_writes import label_savepoint_rolled_back, takes_label_lock + + +class NativeRows: + label_count_canonical_unique_ids = True + + def __init__(self, rows, roots): + self.rows, self.roots = rows, roots + self.conn = SimpleNamespace() + self.locks = 0 + + def read_label_rows(self, kind, ids): + return [dict(row) for row in self.rows if row["kind"] == kind and identifier(row["id"]) in ids] + + def count_original_label_statuses(self, *args, **kwargs): + return {} + + def iter_label_rows(self, kind, **kwargs): + yield self.roots + + def lock_label_writes(self, **kwargs): + self.locks += 1 + + @takes_label_lock + def update_sensitivity(self, source, sensitivity): + source["sensitivity"] = sensitivity + + +def source_and_root(sensitivity="public"): + source = {"kind": "source", "id": UUID(int=1), "user_id": "label-guard", "domain": "project", + "sensitivity": sensitivity, "metadata_json": {"project_scope": [], "project_floor": []}} + hidden = {**deepcopy(source), "id": UUID(int=3), "sensitivity": "confidential"} + root = {"kind": "memory", "id": UUID(int=0xABCDEF), "user_id": "label-guard", "domain": "project", + "sensitivity": "public", "status": "active", "value": {}, + "metadata_json": {"source_id": str(source["id"]), "project_scope": [], "project_floor": []}} + return NativeRows([source, hidden, root], [root]), source, hidden, root + + +def mixed_rows(): + rng = random.Random(20261007) + sources = [{"kind": "source", "id": UUID(int=i + 1), "user_id": "label-guard", "domain": ("health", "legal", "project")[i % 3], + "sensitivity": "confidential" if i % 2 else "public", + "metadata_json": {"project_scope": [], "project_floor": []}} for i in range(8)] + roots = [] + for i in range(32): + parents = rng.sample(sources + roots, 2) + metadata = with_derived_from({"project_scope": [], "project_floor": [], "observation_index": i}, + {"sources": [row for row in parents if row["kind"] == "source"], + "memories": [row for row in parents if row["kind"] == "memory"]}) + metadata[("workflow", "candidate_kind", "discovered_by")[i % 3]] = ( + "project_auto_update", "memory_consolidation", "vnext_weekly_synthesis")[i % 3] + roots.append({"kind": "memory", "id": UUID(int=100 + i), "user_id": "label-guard", "domain": "project", + "sensitivity": "public", "status": "active", "value": {}, "metadata_json": metadata}) + return sources, roots + + +def full_labels(rows): + return settle_labels([{**row, "user_id": "label-guard"} for row in rows], on_cycle="unverified", + max_nodes=guards.NODE_BOUND, max_hops=guards.HOP_BOUND) + + +def assert_only_full_labels(state): + assert all(isinstance(value, SettledLabel) for value in state.labels.values()) + assert all(isinstance(entry[1], SettledLabel) for entry in state.native_labels.values()) + + +def test_native_rank_dag_matches_complete_kernel_and_keeps_full_projections(): + sources, roots = mixed_rows() + expected = full_labels(sources + roots) + by_id = {label.normalized_id: label for label in expected.rows if label.kind == "memory"} + visible = {row["id"] for row in roots if not by_id[str(row["id"])].unverified and by_id[str(row["id"])].sensitivity == "public"} + store = NativeRows(sources + roots, roots) + with guards.label_read_scope(store): + guard = guards.LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard.readable_status_counts("memory") == ({"active": len(visible)} if visible else {}) + assert {row["id"] for row in guard.admit_rows("memory", roots)} == visible + state = guard._state() + assert len(state.rank_origins) == len(roots) + assert not state.labels and not state.native_labels + for row in roots: + label = by_id[str(row["id"])] + assert state.rank_origins[("memory", str(row["id"]))][1] == SENSITIVITY_RANK[label.sensitivity] + effective = guard.effective_row("memory", row) + assert (effective["domain"], effective["sensitivity"], tuple(effective["project_scope"]), + tuple(effective["project_floor"]), effective["unverified"]) == ( + label.domain, label.sensitivity, label.project_scope, label.project_floor, label.unverified) + assert_only_full_labels(state) + + +CEILINGS = [tuple(value for value, rank in SENSITIVITY_RANK.items() if rank <= high) for high in range(1, 7)] +CEILINGS += [("public", "internal"), ("public", "unknown"), ("public", "confidential"), + ("public", "internal", "unknown", "private", "confidential", "highly_sensitive", "sacred"), + ("public", "internal", "unknown", "private", "confidential", "highly_sensitive", "regulated"), + ("invalid",), ()] + + +@pytest.mark.parametrize("sensitivity,allowed", list(product((*SENSITIVITY_RANK, "invalid"), CEILINGS))) +def test_all_rank_boundaries_and_split_allowances_match_full_labels(sensitivity, allowed): + store, source, _hidden, root = source_and_root(sensitivity) + label = full_labels(store.rows).by_stored("memory", str(root["id"]), user_id="label-guard") + expected = not allowed or label.sensitivity in allowed + with guards.label_read_scope(store): + guard = guards.LabelGuard(store, active=True, sensitivity_allowed=allowed) + assert guard.readable_status_counts("memory") == ({"active": 1} if expected else {}) + assert bool(guard.admit_rows("memory", [root])) == expected + if guard._rank_ceiling() is None or source["sensitivity"] == "invalid": + assert not guard._state().rank_origins + assert_only_full_labels(guard._state()) + + +@pytest.mark.parametrize("variant", ("missing", "malformed-counts", "malformed-marker", "floor-null", "scope-null", + "scoped", "cycle", "alias", "implicit-weekly", "implicit-weekly-json", "node-bound", "hop-bound")) +def test_unsupported_graphs_use_full_kernel_without_publishing_ranks(monkeypatch, variant): + sources, roots = mixed_rows() + rows = sources + roots + if variant == "missing": + rows = roots + elif variant == "malformed-counts": + roots[0]["metadata_json"]["derived_from"]["counts"]["sources"] += 1 + elif variant == "malformed-marker": + roots[0]["metadata_json"]["candidate_kind"] = [] + elif variant == "floor-null": + roots[0]["metadata_json"]["project_floor"] = None + elif variant == "scope-null": + roots[0]["metadata_json"]["project_scope"] = None + elif variant == "scoped": + sources[0]["metadata_json"]["project_scope"] = ["P1"] + elif variant == "cycle": + roots[0]["metadata_json"] = with_derived_from({}, {"memories": [roots[1]]}) + roots[1]["metadata_json"] = with_derived_from({}, {"memories": [roots[0]], "sources": [sources[1]]}) + elif variant == "alias": + rows.append({**deepcopy(sources[0]), "id": "urn:uuid:" + str(sources[0]["id"])}) + elif variant.startswith("implicit-weekly"): + artifact = {"kind": "artifact", "id": UUID(int=400), "user_id": "label-guard", "domain": "project", + "sensitivity": "public", "artifact_type": "weekly_synthesis", + "metadata_json": {"candidate_memory_ids": [str(roots[0]["id"])], + "input_summary": {"source_ids": [str(sources[1]["id"])]}}} + roots[0]["metadata_json"] = {"discovered_by": "vnext_weekly_synthesis", "source_artifact_id": str(artifact["id"])} + if variant.endswith("-json"): + artifact["metadata_json"] = json.dumps(artifact["metadata_json"]) + rows.append(artifact) + elif variant == "node-bound": + monkeypatch.setattr(guards, "NODE_BOUND", 3) + elif variant == "hop-bound": + monkeypatch.setattr(guards, "HOP_BOUND", 1) + expected = full_labels(rows) + labels = {label.normalized_id: label for label in expected.rows if label.kind == "memory"} + visible = {row["id"] for row in roots if not labels[str(row["id"])].unverified and labels[str(row["id"])].sensitivity == "public"} + full_calls = [] + def counted_full(*args, **kwargs): + full_calls.append(True) + return settle_labels(*args, **kwargs) + monkeypatch.setattr(guards, "settle_labels", counted_full) + store = NativeRows(rows, roots) + with guards.label_read_scope(store): + guard = guards.LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard.readable_status_counts("memory") == ({"active": len(visible)} if visible else {}) + assert {row["id"] for row in guard.admit_rows("memory", roots)} == visible + assert not guard._state().rank_origins and not guard._state().rank_rows + assert full_calls + assert_only_full_labels(guard._state()) + + +@pytest.mark.parametrize("variant", ("same", "scope-null", "floor-null", "legacy-project", "legacy-scope-json", + "nested-agent", "metadata-json-string", "missing-scope-floor", "value-hidden-parent", "source-hidden-parent", + "source-equivalent-uuid-spelling", "row-uuid-alias", "domain-changed", "sensitivity-changed", "missing-user", "count-disagreement")) +def test_reloaded_projections_require_supported_shape_and_canonical_signature(variant): + store, source, hidden, root = source_and_root() + with guards.label_read_scope(store): + guard = guards.LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard.readable_status_counts("memory") == {"active": 1} + reloaded = deepcopy(root) + if variant == "scope-null": + reloaded["metadata_json"]["project_scope"] = None + elif variant == "floor-null": + reloaded["project_floor"] = None + elif variant == "legacy-project": + reloaded["project_id"] = "P1" + elif variant == "legacy-scope-json": + reloaded["scope_json"] = {"project_scope": []} + elif variant == "nested-agent": + reloaded["metadata_json"]["agentic_memory"] = {"project_scope": []} + elif variant == "metadata-json-string": + reloaded["metadata_json"] = json.dumps(reloaded["metadata_json"]) + elif variant == "missing-scope-floor": + reloaded["metadata_json"].pop("project_scope") + reloaded["metadata_json"].pop("project_floor") + elif variant == "value-hidden-parent": + reloaded["value"] = {"source_id": str(hidden["id"])} + elif variant == "source-hidden-parent": + reloaded["metadata_json"]["source_id"] = str(hidden["id"]) + elif variant == "source-equivalent-uuid-spelling": + reloaded["metadata_json"]["source_id"] = "SOURCE:" + str(source["id"]).upper() + elif variant == "row-uuid-alias": + reloaded["id"] = "urn:uuid:" + str(root["id"]) + elif variant == "domain-changed": + reloaded["domain"] = "health" + elif variant == "sensitivity-changed": + reloaded["sensitivity"] = "confidential" + elif variant == "missing-user": + reloaded.pop("user_id") + elif variant == "count-disagreement": + reloaded["metadata_json"] = with_derived_from(reloaded["metadata_json"], {"sources": [source]}) + reloaded["metadata_json"]["derived_from"]["counts"]["sources"] += 1 + rank = guard._native_rank_for_projection("memory", reloaded) + if variant not in ("same", "missing-scope-floor", "source-equivalent-uuid-spelling"): + assert rank is None + for allowed in (("public",), ("public", "internal", "unknown", "private"), ("public", "confidential")): + independent = guards.LabelGuard(NativeRows(store.rows, [reloaded]), active=True, sensitivity_allowed=allowed) + expected = bool(independent.admit_rows("memory", [reloaded])) + assert bool(replace(guard, sensitivity_allowed=allowed).admit_rows("memory", [reloaded])) == expected + assert_only_full_labels(guard._state()) + + +@pytest.mark.parametrize("source_ref", ("canonical", "prefixed", "encoded")) +def test_source_kind_and_source_grammar_stay_canonical(source_ref): + store, source, hidden, root = source_and_root() + hidden["id"] = UUID("123e4567-e89b-42d3-a456-426614174000") + source["metadata_json"]["source_id"] = str(UUID(int=9999)) + hidden["metadata_json"]["source_id"] = str(UUID(int=9999)) + ref = str(hidden["id"]) + if source_ref == "prefixed": + ref = "SOURCE:" + ref.upper() + elif source_ref == "encoded": + escaped = "".join("\\u%04x" % ord(character) for character in ref) + ref = '{"source_id":"' + escaped + '"}' + root["metadata_json"] = {"source_refs": [ref], "derived_from": {"v": 1, "sources": [], "counts": {}}, + "project_scope": [], "project_floor": []} + label = full_labels(store.rows).by_stored("memory", str(root["id"]), user_id="label-guard") + with guards.label_read_scope(store): + guard = guards.LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard.readable_status_counts("memory") == {} + assert label.sensitivity == "confidential" and not label.unverified + assert len(guard._state().rank_origins) == 1 + assert guard.admit_rows("memory", [root]) == [] + + +def test_event_spelling_filter_grants_and_writer_rollback_invalidation(): + store, source, _hidden, root = source_and_root() + canonical = {"target_type": "memory", "target_id": str(root["id"]), "event_type": "memory.created"} + alias = {**canonical, "target_id": str(root["id"]).upper()} + source_event = {"target_type": "source", "target_id": str(source["id"]), "event_type": "source.created"} + with guards.label_read_scope(store): + guard = guards.LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard.readable_status_counts("memory") == {"active": 1} + assert guard.admit_events([canonical, alias]) == [canonical] + assert guard.admit_events([source_event]) == [source_event] + assert replace(guard, domains=("health",)).admit_events([canonical]) == [] + assert replace(guard, projects=("P1",), all_of=("P1",)).admit_events([canonical]) == [] + assert replace(guard, sensitivity_allowed=("public", "internal", "unknown", "private")).admit_events([canonical]) == [canonical] + assert len(guard._state().rank_admission) == 2 + store.update_sensitivity(source, "confidential") + assert store.locks == 1 + assert not guard._state().rank_rows and not guard._state().rank_origins and not guard._state().rank_admission + assert guard.admit_events([canonical]) == [] + source["sensitivity"] = "public" + label_savepoint_rolled_back(store) + assert store.conn._alice_label_rollback_counter == 1 + assert not guard._state().rank_rows and not guard._state().rank_origins and not guard._state().rank_admission + assert guard.readable_status_counts("memory") == {"active": 1} + assert guard.admit_events([canonical]) == [canonical] + assert_only_full_labels(guard._state()) + + +@pytest.mark.parametrize("variant,expected_rank_walks", (("acyclic", 1), ("node-bound", 1), ("cycle", 0))) +def test_cyclic_fallback_skips_the_extra_rank_bound_walk(monkeypatch, variant, expected_rank_walks): + store, _source, _hidden, root = source_and_root() + if variant == "cycle": + second = {**deepcopy(root), "id": UUID(int=100)} + root["metadata_json"] = with_derived_from({}, {"memories": [second]}) + second["metadata_json"] = with_derived_from({}, {"memories": [root]}) + store.rows.append(second) + store.roots.append(second) + elif variant == "node-bound": + monkeypatch.setattr(guards, "NODE_BOUND", 1) + labels = full_labels(store.rows) + visible = sum(not labels.by_stored("memory", str(row["id"])).unverified + and labels.by_stored("memory", str(row["id"])).sensitivity == "public" for row in store.roots) + original = guards._mark_dependency_bounds + walks = [] + + def counted(*args, **kwargs): + walks.append(True) + return original(*args, **kwargs) + + monkeypatch.setattr(guards, "_mark_dependency_bounds", counted) + with guards.label_read_scope(store): + guard = guards.LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard.readable_status_counts("memory") == ({"active": visible} if visible else {}) + assert len(walks) == expected_rank_walks + if variant != "acyclic": + assert not guard._state().rank_origins + assert_only_full_labels(guard._state()) + + +def test_rank_proofs_require_native_locked_store_and_active_guard(): + store, _source, _hidden, root = source_and_root() + guard = guards.LabelGuard(store, active=True, sensitivity_allowed=("public",)) + assert guard._rank_ceiling() is None + with guards.label_read_scope(store): + assert replace(guard, active=False)._rank_ceiling() is None + store.label_count_canonical_unique_ids = False + # The capability belongs to the store type, not a mutable instance. + assert guard._rank_ceiling() == 1 + class TextRows(NativeRows): + label_count_canonical_unique_ids = False + other = TextRows(store.rows, [root]) + with guards.label_read_scope(other): + assert guards.LabelGuard(other, active=True, sensitivity_allowed=("public",))._rank_ceiling() is None + + +def test_same_origin_keeps_counts_and_event_grants_separate_across_rank_ceilings(): + store, _source, _hidden, root = source_and_root("private") + canonical = {"target_type": "memory", "target_id": str(root["id"]), "event_type": "memory.created"} + alias = {**canonical, "target_id": str(root["id"]).upper()} + label = full_labels(store.rows).by_stored("memory", str(root["id"]), user_id="label-guard") + assert not label.unverified and label.sensitivity == "private" + with guards.label_read_scope(store): + public = guards.LabelGuard(store, active=True, sensitivity_allowed=("public",)) + private = replace(public, sensitivity_allowed=("public", "internal", "unknown", "private")) + for caller in (public, private, public): + readable = label.sensitivity in caller.sensitivity_allowed + assert caller.readable_status_counts("memory") == ({"active": 1} if readable else {}) + assert caller.admit_rows("memory", [deepcopy(root)]) == ([root] if readable else []) + assert caller.admit_events([canonical, alias]) == ([canonical] if readable else []) + assert len(public._state().rank_origins) == 1 + assert_only_full_labels(public._state()) diff --git a/tests/unit/test_round3_budget_profile_protocol.py b/tests/unit/test_round3_budget_profile_protocol.py new file mode 100644 index 000000000..d4770d58f --- /dev/null +++ b/tests/unit/test_round3_budget_profile_protocol.py @@ -0,0 +1,252 @@ +"""Diagnostics retain useful evidence without becoming benchmark samples.""" + +import io +import json +from pathlib import Path +import pstats +import subprocess +from types import SimpleNamespace +import threading + +import pytest +import yaml + +from tests.performance import round3_budget_probe as probe +from tests.performance import test_label_round3_read_budget as budgets + + +REVISIONS = {"main": "a" * 40, "head": "b" * 40} +KEYS = {"trusted_local_agent": "synthetic-credential", "admin_agent": "synthetic-credential"} + + +def fake_children(monkeypatch, tmp_path, *, gate=None, capture_status="complete"): + events = [] + evidence = tmp_path / "samples.jsonl" + monkeypatch.setenv("ALICE_READ_MAIN_CHECKOUT", "paired-main") + monkeypatch.setenv("ALICE_READ_BUDGET_EVIDENCE", str(evidence)) + monkeypatch.setattr(budgets.subprocess, "check_output", lambda args, **kwargs: REVISIONS["main" if args[2] == "paired-main" else "head"]) + + class Child: + def __init__(self, args, **kwargs): + self.revision = "main" if args[2] == "paired-main" else "head" + self.profile = args[-2] + self.commands = [] + self.stdin = self.stdout = self.stderr = self + self.stopped = False + + def write(self, command): + action = command.strip() + events.append(("measurement", self.profile, self.revision, action)) + self.commands.append(action) + self.stopped = action == "stop" + + def flush(self): + pass + + def readline(self): + action = self.commands.pop(0) + wall = cpu = .1 + if self.revision == "head": + if gate == "absolute" and action == "workspace": + wall = 1.01 + if gate == "relative" and action == "pack": + wall = cpu = .31 + return json.dumps({"wall": wall, "cpu": cpu, "memories": 7}) + "\n" + + def read(self): + return "" + + def poll(self): + return 0 if self.stopped else None + + def communicate(self, **kwargs): + return "", "" + + def diagnostic(args, *, input, env, **kwargs): + # Both profiles and all four actions were retained before any capture. + assert len(evidence.read_text().splitlines()) == 2 + assert sum(event[-1] == "stop" for event in events if event[0] == "measurement") == 4 + assert input == "workspace\nprofile:workspace\ndogfooding\nprofile:dogfooding\nstop\n" + context = json.loads(env["ALICE_READ_PROFILE_CONTEXT"]) + events.append(("diagnostic", context["profile"], context["revision_label"], context["case"])) + captures = [{**context, "diagnostic": True, "action": action, "status": capture_status, + "wall": 999, "cpu": 999} for action in ("workspace", "dogfooding")] + return SimpleNamespace(returncode=0, stdout="\n".join(json.dumps(item) for item in captures), stderr="sensitive-test-detail") + + monkeypatch.setattr(budgets.subprocess, "Popen", Child) + monkeypatch.setattr(budgets.subprocess, "run", diagnostic) + return evidence, events + + +@pytest.mark.parametrize("gate", [None, "absolute", "relative"]) +def test_diagnostics_do_not_change_samples_or_budget_verdict(monkeypatch, tmp_path, gate): + evidence, events = fake_children(monkeypatch, tmp_path, gate=gate) + rows = [] + for profiling in (False, True): + evidence.unlink(missing_ok=True) + events.clear() + if profiling: + monkeypatch.setenv("ALICE_READ_PROFILE", str(tmp_path / "profiles")) + else: + monkeypatch.delenv("ALICE_READ_PROFILE", raising=False) + if gate: + with pytest.raises(AssertionError) as caught: + budgets.paired_budgets("postgres", "synthetic-database", budgets.USER, KEYS, + case="all-visible", source_count=3000, repaired=False) + message = str(caught.value) + assert ("workspace" if gate == "absolute" else "pack") in message + assert "minimum_wall" in message and "captures" not in message + else: + assert budgets.paired_budgets("postgres", "synthetic-database", budgets.USER, KEYS, + case="all-visible", source_count=3000, repaired=False) == [] + rows.append([json.loads(line) for line in evidence.read_text().splitlines()]) + measured = [event for event in events if event[0] == "measurement"] + expected = [] + for profile in KEYS: + for action in ("pack", "recall", "workspace", "dogfooding"): + expected.extend(("measurement", profile, revision, action) for revision in ("main", "head")) + for sample in range(10): + expected.extend(("measurement", profile, revision, action) + for revision in (("main", "head") if sample % 2 == 0 else ("head", "main"))) + expected.extend(("measurement", profile, revision, "stop") for revision in ("main", "head")) + assert measured == expected + diagnostics = [event for event in events if event[0] == "diagnostic"] + assert len(diagnostics) == (4 if profiling else 0) + if profiling: + assert all(event[0] == "measurement" for event in events[:-4]) + assert rows[0] == rows[1] + for row in rows[1]: + for revision in REVISIONS: + for action in ("pack", "recall", "workspace", "dogfooding"): + assert len(row[revision][action]["wall"]) == len(row[revision][action]["cpu"]) == 10 + assert 999 not in row[revision][action]["wall"] + + +def test_failed_budget_keeps_samples_and_incomplete_capture_receipts(monkeypatch, tmp_path): + evidence, _events = fake_children(monkeypatch, tmp_path, gate="absolute", capture_status="incomplete") + monkeypatch.setenv("ALICE_READ_PROFILE", str(tmp_path / "profiles")) + with pytest.raises(AssertionError) as caught: + budgets.paired_budgets("postgres", "synthetic-database", budgets.USER, KEYS, + case="extra-rows", source_count=3000, repaired=True) + assert "workspace" in str(caught.value) and "captures" not in str(caught.value) + assert len(evidence.read_text().splitlines()) == 2 + summaries = list((tmp_path / "profiles").rglob("capture-summary.json")) + assert len(summaries) == 4 + for path in summaries: + receipt = json.loads(path.read_text()) + assert receipt["status"] == "incomplete" + assert len(receipt["captures"]) == 2 + assert "synthetic-credential" not in path.read_text() + assert "sensitive-test-detail" not in path.read_text() + + +def test_profile_cases_have_distinct_directories_and_incomplete_captures_fail(monkeypatch, tmp_path): + evidence, events = fake_children(monkeypatch, tmp_path, capture_status="incomplete") + monkeypatch.setenv("ALICE_READ_PROFILE", str(tmp_path / "profiles")) + for case, repaired in (("all-visible", False), ("extra-rows", True)): + evidence.unlink(missing_ok=True) + events.clear() + with pytest.raises(AssertionError) as caught: + budgets.paired_budgets("postgres", "synthetic-database", budgets.USER, KEYS, + case=case, source_count=3000, repaired=repaired) + assert "captures" in str(caught.value) + summaries = list((tmp_path / "profiles").rglob("capture-summary.json")) + assert len(summaries) == 8 + assert all(json.loads(path.read_text())["status"] == "incomplete" for path in summaries) + assert {path.parent.parent.name for path in summaries} == set(KEYS) + assert {path.parent.name for path in summaries} == set(REVISIONS) + assert {path.parent.parent.parent.name for path in summaries} == { + "all-visible-3000-unrepaired", "extra-rows-3000-repaired", + } + + +@pytest.mark.parametrize("failure", ["exit", "timeout"]) +def test_diagnostic_process_failure_is_retained_without_sensitive_output(monkeypatch, tmp_path, failure): + evidence, _events = fake_children(monkeypatch, tmp_path) + monkeypatch.setenv("ALICE_READ_PROFILE", str(tmp_path / "profiles")) + + def broken(args, **kwargs): + if failure == "timeout": + raise subprocess.TimeoutExpired(args, 60, output="sensitive-test-detail", stderr="synthetic-credential") + return SimpleNamespace(returncode=3, stdout="sensitive-test-detail", stderr="synthetic-credential") + + monkeypatch.setattr(budgets.subprocess, "run", broken) + with pytest.raises(AssertionError): + budgets.paired_budgets("postgres", "synthetic-database", budgets.USER, KEYS, + case="all-visible", source_count=3000, repaired=False) + assert len(evidence.read_text().splitlines()) == 2 + summaries = list((tmp_path / "profiles").rglob("capture-summary.json")) + assert len(summaries) == 4 + for path in summaries: + assert json.loads(path.read_text())["status"] == "error" + assert "synthetic-credential" not in path.read_text() + assert "sensitive-test-detail" not in path.read_text() + + +@pytest.mark.parametrize("name", ["workspace", "dogfooding"]) +@pytest.mark.parametrize("capture", ["complete", "incomplete", "error"]) +def test_explicit_profile_protocol_and_capture_completeness(tmp_path, name, capture): + def _vnext_workspace_payload(): + return {} + + def get_vnext_workspace(): + return _vnext_workspace_payload() + + def dashboard(): + return {} + + def get_vnext_dogfooding_dashboard(): + return dashboard() + + def incomplete(): + return {} + + def failed(): + raise ValueError("sensitive-test-detail") + + def worker(): + thread = threading.Thread(target=get_vnext_workspace if name == "workspace" else get_vnext_dogfooding_dashboard) + thread.start() + thread.join() + return {} + + action = {"complete": worker, "incomplete": incomplete, "error": failed}[capture] + output = io.StringIO() + trace = tmp_path / "capture" + probe.serve({name: incomplete}, [name + "\n", "stop\n"], output, trace=trace) + assert len(json.loads(output.getvalue())) == 2 + assert not trace.exists(), "normal commands must never profile the warmup or gate samples" + output = io.StringIO() + probe.serve({name: action}, ["profile:" + name + "\n", "stop\n"], output, + trace=trace, profile_context=lambda: {"revision": REVISIONS["head"]}) + receipt = json.loads(output.getvalue()) + assert receipt["diagnostic"] is True + assert receipt["status"] == capture + assert "wall" not in receipt and "cpu" not in receipt + assert receipt["runtime"]["python"] + assert receipt["revision"] == REVISIONS["head"] + assert receipt["missing_functions"] == ([] if capture == "complete" else sorted(probe.EXPECTED_CAPTURE_FUNCTIONS[name])) + assert json.loads((trace / (name + ".json")).read_text()) == receipt + assert "sensitive-test-detail" not in output.getvalue() + stats = pstats.Stats(str(trace / (name + ".prof"))) + assert all("/" not in key[0] for key in stats.stats) + report = (trace / (name + ".txt")).read_text() + assert "Ordered by: cumulative time" in report + assert "Ordered by: internal time" in report + assert str(tmp_path) not in report + + +def test_ci_profiles_only_the_maximum_smoke_and_retains_attempts(): + workflow = yaml.safe_load((Path(__file__).resolve().parents[2] / ".github/workflows/tests.yml").read_text()) + steps = workflow["jobs"]["python-integration"]["steps"] + profile_steps = [step for step in steps if "ALICE_READ_PROFILE" in step.get("env", {})] + assert len(profile_steps) == 1 + assert profile_steps[0]["name"] == "Maximum-workload read-budget controls" + assert "[False-3000-all-visible]" in profile_steps[0]["run"] + assert "[True-3000-extra-rows]" in profile_steps[0]["run"] + upload = next(step for step in steps if step.get("name") == "Retain paired read-budget samples, including failed runs") + assert upload["if"].startswith("always()") + assert "${{ github.run_attempt }}" in upload["with"]["name"] + assert "round3-read-profiles/" in upload["with"]["path"] + assert "round3-postgres-read-budgets.jsonl" in upload["with"]["path"] + assert "round3-sqlite-read-budgets.jsonl" in upload["with"]["path"] From e58fb932362d0edcc04f51accdae123a71bcee43 Mon Sep 17 00:00:00 2001 From: Sami Rusani <samrusani@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:23:12 +0200 Subject: [PATCH 270/270] Check retained profile receipts without pytest error formatting --- tests/unit/test_round3_budget_profile_protocol.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_round3_budget_profile_protocol.py b/tests/unit/test_round3_budget_profile_protocol.py index d4770d58f..d6d6eec73 100644 --- a/tests/unit/test_round3_budget_profile_protocol.py +++ b/tests/unit/test_round3_budget_profile_protocol.py @@ -146,13 +146,14 @@ def test_profile_cases_have_distinct_directories_and_incomplete_captures_fail(mo for case, repaired in (("all-visible", False), ("extra-rows", True)): evidence.unlink(missing_ok=True) events.clear() - with pytest.raises(AssertionError) as caught: + with pytest.raises(AssertionError): budgets.paired_budgets("postgres", "synthetic-database", budgets.USER, KEYS, case=case, source_count=3000, repaired=repaired) - assert "captures" in str(caught.value) + assert len(evidence.read_text().splitlines()) == 2 summaries = list((tmp_path / "profiles").rglob("capture-summary.json")) assert len(summaries) == 8 assert all(json.loads(path.read_text())["status"] == "incomplete" for path in summaries) + assert all(len(json.loads(path.read_text())["captures"]) == 2 for path in summaries) assert {path.parent.parent.name for path in summaries} == set(KEYS) assert {path.parent.name for path in summaries} == set(REVISIONS) assert {path.parent.parent.parent.name for path in summaries} == {