From e31ce3bcfa9794366592b44388e683507a1411a9 Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Sat, 31 Oct 2026 14:41:52 +0800 Subject: [PATCH 1/8] feat(storage): encode semantic directories once Generated-by: Codex --- Cargo.lock | 1 + Cargo.toml | 1 + platform/event-log/Cargo.toml | 1 + platform/event-log/README.md | 4 + platform/event-log/README.zh-CN.md | 4 + .../event-log/examples/event_storage_bench.rs | 681 ++++++++++++++++++ .../0013_semantic_directory_bytes.sql | 26 + .../event-log/src/append/ingress/tests.rs | 3 + platform/event-log/src/connection.rs | 26 + platform/event-log/src/context/semantic.rs | 1 + .../context/semantic/captured/directory.rs | 268 ++++--- .../src/context/semantic/directory.rs | 3 +- .../src/context/semantic/directory/codec.rs | 38 + .../context/semantic/directory/codec/tests.rs | 314 ++++++++ .../src/context/semantic/migration.rs | 216 ++++++ .../event-log/src/context/semantic/write.rs | 8 +- platform/event-log/src/schema.rs | 17 +- platform/event-log/src/schema/fixtures.rs | 5 + platform/event-log/src/sqlite_functions.rs | 46 +- .../event-log/tests/integration/startup.rs | 2 +- .../facts/src/history/semantic/directory.rs | 4 + scripts/bench-event-storage.py | 144 ++++ 22 files changed, 1674 insertions(+), 139 deletions(-) create mode 100644 platform/event-log/examples/event_storage_bench.rs create mode 100644 platform/event-log/migrations/0013_semantic_directory_bytes.sql create mode 100644 platform/event-log/src/context/semantic/directory/codec.rs create mode 100644 platform/event-log/src/context/semantic/directory/codec/tests.rs create mode 100644 platform/event-log/src/context/semantic/migration.rs create mode 100644 scripts/bench-event-storage.py diff --git a/Cargo.lock b/Cargo.lock index 4229815ecb..cc0a5dc5a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7773,6 +7773,7 @@ dependencies = [ "maka-plugin-core", "maka-presentation", "maka-sandbox", + "postcard", "regex", "rusqlite", "schemars 1.2.3", diff --git a/Cargo.toml b/Cargo.toml index cd4ba81846..5eadf079d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,6 +92,7 @@ publish = false [workspace.dependencies] mxc-sdk = { git = "https://github.com/microsoft/mxc", rev = "05a430bc17a3e1b3fa2131524fc52e4319e07298" } flatbuffers = "25.12.19" +postcard = { version = "1.1.3", default-features = false, features = ["alloc"] } maka-python-runtime = { path = "platform/python-runtime" } maka-mcp = { path = "plugins/agent/mcp" } hex = "0.4.3" diff --git a/platform/event-log/Cargo.toml b/platform/event-log/Cargo.toml index 48fbb1d0c8..e30bc7c7b3 100644 --- a/platform/event-log/Cargo.toml +++ b/platform/event-log/Cargo.toml @@ -25,6 +25,7 @@ autotests = false [dependencies] hex.workspace = true +postcard.workspace = true maka-agent-facts.workspace = true maka-packages.workspace = true maka-agent-ports.workspace = true diff --git a/platform/event-log/README.md b/platform/event-log/README.md index 2256d78048..21c64efc0f 100644 --- a/platform/event-log/README.md +++ b/platform/event-log/README.md @@ -71,6 +71,10 @@ Canonical runtime-event ingress inserts each event, captures its sequence, then Canonical databases use UTF-8. Semantic fragments for successful tool results reference checked byte spans in their original event JSON. Resolution reads and decodes the selected projection on the original SQLite worker within the caller's read snapshot; the source collector retains it for live history owners. Derived archive placeholders and other fragment bodies keep their chunked storage. +Semantic row, payload and child directories are finite derived descriptors encoded with Postcard. Writers encode them once in the publication transaction. Pages copy their stored bytes alongside fixed little-endian locator fields into the caller's charged allocation; consumers decode the original typed descriptors without navigating whole bodies. Original byte positions and leaf commitments remain unchanged. Directory encoding never supplies a canonical fact or source identity. The SQL migration converts former JSON descriptors atomically; its conversion functions exist only during migration. + +`event_storage_bench` exercises real Root writes, captured directories, child pages, first/tail windows, sequential bodies, event prefixes and bundle export. Build the same example in release against both revisions and retain each binary, then run `python3 scripts/bench-event-storage.py --output /tmp/storage-comparison original=/path/to/original candidate=/path/to/candidate`. It alternates at least five runs and validates event, descriptor, body and export digests. The first read pass validates and warms caches; reopened measurements are OS-cache-warm. Raw distributions, process CPU/RSS/I/O availability, SQLite cache counters and separate main/WAL/SHM file sizes are recorded. Live prepared-statement snapshots do not represent total SQL work, and mixed projection timings require per-kind follow-up when evaluating individual-path regressions. + A statement failure rolls back the entire group unless it is a domain rejection isolated by its own savepoint. Former ordinary writer requests can continue recovery on the same owner after acknowledged rollback and fresh authority validation; pure observation batches retain their stricter retirement policy. Failed commits always return unknown outcomes without execution claims. Reusing a former ordinary writer group after a failed commit additionally requires acknowledged cleanup and zero transaction depth. Savepoint, authority and unconfirmed cleanup failures retire the writer. Single-key and paged plugin-data reads use the independent reader owner and see committed snapshots; dependent reads must still await their preceding write receipt. Saved application inspection uses `RootOwner::open_existing` and `EventLog::for_application_inspection`. It requires existing private coordination state and the original exclusive Root/database leases, reads committed application facts including retained WAL content, and closes without changing stored members or bytes. It performs no migration or derived initialization; missing state fails without repair. Only application queries on the owned connection belong to this opening. CLI application read and default-update preview share it; applying an update opens a fresh ordinary writer and recomputes the change. diff --git a/platform/event-log/README.zh-CN.md b/platform/event-log/README.zh-CN.md index 3c659cf861..95d0869f0a 100644 --- a/platform/event-log/README.zh-CN.md +++ b/platform/event-log/README.zh-CN.md @@ -65,6 +65,10 @@ Control 中的调用意图、项目登记、workspace attachment、Session 路 规范数据库使用 UTF-8。成功工具结果的语义片段引用原始事件 JSON 中经过校验的字节区间。解析在调用者的读取快照内，由原 SQLite worker 读取并解码选中的工具投影；来源收集器为仍有历史所有者的正文保留存活期。归档占位内容和其他片段正文继续使用分块存储。 +语义行、payload 和子项目录是有限的派生描述符，采用 Postcard 编码。writer 在发布事务内编码一次；分页把已存字节和固定小端序定位字段直接复制到调用者计费的缓冲区，消费端恢复原有类型，无需读取整个正文。原始字节位置与叶子摘要保持不变，目录编码不参与规范事实或来源身份。SQL 迁移在同一事务内转换原 JSON 描述符；转换函数仅在迁移期间存在。 + +`event_storage_bench` 测量真实 Root 写入、冻结目录、子项分页、首尾窗口、顺序正文、事件前缀和 bundle 导出。在两个版本上用 release 构建同一 example 并各自保存二进制后，运行 `python3 scripts/bench-event-storage.py --output /tmp/storage-comparison original=/path/to/original candidate=/path/to/candidate`。脚本交错执行至少五轮，校验事件、目录、正文与导出摘要。首次读取用于校验和预热；重开测量保留操作系统缓存。记录保留原始分布、进程 CPU/RSS/I/O 可用性、SQLite 缓存计数，以及分列的主库、WAL、SHM 大小。存活 prepared statement 的快照不能代表全部 SQL 工作；评估单独路径是否退化时，需要按投影类型复核混合计时。 + 语句失败会回滚整组，只有已由自身保存点隔离的领域拒绝例外。原先使用普通写入路径的请求，在确认回滚并重新校验所有权后，可以继续通过同一 owner 恢复；纯 observation 批次保留更严格的退出策略。提交失败始终返回结果未知，不产生执行凭据；此后复用原普通写入组，还要求确认清理完成且事务深度为零。保存点、所有权或无法确认的清理失败都会让写入器退出。单键与分页插件数据读取使用独立 reader，读取已提交的快照；依赖写入的读取仍需先等待相应回执。 已保存应用的检查使用 `RootOwner::open_existing` 和 `EventLog::for_application_inspection`，要求已有的私有协调状态，并持有原有的独占 Root／数据库租约。检查读取已提交的应用事实，包括保留在 WAL 中的内容，关闭后不改变存储成员或字节；不执行迁移或派生初始化，缺失状态直接报错，不做修复。该打开方式仅供原连接上的应用查询使用。CLI 的 application read 与默认更新预览共用此入口；实际应用更新时重新打开普通写入所有者，并重新计算变更。 diff --git a/platform/event-log/examples/event_storage_bench.rs b/platform/event-log/examples/event_storage_bench.rs new file mode 100644 index 0000000000..6b7a01d6c7 --- /dev/null +++ b/platform/event-log/examples/event_storage_bench.rs @@ -0,0 +1,681 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Repeatable storage measurements through real Root and frozen semantic-source APIs. +use maka_agent_facts::{ + event::{ + EventWrite, Fact, Invocation, InvocationInput, InvocationOutcome, LogScope, RuntimeEvent, + }, + history::semantic::directory::{ + ChildDirectory, JsonSpan, ListKind, PayloadDirectory, RowDirectory, TextSpan, + ToolDirectory, ToolPartDirectory, + }, + model::prompt_source::source::{Allocate, ChargedBuffer, SourceBudget, SourceBytes}, + tool_call::ToolCallIdentity, + tool_output::{DurableToolProjection, ProjectionPart, ToolOutput, ToolSuccess}, +}; +use maka_event_log::{ + root::{RootNamespaces, RootOwner}, + root_store::{RootStore, SessionStore}, +}; +use serde_json::json; +use sha2::{Digest, Sha256}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::{ + collections::BTreeMap, + error::Error, + path::PathBuf, + sync::Arc, + time::{Instant, SystemTime, UNIX_EPOCH}, +}; + +type Result = std::result::Result>; +const PAGE_BYTES: usize = 64 * 1024; +struct Budget; +static LIVE: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); +static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); +struct Charge(usize); +impl Drop for Charge { + fn drop(&mut self) { + LIVE.fetch_sub(self.0, Ordering::Relaxed); + } +} +impl SourceBudget for Budget { + fn allocate(&self, minimum: usize) -> Allocate<'_> { + Box::pin(async move { Ok(allocation(minimum)) }) + } +} +fn allocation(bytes: usize) -> ChargedBuffer { + let live = LIVE.fetch_add(bytes, Ordering::Relaxed) + bytes; + PEAK.fetch_max(live, Ordering::Relaxed); + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + ChargedBuffer::new(vec![0; bytes], Charge(bytes)) +} +fn digest(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} +async fn consume(bytes: &SourceBytes) -> Result> { + let mut cursor = bytes.open(Arc::new(Budget)); + let mut buffer = allocation(PAGE_BYTES); + let mut result = Vec::new(); + loop { + let chunk = cursor.read_chunk(buffer).await?; + result.extend_from_slice(&chunk.buffer.as_ref()[..chunk.bytes_read]); + buffer = chunk.buffer; + if chunk.bytes_read == 0 { + break; + } + } + Ok(result) +} +async fn check_text( + captured: &maka_event_log::context::semantic::CapturedSource, + body: &maka_event_log::context::semantic::PayloadSource, + span: TextSpan, +) -> Result<()> { + let source = captured.text(body, span)?; + let raw = consume(source.bytes()).await?; + let decoded: String = serde_json::from_slice(&raw)?; + if decoded.len() as u64 != span.measure.bytes + || Sha256::digest(decoded.as_bytes()).as_slice() != span.measure.sha256 + { + return Err("text descriptor commitment changed".into()); + } + Ok(()) +} +async fn check_json( + captured: &maka_event_log::context::semantic::CapturedSource, + body: &maka_event_log::context::semantic::PayloadSource, + span: JsonSpan, +) -> Result<()> { + let source = captured.json(body, span)?; + let raw = consume(source.bytes()).await?; + if raw.len() as u64 != span.measure.bytes + || Sha256::digest(&raw).as_slice() != span.measure.sha256 + { + return Err("JSON descriptor commitment changed".into()); + } + let _: serde_json::Value = serde_json::from_slice(&raw)?; + Ok(()) +} +fn plain(invocation: &Invocation, fact: Fact, seconds: u64) -> Result { + let identity = format!( + "bench-{}-{}", + fact.kind(), + fact.operation_id().unwrap_or("boundary") + ); + let mut event = RuntimeEvent::new(invocation.clone(), fact); + event.id = identity; + event.recorded_at = UNIX_EPOCH + std::time::Duration::from_secs(seconds); + Ok(EventWrite::plain(event)?) +} +async fn append_tool( + session: &SessionStore, + invocation: &Invocation, + step: &str, + text: &str, + index: u64, +) -> Result<(EventWrite, u64, u64)> { + use maka_agent_facts::{ + context::ModelPurpose, + model::{ + ModelEvent, ModelFinishReason, + assembly::{AttemptBinding, AttemptWriter}, + }, + }; + let time = UNIX_EPOCH + std::time::Duration::from_secs(index * 2 + 1); + let source = session + .read_model_view(&invocation.session_id, Some(&invocation.invocation_id)) + .await?; + session + .append( + &plain( + invocation, + Fact::ModelRequested { + source_capture: maka_agent_facts::event::SourceCapture { + publication_watermark: source.selection.captured_through, + prior_unknown: false, + }, + context_breakdown: None, + item_acceptance: false, + step_id: step.into(), + model_id: "test".into(), + purpose: ModelPurpose::Main, + context: None, + source_scope: source.source_evidence.scope, + source_high_water: source.source_evidence.high_water, + source_digest: source.source_evidence.digest, + input_digest: "fixture".into(), + route_identity: format!("sha256:{}", "a".repeat(64)), + checkpoint_event_id: source.baseline.map(|baseline| baseline.event_id), + effective_source_digest: Some(source.effective_source_digest), + }, + index * 2 + 1, + )? + .with_quote(maka_agent_facts::pricing::Quote { + provider_id: "fixture".into(), + revision: 1, + pricing: Some(maka_agent_facts::pricing::Pricing { + model_key: "fixture:test".into(), + input_usd_per_million: 1.0, + output_usd_per_million: 2.0, + cache_read_usd_per_million: None, + cache_write_usd_per_million: None, + }), + })?, + ) + .await?; + let mut writer = AttemptWriter::new( + AttemptBinding { + invocation: invocation.clone(), + step_id: step.into(), + purpose: ModelPurpose::Main, + }, + false, + )?; + writer.push(ModelEvent::ToolCall(serde_json::from_value(json!({"id":step,"name":"Read","input":{"kind":"json","data":{"path":"fixture"}},"provider_executed":false}))?),time)?; + let observations = writer.push( + ModelEvent::Finished { + reason: ModelFinishReason::ToolCalls, + usage: Default::default(), + provider_options: None, + }, + time, + )?; + for observation in observations { + let mut event = RuntimeEvent::new( + invocation.clone(), + Fact::ModelObserved { + step_id: step.into(), + event: observation, + }, + ); + // The observation kind differentiates the part and terminal witness. + event.id = format!( + "observed-{step}-{}", + digest(&serde_json::to_vec(&event.fact)?) + ); + event.recorded_at = time; + session.append(&EventWrite::plain(event)?).await?; + } + let output = writer.completed()?.completion(); + session + .append(&plain( + invocation, + Fact::ModelCompleted { + step_id: step.into(), + output, + }, + index * 2 + 1, + )?) + .await?; + let operation = format!("{step}:{step}"); + session + .append(&plain( + invocation, + Fact::ToolDispatched { + presentation: None, + title: None, + operation_id: operation.clone(), + call: ToolCallIdentity::provider(step.into(), step.into()), + name: "Read".into(), + input: json!({"path":"fixture"}), + }, + index * 2 + 1, + )?) + .await?; + let output = ToolOutput::Text(text.into()); + let projection = match index % 3 { + 0 => DurableToolProjection::Json { + value: json!({"event":index,"text":text}), + }, + 1 => DurableToolProjection::Text { text: text.into() }, + _ => DurableToolProjection::Content { + parts: (0..9) + .map(|part| ProjectionPart::Text { + text: format!("part-{part}:{text}"), + }) + .collect(), + }, + }; + let (write, _) = EventWrite::tool_success( + format!("result-{step}"), + UNIX_EPOCH + std::time::Duration::from_secs(index * 2 + 2), + invocation.clone(), + operation, + ToolSuccess::projected(output, projection), + )?; + let start = Instant::now(); + let sequence = session.append(&write).await?; + Ok((write, sequence, start.elapsed().as_nanos() as u64)) +} +#[derive(Default)] +struct Samples(BTreeMap<&'static str, Vec>); +impl Samples { + fn add(&mut self, operation: &'static str, start: Instant) { + self.0 + .entry(operation) + .or_default() + .push(start.elapsed().as_nanos() as u64); + } + fn report(self) -> serde_json::Value { + let mut output = serde_json::Map::new(); + for (operation, mut values) in self.0 { + values.sort_unstable(); + let at = |quantile: usize| values[((values.len() - 1) * quantile) / 100]; + output.insert(operation.into(),json!({"samples":values.len(),"min_ns":values[0],"p50_ns":at(50),"p95_ns":at(95),"p99_ns":at(99),"max_ns":values[values.len()-1],"values_ns":values})); + } + output.into() + } +} +fn io_stats() -> serde_json::Value { + match std::fs::read_to_string("/proc/self/io") { + Ok(value) => json!({"value":value}), + Err(error) => json!({"unavailable":error.to_string()}), + } +} +fn usage() -> serde_json::Value { + #[cfg(unix)] + { + let mut value = std::mem::MaybeUninit::::uninit(); + // SAFETY: getrusage initializes this correctly sized output on success. + if unsafe { libc::getrusage(libc::RUSAGE_SELF, value.as_mut_ptr()) } == 0 { + // SAFETY: the successful call initialized every field. + let value = unsafe { value.assume_init() }; + let micros = |time: libc::timeval| { + i128::from(time.tv_sec) * 1_000_000 + i128::from(time.tv_usec) + }; + return json!({"user_us":micros(value.ru_utime),"system_us":micros(value.ru_stime),"max_rss_native_units":value.ru_maxrss,"minor_faults":value.ru_minflt,"major_faults":value.ru_majflt,"block_inputs":value.ru_inblock,"block_outputs":value.ru_oublock}); + } + } + serde_json::Value::Null +} +fn resource_summary(stats: &maka_event_log::connection::SqliteResourceStats) -> serde_json::Value { + json!({"id":stats.connection_id,"cache_bytes":stats.cache_bytes,"cache_hits":stats.cache_hits,"cache_misses":stats.cache_misses,"cache_spills":stats.cache_spills,"cached_statements":stats.cached_statements}) +} +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<()> { + let arguments: Vec = std::env::args().skip(1).collect(); + let value = |name: &str, default: &str| { + arguments + .iter() + .position(|arg| arg == name) + .and_then(|index| arguments.get(index + 1)) + .cloned() + .unwrap_or_else(|| default.into()) + }; + let directory = PathBuf::from(value("--database", "/tmp/maka-event-storage-bench")); + let mode = value("--mode", "create"); + if !matches!(mode.as_str(), "create" | "read") { + return Err("mode must be create or read".into()); + } + let events: usize = value("--events", "128").parse()?; + let body_bytes: usize = value("--body-bytes", "65536").parse()?; + let iterations: usize = value("--iterations", "10").parse()?; + if events == 0 || iterations == 0 { + return Err("events and iterations must be positive".into()); + } + std::fs::create_dir_all(&directory)?; + let namespaces = RootNamespaces { + ownership: directory.join("ownership"), + control: directory.join("control"), + }; + let owner = if mode == "create" { + RootOwner::create(&directory.join("root"), &namespaces)? + } else { + RootOwner::open(&directory.join("root"), &namespaces)? + }; + let root = RootStore::open(Arc::new(owner)).await?; + let mut samples = Samples::default(); + let before_io = io_stats(); + let before_usage = usage(); + let mut manifest = serde_json::Value::Null; + if mode == "create" { + let session = root.create_session("bench", "bench", &json!({}), 1).await?; + let invocation = Invocation { + session_id: "bench".into(), + turn_id: "turn".into(), + run_id: "run".into(), + invocation_id: "invocation".into(), + }; + session + .append(&plain( + &invocation, + Fact::InvocationOpened { + configuration: None, + input: InvocationInput::Message { + content: "storage measurement".into(), + source_messages: vec![], + request_fingerprint: None, + }, + }, + 0, + )?) + .await?; + let pattern = "line: storage \\\"payload\\\"\n"; + let mut body = pattern.repeat(body_bytes.div_ceil(pattern.len())); + body.truncate(body_bytes); + let mut payloads = serde_json::Map::new(); + for index in 0..events { + let operation = format!("tool-{index}"); + let text = format!("event-{index:016x}:{body}:end-{index:016x}"); + let start = Instant::now(); + let (write, sequence, settled_ns) = + append_tool(&session, &invocation, &operation, &text, index as u64).await?; + samples.add("append_attempt", start); + samples + .0 + .entry("append_tool_settled") + .or_default() + .push(settled_ns); + if let Fact::ToolSettled { + outcome: + maka_agent_facts::event::ToolOutcome::Succeeded { + model_projection, .. + }, + .. + } = &write.event().fact + { + let projection = serde_json::to_vec(model_projection)?; + payloads.insert(sequence.to_string(),json!({"bytes":projection.len(),"digest":digest(&projection),"first_digest":digest(&projection[..projection.len().min(64)]),"tail_digest":digest(&projection[projection.len().saturating_sub(64)..])})); + } + } + session + .append(&plain( + &invocation, + Fact::InvocationEnded { + outcome: InvocationOutcome::Completed, + }, + events as u64 * 2 + 1, + )?) + .await?; + let prefix = session + .scoped_prefix( + LogScope::Session { id: "bench".into() }, + usize::MAX, + usize::MAX, + ) + .await?; + let inventory = root.preview_bundle("bench").await?; + let (_, export) = root + .export_bundle("bench", &inventory.subtree_digest, tokio::io::sink()) + .await?; + manifest = json!({"events":events,"body_bytes":body_bytes,"prefix_digest":prefix.digest,"export_digest":export.digest,"events_digest":digest(&serde_json::to_vec(&prefix.events)?),"payloads":payloads,"canonical_events":prefix.events.len()}); + std::fs::write( + directory.join("fixture.json"), + serde_json::to_vec_pretty(&manifest)?, + )?; + } + if manifest.is_null() { + manifest = serde_json::from_slice(&std::fs::read(directory.join("fixture.json"))?)?; + } + let expected = manifest["events"].as_u64().ok_or("fixture lacks count")? as usize; + let session = root.session("bench").await?; + let view = session.read_model_view("bench", None).await?; + let scope = session.execution_scope(None).await?; + let running = scope.store(); + let lease = scope.accept(); + let captured = running.capture_source(view.selection).await?; + let mut completed_bytes = 0u64; + let mut completed_rows = 0u64; + let mut descriptors = Sha256::new(); + let mut directory_buffer = allocation(32 * 1024); + let mut directory_pages = 0u64; + let mut child_pages = 0u64; + // The first pass validates descriptors and warms application/SQLite caches. + for iteration in 0..=iterations { + let start = Instant::now(); + let metadata = session.read_model_view("bench", None).await?; + std::hint::black_box(metadata.source_evidence.digest); + if iteration > 0 { + samples.add("model_view", start); + } + let start = Instant::now(); + let mut cursor = None; + let mut payloads = Vec::new(); + loop { + let mut page = captured.directory_page(cursor, directory_buffer).await?; + directory_pages += 1; + while let Some(row) = page.next_row()? { + completed_rows += 1; + std::hint::black_box((&row.directory, &row.payload)); + if iteration == 0 { + descriptors.update(serde_json::to_vec(&( + &row.row.id, + &row.directory, + &row.payload, + ))?); + } + if matches!(row.directory, RowDirectory::ToolResult { .. }) { + let body = captured + .describe_payload(&row.row)? + .ok_or("tool lacks body")?; + let length = body.byte_len().ok_or("tool lacks length")?; + let evidence = manifest["payloads"] + .get(row.row.id.source_sequence.to_string()) + .ok_or("unexpected payload identity")? + .clone(); + if length != evidence["bytes"].as_u64().ok_or("missing payload length")? { + return Err("payload length changed".into()); + } + payloads.push(( + body, + length, + evidence, + row.payload.ok_or("missing typed payload")?, + )); + } + } + cursor = page.next; + directory_buffer = page.into_buffer(); + if cursor.is_none() { + break; + } + } + if iteration > 0 { + samples.add("directory_page_all", start); + } + if payloads.len() != expected { + return Err(format!("expected {expected} tool results, got {}", payloads.len()).into()); + } + let start = Instant::now(); + for (body, length, evidence, _) in &payloads { + let source = captured.byte_range(body, 0, (*length).min(64))?; + let mut reader = source.open(Arc::new(Budget)); + let read = reader.read_chunk(allocation(64)).await?; + if read.bytes_read != (*length).min(64) as usize { + return Err("short first read".into()); + } + completed_bytes += read.bytes_read as u64; + if digest(&read.buffer.as_ref()[..read.bytes_read]) + != evidence["first_digest"] + .as_str() + .ok_or("missing first digest")? + { + return Err("first window changed".into()); + } + } + if iteration > 0 { + samples.add("first_64_all", start); + } + let start = Instant::now(); + for (body, length, evidence, _) in &payloads { + let count = (*length).min(64); + let source = captured.byte_range(body, length - count, count)?; + let mut reader = source.open(Arc::new(Budget)); + let read = reader.read_chunk(allocation(64)).await?; + if read.bytes_read != count as usize { + return Err("short tail read".into()); + } + completed_bytes += read.bytes_read as u64; + if digest(&read.buffer.as_ref()[..read.bytes_read]) + != evidence["tail_digest"] + .as_str() + .ok_or("missing tail digest")? + { + return Err("tail window changed".into()); + } + } + if iteration > 0 { + samples.add("tail_64_all", start); + } + let start = Instant::now(); + for (body, _, evidence, _) in &payloads { + let (sha, bytes) = captured + .with_sequential_source(body, |reader| { + let mut buffer = [0u8; PAGE_BYTES]; + let mut sha = Sha256::new(); + let mut bytes = 0; + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + sha.update(&buffer[..count]); + bytes += count as u64; + } + Ok((hex::encode(sha.finalize()), bytes)) + }) + .await?; + if sha + != evidence["digest"] + .as_str() + .ok_or("missing expected digest")? + { + return Err("projection bytes changed".into()); + } + completed_bytes += bytes; + } + if iteration > 0 { + samples.add("sequential_all", start); + } + let start = Instant::now(); + for (body, _, _, directory) in &payloads { + match directory { + PayloadDirectory::Tool(ToolDirectory::Text(span)) => { + check_text(&captured, body, *span).await? + } + PayloadDirectory::Tool(ToolDirectory::Json(span)) => { + check_json(&captured, body, *span).await? + } + PayloadDirectory::Tool(ToolDirectory::Content { parts }) => { + let mut after = None; + let mut count = 0; + let mut buffer = allocation(1024); + loop { + let mut page = captured + .child_page(body, ListKind::ToolParts, after, buffer) + .await?; + child_pages += 1; + while let Some(child) = page.next_child()? { + if iteration == 0 { + descriptors.update(serde_json::to_vec(&child)?); + } + let ChildDirectory::ToolPart(ToolPartDirectory::Text(span)) = child + else { + return Err("unexpected child".into()); + }; + check_text(&captured, body, span).await?; + count += 1; + } + after = page.next; + buffer = page.into_buffer(); + if after.is_none() { + break; + } + } + if count != *parts { + return Err("child list changed".into()); + } + } + _ => return Err("unexpected tool directory".into()), + } + } + if iteration > 0 { + samples.add("descriptor_leaves_all", start); + } + let start = Instant::now(); + let prefix = session + .scoped_prefix( + LogScope::Session { id: "bench".into() }, + usize::MAX, + usize::MAX, + ) + .await?; + if prefix.digest + != manifest["prefix_digest"] + .as_str() + .ok_or("missing prefix digest")? + { + return Err("event proof changed".into()); + } + if prefix.events.len() + != manifest["canonical_events"] + .as_u64() + .ok_or("missing event count")? as usize + { + return Err("event count changed".into()); + } + if digest(&serde_json::to_vec(&prefix.events)?) + != manifest["events_digest"] + .as_str() + .ok_or("missing event content digest")? + { + return Err("event content changed".into()); + } + if iteration > 0 { + samples.add("prefix_all", start); + } + let start = Instant::now(); + let inventory = root.preview_bundle("bench").await?; + let (_, export) = root + .export_bundle("bench", &inventory.subtree_digest, tokio::io::sink()) + .await?; + if export.digest + != manifest["export_digest"] + .as_str() + .ok_or("missing root digest")? + { + return Err("original canonical bytes changed".into()); + } + if iteration > 0 { + samples.add("export_all", start); + } + } + manifest["descriptor_digest"] = hex::encode(descriptors.finalize()).into(); + let resources: Vec<_>=root.sqlite_resource_stats().await?.iter().map(|owner|json!({"writer":resource_summary(&owner.writer),"readers":owner.readers.iter().map(|reader|reader.as_ref().map(resource_summary)).collect::>()})).collect(); + let after_io = io_stats(); + drop(captured); + drop(running); + lease.finish(); + root.close().await?; + let mut database_bytes = 0; + for entry in std::fs::read_dir(directory.join("root/runtime-partitions"))? { + let entry = entry?; + if entry.file_type()?.is_file() { + database_bytes += entry.metadata()?.len(); + } + } + println!( + "{}", + json!({"mode":mode,"fixture":manifest,"iterations":iterations,"profile":if cfg!(debug_assertions){"debug"}else{"release"},"samples":samples.report(),"completed_rows":completed_rows,"completed_bytes":completed_bytes,"directory_pages":directory_pages,"child_pages":child_pages,"peak_charged_bytes":PEAK.load(Ordering::Relaxed),"charged_allocations":ALLOCATIONS.load(Ordering::Relaxed),"errors":0,"database_bytes_after_close":database_bytes,"io_before":before_io,"io_after":after_io,"sqlite_resources":resources,"usage_before":before_usage,"usage_after":usage(),"timestamp":SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs()}) + ); + Ok(()) +} diff --git a/platform/event-log/migrations/0013_semantic_directory_bytes.sql b/platform/event-log/migrations/0013_semantic_directory_bytes.sql new file mode 100644 index 0000000000..3d6701c08e --- /dev/null +++ b/platform/event-log/migrations/0013_semantic_directory_bytes.sql @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +ALTER TABLE context_semantic_rows RENAME COLUMN directory_json TO directory_bytes; +ALTER TABLE context_semantic_fragments RENAME COLUMN directory_json TO directory_bytes; +ALTER TABLE context_semantic_directory_children RENAME COLUMN directory_json TO directory_bytes; + +UPDATE context_semantic_rows SET directory_bytes=maka_directory_row(directory_bytes); +UPDATE context_semantic_fragments SET directory_bytes=maka_directory_payload(directory_bytes); +UPDATE context_semantic_directory_children SET directory_bytes=maka_directory_child(directory_bytes); diff --git a/platform/event-log/src/append/ingress/tests.rs b/platform/event-log/src/append/ingress/tests.rs index e202cdbb4a..17c423120d 100644 --- a/platform/event-log/src/append/ingress/tests.rs +++ b/platform/event-log/src/append/ingress/tests.rs @@ -431,6 +431,9 @@ async fn canonical_open_rejects_utf16_and_retains_utf8() { .await .unwrap(); crate::sqlite_functions::register(&mut db).await.unwrap(); + crate::sqlite_functions::register_migrations(&mut db) + .await + .unwrap(); sqlx::migrate!("./migrations") .run_direct(None, &mut db, false) .await diff --git a/platform/event-log/src/connection.rs b/platform/event-log/src/connection.rs index c8b29de0d8..cd70d963f5 100644 --- a/platform/event-log/src/connection.rs +++ b/platform/event-log/src/connection.rs @@ -1030,6 +1030,32 @@ mod tests { time::Duration, }; + #[tokio::test] + async fn long_lived_readers_do_not_install_startup_migration_functions() { + let directory = tempfile::tempdir().unwrap(); + let options = SqliteConnectOptions::new() + .filename(directory.path().join("runtime.sqlite")) + .create_if_missing(true) + .foreign_keys(true); + let mut writer = connect_owned(&options).await.unwrap(); + crate::schema::initialize_connection(&mut writer) + .await + .unwrap(); + writer.close().await.unwrap(); + for role in [ReaderRole::Read, ReaderRole::Observation] { + let mut reader = connect_reader(&options, None, role).await.unwrap(); + let functions: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM pragma_function_list + WHERE name IN ('maka_directory_row','maka_directory_payload','maka_directory_child')", + ).fetch_one(&mut reader).await.unwrap(); + assert_eq!( + functions, 0, + "readers must never retain migration-only functions" + ); + close_reader(reader).await.unwrap(); + } + } + #[tokio::test] async fn canonical_session_reads_bypass_a_blocked_writer() { let directory = tempfile::tempdir().unwrap(); diff --git a/platform/event-log/src/context/semantic.rs b/platform/event-log/src/context/semantic.rs index 3c6ec10bca..7467653bfb 100644 --- a/platform/event-log/src/context/semantic.rs +++ b/platform/event-log/src/context/semantic.rs @@ -21,6 +21,7 @@ mod captured; mod directory; pub(crate) mod gc; +pub(crate) mod migration; pub use captured::{ ArtifactCapture, ArtifactIdentityValidity, ArtifactLookup, ArtifactLookupResult, CapturedDirectoryRow, CapturedLocalResource, CapturedRow, CapturedSource, ChildPage, diff --git a/platform/event-log/src/context/semantic/captured/directory.rs b/platform/event-log/src/context/semantic/captured/directory.rs index 315f2a58ff..9c85958801 100644 --- a/platform/event-log/src/context/semantic/captured/directory.rs +++ b/platform/event-log/src/context/semantic/captured/directory.rs @@ -18,13 +18,12 @@ */ //! Bounded pages of closed semantic records; every source stays under its capture. +use super::super::directory::codec; use super::*; use maka_agent_facts::history::semantic::directory::*; use maka_agent_facts::model::prompt_source::source::{ JsonSource, SourceBytes, TextEncoding, TextSource, }; -use serde::{Deserialize, Serialize, de::DeserializeOwned}; -use std::io::{self, Write}; struct Packed { buffer: B, @@ -32,19 +31,8 @@ struct Packed { read: usize, count: usize, } -struct Size(usize); -impl Write for Size { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.0 = self - .0 - .checked_add(bytes.len()) - .ok_or_else(|| io::Error::other("directory record size overflow"))?; - Ok(bytes.len()) - } - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} +// The record prefix carries its exact byte length. Descriptor bytes are copied +// from SQLite directly into the caller's charged page allocation. impl Packed { fn new(buffer: B) -> Self { Self { @@ -54,11 +42,12 @@ impl Packed { count: 0, } } - fn push(&mut self, value: &impl Serialize) -> Result { - let mut size = Size(0); - serde_json::to_writer(&mut size, value)?; - let needed = size - .0 + fn push(&mut self, parts: &[&[u8]]) -> Result { + let bytes = parts.iter().try_fold(0usize, |sum, part| { + sum.checked_add(part.len()) + .ok_or_else(|| invalid("directory record size overflow")) + })?; + let needed = bytes .checked_add(8) .ok_or_else(|| invalid("directory record size overflow"))?; if needed > self.buffer.as_mut().len().saturating_sub(self.written) { @@ -69,66 +58,56 @@ impl Packed { } return Ok(false); } - let start = self.written; - let end = start + needed; - self.buffer.as_mut()[start..start + 8].copy_from_slice(&(size.0 as u64).to_le_bytes()); - serde_json::to_writer(&mut &mut self.buffer.as_mut()[start + 8..end], value)?; - self.written = end; + let target = &mut self.buffer.as_mut()[self.written..self.written + needed]; + target[..8].copy_from_slice(&(bytes as u64).to_le_bytes()); + let mut position = 8; + for part in parts { + target[position..position + part.len()].copy_from_slice(part); + position += part.len(); + } + self.written += needed; self.count += 1; Ok(true) } - fn next(&mut self) -> Result, StoreError> { + fn next(&mut self) -> Result, StoreError> { if self.read == self.written { return Ok(None); } - let start = self.read; - let mut len = [0; 8]; - len.copy_from_slice(&self.buffer.as_mut()[start..start + 8]); + let prefix = self + .buffer + .as_mut() + .get(self.read..self.read + 8) + .ok_or_else(|| invalid("incomplete directory record"))?; + let len = usize::try_from(u64::from_le_bytes( + prefix.try_into().expect("eight-byte length prefix"), + )) + .map_err(|_| invalid("directory record size overflow"))?; + let start = self.read + 8; let end = start - + 8 - + usize::try_from(u64::from_le_bytes(len)) - .map_err(|_| invalid("directory record size overflow"))?; - let value = serde_json::from_slice(&self.buffer.as_mut()[start + 8..end])?; + .checked_add(len) + .filter(|end| *end <= self.written) + .ok_or_else(|| invalid("incomplete directory record"))?; self.read = end; - Ok(Some(value)) + Ok(Some(&self.buffer.as_mut()[start..end])) } } -#[derive(Serialize, Deserialize)] -enum FragmentStorage { - Pieces, - Canonical { offset: u64 }, -} -#[derive(Serialize, Deserialize)] -struct FragmentRecord { - key: RowPosition, - storage: FragmentStorage, - bytes: u64, - directory: PayloadDirectory, -} -impl FragmentRecord { - fn backing(&self) -> Backing { - match self.storage { - FragmentStorage::Pieces => Backing::Pieces { - key: self.key, - bytes: self.bytes, - }, - FragmentStorage::Canonical { offset } => Backing::Canonical { - key: self.key, - offset, - bytes: self.bytes, - }, - } +const ROW_WORDS: usize = 13; +const ROW_HEADER_BYTES: usize = ROW_WORDS * 8; +fn push_row( + packed: &mut Packed, + metadata: [u64; 12], + directory: &[u8], + payload: &[u8], +) -> Result { + let mut header = [0u8; ROW_HEADER_BYTES]; + for (index, value) in metadata + .into_iter() + .chain(std::iter::once(directory.len() as u64)) + .enumerate() + { + header[index * 8..index * 8 + 8].copy_from_slice(&value.to_le_bytes()); } -} -#[derive(Serialize, Deserialize)] -struct RowRecord { - rowid: i64, - id: RowPosition, - order: RowPosition, - revision: u64, - bytes: u64, - directory: RowDirectory, - payload: Option, + packed.push(&[&header, directory, payload]) } /// One finite row header and its complete payload's finite header, under the same source cut. pub struct CapturedDirectoryRow { @@ -153,31 +132,73 @@ impl DirectoryPage { self.packed.buffer } pub fn next_row(&mut self) -> Result, StoreError> { - let Some(record) = self.packed.next::()? else { + let Some(record) = self.packed.next()? else { return Ok(None); }; - let backing = match record.directory.payload() { - Some(PartSource::Complete(_)) => Some( - record - .payload - .as_ref() - .ok_or_else(|| invalid("complete row lacks typed payload"))? - .backing(), - ), - None => None, + let header = record + .get(..ROW_HEADER_BYTES) + .ok_or_else(|| invalid("incomplete directory row header"))?; + let words: [u64; ROW_WORDS] = std::array::from_fn(|index| { + u64::from_le_bytes( + header[index * 8..index * 8 + 8] + .try_into() + .expect("eight-byte directory field"), + ) + }); + let directory_len = usize::try_from(words[12]) + .map_err(|_| invalid("directory length exceeds memory range"))?; + let directory_end = ROW_HEADER_BYTES + .checked_add(directory_len) + .filter(|end| *end <= record.len()) + .ok_or_else(|| invalid("incomplete row descriptor"))?; + let directory: RowDirectory = codec::decode(&record[ROW_HEADER_BYTES..directory_end])?; + let payload_bytes = &record[directory_end..]; + let payload = if words[7] == 0 { + if !payload_bytes.is_empty() { + return Err(invalid("payload descriptor without backing")); + } + None + } else { + Some(codec::decode::(payload_bytes)?) + }; + let key = RowPosition { + source_sequence: words[8], + item_index: words[9], }; - let payload = record.payload.map(|p| p.directory); + let backing = match words[7] { + 0 => None, + 1 => Some(Backing::Pieces { + key, + bytes: words[11], + }), + 2 => Some(Backing::Canonical { + key, + offset: words[10], + bytes: words[11], + }), + _ => return Err(invalid("invalid directory payload backing")), + }; + if directory.payload().is_some() != backing.is_some() { + return Err(invalid("directory payload presence differs from backing")); + } Ok(Some(CapturedDirectoryRow { row: CapturedRow { domain: self.domain.clone(), - rowid: record.rowid, - id: record.id, - order: record.order, - revision: record.revision, - bytes: record.bytes, + rowid: i64::try_from(words[0]) + .map_err(|_| invalid("directory rowid exceeds SQLite range"))?, + id: RowPosition { + source_sequence: words[1], + item_index: words[2], + }, + order: RowPosition { + source_sequence: words[3], + item_index: words[4], + }, + revision: words[5], + bytes: words[6], payload: backing, }, - directory: record.directory, + directory, payload, })) } @@ -189,7 +210,7 @@ pub struct ChildPage { } impl ChildPage { pub fn next_child(&mut self) -> Result, StoreError> { - self.packed.next() + self.packed.next()?.map(codec::decode).transpose() } pub fn into_buffer(self) -> B { self.packed.buffer @@ -201,61 +222,70 @@ impl ChildPage { self.packed.count == 0 } } -fn column_text<'row>(row: &'row rusqlite::Row<'_>, index: usize) -> Result<&'row str, StoreError> { - row.get_ref(index)?.as_str().map_err(|error| { +fn column_bytes<'row>( + row: &'row rusqlite::Row<'_>, + index: usize, +) -> Result<&'row [u8], StoreError> { + row.get_ref(index)?.as_bytes().map_err(|error| { rusqlite::Error::FromSqlConversionFailure( index, - rusqlite::types::Type::Text, + rusqlite::types::Type::Blob, Box::new(error), ) .into() }) } -fn fragment_record(raw: &RawConnection, key: RowPosition) -> Result { - let mut statement=raw.prepare("SELECT storage_kind,byte_offset,byte_len,payload_bytes,directory_json FROM context_semantic_fragments WHERE source_sequence=? AND source_index=?")?; +fn with_fragment( + raw: &RawConnection, + key: RowPosition, + use_fragment: impl FnOnce([u64; 5], &[u8]) -> Result, +) -> Result { + let mut statement=raw.prepare("SELECT storage_kind,byte_offset,byte_len,payload_bytes,directory_bytes FROM context_semantic_fragments WHERE source_sequence=? AND source_index=?")?; let mut rows = statement.query(params![key.source_sequence as i64, key.item_index as i64])?; let row = rows.next()?.ok_or(rusqlite::Error::QueryReturnedNoRows)?; - let kind = column_text(row, 0)?; + let kind: &str = row + .get_ref(0)? + .as_str() + .map_err(|error| invalid(error.to_string()))?; let offset: Option = row.get(1)?; let len: Option = row.get(2)?; let bytes: i64 = row.get(3)?; if bytes <= 0 { return Err(invalid("typed fragment has no stored bytes")); } - let storage = match super::super::source::SemanticPayloadSource::from_metadata( + let source = super::super::source::SemanticPayloadSource::from_metadata( key.source_sequence as i64, key.item_index as i64, kind, offset, len, - )? { - super::super::source::SemanticPayloadSource::Pieces => FragmentStorage::Pieces, + )?; + let (tag, offset) = match source { + super::super::source::SemanticPayloadSource::Pieces => (1, 0), super::super::source::SemanticPayloadSource::CanonicalToolProjection(span) => { if span.len() != bytes { return Err(invalid("canonical typed fragment length changed")); } - FragmentStorage::Canonical { - offset: span.offset() as u64, - } + (2, span.offset() as u64) } - super::super::source::SemanticPayloadSource::CanonicalModelPart(span) => { if span.len() != bytes { return Err(invalid("canonical typed fragment length changed")); } - FragmentStorage::Canonical { - offset: span.offset() as u64, - } + (2, span.offset() as u64) } }; - Ok(FragmentRecord { - key, - storage, - bytes: bytes as u64, - directory: serde_json::from_str(column_text(row, 4)?)?, - }) + use_fragment( + [ + tag, + key.source_sequence, + key.item_index, + offset, + bytes as u64, + ], + column_bytes(row, 4)?, + ) } - impl CapturedSource { /// Restore an original quoted token with its write-time decoded-content measure. pub fn text(&self, root: &PayloadSource, span: TextSpan) -> Result { @@ -289,17 +319,21 @@ impl CapturedSource { let result=crate::connection::worker_query(&mut tx,move|worker|Box::pin(async move{with_raw(worker,|raw|{ let mut packed=Packed::new(buffer); let zero=RowPosition{source_sequence:0,item_index:0};let point=cursor.unwrap_or(SemanticCursor{order:zero,id:zero}); - let query=rows_query(true).replace("r.order_sequence=?11","?11 IS NULL").replace("?13 IS NULL","?15 IS NULL").replace("json_extract(?13,'$.base')","?15").replace("json_extract(?13,'$.current')","?16").replace("SELECT r.row_json FROM","SELECT r.rowid,r.source_sequence,r.source_index,r.revision,r.order_sequence,r.order_index,length(CAST(r.row_json AS BLOB)),r.directory_json FROM"); + let query=rows_query(true).replace("r.order_sequence=?11","?11 IS NULL").replace("?13 IS NULL","?15 IS NULL").replace("json_extract(?13,'$.base')","?15").replace("json_extract(?13,'$.current')","?16").replace("SELECT r.row_json FROM","SELECT r.rowid,r.source_sequence,r.source_index,r.revision,r.order_sequence,r.order_index,length(CAST(r.row_json AS BLOB)),r.directory_bytes FROM"); let mut statement=raw.prepare(&query)?; let mut rows=statement.query(params![&selection.session,selection.after as i64,selection.through as i64,selection.captured_through as i64,selection.before.min(i64::MAX as u64) as i64,&selection.lineage,point.order.source_sequence as i64,point.order.item_index as i64,point.id.source_sequence as i64,point.id.item_index as i64,None::,i64::MAX,None::,selection.prior_unknown,selection.replay.as_ref().map(|r|r.base as i64),selection.replay.as_ref().map(|r|r.current.as_str())])?; let mut last=None;let mut more=false; while let Some(r)=rows.next()?{ - let directory:RowDirectory=serde_json::from_str(column_text(r,7)?)?; - let payload=match directory.payload(){Some(PartSource::Complete(key))=>Some(fragment_record(raw,key)?),_=>None}; + let encoded=column_bytes(r,7)?; + let directory:RowDirectory=codec::decode(encoded)?; let id=RowPosition{source_sequence:crate::sequence_number(r.get(1)?)?,item_index:crate::sequence_number(r.get(2)?)?}; let order=RowPosition{source_sequence:crate::sequence_number(r.get(4)?)?,item_index:crate::sequence_number(r.get(5)?)?}; - let record=RowRecord{rowid:r.get(0)?,id,order,revision:crate::sequence_number(r.get(3)?)?,bytes:crate::sequence_number(r.get(6)?)?,directory,payload}; - if !packed.push(&record)?{more=true;break;}last=Some(SemanticCursor{order,id}); + let mut metadata=[u64::try_from(r.get::<_,i64>(0)?).map_err(|_|invalid("negative directory rowid"))?,id.source_sequence,id.item_index,order.source_sequence,order.item_index,crate::sequence_number(r.get(3)?)?,crate::sequence_number(r.get(6)?)?,0,0,0,0,0]; + let pushed=match directory.payload() { + Some(PartSource::Complete(key))=>with_fragment(raw,key,|fragment,payload|{metadata[7..12].copy_from_slice(&fragment);push_row(&mut packed,metadata,encoded,payload)})?, + None=>push_row(&mut packed,metadata,encoded,&[])?, + }; + if !pushed {more=true;break;}last=Some(SemanticCursor{order,id}); } Ok(DirectoryPage{domain,packed,next:if more{last}else{None}}) })})).await; @@ -325,10 +359,10 @@ impl CapturedSource { .map_err(|_| invalid("directory ordinal exceeds SQLite range"))?; self.0.log.connection.read(move|db|Box::pin(async move{ let mut tx=db.begin().await?;let result=crate::connection::worker_query(&mut tx,move|worker|Box::pin(async move{with_raw(worker,|raw|{ - let mut statement=raw.prepare("SELECT ordinal,directory_json FROM context_semantic_directory_children WHERE source_sequence=? AND source_index=? AND list_kind=? AND ordinal>? ORDER BY ordinal")?; + let mut statement=raw.prepare("SELECT ordinal,directory_bytes FROM context_semantic_directory_children WHERE source_sequence=? AND source_index=? AND list_kind=? AND ordinal>? ORDER BY ordinal")?; let mut rows=statement.query(params![key.source_sequence as i64,key.item_index as i64,kind.as_str(),after.unwrap_or(-1)])?; let mut packed=Packed::new(buffer);let mut last=None;let mut more=false; - while let Some(r)=rows.next()?{let child:ChildDirectory=serde_json::from_str(column_text(r,1)?)?;if !packed.push(&child)?{more=true;break;}last=Some(crate::sequence_number(r.get(0)?)?);} + while let Some(r)=rows.next()?{if !packed.push(&[column_bytes(r,1)?])?{more=true;break;}last=Some(crate::sequence_number(r.get(0)?)?);} Ok(ChildPage{packed,next:if more{last}else{None}}) })})).await;let closed=tx.rollback().await;settle_read(result,closed) })).await diff --git a/platform/event-log/src/context/semantic/directory.rs b/platform/event-log/src/context/semantic/directory.rs index 495f1b86ac..2922708fd3 100644 --- a/platform/event-log/src/context/semantic/directory.rs +++ b/platform/event-log/src/context/semantic/directory.rs @@ -18,6 +18,7 @@ */ //! Same-transaction typed metadata over the actual immutable payload encoding. +pub(super) mod codec; mod scan; use super::{captured, invalid}; use crate::StoreError; @@ -52,7 +53,7 @@ pub(super) fn prepare_fragment( }; captured::with_fragment_reader(raw, key, source, |reader| { scan::payload(reader, root, |kind, ordinal, child| { - raw.execute("INSERT INTO context_semantic_directory_children(source_sequence,source_index,list_kind,ordinal,directory_json) VALUES(?,?,?,?,?)",params![key.source_sequence as i64,key.item_index as i64,kind.as_str(),i64::try_from(ordinal).map_err(|_|invalid("directory ordinal exceeds SQLite range"))?,serde_json::to_string(&child)?])?; + raw.execute("INSERT INTO context_semantic_directory_children(source_sequence,source_index,list_kind,ordinal,directory_bytes) VALUES(?,?,?,?,?)",params![key.source_sequence as i64,key.item_index as i64,kind.as_str(),i64::try_from(ordinal).map_err(|_|invalid("directory ordinal exceeds SQLite range"))?,codec::encode(&child)?])?; Ok(()) }) }) diff --git a/platform/event-log/src/context/semantic/directory/codec.rs b/platform/event-log/src/context/semantic/directory/codec.rs new file mode 100644 index 0000000000..ef1e8154ea --- /dev/null +++ b/platform/event-log/src/context/semantic/directory/codec.rs @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Compact derived descriptors using Postcard's typed enum and integer encoding. +//! Descriptor declarations and their Serde layout define the storage schema. +use super::super::invalid; +use crate::StoreError; +use serde::{Serialize, de::DeserializeOwned}; +pub(crate) fn encode(value: &T) -> Result, StoreError> { + postcard::to_allocvec(value).map_err(|error| invalid(error.to_string())) +} +pub(crate) fn decode(bytes: &[u8]) -> Result { + let (value, remaining) = + postcard::take_from_bytes(bytes).map_err(|error| invalid(error.to_string()))?; + if !remaining.is_empty() { + return Err(invalid("trailing directory descriptor bytes")); + } + Ok(value) +} + +#[cfg(test)] +mod tests; diff --git a/platform/event-log/src/context/semantic/directory/codec/tests.rs b/platform/event-log/src/context/semantic/directory/codec/tests.rs new file mode 100644 index 0000000000..5331806105 --- /dev/null +++ b/platform/event-log/src/context/semantic/directory/codec/tests.rs @@ -0,0 +1,314 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use super::*; +use maka_agent_facts::{ + history::{RowPosition, semantic::directory::*}, + model::{ + TextKind, + prompt_source::source::{DecodedTextMeasure, RawJsonMeasure}, + }, +}; +fn text(seed: u64) -> TextSpan { + TextSpan { + range: ByteRange { + start: u64::MAX - seed, + len: seed, + }, + measure: DecodedTextMeasure { + bytes: u64::MAX - seed * 2, + sha256: std::array::from_fn(|index| (index as u8).wrapping_mul(seed as u8)), + }, + } +} +fn json(seed: u64, kind: ValueKind) -> JsonSpan { + JsonSpan { + range: ByteRange { + start: seed, + len: u64::MAX - seed, + }, + measure: RawJsonMeasure { + bytes: u64::MAX - seed, + sha256: std::array::from_fn(|index| (index as u8).wrapping_add(seed as u8)), + }, + kind, + } +} +macro_rules! assert_roundtrip { + ($value:expr,$ty:ty) => {{ + let value = $value; + let encoded = encode(&value).unwrap(); + let decoded = decode::<$ty>(&encoded).unwrap(); + assert_eq!( + serde_json::to_value(decoded).unwrap(), + serde_json::to_value(value).unwrap() + ); + for length in 0..encoded.len() { + assert!( + decode::<$ty>(&encoded[..length]).is_err(), + "accepted truncated descriptor at {length}/{}", + encoded.len() + ); + } + let mut trailing = encoded; + trailing.push(0); + assert!( + decode::<$ty>(&trailing).is_err(), + "accepted descriptor with an unconsumed suffix" + ); + }}; +} +#[test] +fn every_row_kind_preserves_positions_measures_and_independent_flags() { + let position = RowPosition { + source_sequence: 9007199254740991, + item_index: 47, + }; + for value in [ + RowDirectory::User { + payload: position, + steering: true, + }, + RowDirectory::User { + payload: position, + steering: false, + }, + RowDirectory::ImportedUser { payload: position }, + RowDirectory::ImportedAssistant { payload: position }, + RowDirectory::ExecutorText { payload: position }, + RowDirectory::AssistantPart { + model_id: text(3), + route_identity: text(7), + payload: PartSource::Complete(position), + }, + RowDirectory::ToolResult { + payload: position, + call_id: text(11), + name: text(13), + custom: true, + is_error: false, + archived: false, + }, + RowDirectory::ToolResult { + payload: position, + call_id: text(17), + name: text(19), + custom: false, + is_error: true, + archived: false, + }, + RowDirectory::ToolResult { + payload: position, + call_id: text(23), + name: text(29), + custom: false, + is_error: false, + archived: true, + }, + RowDirectory::Notification { + payload: position, + call_id: text(31), + custom: true, + }, + RowDirectory::UnknownTool { + call_id: text(37), + name: text(41), + custom: false, + }, + ] { + assert_roundtrip!(value, RowDirectory); + } +} +#[test] +fn every_payload_kind_preserves_optional_tokens_and_u64_boundaries() { + for kind in [ + ValueKind::Null, + ValueKind::Bool, + ValueKind::Number, + ValueKind::String, + ValueKind::Array, + ValueKind::Object, + ] { + assert_roundtrip!( + PayloadDirectory::Tool(ToolDirectory::Json(json(3, kind))), + PayloadDirectory + ); + } + for value in [ + PayloadDirectory::User { + text: text(5), + quotes: u64::MAX, + attachments: 7, + directory_references: Some(json(11, ValueKind::Array)), + }, + PayloadDirectory::User { + text: text(13), + quotes: 0, + attachments: u64::MAX, + directory_references: None, + }, + PayloadDirectory::Text(text(17)), + PayloadDirectory::Part(PartDirectory::Source), + PayloadDirectory::Part(PartDirectory::Text { + text_kind: TextKind::Thinking, + text: text(19), + provider_options: Some(json(23, ValueKind::Object)), + }), + PayloadDirectory::Part(PartDirectory::Text { + text_kind: TextKind::Text, + text: text(29), + provider_options: None, + }), + PayloadDirectory::Part(PartDirectory::ToolCall { + id: text(31), + name: text(37), + input: InputDirectory::Json(json(41, ValueKind::Array)), + provider_executed: true, + provider_options: None, + }), + PayloadDirectory::Part(PartDirectory::ToolCall { + id: text(43), + name: text(47), + input: InputDirectory::Text(text(53)), + provider_executed: false, + provider_options: Some(json(59, ValueKind::Number)), + }), + PayloadDirectory::Part(PartDirectory::ToolResult { + id: text(61), + name: text(67), + output: json(71, ValueKind::Object), + is_error: true, + provider_options: None, + }), + PayloadDirectory::Part(PartDirectory::ToolResult { + id: text(73), + name: text(79), + output: json(83, ValueKind::String), + is_error: false, + provider_options: Some(json(89, ValueKind::Null)), + }), + PayloadDirectory::Tool(ToolDirectory::Text(text(97))), + PayloadDirectory::Tool(ToolDirectory::Content { parts: u64::MAX }), + PayloadDirectory::Tool(ToolDirectory::Failure), + ] { + assert_roundtrip!(value, PayloadDirectory); + } +} +#[test] +fn every_child_and_storage_kind_preserves_its_complete_descriptor() { + let references = [ + StorageDirectory::SessionFile { + session_id: text(3), + relative_path: text(5), + }, + StorageDirectory::WorkspaceFile { + relative_path: text(7), + }, + StorageDirectory::ExternalFile { + absolute_path: text(11), + }, + StorageDirectory::SessionContext { + session_id: text(13), + ref_id: text(17), + }, + ]; + for reference in references { + for image in [false, true] { + assert_roundtrip!( + ChildDirectory::Attachment(AttachmentDirectory { + image, + name: text(19), + mime_type: text(23), + reference + }), + ChildDirectory + ); + } + for detail in [None, Some(text(29))] { + let media = MediaDirectory { + reference, + mime_type: text(31), + detail, + }; + assert_roundtrip!( + ChildDirectory::ToolPart(ToolPartDirectory::Artifact(media)), + ChildDirectory + ); + assert_roundtrip!( + ChildDirectory::ToolPart(ToolPartDirectory::Audio(media)), + ChildDirectory + ); + } + } + for label in [None, Some(text(37))] { + for source in [None, Some(json(41, ValueKind::Array))] { + assert_roundtrip!( + ChildDirectory::Quote { + label, + source, + text: text(43) + }, + ChildDirectory + ); + } + } + assert_roundtrip!( + ChildDirectory::ToolPart(ToolPartDirectory::Text(text(47))), + ChildDirectory + ); +} + +// Frozen records from the real Root fixture: decoding and encoding must retain +// their existing bytes when the Rust descriptor declarations change. +fn stored_fixture( + encoded: &str, + expected: &str, +) { + let bytes = hex::decode(encoded).unwrap(); + let decoded = decode::(&bytes).unwrap(); + assert_eq!( + serde_json::to_value(&decoded).unwrap(), + serde_json::from_str::(expected).unwrap() + ); + assert_eq!(encode(&decoded).unwrap(), bytes); +} + +#[test] +fn frozen_rowdirectory_bytes_keep_their_meaning() { + stored_fixture::( + "04f90106049f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08910249479f447b1b473d359884f2b8541a1ad0b7d194ed59b79e8eee48047ec4830564fd000400", + r#"{"AssistantPart":{"model_id":{"range":{"start":249,"len":6},"measure":{"bytes":4,"sha256":[159,134,208,129,136,76,125,101,154,47,234,160,197,90,208,21,163,191,79,27,43,11,130,44,209,93,108,21,176,240,10,8]}},"route_identity":{"range":{"start":273,"len":73},"measure":{"bytes":71,"sha256":[159,68,123,27,71,61,53,152,132,242,184,84,26,26,208,183,209,148,237,89,183,158,142,238,72,4,126,196,131,5,100,253]}},"payload":{"Complete":{"sourceSequence":4,"itemIndex":0}}}}"#, + ); +} + +#[test] +fn frozen_payloaddirectory_bytes_keep_their_meaning() { + stored_fixture::( + "0202210806e9c8871e2ef30b1e62c50b20c4b6402d9729353e677240d2eeb211241918315a3106049b9a8d05a7ec353bda84f9c1bb3178c299de3001b5e970508ddc889c487f92ca00561212a882010639b4186ec8f8988815fa25bd56d926099aec3096bb5b073f799d83e8050000", + r#"{"Part":{"ToolCall":{"id":{"range":{"start":33,"len":8},"measure":{"bytes":6,"sha256":[233,200,135,30,46,243,11,30,98,197,11,32,196,182,64,45,151,41,53,62,103,114,64,210,238,178,17,36,25,24,49,90]}},"name":{"range":{"start":49,"len":6},"measure":{"bytes":4,"sha256":[155,154,141,5,167,236,53,59,218,132,249,193,187,49,120,194,153,222,48,1,181,233,112,80,141,220,136,156,72,127,146,202]}},"input":{"Json":{"range":{"start":86,"len":18},"measure":{"bytes":18,"sha256":[168,130,1,6,57,180,24,110,200,248,152,136,21,250,37,189,86,217,38,9,154,236,48,150,187,91,7,63,121,157,131,232]},"kind":"Object"}},"provider_executed":false,"provider_options":null}}}"#, + ); +} + +#[test] +fn frozen_childdirectory_bytes_keep_their_meaning() { + stored_fixture::( + "020031f809b30806e91925212eb903733201aaca81d91041147cbfdc01bde65b3967c34cc6cfb5", + r#"{"ToolPart":{"Text":{"range":{"start":49,"len":1272},"measure":{"bytes":1075,"sha256":[6,233,25,37,33,46,185,3,115,50,1,170,202,129,217,16,65,20,124,191,220,1,189,230,91,57,103,195,76,198,207,181]}}}}"#, + ); +} diff --git a/platform/event-log/src/context/semantic/migration.rs b/platform/event-log/src/context/semantic/migration.rs new file mode 100644 index 0000000000..b8508797aa --- /dev/null +++ b/platform/event-log/src/context/semantic/migration.rs @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Startup-only conversion from the former JSON directory representation. +use super::directory::codec; +use crate::StoreError; +use maka_agent_facts::history::semantic::directory::{ + ChildDirectory, PayloadDirectory, RowDirectory, +}; +use rusqlite::{ + Connection, + functions::{Context, FunctionFlags}, +}; +use serde::de::DeserializeOwned; + +fn transcode( + context: &Context<'_>, + encode: impl FnOnce(&T) -> Result, StoreError>, +) -> rusqlite::Result> { + let value: T = serde_json::from_slice(context.get_raw(0).as_bytes()?) + .map_err(|error| rusqlite::Error::UserFunctionError(Box::new(error)))?; + encode(&value).map_err(|error| rusqlite::Error::UserFunctionError(Box::new(error))) +} +pub(crate) fn register(db: &Connection) -> Result<(), StoreError> { + let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC; + db.create_scalar_function("maka_directory_row", 1, flags, |context| { + transcode::(context, codec::encode) + })?; + db.create_scalar_function("maka_directory_payload", 1, flags, |context| { + transcode::(context, codec::encode) + })?; + db.create_scalar_function("maka_directory_child", 1, flags, |context| { + transcode::(context, codec::encode) + })?; + Ok(()) +} +pub(crate) fn unregister(db: &Connection) -> Result<(), StoreError> { + for name in [ + "maka_directory_row", + "maka_directory_payload", + "maka_directory_child", + ] { + db.remove_function(name, 1)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use maka_agent_facts::{ + history::{ + RowPosition, + semantic::directory::{ByteRange, JsonSpan, TextSpan, ValueKind}, + }, + model::prompt_source::source::{DecodedTextMeasure, RawJsonMeasure}, + }; + use sqlx::Connection; + + #[tokio::test] + async fn migration_is_atomic_preserves_commitments_and_does_not_repeat() { + let directory = tempfile::tempdir().unwrap(); + let options = sqlx::sqlite::SqliteConnectOptions::new() + .filename(directory.path().join("runtime.sqlite")) + .create_if_missing(true) + .foreign_keys(true); + let mut db = sqlx::SqliteConnection::connect_with(&options) + .await + .unwrap(); + sqlx::query("PRAGMA application_id=1296124754") + .execute(&mut db) + .await + .unwrap(); + crate::sqlite_functions::register(&mut db).await.unwrap(); + let all = sqlx::migrate!("./migrations"); + let previous = sqlx::migrate::Migrator { + migrations: all + .iter() + .filter(|migration| migration.version < 13) + .cloned() + .collect::>() + .into(), + ..sqlx::migrate::Migrator::DEFAULT + }; + previous.run_direct(None, &mut db, false).await.unwrap(); + let text = TextSpan { + range: ByteRange { start: 7, len: 15 }, + measure: DecodedTextMeasure { + bytes: 9, + sha256: std::array::from_fn(|index| (index * 7) as u8), + }, + }; + let json = JsonSpan { + range: ByteRange { start: 23, len: 29 }, + measure: RawJsonMeasure { + bytes: 29, + sha256: std::array::from_fn(|index| 255 - index as u8), + }, + kind: ValueKind::Object, + }; + let row = RowDirectory::User { + payload: RowPosition { + source_sequence: 1, + item_index: 0, + }, + steering: true, + }; + let payload = PayloadDirectory::User { + text, + quotes: 1, + attachments: 0, + directory_references: Some(json), + }; + let child = ChildDirectory::Quote { + label: Some(text), + source: Some(json), + text, + }; + let row_json = serde_json::to_string_pretty(&row).unwrap(); + sqlx::query("INSERT INTO context_semantic_rows(owner_session_id,source_sequence,source_index,revision,order_sequence,order_index,invocation_id,revision_invocation,event_id,kind,archived,inherited,visible,unknown_only,row_json,directory_json) VALUES('session',1,0,1,1,0,'invocation','invocation','event','user',0,0,1,0,'{}',?)").bind(&row_json).execute(&mut db).await.unwrap(); + sqlx::query("INSERT INTO context_semantic_fragments(source_sequence,source_index,storage_kind,byte_offset,byte_len,directory_json,payload_bytes) VALUES(1,0,'pieces',NULL,NULL,'corrupt descriptor',64)").execute(&mut db).await.unwrap(); + sqlx::query("INSERT INTO context_semantic_directory_children(source_sequence,source_index,list_kind,ordinal,directory_json) VALUES(1,0,'quotes',0,?)").bind(serde_json::to_string_pretty(&child).unwrap()).execute(&mut db).await.unwrap(); + assert!(crate::schema::initialize_connection(&mut db).await.is_err()); + let saved: String = sqlx::query_scalar("SELECT directory_json FROM context_semantic_rows") + .fetch_one(&mut db) + .await + .unwrap(); + assert_eq!( + saved, row_json, + "the earlier successful row conversion must roll back with the later failure" + ); + let applied: bool = + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM _sqlx_migrations WHERE version=13)") + .fetch_one(&mut db) + .await + .unwrap(); + assert!( + !applied, + "a failed conversion must not publish its migration ledger entry" + ); + let functions:i64=sqlx::query_scalar("SELECT COUNT(*) FROM pragma_function_list WHERE name IN ('maka_directory_row','maka_directory_payload','maka_directory_child')").fetch_one(&mut db).await.unwrap(); + assert_eq!( + functions, 0, + "migration functions must be removed after failure" + ); + sqlx::query("UPDATE context_semantic_fragments SET directory_json=?") + .bind(serde_json::to_string_pretty(&payload).unwrap()) + .execute(&mut db) + .await + .unwrap(); + crate::schema::initialize_connection(&mut db).await.unwrap(); + let row_bytes: Vec = + sqlx::query_scalar("SELECT directory_bytes FROM context_semantic_rows") + .fetch_one(&mut db) + .await + .unwrap(); + let payload_bytes: Vec = + sqlx::query_scalar("SELECT directory_bytes FROM context_semantic_fragments") + .fetch_one(&mut db) + .await + .unwrap(); + let child_bytes: Vec = + sqlx::query_scalar("SELECT directory_bytes FROM context_semantic_directory_children") + .fetch_one(&mut db) + .await + .unwrap(); + assert_eq!( + serde_json::to_value(codec::decode::(&row_bytes).unwrap()).unwrap(), + serde_json::to_value(row).unwrap() + ); + assert_eq!( + serde_json::to_value(codec::decode::(&payload_bytes).unwrap()) + .unwrap(), + serde_json::to_value(payload).unwrap() + ); + assert_eq!( + serde_json::to_value(codec::decode::(&child_bytes).unwrap()).unwrap(), + serde_json::to_value(child).unwrap() + ); + db.close().await.unwrap(); + let mut db = sqlx::SqliteConnection::connect_with(&options) + .await + .unwrap(); + crate::schema::initialize_connection(&mut db).await.unwrap(); + let repeated: Vec = + sqlx::query_scalar("SELECT directory_bytes FROM context_semantic_rows") + .fetch_one(&mut db) + .await + .unwrap(); + assert_eq!( + repeated, row_bytes, + "reopening must not convert binary descriptors a second time" + ); + let functions:i64=sqlx::query_scalar("SELECT COUNT(*) FROM pragma_function_list WHERE name IN ('maka_directory_row','maka_directory_payload','maka_directory_child')").fetch_one(&mut db).await.unwrap(); + assert_eq!( + functions, 0, + "migration functions must not remain on live query connections" + ); + } +} diff --git a/platform/event-log/src/context/semantic/write.rs b/platform/event-log/src/context/semantic/write.rs index ebcf13bc05..814723c8c4 100644 --- a/platform/event-log/src/context/semantic/write.rs +++ b/platform/event-log/src/context/semantic/write.rs @@ -185,13 +185,13 @@ async fn fragment( flushed?; } let (directory, payload_bytes) = super::directory::prepare_fragment(db, fragment.key, source)?; - sqlx::query("INSERT INTO context_semantic_fragments(source_sequence,source_index,storage_kind,byte_offset,byte_len,directory_json,payload_bytes) VALUES(?,?,?,?,?,?,?)") + sqlx::query("INSERT INTO context_semantic_fragments(source_sequence,source_index,storage_kind,byte_offset,byte_len,directory_bytes,payload_bytes) VALUES(?,?,?,?,?,?,?)") .bind(fragment.key.source_sequence as i64) .bind(fragment.key.item_index as i64) .bind(storage_kind) .bind(offset) .bind(len) - .bind(json(&directory)?) + .bind(super::directory::codec::encode(&directory)?) .bind(i64::try_from(payload_bytes).map_err(|_|invalid("payload length exceeds SQLite range"))?) .execute(&mut *db).await?; Ok(()) @@ -266,7 +266,7 @@ async fn save_row_mode( } let encoded_row = json(row)?; let row_directory = super::directory::row(&encoded_row)?; - sqlx::query("INSERT INTO context_semantic_rows(owner_session_id,source_sequence,source_index,revision,order_sequence,order_index,invocation_id,revision_invocation,event_id,kind,archived,inherited,visible,unknown_only,row_json,directory_json) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") + sqlx::query("INSERT INTO context_semantic_rows(owner_session_id,source_sequence,source_index,revision,order_sequence,order_index,invocation_id,revision_invocation,event_id,kind,archived,inherited,visible,unknown_only,row_json,directory_bytes) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") .bind(owner) .bind(row.id.source_sequence as i64) .bind(row.id.item_index as i64) @@ -282,7 +282,7 @@ async fn save_row_mode( .bind(visible) .bind(unknown_only) .bind(encoded_row) - .bind(json(&row_directory)?) + .bind(super::directory::codec::encode(&row_directory)?) .execute(&mut *db) .await?; Ok(()) diff --git a/platform/event-log/src/schema.rs b/platform/event-log/src/schema.rs index 4aa750517e..3ce9ab02a9 100644 --- a/platform/event-log/src/schema.rs +++ b/platform/event-log/src/schema.rs @@ -112,7 +112,22 @@ pub(crate) async fn initialize_connection_with_control( .await?; } crate::sqlite_functions::register(connection).await?; - MIGRATIONS.run_direct(None, &mut *connection, false).await?; + crate::sqlite_functions::register_migrations(connection).await?; + let migrated = MIGRATIONS + .run_direct(None, &mut *connection, false) + .await + .map_err(StoreError::from); + let unregistered = crate::sqlite_functions::unregister_migrations(connection).await; + match (migrated, unregistered) { + (Ok(()), Ok(())) => {} + (Err(cause), Err(cleanup)) => { + return Err(StoreError::StartupCleanupFailed { + cause: Box::new(cause), + cleanup: Box::new(cleanup), + }); + } + (Err(error), Ok(())) | (Ok(()), Err(error)) => return Err(error), + } sqlx::raw_sql( "PRAGMA journal_mode = WAL; PRAGMA synchronous = FULL; PRAGMA temp_store = FILE;", ) diff --git a/platform/event-log/src/schema/fixtures.rs b/platform/event-log/src/schema/fixtures.rs index 9c6386f0d7..0d87ffbcd3 100644 --- a/platform/event-log/src/schema/fixtures.rs +++ b/platform/event-log/src/schema/fixtures.rs @@ -22,9 +22,11 @@ use rusqlite::Connection; pub(crate) fn schema(include: impl Fn(&str, &str) -> bool) -> String { let db = Connection::open_in_memory().unwrap(); + crate::context::semantic::migration::register(&db).unwrap(); for migration in super::MIGRATIONS.iter() { db.execute_batch(migration.sql.as_ref()).unwrap(); } + crate::context::semantic::migration::unregister(&db).unwrap(); let mut query = db.prepare( "SELECT type,name,sql FROM sqlite_schema WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%' AND tbl_name NOT IN (SELECT name FROM pragma_table_list WHERE schema='main' AND type='shadow') ORDER BY rowid", ).unwrap(); @@ -103,6 +105,9 @@ async fn existing_artifact_rows_migrate_without_losing_bytes_or_history_aliases( .execute(&mut db) .await .unwrap(); + crate::sqlite_functions::register_migrations(&mut db) + .await + .unwrap(); let error = super::MIGRATIONS .run_direct(None, &mut db, false) .await diff --git a/platform/event-log/src/sqlite_functions.rs b/platform/event-log/src/sqlite_functions.rs index 5534c19ee5..dc159ca0d4 100644 --- a/platform/event-log/src/sqlite_functions.rs +++ b/platform/event-log/src/sqlite_functions.rs @@ -21,23 +21,39 @@ use sqlx::SqliteConnection; use crate::StoreError; -/// Startup only, before installing application hooks. rusqlite supplies its -/// safe scalar-function trampoline; SQLx retains sole connection ownership and -/// executes the SQL. No borrowed wrapper or statement may escape this scope. +/// Register the functions required by ordinary writer and reader connections. pub(crate) async fn register(connection: &mut SqliteConnection) -> Result<(), StoreError> { + on_handle(connection, |db| { + crate::sessions::register_functions(db) + .and_then(|()| crate::message_identity::register(db)) + .and_then(|()| crate::transcript::navigation::register(db)) + .and_then(|()| crate::context::evidence::register(db)) + }) + .await +} +pub(crate) async fn register_migrations( + connection: &mut SqliteConnection, +) -> Result<(), StoreError> { + on_handle(connection, crate::context::semantic::migration::register).await +} +pub(crate) async fn unregister_migrations( + connection: &mut SqliteConnection, +) -> Result<(), StoreError> { + on_handle(connection, crate::context::semantic::migration::unregister).await +} + +async fn on_handle( + connection: &mut SqliteConnection, + apply: impl FnOnce(&rusqlite::Connection) -> Result<(), StoreError> + Send, +) -> Result<(), StoreError> { let mut locked = connection.lock_handle().await?; - // SAFETY: SQLx's guard prevents every worker FFI call while this borrowed - // non-owning handle exists. Both crates share libsqlite3-sys's sqlite3 type - // (no cast); from_handle does not close the underlying connection. Only - // Send + 'static UDFs are registered, and no await occurs before wrapper - // drop. SQLite owns their closures until replacement or connection close. + // SAFETY: SQLx's guard excludes worker FFI calls while this borrowed, + // non-owning connection exists. Both crates use the same libsqlite3-sys. + // The synchronous callback cannot retain a wrapper or statement, and the + // wrapper drops before the guard. Registered functions are Send + 'static. let borrowed = unsafe { rusqlite::Connection::from_handle(locked.as_raw_handle().as_ptr()) }?; - let registered = crate::sessions::register_functions(&borrowed) - .and_then(|()| crate::message_identity::register(&borrowed)) - .and_then(|()| crate::transcript::navigation::register(&borrowed)) - .and_then(|()| crate::context::evidence::register(&borrowed)); - // rusqlite 0.39 removes hooks only for an owned connection; this borrowed - // wrapper leaves SQLx's connection and hooks intact. + let result = apply(&borrowed); + // A borrowed rusqlite connection leaves SQLx's live connection and hooks intact. drop(borrowed); - registered + result } diff --git a/platform/event-log/tests/integration/startup.rs b/platform/event-log/tests/integration/startup.rs index decb4cba07..a51d9ea9fe 100644 --- a/platform/event-log/tests/integration/startup.rs +++ b/platform/event-log/tests/integration/startup.rs @@ -158,7 +158,7 @@ async fn catalog_migration_preserves_existing_history_and_backfills_cached_activ } let checksums = migration_checksums(&Connection::open(&path).unwrap()); assert_eq!(&checksums[..old_checksums.len()], old_checksums); - assert_eq!(checksums.last().unwrap().0, 12); + assert_eq!(checksums.last().unwrap().0, 13); } #[tokio::test] diff --git a/plugins/agent/facts/src/history/semantic/directory.rs b/plugins/agent/facts/src/history/semantic/directory.rs index b569c93b8e..75e77b7d84 100644 --- a/plugins/agent/facts/src/history/semantic/directory.rs +++ b/plugins/agent/facts/src/history/semantic/directory.rs @@ -18,6 +18,10 @@ */ //! Closed semantic schema positions and leaf commitments in actual stored bytes. +//! +//! Serde variant order, field order and attributes form the persisted directory +//! schema, including `RowPosition`, `TextKind` and the leaf measure types. Changes +//! require a storage migration or an explicit rebuild decision. use super::super::RowPosition; use crate::model::{ TextKind, diff --git a/scripts/bench-event-storage.py b/scripts/bench-event-storage.py new file mode 100644 index 0000000000..9269854530 --- /dev/null +++ b/scripts/bench-event-storage.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Alternate frozen release binaries and validate their actual storage work. + +Example: scripts/bench-event-storage.py --output /tmp/storage-comparison \ + original=/path/to/original candidate=/path/to/candidate +The output directory must be new. Datasets and every raw result are retained. +Reopened reads are OS-cache-warm, not cold-cache measurements. +""" + +import argparse +import hashlib +import json +import os +import platform +from pathlib import Path +import shutil +import sqlite3 +import statistics +import subprocess +import time + + +def storage_stats(directory): + result = [] + for path in sorted((directory.resolve() / "root" / "runtime-partitions").glob("*.sqlite")): + if not path.is_file(): + continue + sidecars = {} + for suffix in ("-wal", "-shm"): + sidecar = path.with_name(path.name + suffix) + sidecars[suffix] = sidecar.stat().st_size if sidecar.exists() else 0 + with sqlite3.connect(f"{path.as_uri()}?mode=ro", uri=True) as db: + tables = {name for (name,) in db.execute("SELECT name FROM sqlite_schema WHERE type='table'")} + directories = {} + for table in ("context_semantic_rows", "context_semantic_fragments", "context_semantic_directory_children"): + if table in tables: + columns = {row[1] for row in db.execute(f"PRAGMA table_info({table})")} + column = "directory_bytes" if "directory_bytes" in columns else "directory_json" + count, size = db.execute(f"SELECT COUNT(*), COALESCE(SUM(length(CAST({column} AS BLOB))),0) FROM {table}").fetchone() + directories[table] = dict(records=count, encoded_bytes=size) + result.append(dict(file=path.name, file_bytes=path.stat().st_size, + wal_bytes=sidecars["-wal"], shm_bytes=sidecars["-shm"], + page_size=db.execute("PRAGMA page_size").fetchone()[0], + pages=db.execute("PRAGMA page_count").fetchone()[0], + free_pages=db.execute("PRAGMA freelist_count").fetchone()[0], + directories=directories)) + return result + + +def run(binary, directory, mode, events, body_bytes, iterations): + command = [str(binary), "--database", str(directory), "--mode", mode, + "--events", str(events), "--body-bytes", str(body_bytes), + "--iterations", str(iterations)] + before = {"load": os.getloadavg(), "free_bytes": shutil.disk_usage(directory.parent).free} + start = time.monotonic_ns() + result = subprocess.run(command, check=True, text=True, capture_output=True) + report = json.loads(result.stdout) + if (report["errors"] or report["profile"] != "release" + or report["completed_rows"] <= 0 or report["completed_bytes"] <= 0): + raise RuntimeError(f"invalid or empty benchmark: {report}") + report.update(environment=before, wall_ns=time.monotonic_ns() - start, + stderr=result.stderr, command=command) + report["storage_after_close"] = storage_stats(directory) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("candidates", nargs="+", metavar="LABEL=BINARY") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--iterations", type=int, default=20) + parser.add_argument("--case", action="append", metavar="NAME:EVENTS:BODY_BYTES") + args = parser.parse_args() + args.output = args.output.resolve() + if args.runs < 5 or args.iterations < 1: + parser.error("at least five runs and one iteration are required") + candidates = [(label, Path(binary).resolve()) for label, binary in + (item.split("=", 1) for item in args.candidates)] + if len({label for label, _ in candidates}) != len(candidates): + parser.error("candidate labels must be unique") + cases = [item.split(":") for item in (args.case or + ["small:64:256", "large:4:2097152", "dense:128:16384"])] + args.output.mkdir(parents=True, exist_ok=False) + binaries = {} + for label, binary in candidates: + with binary.open("rb") as stream: + binaries[label] = dict(path=str(binary), sha256=hashlib.file_digest(stream, "sha256").hexdigest()) + (args.output / "manifest.json").write_text(json.dumps(dict( + binaries=binaries, platform=platform.platform(), cpu_count=os.cpu_count(), + runs=args.runs, iterations=args.iterations, cases=cases, + cache_state="one validation/warmup pass; steady state and OS-warm reopen"), indent=2) + "\n") + summaries = {} + fixtures = {} + for repeat in range(args.runs): + # Rotate order so each candidate sees a different position between runs. + order = candidates[repeat % len(candidates):] + candidates[:repeat % len(candidates)] + for case, event_count, body_size in cases: + for label, binary in order: + directory = args.output / f"{case}-{repeat}-{label}" + for mode in ("create", "read"): + report = run(binary, directory, mode, int(event_count), + int(body_size), args.iterations) + fixture = report["fixture"] + previous = fixtures.setdefault(case, fixture) + if fixture != previous: + raise RuntimeError(f"canonical fixture changed in {case}/{label}") + report_path = args.output / f"{case}-{repeat}-{label}-{mode}.json" + report_path.write_text(json.dumps(report, indent=2) + "\n") + for operation, sample in report["samples"].items(): + key = (case, mode, operation, label) + summaries.setdefault(key, []).append(sample) + print(f"{case} run={repeat} {label} {mode}: " + f"{report['completed_bytes']} bytes checked", flush=True) + rows = [] + for (case, mode, operation, label), samples in sorted(summaries.items()): + values = [sample["p50_ns"] for sample in samples] + rows.append(dict(case=case, mode=mode, operation=operation, candidate=label, + runs=len(values), median_ns=statistics.median(values), + min_ns=min(values), max_ns=max(values), run_medians_ns=values, + run_p95_ns=[sample["p95_ns"] for sample in samples], + run_p99_ns=[sample["p99_ns"] for sample in samples])) + (args.output / "summary.json").write_text(json.dumps(rows, indent=2) + "\n") + + +if __name__ == "__main__": + main() From 75a44d6eb0d9a68a57d8925a99c3b3d053d357d2 Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Sat, 31 Oct 2026 17:05:46 +0800 Subject: [PATCH 2/8] feat(storage): separate canonical tool bodies from event headers Preserve exact canonical bytes with immutable shared projection bodies and header slots. Resolve frozen ranges on the original SQLite worker using its existing statement cache, and keep full replay, transfer, and collection in the same owner lifecycle. Generated-by: Codex --- platform/event-log/README.md | 8 +- platform/event-log/README.zh-CN.md | 8 +- .../event-log/examples/event_storage_bench.rs | 75 ++- .../0013_canonical_event_bodies.sql | 305 ++++++++++++ platform/event-log/src/append.rs | 16 +- platform/event-log/src/append/history.rs | 4 +- platform/event-log/src/append/ingress.rs | 11 +- .../event-log/src/append/ingress/tests.rs | 20 +- platform/event-log/src/archive/accepted.rs | 4 +- platform/event-log/src/archive/candidates.rs | 15 +- platform/event-log/src/archive/target.rs | 60 ++- platform/event-log/src/artifacts/history.rs | 33 +- platform/event-log/src/bundle/accounting.rs | 12 +- platform/event-log/src/bundle/blobs.rs | 91 ++-- platform/event-log/src/bundle/closure.rs | 24 +- platform/event-log/src/bundle/export.rs | 57 ++- .../event-log/src/bundle/import/collision.rs | 4 +- .../event-log/src/bundle/import/publish.rs | 14 +- platform/event-log/src/bundle/validation.rs | 4 +- .../src/bundle/validation/artifacts.rs | 45 +- .../event-log/src/bundle/validation/copies.rs | 6 +- .../src/bundle/validation/material.rs | 4 +- platform/event-log/src/canonical.rs | 32 ++ platform/event-log/src/canonical/migration.rs | 233 +++++++++ platform/event-log/src/canonical/read.rs | 227 +++++++++ platform/event-log/src/canonical/recipe.rs | 441 ++++++++++++++++++ platform/event-log/src/canonical/source.rs | 240 ++++++++++ platform/event-log/src/canonical/write.rs | 74 +++ platform/event-log/src/canonical_insert.rs | 13 +- platform/event-log/src/catalog_changes.rs | 4 +- platform/event-log/src/composition.rs | 14 +- platform/event-log/src/context/boundary.rs | 28 +- .../event-log/src/context/evidence/roots.rs | 4 +- platform/event-log/src/context/history.rs | 8 +- platform/event-log/src/context/latest_main.rs | 42 +- platform/event-log/src/context/projection.rs | 8 +- platform/event-log/src/context/proof.rs | 64 ++- platform/event-log/src/context/read.rs | 12 +- platform/event-log/src/context/report.rs | 12 +- platform/event-log/src/context/request.rs | 8 +- platform/event-log/src/context/safety.rs | 16 +- platform/event-log/src/context/selection.rs | 4 +- .../src/context/semantic/captured.rs | 182 +++++--- .../semantic/captured/artifact_identity.rs | 13 +- .../context/semantic/captured/canonical.rs | 114 +++++ .../event-log/src/context/semantic/source.rs | 97 +++- platform/event-log/src/context/tail.rs | 30 +- platform/event-log/src/context/usage.rs | 50 +- platform/event-log/src/context/validate.rs | 2 +- platform/event-log/src/continuation.rs | 22 +- platform/event-log/src/diagnostics.rs | 20 +- platform/event-log/src/executor.rs | 2 +- platform/event-log/src/executor/tests.rs | 6 +- .../event-log/src/executor_identity_tests.rs | 2 +- platform/event-log/src/handoff.rs | 24 +- platform/event-log/src/handoff/evidence.rs | 8 +- platform/event-log/src/handoff/owner.rs | 2 +- platform/event-log/src/lib.rs | 1 + .../src/message_admissions/insert.rs | 8 +- platform/event-log/src/message_identity.rs | 18 +- platform/event-log/src/message_interrupts.rs | 4 +- platform/event-log/src/message_queue/edit.rs | 2 +- platform/event-log/src/message_resolution.rs | 2 +- .../event-log/src/message_resolution/owner.rs | 14 +- platform/event-log/src/message_sources.rs | 36 +- platform/event-log/src/model_items.rs | 12 +- platform/event-log/src/model_retries.rs | 12 +- platform/event-log/src/native_approvals.rs | 6 +- platform/event-log/src/native_computer.rs | 12 +- platform/event-log/src/observation.rs | 26 +- .../event-log/src/observation/delivery.rs | 102 ++-- platform/event-log/src/observation/message.rs | 16 +- platform/event-log/src/observation/streams.rs | 22 +- platform/event-log/src/observation/tools.rs | 16 +- platform/event-log/src/prefix.rs | 56 +-- platform/event-log/src/read_resources.rs | 4 +- platform/event-log/src/recovery.rs | 16 +- platform/event-log/src/root_store/catalog.rs | 2 +- .../src/root_store/material/tests.rs | 6 +- .../event-log/src/root_store/usage_meter.rs | 8 +- .../src/root_store/usage_meter/tests.rs | 2 +- platform/event-log/src/run_source.rs | 10 +- platform/event-log/src/schema/fixtures.rs | 2 + platform/event-log/src/sessions/activity.rs | 26 +- .../src/sessions/activity/partial.rs | 6 +- platform/event-log/src/sessions/copy.rs | 10 +- platform/event-log/src/sessions/execution.rs | 16 +- platform/event-log/src/sessions/import.rs | 2 +- platform/event-log/src/sessions/material.rs | 25 +- platform/event-log/src/sessions/read_state.rs | 2 +- platform/event-log/src/sessions/removal.rs | 6 +- platform/event-log/src/sqlite_functions.rs | 12 +- platform/event-log/src/steering.rs | 12 +- platform/event-log/src/tool_calls.rs | 32 +- platform/event-log/src/tool_payloads.rs | 36 +- platform/event-log/src/transcript/active.rs | 2 +- .../event-log/src/transcript/compaction.rs | 20 +- .../event-log/src/transcript/directory.rs | 2 +- platform/event-log/src/transcript/emit.rs | 24 +- platform/event-log/src/transcript/evidence.rs | 45 +- .../event-log/src/transcript/final_output.rs | 6 +- platform/event-log/src/transcript/model.rs | 28 +- .../src/transcript/navigation/queries.rs | 16 +- platform/event-log/src/transcript/tools.rs | 4 +- platform/event-log/src/turns.rs | 32 +- platform/event-log/src/turns/root.rs | 10 +- platform/event-log/src/usage/auxiliary.rs | 20 +- platform/event-log/src/workspace.rs | 2 +- .../tests/integration/archive/mod.rs | 8 +- .../tests/integration/bundle/import.rs | 2 +- .../context_compaction/active/latest.rs | 2 +- .../continuation/handoff_cancel.rs | 2 +- .../tests/integration/continuation/mod.rs | 2 +- .../tests/integration/import_atomic.rs | 8 +- .../tests/integration/session_material.rs | 113 ++++- .../event-log/tests/integration/startup.rs | 2 +- plugins/agent/native/src/owner/tests.rs | 4 +- .../native/src/owner/tests/prompt_language.rs | 2 +- .../native/src/owner/tests/title_tests.rs | 2 +- scripts/bench-event-storage.py | 12 +- 120 files changed, 2988 insertions(+), 860 deletions(-) create mode 100644 platform/event-log/migrations/0013_canonical_event_bodies.sql create mode 100644 platform/event-log/src/canonical.rs create mode 100644 platform/event-log/src/canonical/migration.rs create mode 100644 platform/event-log/src/canonical/read.rs create mode 100644 platform/event-log/src/canonical/recipe.rs create mode 100644 platform/event-log/src/canonical/source.rs create mode 100644 platform/event-log/src/canonical/write.rs create mode 100644 platform/event-log/src/context/semantic/captured/canonical.rs diff --git a/platform/event-log/README.md b/platform/event-log/README.md index 21c64efc0f..dba66b3ca3 100644 --- a/platform/event-log/README.md +++ b/platform/event-log/README.md @@ -69,11 +69,15 @@ New runtime events use UUIDv7 identities to keep event and derived message index Canonical runtime-event ingress inserts each event, captures its sequence, then maintains safety, execution, source and semantic projections as separate statements on the same SQLite worker and writer transaction. The shared insert statement derives indexed accounting and owner fields from those same JSON bytes; collection preserves their published values when removing retired bodies. Typed facts select the safety and execution statements to run. The next event cannot begin before those projections finish. Projection storage failures roll back the entire batch, including imported events; notifications remain tied to successful commit. Copied-prefix projections are maintained by their current copy paths. -Canonical databases use UTF-8. Semantic fragments for successful tool results reference checked byte spans in their original event JSON. Resolution reads and decodes the selected projection on the original SQLite worker within the caller's read snapshot; the source collector retains it for live history owners. Derived archive placeholders and other fragment bodies keep their chunked storage. +Canonical databases use UTF-8. Successful tool results store their original `model_projection` token separately: `event_header` preserves every surrounding byte with a literal `null` placeholder, and an immutable slot records the original byte offset and body identity. Reconstructing the header and original token reproduces the exact canonical event, including imported whitespace, number spelling and Unicode escapes. `canonical_bytes` supplies logical byte budgets; encoded size never substitutes for admitted canonical size. Other event kinds remain inline. + +Bodies are directly seekable raw bytes, shared by SHA-256 of their original token with exact byte comparison on a repeated identity. Reads validate logical lengths and selected ranges. Framed compression was evaluated against raw storage and rejected because it repeatedly slowed supported window and sequential reads; it is not a persisted format. Space savings come from compact directories and sharing identical projection tokens, not from compressing unique bodies. + +Complete-event queries join the slot and body into the selected SQL row, then reconstruct in Rust within the original snapshot. Metadata predicates use the header and retain their indexes. Semantic spans retain original canonical coordinates; tool spans map those logical coordinates to header/body ranges, while inline model parts retain physical SQLite BLOB bounds. Resolution stays on the original SQLite worker. Hot range reads use its existing prepared-statement cache; bounded prefix decoding borrows rows on that same worker without copying every nullable column across the callback. Slots follow the existing history-owner collection proof; deleting the final slot removes its body in the same transaction. Neither a body digest nor a derived directory grants access or extends lifetime. Derived archive placeholders and other fragment bodies keep their chunked storage. Semantic row, payload and child directories are finite derived descriptors encoded with Postcard. Writers encode them once in the publication transaction. Pages copy their stored bytes alongside fixed little-endian locator fields into the caller's charged allocation; consumers decode the original typed descriptors without navigating whole bodies. Original byte positions and leaf commitments remain unchanged. Directory encoding never supplies a canonical fact or source identity. The SQL migration converts former JSON descriptors atomically; its conversion functions exist only during migration. -`event_storage_bench` exercises real Root writes, captured directories, child pages, first/tail windows, sequential bodies, event prefixes and bundle export. Build the same example in release against both revisions and retain each binary, then run `python3 scripts/bench-event-storage.py --output /tmp/storage-comparison original=/path/to/original candidate=/path/to/candidate`. It alternates at least five runs and validates event, descriptor, body and export digests. The first read pass validates and warms caches; reopened measurements are OS-cache-warm. Raw distributions, process CPU/RSS/I/O availability, SQLite cache counters and separate main/WAL/SHM file sizes are recorded. Live prepared-statement snapshots do not represent total SQL work, and mixed projection timings require per-kind follow-up when evaluating individual-path regressions. +`event_storage_bench` exercises real Root writes, captured directories, child pages, first/tail and repeated 64-byte windows, sequential bodies, event prefixes and bundle export. Build the same example in release against both revisions and retain each binary, then run `python3 scripts/bench-event-storage.py --output /tmp/storage-comparison original=/path/to/original candidate=/path/to/candidate`. It alternates at least five runs and validates event, descriptor, body and export digests. The first read pass validates and warms caches; reopened measurements are OS-cache-warm. Raw distributions, process CPU/RSS/I/O availability, SQLite cache counters and separate main/WAL/SHM file sizes are recorded. Live prepared-statement snapshots do not represent total SQL work, and JSON/text/content leaf reads and settlement writes are also reported separately. Use `--body-pattern entropy` to include deterministic low-compressibility printable input. A statement failure rolls back the entire group unless it is a domain rejection isolated by its own savepoint. Former ordinary writer requests can continue recovery on the same owner after acknowledged rollback and fresh authority validation; pure observation batches retain their stricter retirement policy. Failed commits always return unknown outcomes without execution claims. Reusing a former ordinary writer group after a failed commit additionally requires acknowledged cleanup and zero transaction depth. Savepoint, authority and unconfirmed cleanup failures retire the writer. Single-key and paged plugin-data reads use the independent reader owner and see committed snapshots; dependent reads must still await their preceding write receipt. diff --git a/platform/event-log/README.zh-CN.md b/platform/event-log/README.zh-CN.md index 95d0869f0a..c0768b2c00 100644 --- a/platform/event-log/README.zh-CN.md +++ b/platform/event-log/README.zh-CN.md @@ -63,11 +63,15 @@ Control 中的调用意图、项目登记、workspace attachment、Session 路 规范运行时事件入口逐条插入事件并取得序号，再在同一 SQLite worker、同一写事务内，用独立语句维护安全、执行、来源和语义投影。用于索引的记账和所有者字段由统一插入语句从同一 JSON 计算，正文回收后保留其发布值。安全与执行维护由类型化事实选择相应语句。当前事件投影完成后才开始下一条。投影存储错误使整个批次回滚，导入事件同样如此；通知仍只在提交成功后发布。复制前缀投影由当前复制路径维护。 -规范数据库使用 UTF-8。成功工具结果的语义片段引用原始事件 JSON 中经过校验的字节区间。解析在调用者的读取快照内，由原 SQLite worker 读取并解码选中的工具投影；来源收集器为仍有历史所有者的正文保留存活期。归档占位内容和其他片段正文继续使用分块存储。 +规范数据库使用 UTF-8。成功工具结果的原始 `model_projection` token 单独保存：`event_header` 用字面量 `null` 占位，保留周围全部原始字节；不可变 slot 记录原字节偏移和正文身份。两者重建出精确的规范事件，包括导入内容的空白、数字拼写和 Unicode 转义。`canonical_bytes` 用于逻辑字节预算，压缩大小不替代已接受的规范大小。其他事件类型仍内联保存。 + +正文保存为可直接 seek 的原始字节，按原始 token 的 SHA-256 共享，重复身份还需逐字节核对。读取校验逻辑长度与选中区间。对照实验中，分帧压缩稳定拖慢了受支持的窗口和顺序读取，因此未作为持久化格式保留。空间收益来自紧凑目录与相同投影 token 的共享，不能把压缩实验中独有正文的压缩率套到最终实现。 + +完整事件查询在同一 SQL 行 JOIN slot 和正文，在原快照内由 Rust 重建；元数据谓词继续读取 header 并使用原索引。语义区间保留原规范坐标，工具区间映射至实际 header／body，内联 model part 则保留 SQLite BLOB 的物理边界。读取仍由原 SQLite worker 执行。热区间读取复用其已有 statement cache；有界前缀在同一 worker 上借用行并解码，避免把每个可空列复制出回调。Slot 存活由既有历史所有者回收证明决定，最后一个 slot 删除时，在同一事务删除正文；正文摘要和派生目录都不授予访问权，也不延长存活期。归档占位内容和其他片段正文继续使用分块存储。 语义行、payload 和子项目录是有限的派生描述符，采用 Postcard 编码。writer 在发布事务内编码一次；分页把已存字节和固定小端序定位字段直接复制到调用者计费的缓冲区，消费端恢复原有类型，无需读取整个正文。原始字节位置与叶子摘要保持不变，目录编码不参与规范事实或来源身份。SQL 迁移在同一事务内转换原 JSON 描述符；转换函数仅在迁移期间存在。 -`event_storage_bench` 测量真实 Root 写入、冻结目录、子项分页、首尾窗口、顺序正文、事件前缀和 bundle 导出。在两个版本上用 release 构建同一 example 并各自保存二进制后，运行 `python3 scripts/bench-event-storage.py --output /tmp/storage-comparison original=/path/to/original candidate=/path/to/candidate`。脚本交错执行至少五轮，校验事件、目录、正文与导出摘要。首次读取用于校验和预热；重开测量保留操作系统缓存。记录保留原始分布、进程 CPU/RSS/I/O 可用性、SQLite 缓存计数，以及分列的主库、WAL、SHM 大小。存活 prepared statement 的快照不能代表全部 SQL 工作；评估单独路径是否退化时，需要按投影类型复核混合计时。 +`event_storage_bench` 测量真实 Root 写入、冻结目录、子项分页、首尾及重复 64 字节窗口、顺序正文、事件前缀和 bundle 导出。在两个版本上用 release 构建同一 example 并各自保存二进制后，运行 `python3 scripts/bench-event-storage.py --output /tmp/storage-comparison original=/path/to/original candidate=/path/to/candidate`。脚本交错执行至少五轮，校验事件、目录、正文与导出摘要。首次读取用于校验和预热；重开测量保留操作系统缓存。记录保留原始分布、进程 CPU/RSS/I/O 可用性、SQLite 缓存计数，以及分列的主库、WAL、SHM 大小。存活 prepared statement 的快照不能代表全部 SQL 工作；另外分别记录 JSON／文本／多段内容的叶子读取和结算写入耗时。 语句失败会回滚整组，只有已由自身保存点隔离的领域拒绝例外。原先使用普通写入路径的请求，在确认回滚并重新校验所有权后，可以继续通过同一 owner 恢复；纯 observation 批次保留更严格的退出策略。提交失败始终返回结果未知，不产生执行凭据；此后复用原普通写入组，还要求确认清理完成且事务深度为零。保存点、所有权或无法确认的清理失败都会让写入器退出。单键与分页插件数据读取使用独立 reader，读取已提交的快照；依赖写入的读取仍需先等待相应回执。 diff --git a/platform/event-log/examples/event_storage_bench.rs b/platform/event-log/examples/event_storage_bench.rs index 6b7a01d6c7..cd3a459a59 100644 --- a/platform/event-log/examples/event_storage_bench.rs +++ b/platform/event-log/examples/event_storage_bench.rs @@ -328,6 +328,10 @@ async fn main() -> Result<()> { } let events: usize = value("--events", "128").parse()?; let body_bytes: usize = value("--body-bytes", "65536").parse()?; + let body_pattern = value("--body-pattern", "repeated"); + if !matches!(body_pattern.as_str(), "repeated" | "entropy") { + return Err("body pattern must be repeated or entropy".into()); + } let iterations: usize = value("--iterations", "10").parse()?; if events == 0 || iterations == 0 { return Err("events and iterations must be positive".into()); @@ -372,6 +376,17 @@ async fn main() -> Result<()> { let pattern = "line: storage \\\"payload\\\"\n"; let mut body = pattern.repeat(body_bytes.div_ceil(pattern.len())); body.truncate(body_bytes); + if body_pattern == "entropy" { + let mut state = 0x9e3779b97f4a7c15u64; + body = (0..body_bytes) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + char::from(33 + (state % 94) as u8) + }) + .collect(); + } let mut payloads = serde_json::Map::new(); for index in 0..events { let operation = format!("tool-{index}"); @@ -380,6 +395,17 @@ async fn main() -> Result<()> { let (write, sequence, settled_ns) = append_tool(&session, &invocation, &operation, &text, index as u64).await?; samples.add("append_attempt", start); + samples + .0 + .entry( + [ + "append_settled_json", + "append_settled_text", + "append_settled_content", + ][index % 3], + ) + .or_default() + .push(settled_ns); samples .0 .entry("append_tool_settled") @@ -394,7 +420,9 @@ async fn main() -> Result<()> { } = &write.event().fact { let projection = serde_json::to_vec(model_projection)?; - payloads.insert(sequence.to_string(),json!({"bytes":projection.len(),"digest":digest(&projection),"first_digest":digest(&projection[..projection.len().min(64)]),"tail_digest":digest(&projection[projection.len().saturating_sub(64)..])})); + let window_start = (32 * 1024usize - 32).min(projection.len().saturating_sub(256)); + let window_end = (window_start + 256).min(projection.len()); + payloads.insert(sequence.to_string(),json!({"window_start":window_start,"window_digest":digest(&projection[window_start..window_end]),"bytes":projection.len(),"digest":digest(&projection),"first_digest":digest(&projection[..projection.len().min(64)]),"tail_digest":digest(&projection[projection.len().saturating_sub(64)..])})); } } session @@ -417,7 +445,7 @@ async fn main() -> Result<()> { let (_, export) = root .export_bundle("bench", &inventory.subtree_digest, tokio::io::sink()) .await?; - manifest = json!({"events":events,"body_bytes":body_bytes,"prefix_digest":prefix.digest,"export_digest":export.digest,"events_digest":digest(&serde_json::to_vec(&prefix.events)?),"payloads":payloads,"canonical_events":prefix.events.len()}); + manifest = json!({"events":events,"body_bytes":body_bytes,"body_pattern":body_pattern,"prefix_digest":prefix.digest,"export_digest":export.digest,"events_digest":digest(&serde_json::to_vec(&prefix.events)?),"payloads":payloads,"canonical_events":prefix.events.len()}); std::fs::write( directory.join("fixture.json"), serde_json::to_vec_pretty(&manifest)?, @@ -537,6 +565,39 @@ async fn main() -> Result<()> { samples.add("tail_64_all", start); } let start = Instant::now(); + for (body, length, evidence, _) in &payloads { + let offset = evidence["window_start"] + .as_u64() + .ok_or("missing window start")?; + let count = (*length - offset).min(256); + let source = captured.byte_range(body, offset, count)?; + let mut reader = source.open(Arc::new(Budget)); + let mut buffer = allocation(64); + let mut sha = Sha256::new(); + let mut bytes = 0; + loop { + let chunk = reader.read_chunk(buffer).await?; + sha.update(&chunk.buffer.as_ref()[..chunk.bytes_read]); + bytes += chunk.bytes_read as u64; + buffer = chunk.buffer; + if chunk.bytes_read == 0 { + break; + } + } + if bytes != count + || hex::encode(sha.finalize()) + != evidence["window_digest"] + .as_str() + .ok_or("missing window digest")? + { + return Err("cross-frame repeated windows changed".into()); + } + completed_bytes += bytes; + } + if iteration > 0 { + samples.add("cross_frame_repeated_64_all", start); + } + let start = Instant::now(); for (body, _, evidence, _) in &payloads { let (sha, bytes) = captured .with_sequential_source(body, |reader| { @@ -568,6 +629,13 @@ async fn main() -> Result<()> { } let start = Instant::now(); for (body, _, _, directory) in &payloads { + let leaf_start = Instant::now(); + let operation = match directory { + PayloadDirectory::Tool(ToolDirectory::Text(_)) => "descriptor_leaf_text", + PayloadDirectory::Tool(ToolDirectory::Json(_)) => "descriptor_leaf_json", + PayloadDirectory::Tool(ToolDirectory::Content { .. }) => "descriptor_leaf_content", + _ => return Err("unexpected tool directory".into()), + }; match directory { PayloadDirectory::Tool(ToolDirectory::Text(span)) => { check_text(&captured, body, *span).await? @@ -607,6 +675,9 @@ async fn main() -> Result<()> { } _ => return Err("unexpected tool directory".into()), } + if iteration > 0 { + samples.add(operation, leaf_start); + } } if iteration > 0 { samples.add("descriptor_leaves_all", start); diff --git a/platform/event-log/migrations/0013_canonical_event_bodies.sql b/platform/event-log/migrations/0013_canonical_event_bodies.sql new file mode 100644 index 0000000000..e393aa1aec --- /dev/null +++ b/platform/event-log/migrations/0013_canonical_event_bodies.sql @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +-- Dependent SELECT * views expose the renamed canonical header together. +DROP VIEW session_history_events; +DROP VIEW local_runtime_events; +DROP VIEW session_message_sources; +DROP VIEW usage_activity; +DROP VIEW tool_usage; +DROP VIEW model_usage; +DROP VIEW auxiliary_model_usage; +DROP VIEW agent_model_usage; +DROP VIEW accounting_events; +DROP VIEW model_selected_parts; +DROP VIEW model_part_bodies; +DROP VIEW runtime_events; + +ALTER TABLE event_log RENAME COLUMN event_json TO event_header; +ALTER TABLE event_log ADD COLUMN canonical_bytes INTEGER CHECK(canonical_bytes IS NULL OR canonical_bytes>=0); +UPDATE event_log SET canonical_bytes=length(CAST(event_header AS BLOB)) WHERE event_header IS NOT NULL; + +CREATE TABLE event_bodies ( + id INTEGER PRIMARY KEY, + digest BLOB NOT NULL UNIQUE CHECK(length(digest)=32), + raw_bytes INTEGER NOT NULL CHECK(raw_bytes>0), + sqlite_json_bytes INTEGER NOT NULL CHECK(sqlite_json_bytes>0), + payload BLOB NOT NULL, + CHECK(length(payload)=raw_bytes) +); +CREATE TABLE event_body_slots ( + event_sequence INTEGER PRIMARY KEY REFERENCES event_log(sequence) ON DELETE CASCADE, + header_offset INTEGER NOT NULL CHECK(header_offset>=0), + body_id INTEGER NOT NULL REFERENCES event_bodies(id) +); +CREATE INDEX event_body_slot_material ON event_body_slots(body_id); + +CREATE TEMP TABLE canonical_body_stage AS + SELECT sequence, + maka_projection_offset(kind,event_header) AS header_offset, + maka_projection_bytes(kind,event_header) AS raw + FROM event_log WHERE kind='tool_settled' AND event_header IS NOT NULL; +DELETE FROM canonical_body_stage WHERE raw IS NULL; +INSERT INTO event_bodies(digest,raw_bytes,sqlite_json_bytes,payload) + SELECT DISTINCT maka_body_sha(raw),length(raw), + length(CAST(json(CAST(raw AS TEXT)) AS BLOB)),raw + FROM canonical_body_stage; +UPDATE event_log SET event_header=maka_event_header(kind,event_header) + WHERE sequence IN (SELECT sequence FROM canonical_body_stage); +INSERT INTO event_body_slots(event_sequence,header_offset,body_id) + SELECT stage.sequence,stage.header_offset,body.id + FROM canonical_body_stage stage JOIN event_bodies body ON body.digest=maka_body_sha(stage.raw); +DROP TABLE canonical_body_stage; + +CREATE TRIGGER event_body_immutable BEFORE UPDATE ON event_bodies BEGIN + SELECT RAISE(ABORT,'canonical body material is immutable'); +END; +CREATE TRIGGER event_body_slot_immutable BEFORE UPDATE ON event_body_slots BEGIN + SELECT RAISE(ABORT,'canonical body slots are immutable'); +END; +CREATE TRIGGER event_body_released AFTER DELETE ON event_body_slots BEGIN + DELETE FROM event_bodies WHERE id=OLD.body_id + AND NOT EXISTS(SELECT 1 FROM event_body_slots WHERE body_id=OLD.body_id); +END; +CREATE TRIGGER event_live_length_insert BEFORE INSERT ON event_log + WHEN (NEW.event_header IS NULL)!=(NEW.canonical_bytes IS NULL) BEGIN + SELECT RAISE(ABORT,'canonical byte length must follow live material'); +END; +CREATE TRIGGER event_live_length_update BEFORE UPDATE OF event_header,canonical_bytes ON event_log + WHEN (NEW.event_header IS NULL)!=(NEW.canonical_bytes IS NULL) + OR (NEW.event_header IS NULL AND EXISTS(SELECT 1 FROM event_body_slots WHERE event_sequence=NEW.sequence)) BEGIN + SELECT RAISE(ABORT,'collected events cannot retain live canonical material'); +END; +CREATE TRIGGER event_body_slot_valid BEFORE INSERT ON event_body_slots + WHEN NOT EXISTS(SELECT 1 FROM event_log e JOIN event_bodies b ON b.id=NEW.body_id + WHERE e.sequence=NEW.event_sequence AND e.event_header IS NOT NULL + AND e.kind='tool_settled' + AND json_extract(e.event_header,'$.fact.outcome.kind')='succeeded' + AND e.canonical_bytes=length(CAST(e.event_header AS BLOB))-4+b.raw_bytes + AND substr(CAST(e.event_header AS BLOB),NEW.header_offset+1,4)=X'6e756c6c') BEGIN + SELECT RAISE(ABORT,'invalid canonical body slot recipe'); +END; + +CREATE VIEW runtime_events AS + SELECT * FROM event_log WHERE invocation_id IS NOT NULL AND event_header IS NOT NULL; + +CREATE VIEW model_part_bodies AS +SELECT p.*, e.kind AS source_kind, + json_extract(e.event_header, CASE e.kind + WHEN 'model_observed' THEN '$.fact.event.data' + WHEN 'model_completed' THEN '$.fact.output.parts[' || p.record_index || '].data' + WHEN 'model_interrupted' THEN '$.fact.partial.parts[' || p.record_index || '].data' + END) AS record, + json_extract(e.event_header, CASE e.kind + WHEN 'model_observed' THEN '$.fact.event.data.part' + WHEN 'model_completed' THEN '$.fact.output.parts[' || p.record_index || '].data.part' + WHEN 'model_interrupted' THEN '$.fact.partial.parts[' || p.record_index || '].data.part' + END) AS part +FROM model_part_records p JOIN runtime_events e ON e.sequence=p.source_sequence +WHERE p.record_kind='part'; + +CREATE VIEW model_selected_parts AS +SELECT e.invocation_id,e.operation_id AS step_id,e.kind AS terminal_kind,e.sequence,e.event_id, + CAST(json_extract(s.value,'$.data.ordinal') AS INTEGER) AS ordinal, + CASE json_extract(s.value,'$.kind') WHEN 'own' THEN e.event_id + WHEN 'ref' THEN json_extract(s.value,'$.data.event_id') END AS body_event_id +FROM runtime_events e, json_each(e.event_header, CASE e.kind + WHEN 'model_completed' THEN '$.fact.output.parts' + WHEN 'model_interrupted' THEN '$.fact.partial.parts' END) s +WHERE e.kind IN ('model_completed','model_interrupted'); + +CREATE VIEW accounting_events AS + SELECT sequence, event_id, invocation_id, kind, operation_id, + accounting_at, accounting_usage, invocation_model, accounting_tool, + COALESCE(event_header, retained_json) AS event_header + FROM event_log; + +CREATE VIEW agent_model_usage AS +WITH attempts AS ( + SELECT request.sequence, request.event_id, + json_extract(request.event_header, '$.invocation') AS invocation, + opening.invocation_model AS binding, + json_extract(request.event_header, '$.fact.model_id') AS model_id, + json_extract(request.event_header, '$.fact.purpose') AS purpose, + request.accounting_at AS started_at, + COALESCE(completed.accounting_at, interrupted.accounting_at, terminal.accounting_at) AS completed_at, + COALESCE(completed.sequence, interrupted.sequence, terminal.sequence) AS completed_sequence, + CASE + WHEN completed.sequence IS NOT NULL THEN 'success' + WHEN json_extract(interrupted.event_header, '$.fact.status') = 'cancelled' THEN 'aborted' + WHEN interrupted.sequence IS NOT NULL THEN 'error' + WHEN json_extract(terminal.event_header, '$.fact.outcome.kind') = 'cancelled' THEN 'aborted' + WHEN terminal.sequence IS NOT NULL THEN 'unknown' + END AS outcome, + COALESCE(completed.accounting_usage, ( + SELECT observed.accounting_usage + FROM event_log observed INDEXED BY model_usage_observation + WHERE observed.kind = 'model_observed' + AND json_extract(COALESCE(observed.event_header, observed.retained_json), '$.fact.event.kind') = 'finished' + AND observed.invocation_id = request.invocation_id + AND json_extract(COALESCE(observed.event_header, observed.retained_json), '$.fact.step_id') = request.operation_id + ORDER BY observed.sequence DESC LIMIT 1 + ), '{}') AS usage + FROM accounting_events request + JOIN accounting_events opening ON opening.invocation_id = request.invocation_id + AND opening.kind = 'invocation_opened' + LEFT JOIN accounting_events completed ON completed.operation_id = request.operation_id + AND completed.kind = 'model_completed' + LEFT JOIN accounting_events interrupted ON interrupted.operation_id = request.operation_id + AND interrupted.kind = 'model_interrupted' + LEFT JOIN accounting_events terminal ON terminal.invocation_id = request.invocation_id + AND terminal.kind = 'invocation_ended' + WHERE request.kind = 'model_requested' +) +SELECT sequence, event_id, + json_object('kind', 'agent', 'invocation', json(invocation), 'purpose', purpose) AS origin, + binding, model_id, outcome, usage, + json_extract(invocation, '$.session_id') AS session_id, + started_at, completed_at, + completed_sequence +FROM attempts; + +CREATE VIEW auxiliary_model_usage AS +SELECT started.sequence, started.event_id, + json_object('kind', 'auxiliary', 'source', json_extract(started.event_header, '$.source')) AS origin, + json_extract(started.event_header, '$.binding') AS binding, + json_extract(started.event_header, '$.binding.model') AS model_id, + json_extract(settled.event_header, '$.outcome') AS outcome, + COALESCE(observed.accounting_usage, '{}') AS usage, + json_extract(started.event_header, '$.session_id') AS session_id, + started.accounting_at AS started_at, settled.accounting_at AS completed_at, + settled.sequence AS completed_sequence +FROM event_log started +LEFT JOIN event_log settled ON settled.kind = 'auxiliary_model_settled' + AND json_extract(settled.event_header, '$.request_id') = started.event_id +LEFT JOIN event_log observed ON observed.kind = 'auxiliary_model_usage' + AND json_extract(COALESCE(observed.event_header, observed.retained_json), '$.request_id') = started.event_id +WHERE started.kind = 'auxiliary_model_started'; + +CREATE VIEW model_usage AS +SELECT calls.*, accounting.quote_json, valuation.usd +FROM (SELECT * FROM agent_model_usage UNION ALL SELECT * FROM auxiliary_model_usage) calls +LEFT JOIN model_accounting accounting ON accounting.request_id = calls.event_id +LEFT JOIN model_valuations valuation ON valuation.request_id = calls.event_id; + +CREATE VIEW tool_usage AS +SELECT request.sequence, request.event_id, + json_extract(request.accounting_tool, '$.invocation') AS invocation, + json_extract(request.accounting_tool, '$.invocation.session_id') AS session_id, + json_extract(request.accounting_tool, '$.call') AS call, + json_extract(request.accounting_tool, '$.name') AS name, + opening.invocation_model AS binding, + CASE WHEN request.kind = 'tool_rejected' THEN request.accounting_at + ELSE COALESCE(settled.accounting_at, terminal.accounting_at) END AS completed_at, + CASE WHEN request.kind = 'tool_rejected' THEN request.sequence + ELSE COALESCE(settled.sequence, terminal.sequence) END AS completed_sequence, + CASE WHEN request.kind = 'tool_rejected' THEN json_object( + 'kind', 'rejected', 'reason', json_extract(request.accounting_tool, '$.rejection')) + ELSE json_object('kind', 'settled', 'startedAt', request.accounting_at, + 'outcome', CASE json_extract(settled.accounting_tool, '$.outcome') + WHEN 'succeeded' THEN 'success' WHEN 'failed' THEN 'error' ELSE 'unknown' END) + END AS result +FROM accounting_events request +JOIN accounting_events opening ON opening.invocation_id = request.invocation_id + AND opening.kind = 'invocation_opened' +LEFT JOIN accounting_events settled ON settled.invocation_id = request.invocation_id + AND settled.operation_id = request.operation_id AND settled.kind = 'tool_settled' +LEFT JOIN accounting_events terminal ON terminal.invocation_id = request.invocation_id + AND terminal.kind = 'invocation_ended' +WHERE request.kind IN ('tool_dispatched', 'tool_rejected'); + +CREATE VIEW usage_activity AS +SELECT 'model' AS kind, sequence, event_id, session_id, completed_sequence, + binding, completed_at, origin, model_id, started_at, outcome, usage, quote_json, usd, + NULL AS invocation, NULL AS call, NULL AS name, NULL AS result, + outcome AS status, + COALESCE(session_id, '') || ' ' || model_id || ' ' || + COALESCE(json_extract(binding, '$.connection_slug'), '') || ' ' || + COALESCE(json_extract(quote_json, '$.providerId'), '') AS search +FROM model_usage +UNION ALL +SELECT 'tool', sequence, event_id, session_id, completed_sequence, + binding, completed_at, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + invocation, call, name, result, + CASE WHEN json_extract(result, '$.kind') = 'rejected' THEN 'rejected' + ELSE json_extract(result, '$.outcome') END, + session_id || ' ' || name || ' ' || + COALESCE(json_extract(binding, '$.connection_slug'), '') || ' ' || + COALESCE(json_extract(binding, '$.model'), '') +FROM tool_usage; + +CREATE VIEW session_message_sources AS + SELECT session_id AS owner_session_id, session_id AS source_session_id, message_id, event_id + FROM message_sources + UNION ALL + SELECT h.session_id, s.session_id, s.message_id, s.event_id FROM session_history_members h + JOIN runtime_events e ON e.sequence=h.sequence JOIN message_sources s ON s.event_id=e.event_id + UNION ALL + SELECT r.session_id, s.session_id, s.message_id, s.event_id FROM session_revision_sources r + JOIN runtime_events e ON e.sequence=r.sequence JOIN message_sources s ON s.event_id=e.event_id; + +CREATE VIEW local_runtime_events AS + SELECT e.* FROM runtime_events e + WHERE NOT EXISTS(SELECT 1 FROM imported_invocations i WHERE i.invocation_id=e.invocation_id); + +CREATE VIEW session_history_events AS + SELECT e.event_session AS owner_session_id, + 0 AS inherited, NULL AS archive_sequence, e.* FROM runtime_events e + UNION ALL + SELECT h.session_id, 1, h.archive_sequence, e.* + FROM session_history_members h JOIN runtime_events e ON e.sequence = h.sequence; + + +-- Tool spans address logical canonical bytes, while model parts still use a +-- physical SQLite header BLOB. Keep fragment pieces (and their rowids) intact. +CREATE TABLE context_semantic_fragments_new ( + source_sequence INTEGER NOT NULL, + source_index INTEGER NOT NULL, + storage_kind TEXT NOT NULL CHECK(storage_kind IN ('pieces','canonical_tool_projection','canonical_model_part')), + byte_offset INTEGER, + byte_len INTEGER, + directory_bytes BLOB NOT NULL, + payload_bytes INTEGER NOT NULL CHECK(payload_bytes>0), + CHECK((storage_kind='pieces' AND byte_offset IS NULL AND byte_len IS NULL) + OR (storage_kind IN ('canonical_tool_projection','canonical_model_part') AND source_sequence>0 + AND (storage_kind='canonical_model_part' OR source_index=0) + AND byte_offset IS NOT NULL AND byte_offset>=0 + AND byte_len IS NOT NULL AND byte_len>0 + AND byte_offset<=CASE storage_kind WHEN 'canonical_model_part' THEN 2147483647 ELSE 9223372036854775807 END-byte_len)), + PRIMARY KEY(source_sequence,source_index) +) WITHOUT ROWID; + +CREATE TABLE context_semantic_directory_children_new ( + source_sequence INTEGER NOT NULL, + source_index INTEGER NOT NULL, + list_kind TEXT NOT NULL CHECK(list_kind IN ('quotes','attachments','tool_parts')), + ordinal INTEGER NOT NULL CHECK(ordinal>=0), + directory_bytes BLOB NOT NULL, + PRIMARY KEY(source_sequence,source_index,list_kind,ordinal), + FOREIGN KEY(source_sequence,source_index) + REFERENCES context_semantic_fragments_new(source_sequence,source_index) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED +) WITHOUT ROWID; + +INSERT INTO context_semantic_fragments_new SELECT * FROM context_semantic_fragments; +INSERT INTO context_semantic_directory_children_new SELECT * FROM context_semantic_directory_children; +DROP TABLE context_semantic_directory_children; +DROP TABLE context_semantic_fragments; +ALTER TABLE context_semantic_fragments_new RENAME TO context_semantic_fragments; +ALTER TABLE context_semantic_directory_children_new RENAME TO context_semantic_directory_children; diff --git a/platform/event-log/src/append.rs b/platform/event-log/src/append.rs index d0fbe8371c..cea5f932d3 100644 --- a/platform/event-log/src/append.rs +++ b/platform/event-log/src/append.rs @@ -292,15 +292,17 @@ impl EventLog { return Err(StoreError::SessionRetired); } let json = serde_json::to_string(event)?; - let previous: Option<(i64, String)> = - sqlx::query_as("SELECT sequence, event_json FROM runtime_events WHERE event_id = ?") - .bind(&event.id) - .fetch_optional(&mut *transaction) - .await?; - if let Some((sequence, previous)) = previous { - if previous != json { + let previous=sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.sequence,e.event_header,{columns} FROM runtime_events e {joins} WHERE e.event_id=?", + columns=crate::canonical::COLUMNS,joins=crate::canonical::JOINS, + ))).bind(&event.id).fetch_optional(&mut *transaction).await?; + if let Some(previous) = previous { + use sqlx::Row; + let sequence: i64 = previous.try_get(0)?; + if crate::canonical::json(&previous)?.as_ref() != json { return Err(StoreError::EventConflict); } + drop(previous); crate::tool_payloads::verify_replay(transaction, write).await?; crate::composition::verify_replay(transaction, write).await?; crate::usage::valuation::verify_replay(transaction, write).await?; diff --git a/platform/event-log/src/append/history.rs b/platform/event-log/src/append/history.rs index cbbf8e2a5f..14fe1a4be1 100644 --- a/platform/event-log/src/append/history.rs +++ b/platform/event-log/src/append/history.rs @@ -79,7 +79,7 @@ async fn identity( return Err(StoreError::Sealed); } let opening: Option = sqlx::query_scalar( - "SELECT json_extract(event_json, '$.invocation') FROM runtime_events WHERE invocation_id=? AND kind='invocation_opened'", + "SELECT json_extract(event_header, '$.invocation') FROM runtime_events WHERE invocation_id=? AND kind='invocation_opened'", ) .bind(id) .fetch_optional(connection) @@ -117,7 +117,7 @@ async fn outcomes( | Fact::ModelObserved { step_id, .. } = &event.fact { let summary: Option = sqlx::query_scalar( - "SELECT json_extract(event_json,'$.fact.purpose')='summary' FROM runtime_events WHERE invocation_id=? AND operation_id=? AND kind='model_requested'", + "SELECT json_extract(event_header,'$.fact.purpose')='summary' FROM runtime_events WHERE invocation_id=? AND operation_id=? AND kind='model_requested'", ).bind(id).bind(step_id).fetch_optional(&mut *connection).await?; let summary = summary.ok_or_else(|| invalid("model output without request"))?; if summary diff --git a/platform/event-log/src/append/ingress.rs b/platform/event-log/src/append/ingress.rs index 1cd6968fc2..0e436f3a5c 100644 --- a/platform/event-log/src/append/ingress.rs +++ b/platform/event-log/src/append/ingress.rs @@ -63,19 +63,24 @@ pub(crate) async fn insert( Position::Imported(sequence) => sequence.map(i64::try_from).transpose() .map_err(|_| StoreError::PrefixTooLarge)?, }; - let inserted = crate::canonical_insert::insert(crate::canonical_insert::Record { + let projection = crate::canonical::projection(event.fact.kind(), &json)?; + let header = projection.as_ref().map(|projection| projection.header(&json)).transpose()?; + let inserted = crate::canonical_insert::insert_header(crate::canonical_insert::Record { sequence: requested_sequence, event_id: &event.id, invocation_id: Some(&event.invocation.invocation_id), kind: event.fact.kind(), operation_id: event.fact.operation_id(), - json: &json, - }).execute(&mut *worker).await?; + json: header.as_deref().unwrap_or(&json), + }, json.len()).execute(&mut *worker).await?; if inserted.rows_affected() != 1 { return Err(StoreError::EventConflict); } // Capture before the projections issue INSERTs on this same handle. let sequence = sequence_number(inserted.last_insert_rowid())?; + if let Some(projection) = &projection { + crate::canonical::publish(worker, sequence, projection).await?; + } let stored = StoredEvent { sequence, event }; crate::model_items::record(worker, &stored).await?; crate::context::projection::record(worker, &stored, &json).await?; diff --git a/platform/event-log/src/append/ingress/tests.rs b/platform/event-log/src/append/ingress/tests.rs index 17c423120d..967dfe1428 100644 --- a/platform/event-log/src/append/ingress/tests.rs +++ b/platform/event-log/src/append/ingress/tests.rs @@ -329,15 +329,7 @@ async fn canonical_tool_spans_preserve_projection_raw_and_replay_after_reopen_an } let canonical: String = reopened .connection - .read_interruptible(|db| { - Box::pin(async move { - Ok( - sqlx::query_scalar("SELECT event_json FROM event_log WHERE sequence=6") - .fetch_one(db) - .await?, - ) - }) - }) + .read_interruptible(|db| Box::pin(async move { crate::canonical::one(db, 6).await })) .await .unwrap(); assert_eq!(canonical, expected_json); @@ -394,15 +386,7 @@ async fn canonical_tool_spans_preserve_projection_raw_and_replay_after_reopen_an ); let stored_json: String = imported .connection - .read_interruptible(|db| { - Box::pin(async move { - Ok( - sqlx::query_scalar("SELECT event_json FROM event_log WHERE sequence=46") - .fetch_one(db) - .await?, - ) - }) - }) + .read_interruptible(|db| Box::pin(async move { crate::canonical::one(db, 46).await })) .await .unwrap(); assert_eq!(stored_json, imported_json); diff --git a/platform/event-log/src/archive/accepted.rs b/platform/event-log/src/archive/accepted.rs index e273b187e5..ce50fe667e 100644 --- a/platform/event-log/src/archive/accepted.rs +++ b/platform/event-log/src/archive/accepted.rs @@ -200,10 +200,10 @@ async fn selected( let filter = Selection::archive_predicate("t", "a", "9223372036854775807", "NULL"); let rows: Vec<(Option, Option, Option)> = sqlx::query_as(sqlx::AssertSqlSafe(format!( - "SELECT json_extract(a.event_json,'$.invocation.session_id'), a.sequence, t.archive_sequence + "SELECT json_extract(a.event_header,'$.invocation.session_id'), a.sequence, t.archive_sequence FROM session_history_events t LEFT JOIN runtime_events a ON a.kind='tool_result_archived' - AND CAST(json_extract(a.event_json,'$.fact.placeholder.identity.runtime_event_id') AS TEXT)=t.event_id + AND CAST(json_extract(a.event_header,'$.fact.placeholder.identity.runtime_event_id') AS TEXT)=t.event_id AND {filter} WHERE t.owner_session_id=?1 AND t.event_id=?2 LIMIT 2" ))) diff --git a/platform/event-log/src/archive/candidates.rs b/platform/event-log/src/archive/candidates.rs index d76d13d9d1..d946da6485 100644 --- a/platform/event-log/src/archive/candidates.rs +++ b/platform/event-log/src/archive/candidates.rs @@ -45,7 +45,7 @@ impl EventLog { let mut tx = connection.begin().await?; if let Some(id) = ¤t { let live: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM runtime_events o WHERE invocation_id=? AND kind='invocation_opened' - AND json_extract(event_json,'$.invocation.session_id')=? AND NOT EXISTS(SELECT 1 FROM runtime_events t WHERE t.invocation_id=o.invocation_id AND t.kind='invocation_ended'))") + AND json_extract(event_header,'$.invocation.session_id')=? AND NOT EXISTS(SELECT 1 FROM runtime_events t WHERE t.invocation_id=o.invocation_id AND t.kind='invocation_ended'))") .bind(id).bind(&session).fetch_one(&mut *tx).await?; if !live { return Err(StoreError::InvalidTransition("prune source invocation is not live".into())); } safety::settled_boundary(&mut tx, id, i64::MAX as u64).await?; @@ -55,7 +55,7 @@ impl EventLog { // The admission preflight checks safety without materializing history. if max_candidates > 0 { let selection = if let Some(id) = ¤t { - let json: String = sqlx::query_scalar("SELECT event_json FROM runtime_events WHERE invocation_id=? AND kind='invocation_opened'") + let json: String = sqlx::query_scalar("SELECT event_header FROM runtime_events WHERE invocation_id=? AND kind='invocation_opened'") .bind(id).fetch_one(&mut *tx).await?; let opening: RuntimeEvent = serde_json::from_str(&json)?; Selection::for_opening(&mut tx, &opening).await? @@ -99,13 +99,14 @@ async fn scan( let archive_filter = Selection::archive_predicate("t", "a", "9223372036854775807", "?4"); let branches = target::SOURCES.map(|source| format!( "SELECT t.sequence AS sequence,t.event_id, - COALESCE(length(CAST(json_extract(t.event_json,'$.fact.outcome.model_projection') AS BLOB)), - length(CAST(json_extract(t.event_json,'$.fact.outcome.message') AS BLOB)),0) - FROM ({source}) t CROSS JOIN runtime_events d + COALESCE(b.sqlite_json_bytes, + length(CAST(json_extract(t.event_header,'$.fact.outcome.message') AS BLOB)),0) + FROM ({source}) t LEFT JOIN event_body_slots slot ON slot.event_sequence=t.sequence + LEFT JOIN event_bodies b ON b.id=slot.body_id CROSS JOIN runtime_events d WHERE d.invocation_id=t.invocation_id AND d.operation_id=t.operation_id AND d.kind='tool_dispatched' AND t.sequence>?1 AND t.sequence<=?3 AND t.owner_session_id=?2 - AND {filter} AND json_extract(d.event_json,'$.fact.call.origin.kind')='provider' - AND NOT EXISTS(SELECT 1 FROM runtime_events a WHERE a.kind='tool_result_archived' AND CAST(json_extract(a.event_json,'$.fact.placeholder.identity.runtime_event_id') AS TEXT)=t.event_id AND {archive_filter})" + AND {filter} AND json_extract(d.event_header,'$.fact.call.origin.kind')='provider' + AND NOT EXISTS(SELECT 1 FROM runtime_events a WHERE a.kind='tool_result_archived' AND CAST(json_extract(a.event_header,'$.fact.placeholder.identity.runtime_event_id') AS TEXT)=t.event_id AND {archive_filter})" )); let query = format!( "{} ORDER BY sequence LIMIT ?5", diff --git a/platform/event-log/src/archive/target.rs b/platform/event-log/src/archive/target.rs index 7ae4f0df0e..3abb9199f7 100644 --- a/platform/event-log/src/archive/target.rs +++ b/platform/event-log/src/archive/target.rs @@ -29,11 +29,11 @@ use sqlx::{Row, SqliteConnection}; // combined history view makes SQLite sort the entire remainder on every page. // The copied branch exposes h.sequence so its membership index supplies order. pub(super) const SOURCES: [&str; 2] = [ - "SELECT e.sequence,e.event_id,e.invocation_id,e.operation_id,e.event_json, - json_extract(e.event_json,'$.invocation.session_id') AS owner_session_id, + "SELECT e.sequence,e.event_id,e.invocation_id,e.operation_id,e.event_header, + json_extract(e.event_header,'$.invocation.session_id') AS owner_session_id, 0 AS inherited,NULL AS archive_sequence FROM runtime_events e WHERE e.kind='tool_settled'", - "SELECT h.sequence,e.event_id,e.invocation_id,e.operation_id,e.event_json, + "SELECT h.sequence,e.event_id,e.invocation_id,e.operation_id,e.event_header, h.session_id AS owner_session_id,1 AS inherited,h.archive_sequence FROM session_history_members h CROSS JOIN runtime_events e WHERE e.sequence=h.sequence AND e.kind='tool_settled'", @@ -56,23 +56,22 @@ pub(super) async fn read( ) -> Result, StoreError> { // Filter before SQLite materializes the compound acceptance query, so only // matching calls carry their full inputs into its temporary rows. - let row = sqlx::query(concat!("WITH ", crate::model_items::accepted_calls!(), - " SELECT t.sequence, t.event_id, t.operation_id, json_extract(t.event_json, '$.invocation') AS ti, - json_extract(d.event_json, '$.invocation') AS di, json_extract(d.event_json, '$.fact.call') AS call, - json_extract(d.event_json, '$.fact.name') AS name, - json_extract(t.event_json, '$.fact.outcome.kind') AS outcome, - json_extract(t.event_json, '$.fact.outcome.model_projection') AS projection, - json_extract(t.event_json, '$.fact.outcome.message') AS message, + let row = sqlx::query(sqlx::AssertSqlSafe(format!(concat!("WITH ", crate::model_items::accepted_calls!(), + " SELECT e.event_header,{columns},t.sequence, t.event_id, t.operation_id, json_extract(t.event_header, '$.invocation') AS ti, + json_extract(d.event_header, '$.invocation') AS di, json_extract(d.event_header, '$.fact.call') AS call, + json_extract(d.event_header, '$.fact.name') AS name, + json_extract(t.event_header, '$.fact.outcome.kind') AS outcome, + json_extract(t.event_header, '$.fact.outcome.message') AS message, c.sequence AS completed, - (json_extract(t.event_json,'$.id')=t.event_id - AND json_extract(t.event_json,'$.invocation.invocation_id')=t.invocation_id - AND json_extract(t.event_json,'$.fact.operation_id')=t.operation_id - AND json_extract(t.event_json,'$.fact.kind')='tool_settled' - AND json_extract(d.event_json,'$.id')=d.event_id - AND json_extract(d.event_json,'$.fact.operation_id')=d.operation_id - AND json_extract(d.event_json,'$.fact.kind')='tool_dispatched' + (json_extract(t.event_header,'$.id')=t.event_id + AND json_extract(t.event_header,'$.invocation.invocation_id')=t.invocation_id + AND json_extract(t.event_header,'$.fact.operation_id')=t.operation_id + AND json_extract(t.event_header,'$.fact.kind')='tool_settled' + AND json_extract(d.event_header,'$.id')=d.event_id + AND json_extract(d.event_header,'$.fact.operation_id')=d.operation_id + AND json_extract(d.event_header,'$.fact.kind')='tool_dispatched' AND c.invocation_id=t.invocation_id - AND json_extract(c.call,'$.name')=json_extract(d.event_json,'$.fact.name') + AND json_extract(c.call,'$.name')=json_extract(d.event_header,'$.fact.name') AND json_extract(c.call,'$.provider_executed')=0 AND c.sequence Some(model_projection), + _ => None, + }; let decoded = (|| -> Option { if !row.try_get::("identities").ok()? { return None; @@ -110,7 +122,7 @@ pub(super) async fn read( } let outcome: &str = row.try_get("outcome").ok()?; let projection = match outcome { - "succeeded" => serde_json::from_str(row.try_get::<&str, _>("projection").ok()?).ok()?, + "succeeded" => durable.take()?, "failed" | "unknown" => DurableToolProjection::Text { text: row.try_get("message").ok()?, }, diff --git a/platform/event-log/src/artifacts/history.rs b/platform/event-log/src/artifacts/history.rs index 09e5cd10d3..a4506702eb 100644 --- a/platform/event-log/src/artifacts/history.rs +++ b/platform/event-log/src/artifacts/history.rs @@ -22,7 +22,7 @@ use crate::StoreError; use maka_agent_facts::attachment::AttachmentRef; use maka_agent_facts::attachment::StorageRef; use maka_agent_facts::event::RuntimeEvent; -use sqlx::SqliteConnection; +use sqlx::{Row, SqliteConnection}; pub(crate) const RESOLVE_SQL: &str = r#"SELECT h.artifact_id FROM session_history_artifacts h JOIN artifacts a ON a.session_id=h.session_id AND a.id=h.artifact_id @@ -84,16 +84,18 @@ pub(crate) async fn retain( ) -> Result<(), StoreError> { let mut after = 0_i64; loop { - let row: Option<(i64, String)> = sqlx::query_as( - "SELECT e.sequence, e.event_json FROM session_history_members h - JOIN runtime_events e ON e.sequence = h.sequence + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.sequence, e.event_header,{} FROM session_history_members h + JOIN runtime_events e ON e.sequence = h.sequence {} WHERE h.session_id = ? AND e.sequence > ? AND (e.kind IN ('message_steered', 'tool_settled') OR - (e.kind = 'invocation_opened' AND json_extract(e.event_json,'$.fact.input.kind') = 'message')) + (e.kind = 'invocation_opened' AND json_extract(e.event_header,'$.fact.input.kind') = 'message')) ORDER BY e.sequence LIMIT 1", - ).bind(target).bind(after).fetch_optional(&mut *tx).await?; - let Some((sequence, json)) = row else { break }; - let event: RuntimeEvent = serde_json::from_str(&json)?; + crate::canonical::COLUMNS, crate::canonical::JOINS + ))).bind(target).bind(after).fetch_optional(&mut *tx).await?; + let Some(row) = row else { break }; + let sequence: i64 = row.try_get("sequence")?; + let event: RuntimeEvent = serde_json::from_str(&crate::canonical::json(&row)?)?; for (reference, attachment) in super::references::from_event(&event) { retain_one(tx, source, target, reference, attachment, now).await?; } @@ -110,17 +112,20 @@ pub(crate) async fn retain_revision( ) -> Result<(), StoreError> { let mut after = 0_i64; loop { - let row: Option<(i64, String)> = sqlx::query_as( - "SELECT e.sequence, e.event_json FROM session_revision_sources s - JOIN runtime_events e ON e.sequence=s.sequence + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.sequence, e.event_header,{} FROM session_revision_sources s + JOIN runtime_events e ON e.sequence=s.sequence {} WHERE s.session_id=? AND s.sequence>? ORDER BY s.sequence LIMIT 1", - ) + crate::canonical::COLUMNS, + crate::canonical::JOINS + ))) .bind(target) .bind(after) .fetch_optional(&mut *tx) .await?; - let Some((sequence, json)) = row else { break }; - let event: RuntimeEvent = serde_json::from_str(&json)?; + let Some(row) = row else { break }; + let sequence: i64 = row.try_get("sequence")?; + let event: RuntimeEvent = serde_json::from_str(&crate::canonical::json(&row)?)?; for attachment in super::references::revision(&event)? { retain_one( tx, diff --git a/platform/event-log/src/bundle/accounting.rs b/platform/event-log/src/bundle/accounting.rs index 545a063fd3..4d53048d38 100644 --- a/platform/event-log/src/bundle/accounting.rs +++ b/platform/event-log/src/bundle/accounting.rs @@ -32,9 +32,9 @@ pub(super) async fn witness( ) -> Result, StoreError> { let row: Option = sqlx::query_scalar( "SELECT MIN(e.sequence) FROM runtime_events r JOIN runtime_events e ON e.invocation_id=r.invocation_id - AND json_extract(e.event_json,'$.fact.step_id')=json_extract(r.event_json,'$.fact.step_id') + AND json_extract(e.event_header,'$.fact.step_id')=json_extract(r.event_header,'$.fact.step_id') WHERE r.event_id=? AND (e.kind='model_completed' OR - (e.kind='model_observed' AND json_extract(e.event_json,'$.fact.event.kind')='finished'))" + (e.kind='model_observed' AND json_extract(e.event_header,'$.fact.event.kind')='finished'))" ).bind(request).fetch_one(db).await?; row.map(crate::sequence_number).transpose() } @@ -58,7 +58,7 @@ pub(super) async fn validate( unreachable!() }; let request: Option = sqlx::query_scalar( - "SELECT event_json FROM runtime_events WHERE event_id=? AND kind='model_requested'", + "SELECT event_header FROM runtime_events WHERE event_id=? AND kind='model_requested'", ) .bind(&event_id) .fetch_optional(&mut *original) @@ -77,9 +77,9 @@ pub(super) async fn validate( let mut sequence = 0i64; loop { let row: Option<(i64, String)> = sqlx::query_as( - "SELECT sequence,event_json FROM runtime_events WHERE invocation_id=?1 AND sequence>?2 - AND json_extract(event_json,'$.fact.step_id')=?3 AND (kind='model_completed' OR - (kind='model_observed' AND json_extract(event_json,'$.fact.event.kind')='finished')) + "SELECT sequence,event_header FROM runtime_events WHERE invocation_id=?1 AND sequence>?2 + AND json_extract(event_header,'$.fact.step_id')=?3 AND (kind='model_completed' OR + (kind='model_observed' AND json_extract(event_header,'$.fact.event.kind')='finished')) ORDER BY sequence LIMIT 1" ).bind(&event.invocation.invocation_id).bind(sequence).bind(step_id).fetch_optional(&mut *original).await?; let Some((next, json)) = row else { break }; diff --git a/platform/event-log/src/bundle/blobs.rs b/platform/event-log/src/bundle/blobs.rs index c0e7bed819..d8f9f8a478 100644 --- a/platform/event-log/src/bundle/blobs.rs +++ b/platform/event-log/src/bundle/blobs.rs @@ -23,8 +23,9 @@ use super::{ format::{Blob, CHUNK, Record, Writer}, }; use crate::{StoreError, sequence_number}; +use futures_util::TryStreamExt; use sha2::{Digest, Sha256}; -use sqlx::SqliteConnection; +use sqlx::{Row, SqliteConnection}; use tokio::io::AsyncWrite; pub(super) async fn export( @@ -35,8 +36,8 @@ pub(super) async fn export( ) -> Result<(), StoreError> { for sequence in &closure.events { let tool: Option<(String, i64, Option, Option)> = sqlx::query_as( - "SELECT e.event_id,length(p.payload),json_extract(e.event_json,'$.fact.outcome.raw.digest'), - json_extract(e.event_json,'$.fact.outcome.raw.bytes') + "SELECT e.event_id,length(p.payload),json_extract(e.event_header,'$.fact.outcome.raw.digest'), + json_extract(e.event_header,'$.fact.outcome.raw.bytes') FROM runtime_events e JOIN tool_result_payloads p ON p.event_id=e.event_id WHERE e.sequence=?" ).bind(*sequence as i64).fetch_optional(&mut *db).await?; if let Some((event_id, bytes, digest, expected_bytes)) = tool { @@ -155,35 +156,61 @@ async fn frozen_witness( if metadata.kind != kind { return Ok(None); } - let sequence: Option = sqlx::query_scalar( - "WITH media AS ( - SELECT e.sequence,e.inherited, - CASE json_extract(p.value,'$.kind') - WHEN 'artifact' THEN json_extract(p.value,'$.image') - WHEN 'audio' THEN json_extract(p.value,'$.audio') END AS value - FROM session_history_events e - JOIN json_each(e.event_json,'$.fact.outcome.model_projection.parts') p - JOIN artifacts a ON a.session_id=?1 AND a.id=?2 - WHERE e.owner_session_id=?1 AND e.kind='tool_settled' - AND json_extract(e.event_json,'$.fact.outcome.kind')='succeeded' - AND json_extract(e.event_json,'$.fact.outcome.model_projection.kind')='content' - AND a.canonical_at>0 AND a.canonical_at<=e.sequence) - SELECT MIN(sequence) FROM media - WHERE json_extract(value,'$.ref.kind')='session_file' - AND json_extract(value,'$.mimeType')=?3 - AND ((inherited=0 AND json_extract(value,'$.ref.sessionId')=?1 - AND json_extract(value,'$.ref.relativePath')=?2) - OR (inherited=1 AND EXISTS( - SELECT 1 FROM session_history_artifacts h WHERE h.session_id=?1 AND h.artifact_id=?2 - AND h.source_session_id=json_extract(value,'$.ref.sessionId') - AND h.source_artifact_id=json_extract(value,'$.ref.relativePath'))))", - ) + let mut rows = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.sequence,e.inherited,e.event_header,{},h.source_session_id,h.source_artifact_id + FROM session_history_events e {} + JOIN artifacts a ON a.session_id=?1 AND a.id=?2 + LEFT JOIN session_history_artifacts h ON h.session_id=?1 AND h.artifact_id=?2 + WHERE e.owner_session_id=?1 AND e.kind='tool_settled' + AND json_extract(e.event_header,'$.fact.outcome.kind')='succeeded' + AND a.canonical_at>0 AND a.canonical_at<=e.sequence ORDER BY e.sequence", + crate::canonical::COLUMNS, + crate::canonical::JOINS + ))) .bind(&metadata.session_id) .bind(&metadata.id) - .bind(&metadata.mime_type) - .fetch_one(db) - .await?; - sequence.map(sequence_number).transpose() + .fetch(db); + while let Some(row) = rows.try_next().await? { + let event: maka_agent_facts::event::RuntimeEvent = + serde_json::from_str(&crate::canonical::json(&row)?)?; + let maka_agent_facts::event::Fact::ToolSettled { + outcome: + maka_agent_facts::event::ToolOutcome::Succeeded { + model_projection: + maka_agent_facts::tool_output::DurableToolProjection::Content { parts }, + .. + }, + .. + } = event.fact + else { + continue; + }; + let inherited: bool = row.try_get("inherited")?; + let source_session: Option<&str> = row.try_get("source_session_id")?; + let source_artifact: Option<&str> = row.try_get("source_artifact_id")?; + for part in parts { + let Some(( + maka_agent_facts::attachment::StorageRef::SessionFile { + session_id, + relative_path, + }, + part_mime, + )) = part.media() + else { + continue; + }; + let owned = if inherited { + source_session == Some(session_id.as_str()) + && source_artifact == Some(relative_path.as_str()) + } else { + session_id == &metadata.session_id && relative_path == &metadata.id + }; + if owned && part_mime == mime { + return Ok(Some(sequence_number(row.try_get("sequence")?)?)); + } + } + } + Ok(None) } async fn required(db: &mut SqliteConnection, session: &str) -> Result<(), StoreError> { @@ -191,7 +218,7 @@ async fn required(db: &mut SqliteConnection, session: &str) -> Result<(), StoreE // a live lookup into an ancestor's artifact namespace. let broken: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM session_history_events e - JOIN json_each(e.event_json,'$.fact.outcome.artifacts') proof + JOIN json_each(e.event_header,'$.fact.outcome.artifacts') proof LEFT JOIN session_history_artifacts h ON h.session_id=e.owner_session_id AND h.source_session_id=e.event_session AND h.source_artifact_id=json_extract(proof.value,'$.id') LEFT JOIN artifacts a ON a.session_id=e.owner_session_id AND @@ -201,7 +228,7 @@ async fn required(db: &mut SqliteConnection, session: &str) -> Result<(), StoreE (a.id IS NULL OR c.content_sha256 IS NULL OR length(c.payload)!=json_extract(proof.value,'$.bytes') OR a.content_sha256!=json_extract(proof.value,'$.digest') OR json_extract(a.record_json,'$.source')!='tool_result_projection' - OR json_extract(a.record_json,'$.turnId')!=json_extract(e.event_json,'$.invocation.turn_id')))" + OR json_extract(a.record_json,'$.turnId')!=json_extract(e.event_header,'$.invocation.turn_id')))" ).bind(session).fetch_one(db).await?; if broken { return Err(invalid("bundle lacks generated tool material")); diff --git a/platform/event-log/src/bundle/closure.rs b/platform/event-log/src/bundle/closure.rs index 80d003cca6..29017beed5 100644 --- a/platform/event-log/src/bundle/closure.rs +++ b/platform/event-log/src/bundle/closure.rs @@ -81,16 +81,14 @@ impl Closure { } } while let Some(sequence) = selected.queue.pop_front() { - let row: Option<(Option, Option)> = sqlx::query_as( - "SELECT event_json, - (SELECT length(payload) FROM tool_result_payloads WHERE event_id=e.event_id) - FROM runtime_events e WHERE sequence=?1", - ) - .bind(sequence as i64) - .fetch_optional(&mut *db) - .await?; - let (json, payload_bytes) = row.ok_or(StoreError::MaterialCollected)?; - let json = json.ok_or(StoreError::PrefixTooLarge)?; + let row=sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.event_header,(SELECT length(payload) FROM tool_result_payloads WHERE event_id=e.event_id) AS tool_payload_bytes,{columns} + FROM runtime_events e {joins} WHERE e.sequence=?1",columns=crate::canonical::COLUMNS,joins=crate::canonical::JOINS, + ))).bind(sequence as i64).fetch_optional(&mut *db).await?; + let row = row.ok_or(StoreError::MaterialCollected)?; + use sqlx::Row; + let payload_bytes: Option = row.try_get("tool_payload_bytes")?; + let json = crate::canonical::json(&row)?; let event: RuntimeEvent = serde_json::from_str(&json)?; crate::tool_payloads::verify_binding(&event, payload_bytes)?; if !matches!(event.fact, Fact::MessageImported { .. }) { @@ -233,9 +231,9 @@ impl Closure { "SELECT EXISTS(SELECT 1 FROM context_source_prefixes p CROSS JOIN event_log e ON e.sequence=p.sequence WHERE p.kind='session' AND p.owner=?1 AND p.sequence<=?2 - AND e.invocation_id IS NOT NULL AND e.event_json IS NULL AND {filter} + AND e.invocation_id IS NOT NULL AND e.event_header IS NULL AND {filter} UNION ALL SELECT 1 FROM session_history_members h JOIN event_log e ON e.sequence=h.sequence - WHERE h.session_id=?1 AND e.sequence<=?2 AND e.event_json IS NULL AND {filter})" + WHERE h.session_id=?1 AND e.sequence<=?2 AND e.event_header IS NULL AND {filter})" ))).bind(session).bind(through as i64).bind(&selection.lineage).fetch_one(&mut *db).await?; if collected { return Err(StoreError::MaterialCollected); @@ -275,7 +273,7 @@ impl Closure { let target = Selection::predicate("t", "?4"); let rows: Vec = sqlx::query_scalar(sqlx::AssertSqlSafe(format!( "SELECT a.sequence FROM session_history_events t JOIN runtime_events a - ON a.kind='tool_result_archived' AND json_extract(a.event_json,'$.fact.placeholder.identity.runtime_event_id')=t.event_id + ON a.kind='tool_result_archived' AND json_extract(a.event_header,'$.fact.placeholder.identity.runtime_event_id')=t.event_id WHERE t.owner_session_id=?1 AND t.sequence<=?2 AND {filter} AND {target} ORDER BY a.sequence" ))).bind(&selection.session).bind(through as i64).bind(before as i64).bind(&selection.lineage) diff --git a/platform/event-log/src/bundle/export.rs b/platform/event-log/src/bundle/export.rs index e4ae5ac47a..880f83810b 100644 --- a/platform/event-log/src/bundle/export.rs +++ b/platform/event-log/src/bundle/export.rs @@ -23,7 +23,8 @@ use super::{ format::{Copy, Record, Valuation, Writer}, }; use crate::{EventLog, StoreError, sequence_number}; -use sqlx::{Connection, SqliteConnection}; +use futures_util::TryStreamExt; +use sqlx::{Connection, Row, SqliteConnection}; use tokio::io::AsyncWrite; /// Inventory and checksum of a complete uncompressed transfer, not permission @@ -71,21 +72,7 @@ impl EventLog { .await?; catalog(&mut tx, &inventory, &mut writer).await?; histories(&mut tx, &closure, &mut writer).await?; - for sequence in &closure.events { - let json: String = sqlx::query_scalar( - "SELECT event_json FROM runtime_events WHERE sequence=?", - ) - .bind(*sequence as i64) - .fetch_one(&mut *tx) - .await - .map_err(StoreError::from)?; - writer - .record(&Record::Event { - sequence: *sequence, - json, - }) - .await?; - } + events(&mut tx, &closure, &mut writer).await?; super::blobs::export(&mut tx, &inventory, &closure, &mut writer).await?; accounting(&mut tx, &closure, &mut writer).await?; let (output, bytes, digest) = writer.finish().await?; @@ -107,6 +94,44 @@ impl EventLog { } } +async fn events( + db: &mut SqliteConnection, + closure: &Closure, + writer: &mut Writer, +) -> Result<(), StoreError> { + // Batch only identities; stream each joined body in the caller's snapshot. + // The batch size bounds query parameters, never the accepted event size. + let mut selected = closure.events.iter(); + loop { + let batch: Vec = selected.by_ref().take(128).copied().collect(); + if batch.is_empty() { + return Ok(()); + } + let mut rows = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.sequence,e.event_header,{} FROM runtime_events e {} + WHERE e.sequence IN(SELECT value FROM json_each(?)) ORDER BY e.sequence", + crate::canonical::COLUMNS, + crate::canonical::JOINS + ))) + .bind(serde_json::to_string(&batch)?) + .fetch(&mut *db); + let mut count = 0; + while let Some(row) = rows.try_next().await? { + let json = crate::canonical::json(&row)?.into_owned(); + writer + .record(&Record::Event { + sequence: sequence_number(row.try_get("sequence")?)?, + json, + }) + .await?; + count += 1; + } + if count != batch.len() { + return Err(StoreError::InvalidTransition("missing bundle event".into())); + } + } +} + async fn require_idle(db: &mut SqliteConnection, inventory: &Inventory) -> Result<(), StoreError> { for session in &inventory.sessions { if crate::sessions::has_active_invocation(db, &session.id).await? { diff --git a/platform/event-log/src/bundle/import/collision.rs b/platform/event-log/src/bundle/import/collision.rs index c1f80dab22..4ba0b9e17f 100644 --- a/platform/event-log/src/bundle/import/collision.rs +++ b/platform/event-log/src/bundle/import/collision.rs @@ -133,8 +133,8 @@ async fn identities( let occupied: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM event_log WHERE invocation_id=?1) OR EXISTS(SELECT 1 FROM event_log WHERE kind='invocation_opened' AND - (json_extract(COALESCE(event_json,retained_json),'$.invocation.run_id')=?2 - OR json_extract(event_json,'$.fact.input.claim.id')=?3))", + (json_extract(COALESCE(event_header,retained_json),'$.invocation.run_id')=?2 + OR json_extract(event_header,'$.fact.input.claim.id')=?3))", ) .bind(invocation) .bind(run) diff --git a/platform/event-log/src/bundle/import/publish.rs b/platform/event-log/src/bundle/import/publish.rs index ffdf4ac0a1..2c15233cf9 100644 --- a/platform/event-log/src/bundle/import/publish.rs +++ b/platform/event-log/src/bundle/import/publish.rs @@ -138,10 +138,16 @@ async fn events( ) -> Result<(), StoreError> { let mut after = 0i64; loop { - let row: Option<(i64, String)> = sqlx::query_as( - "SELECT sequence,event_json FROM runtime_events WHERE sequence>? ORDER BY sequence LIMIT 1", - ).bind(after).fetch_optional(&mut *relocated).await?; - let Some((sequence, json)) = row else { break }; + let row=sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.sequence,e.event_header,{columns} FROM runtime_events e {joins} WHERE e.sequence>? ORDER BY e.sequence LIMIT 1", + columns=crate::canonical::COLUMNS,joins=crate::canonical::JOINS, + ))).bind(after).fetch_optional(&mut *relocated).await?; + let Some(row) = row else { + break; + }; + use sqlx::Row; + let sequence: i64 = row.try_get(0)?; + let json = crate::canonical::json(&row)?.into_owned(); let event: RuntimeEvent = serde_json::from_str(&json)?; sqlx::query("INSERT INTO imported_invocations VALUES(?,?) ON CONFLICT DO NOTHING") .bind(&event.invocation.invocation_id) diff --git a/platform/event-log/src/bundle/validation.rs b/platform/event-log/src/bundle/validation.rs index 2025087b8f..9d7b8b04b0 100644 --- a/platform/event-log/src/bundle/validation.rs +++ b/platform/event-log/src/bundle/validation.rs @@ -171,7 +171,7 @@ async fn replay( let mixed: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM runtime_events GROUP BY invocation_id HAVING SUM(kind='message_imported')>0 AND COUNT(*)>1) - OR EXISTS(SELECT 1 FROM runtime_events GROUP BY event_session,json_extract(event_json,'$.invocation.run_id') + OR EXISTS(SELECT 1 FROM runtime_events GROUP BY event_session,json_extract(event_header,'$.invocation.run_id') HAVING COUNT(DISTINCT invocation_id)>1)", ) .fetch_one(&mut *original) @@ -182,7 +182,7 @@ async fn replay( let mut after = 0i64; loop { let row: Option<(i64,String)> = sqlx::query_as( - "SELECT sequence,event_json FROM runtime_events WHERE kind='context_checkpoint_recorded' AND sequence>? ORDER BY sequence LIMIT 1" + "SELECT sequence,event_header FROM runtime_events WHERE kind='context_checkpoint_recorded' AND sequence>? ORDER BY sequence LIMIT 1" ).bind(after).fetch_optional(&mut *original).await?; let Some((sequence, json)) = row else { break; diff --git a/platform/event-log/src/bundle/validation/artifacts.rs b/platform/event-log/src/bundle/validation/artifacts.rs index a90806d344..74a9645fcd 100644 --- a/platform/event-log/src/bundle/validation/artifacts.rs +++ b/platform/event-log/src/bundle/validation/artifacts.rs @@ -30,7 +30,7 @@ use maka_agent_facts::artifact::Artifact; use maka_agent_facts::attachment::AttachmentRef; use maka_agent_facts::attachment::StorageRef; use maka_agent_facts::event::RuntimeEvent; -use sqlx::SqliteConnection; +use sqlx::{Row, SqliteConnection}; pub(super) async fn validate( staged: &mut SqliteConnection, @@ -96,28 +96,30 @@ pub(super) async fn validate( loop { // Bound each ownership branch before merging so every cursor step // sorts at most three candidates instead of the remaining history. - let row: Option<(i64, bool, bool, String)> = sqlx::query_as( - "SELECT sequence,revision,inherited,event_json FROM ( - SELECT e.sequence,0 AS revision,0 AS inherited,e.event_json FROM runtime_events e - WHERE (e.event_json IS NOT NULL AND e.event_session=?1) AND e.sequence>?2 + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT selected.sequence,selected.revision,selected.inherited,e.event_header,{} FROM (SELECT sequence,revision,inherited FROM ( + SELECT e.sequence,0 AS revision,0 AS inherited FROM runtime_events e + WHERE (e.event_header IS NOT NULL AND e.event_session=?1) AND e.sequence>?2 ORDER BY e.sequence LIMIT 1) UNION ALL - SELECT sequence,revision,inherited,event_json FROM ( - SELECT e.sequence,0 AS revision,1 AS inherited,e.event_json FROM session_history_members h + SELECT sequence,revision,inherited FROM ( + SELECT e.sequence,0 AS revision,1 AS inherited FROM session_history_members h JOIN runtime_events e ON e.sequence=h.sequence WHERE h.session_id=?1 AND h.sequence>?2 ORDER BY h.sequence LIMIT 1) UNION ALL - SELECT sequence,revision,inherited,event_json FROM ( - SELECT r.sequence,1 AS revision,1 AS inherited,e.event_json FROM session_revision_sources r + SELECT sequence,revision,inherited FROM ( + SELECT r.sequence,1 AS revision,1 AS inherited FROM session_revision_sources r JOIN runtime_events e ON e.sequence=r.sequence WHERE r.session_id=?1 AND r.sequence>=?2 AND (r.sequence>?2 OR ?3=0) ORDER BY r.sequence LIMIT 1) - ORDER BY sequence,revision LIMIT 1" - ).bind(&session.id).bind(after.0).bind(after.1).fetch_optional(&mut *original).await?; - let Some((sequence, revision, inherited, json)) = row else { - break; - }; - let event: RuntimeEvent = serde_json::from_str(&json)?; + ORDER BY sequence,revision LIMIT 1) selected JOIN runtime_events e ON e.sequence=selected.sequence {}", + crate::canonical::COLUMNS, crate::canonical::JOINS + ))).bind(&session.id).bind(after.0).bind(after.1).fetch_optional(&mut *original).await?; + let Some(row) = row else { break }; + let sequence: i64 = row.try_get("sequence")?; + let revision: bool = row.try_get("revision")?; + let inherited: bool = row.try_get("inherited")?; + let event: RuntimeEvent = serde_json::from_str(&crate::canonical::json(&row)?)?; let refs = if revision { references::revision(&event)? .into_iter() @@ -162,12 +164,13 @@ async fn frozen_witness( } // A proof-only event's existence is insufficient. Membership belongs to the // exact artifact owner; a copied owner resolves only its validated mapping. - let row: Option<(bool, String)> = sqlx::query_as( - "SELECT inherited,event_json FROM session_history_events WHERE owner_session_id=? AND sequence=?", - ).bind(&artifact.session_id).bind(sequence as i64).fetch_optional(&mut *original).await?; - let (inherited, json) = - row.ok_or_else(|| invalid("bundle media witness is outside its owner history"))?; - let event: RuntimeEvent = serde_json::from_str(&json)?; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.inherited,e.event_header,{} FROM session_history_events e {} WHERE e.owner_session_id=? AND e.sequence=?", + crate::canonical::COLUMNS, crate::canonical::JOINS + ))).bind(&artifact.session_id).bind(sequence as i64).fetch_optional(&mut *original).await? + .ok_or_else(|| invalid("bundle media witness is outside its owner history"))?; + let inherited: bool = row.try_get("inherited")?; + let event: RuntimeEvent = serde_json::from_str(&crate::canonical::json(&row)?)?; let Fact::ToolSettled { outcome: ToolOutcome::Succeeded { diff --git a/platform/event-log/src/bundle/validation/copies.rs b/platform/event-log/src/bundle/validation/copies.rs index 98132090e4..e164ede7c5 100644 --- a/platform/event-log/src/bundle/validation/copies.rs +++ b/platform/event-log/src/bundle/validation/copies.rs @@ -169,10 +169,10 @@ impl Copies { } let selected: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM runtime_events WHERE sequence=? - AND json_extract(event_json,'$.invocation.turn_id')=? + AND json_extract(event_header,'$.invocation.turn_id')=? AND ((kind='invocation_opened' - AND json_extract(event_json,'$.fact.input.kind')='message' - AND json_array_length(event_json,'$.fact.input.source_messages')>0) + AND json_extract(event_header,'$.fact.input.kind')='message' + AND json_array_length(event_header,'$.fact.input.source_messages')>0) OR kind='message_steered'))", ) .bind(sequence as i64) diff --git a/platform/event-log/src/bundle/validation/material.rs b/platform/event-log/src/bundle/validation/material.rs index 1b8c9f699c..7a2760657f 100644 --- a/platform/event-log/src/bundle/validation/material.rs +++ b/platform/event-log/src/bundle/validation/material.rs @@ -134,7 +134,7 @@ pub(super) async fn check_bindings( LEFT JOIN runtime_events a ON a.sequence=h.archive_sequence WHERE t.event_session=h.session_id OR (h.archive_sequence IS NOT NULL AND (a.kind IS NOT 'tool_result_archived' OR a.sequence<=t.sequence OR - json_extract(a.event_json,'$.fact.placeholder.identity.runtime_event_id') IS NOT t.event_id)))" + json_extract(a.event_header,'$.fact.placeholder.identity.runtime_event_id') IS NOT t.event_id)))" ).fetch_one(&mut *db).await?; if invalid_pin { return Err(invalid("copy has a self member or mismatched archive pin")); @@ -147,7 +147,7 @@ pub(super) async fn check_bindings( WHERE h.archive_sequence IS NOT COALESCE( (SELECT a.sequence FROM runtime_events a WHERE a.kind='tool_result_archived' AND a.event_session=c.source_session_id AND a.sequence<=c.observed_through - AND json_extract(a.event_json,'$.fact.placeholder.identity.runtime_event_id')=target.event_id), + AND json_extract(a.event_header,'$.fact.placeholder.identity.runtime_event_id')=target.event_id), parent.archive_sequence))" ).fetch_one(&mut *db).await?; if changed_projection { diff --git a/platform/event-log/src/canonical.rs b/platform/event-log/src/canonical.rs new file mode 100644 index 0000000000..57b0f52564 --- /dev/null +++ b/platform/event-log/src/canonical.rs @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Lossless canonical events over immutable headers and typed body slots. +pub(crate) mod migration; +mod read; +mod recipe; +mod source; +mod write; + +pub(crate) use read::{COLUMNS, JOINS, json, json_on_handle, one}; +pub(crate) use recipe::projection; +pub(crate) use source::{ + SPAN_COLUMNS, SpanMetadata, with_span, with_span_metadata, with_span_on_row, +}; +pub(crate) use write::publish; diff --git a/platform/event-log/src/canonical/migration.rs b/platform/event-log/src/canonical/migration.rs new file mode 100644 index 0000000000..1ade8255f9 --- /dev/null +++ b/platform/event-log/src/canonical/migration.rs @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Pure conversion functions installed only within the schema migration scope. +use super::recipe; +use crate::StoreError; +use rusqlite::{Connection, functions::FunctionFlags}; +use sha2::{Digest, Sha256}; +fn udf(error: impl std::error::Error + Send + Sync + 'static) -> rusqlite::Error { + rusqlite::Error::UserFunctionError(Box::new(error)) +} +pub(crate) fn register(db: &Connection) -> Result<(), StoreError> { + let flags = FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC; + db.create_scalar_function("maka_projection_offset", 2, flags, |context| { + let kind = context.get_raw(0).as_str()?; + let json = context.get_raw(1).as_str()?; + Ok(recipe::projection(kind, json) + .map_err(udf)? + .map(|projection| projection.offset as i64)) + })?; + db.create_scalar_function("maka_projection_bytes", 2, flags, |context| { + let kind = context.get_raw(0).as_str()?; + let json = context.get_raw(1).as_str()?; + Ok(recipe::projection(kind, json) + .map_err(udf)? + .map(|projection| projection.json.as_bytes().to_vec())) + })?; + db.create_scalar_function("maka_event_header", 2, flags, |context| { + let kind = context.get_raw(0).as_str()?; + let json = context.get_raw(1).as_str()?; + match recipe::projection(kind, json).map_err(udf)? { + Some(projection) => projection.header(json).map_err(udf), + None => Ok(json.to_owned()), + } + })?; + db.create_scalar_function("maka_body_sha", 1, flags, |context| { + Ok(Sha256::digest(context.get_raw(0).as_bytes()?).to_vec()) + })?; + Ok(()) +} +pub(crate) fn unregister(db: &Connection) -> Result<(), StoreError> { + for name in [ + "maka_projection_offset", + "maka_projection_bytes", + "maka_event_header", + ] { + db.remove_function(name, 2)?; + } + db.remove_function("maka_body_sha", 1)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use sqlx::Connection; + #[tokio::test] + async fn old_bytes_ranges_lengths_and_shared_last_reference_survive_migration() { + let mut db = sqlx::SqliteConnection::connect("sqlite::memory:") + .await + .unwrap(); + crate::sqlite_functions::register(&mut db).await.unwrap(); + crate::sqlite_functions::register_migrations(&mut db) + .await + .unwrap(); + let all = sqlx::migrate!("./migrations"); + let previous = sqlx::migrate::Migrator { + migrations: all + .iter() + .filter(|m| m.version < 13) + .cloned() + .collect::>() + .into(), + ..sqlx::migrate::Migrator::DEFAULT + }; + previous.run_direct(None, &mut db, false).await.unwrap(); + let text = "汉a escaped \\\" line\n".repeat(4000); + let projection = serde_json::json!({"kind":"json","value":{"n":1.0,"text":text}}); + let original=serde_json::to_string_pretty(&serde_json::json!({"invocation":{"session_id":"session","invocation_id":"invocation"},"fact":{"kind":"tool_settled","operation_id":"operation","outcome":{"kind":"succeeded","model_projection":projection}}})).unwrap().replace("\"n\": 1.0","\"n\": 1e+00").replace("汉a","汉\\u0061"); + for id in ["first", "second"] { + sqlx::query("INSERT INTO event_log(event_id,invocation_id,kind,event_json) VALUES(?,'invocation','tool_settled',?)").bind(id).bind(&original).execute(&mut db).await.unwrap(); + } + sqlx::query("INSERT INTO event_log(event_id,invocation_id,kind,event_json,retained_json,body_digest) VALUES('collected','invocation','tool_settled',NULL,'{}','old-digest')").execute(&mut db).await.unwrap(); + // Rebuilding span constraints must preserve opaque encoded directories, + // child ownership, and rowid-addressed legacy pieces. + sqlx::raw_sql("INSERT INTO context_semantic_fragments VALUES(1,0,'pieces',NULL,NULL,x'1020',2); + INSERT INTO context_semantic_directory_children VALUES(1,0,'tool_parts',0,x'3040'); + INSERT INTO context_semantic_fragment_pieces(rowid,source_sequence,source_index,piece_index,bytes) VALUES(904,1,0,0,x'5060');") + .execute(&mut db).await.unwrap(); + all.run_direct(None, &mut db, false).await.unwrap(); + crate::sqlite_functions::unregister_migrations(&mut db) + .await + .unwrap(); + let directory: Vec = sqlx::query_scalar( + "SELECT directory_bytes FROM context_semantic_fragments WHERE source_sequence=1", + ) + .fetch_one(&mut db) + .await + .unwrap(); + assert_eq!(directory, [0x10, 0x20]); + let child: Vec = sqlx::query_scalar("SELECT directory_bytes FROM context_semantic_directory_children WHERE source_sequence=1").fetch_one(&mut db).await.unwrap(); + assert_eq!(child, [0x30, 0x40]); + let piece: (i64, Vec) = sqlx::query_as( + "SELECT rowid,bytes FROM context_semantic_fragment_pieces WHERE source_sequence=1", + ) + .fetch_one(&mut db) + .await + .unwrap(); + assert_eq!(piece, (904, vec![0x50, 0x60])); + sqlx::query("INSERT INTO context_semantic_fragments VALUES(9,0,'canonical_tool_projection',2147483648,32,x'00',32)").execute(&mut db).await.unwrap(); + assert!(sqlx::query("INSERT INTO context_semantic_fragments VALUES(10,0,'canonical_model_part',2147483648,32,x'00',32)").execute(&mut db).await.is_err()); + assert!(sqlx::query("INSERT INTO context_semantic_fragments VALUES(11,0,'canonical_tool_projection',9223372036854775776,32,x'00',32)").execute(&mut db).await.is_err()); + let foreign_keys = sqlx::query("PRAGMA foreign_key_check") + .fetch_all(&mut db) + .await + .unwrap(); + assert!(foreign_keys.is_empty()); + sqlx::query("DELETE FROM context_semantic_fragments WHERE source_sequence=1") + .execute(&mut db) + .await + .unwrap(); + let children: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM context_semantic_directory_children WHERE source_sequence=1", + ) + .fetch_one(&mut db) + .await + .unwrap(); + assert_eq!(children, 0); + for sequence in [1, 2] { + assert_eq!( + crate::canonical::one(&mut db, sequence).await.unwrap(), + original + ); + } + let bodies: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM event_bodies") + .fetch_one(&mut db) + .await + .unwrap(); + assert_eq!(bodies, 1); + let expected: i64 = sqlx::query_scalar( + "SELECT length(CAST(json_extract(?,'$.fact.outcome.model_projection') AS BLOB))", + ) + .bind(&original) + .fetch_one(&mut db) + .await + .unwrap(); + let measured: i64 = sqlx::query_scalar("SELECT sqlite_json_bytes FROM event_bodies") + .fetch_one(&mut db) + .await + .unwrap(); + assert_eq!(measured, expected); + let length: i64 = + sqlx::query_scalar("SELECT canonical_bytes FROM event_log WHERE sequence=1") + .fetch_one(&mut db) + .await + .unwrap(); + assert_eq!(length as usize, original.len()); + let source = recipe::projection("tool_settled", &original) + .unwrap() + .unwrap(); + let ranges = [ + 0..64, + (source.offset - 2)..(source.offset + 32), + (source.offset + source.json.len() - 2)..(source.offset + source.json.len() + 4), + (original.len() - 64)..original.len(), + ]; + let expected_ranges: Vec<_> = ranges + .iter() + .map(|range| original.as_bytes()[range.clone()].to_vec()) + .collect(); + crate::connection::worker_query(&mut db, move |worker| { + Box::pin(async move { + // SAFETY: the callback exclusively owns this original SQLite worker. + let raw = + unsafe { rusqlite::Connection::from_handle(worker.as_raw_handle().as_ptr()) }?; + for (range, expected) in ranges.into_iter().zip(expected_ranges) { + let actual = crate::canonical::with_span( + &raw, + 1, + range.start as u64, + (range.end - range.start) as u64, + |reader| { + let mut bytes = Vec::new(); + reader.read_to_end(&mut bytes)?; + Ok(bytes) + }, + )?; + assert_eq!(actual, expected); + } + drop(raw); + Ok(()) + }) + }) + .await + .unwrap(); + assert!(matches!( + crate::canonical::one(&mut db, 3).await, + Err(StoreError::MaterialCollected) + )); + assert!(sqlx::query("UPDATE event_log SET event_header=NULL,canonical_bytes=NULL,retained_json='{}',body_digest='digest' WHERE sequence=1").execute(&mut db).await.is_err()); + for (sequence, remaining) in [(1, 1), (2, 0)] { + let mut tx = db.begin().await.unwrap(); + sqlx::query("DELETE FROM event_body_slots WHERE event_sequence=?") + .bind(sequence) + .execute(&mut *tx) + .await + .unwrap(); + sqlx::query("UPDATE event_log SET event_header=NULL,canonical_bytes=NULL,retained_json='{}',body_digest='digest' WHERE sequence=?").bind(sequence).execute(&mut *tx).await.unwrap(); + tx.commit().await.unwrap(); + let bodies: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM event_bodies") + .fetch_one(&mut db) + .await + .unwrap(); + assert_eq!(bodies, remaining); + } + } +} diff --git a/platform/event-log/src/canonical/read.rs b/platform/event-log/src/canonical/read.rs new file mode 100644 index 0000000000..cd1495b294 --- /dev/null +++ b/platform/event-log/src/canonical/read.rs @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Complete canonical reads join one immutable body into the same selected row. +use super::recipe::{Recipe, Slot}; +use crate::StoreError; +use sqlx::{Row, sqlite::SqliteRow}; +use std::{borrow::Cow, io}; + +// Full-event queries bind their selected event as e. Metadata-only queries do +// not join these tables or fetch body payloads. +pub(crate) const COLUMNS: &str = "e.canonical_bytes AS canonical_length, + canonical_slot.event_sequence AS canonical_slot_sequence, + canonical_slot.header_offset AS canonical_offset, + canonical_body.id AS canonical_body_id, + canonical_body.raw_bytes AS canonical_raw_bytes, + canonical_body.payload AS canonical_payload"; +pub(crate) const JOINS: &str = + "LEFT JOIN event_body_slots canonical_slot ON canonical_slot.event_sequence=e.sequence + LEFT JOIN event_bodies canonical_body ON canonical_body.id=canonical_slot.body_id"; +fn invalid(reason: &'static str) -> StoreError { + io::Error::new(io::ErrorKind::InvalidData, reason).into() +} +fn nonnegative(value: i64) -> Result { + u64::try_from(value).map_err(|_| invalid("negative canonical storage metadata")) +} +struct Fields<'a> { + header: Option<&'a str>, + length: Option, + slot: Option, + offset: Option, + body: Option, + raw_bytes: Option, + payload: Option<&'a [u8]>, +} +fn reconstruct(fields: Fields<'_>) -> Result, StoreError> { + let Fields { + header, + length, + slot, + offset, + body, + raw_bytes, + payload, + } = fields; + let Some(header) = header else { + if length.is_some() + || slot.is_some() + || offset.is_some() + || body.is_some() + || raw_bytes.is_some() + || payload.is_some() + { + return Err(invalid("collected event retains live body metadata")); + } + return Err(StoreError::MaterialCollected); + }; + let length = + nonnegative(length.ok_or_else(|| invalid("live event lacks its canonical byte length"))?)?; + match (slot, offset, body, raw_bytes, payload) { + (None, None, None, None, None) => Recipe::new(header, length, None)?.reconstruct(None), + (Some(sequence), Some(offset), Some(id), Some(raw_bytes), Some(payload)) => { + if sequence <= 0 || id <= 0 || raw_bytes <= 0 { + return Err(invalid("invalid canonical body identity or length")); + } + let raw_bytes = nonnegative(raw_bytes)?; + let recipe = Recipe::new( + header, + length, + Some(Slot { + offset: nonnegative(offset)?, + bytes: raw_bytes, + }), + )?; + recipe.reconstruct(Some(payload)) + } + _ => Err(invalid("canonical slot or body is missing joined metadata")), + } +} +pub(crate) fn json(row: &SqliteRow) -> Result, StoreError> { + let header = row.try_get("event_header")?; + let length = row.try_get("canonical_length")?; + let slot: Option = row.try_get("canonical_slot_sequence")?; + // The shared LEFT JOIN cannot expose body fields without a slot. A dangling + // body still has slot presence and takes the complete validation path. + if slot.is_none() { + return reconstruct(Fields { + header, + length, + slot, + offset: None, + body: None, + raw_bytes: None, + payload: None, + }); + } + reconstruct(Fields { + header, + length, + slot, + offset: row.try_get("canonical_offset")?, + body: row.try_get("canonical_body_id")?, + raw_bytes: row.try_get("canonical_raw_bytes")?, + payload: row.try_get("canonical_payload")?, + }) +} +pub(crate) fn json_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result, StoreError> { + let header = row + .get_ref("event_header")? + .as_str_or_null() + .map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + row.as_ref() + .column_index("event_header") + .expect("selected canonical header"), + rusqlite::types::Type::Text, + Box::new(error), + ) + })?; + let length = row.get("canonical_length")?; + let slot: Option = row.get("canonical_slot_sequence")?; + if slot.is_none() { + return reconstruct(Fields { + header, + length, + slot, + offset: None, + body: None, + raw_bytes: None, + payload: None, + }); + } + let payload = row + .get_ref("canonical_payload")? + .as_blob_or_null() + .map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + row.as_ref() + .column_index("canonical_payload") + .expect("selected canonical body"), + rusqlite::types::Type::Blob, + Box::new(error), + ) + })?; + reconstruct(Fields { + header, + length, + slot, + offset: row.get("canonical_offset")?, + body: row.get("canonical_body_id")?, + raw_bytes: row.get("canonical_raw_bytes")?, + payload, + }) +} +pub(crate) async fn one( + connection: &mut sqlx::SqliteConnection, + sequence: u64, +) -> Result { + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.event_header,{COLUMNS} FROM event_log e {JOINS} WHERE e.sequence=?" + ))) + .bind(sequence as i64) + .fetch_one(connection) + .await?; + Ok(json(&row)?.into_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn corrupt_large_length_is_rejected_before_reserving_the_event() { + let result = reconstruct(Fields { + header: Some("null"), + length: Some(i64::MAX), + slot: Some(1), + offset: Some(0), + body: Some(1), + raw_bytes: Some(i64::MAX), + payload: Some(&[0; 8]), + }); + let error = result.unwrap_err(); + assert!( + error.to_string().contains("canonical body length changed"), + "{error}" + ); + } + #[test] + fn incomplete_joins_never_turn_into_inline_events() { + let result = reconstruct(Fields { + header: Some("null"), + length: Some(4), + slot: Some(1), + offset: Some(0), + body: None, + raw_bytes: None, + payload: None, + }); + assert!(result.is_err()); + let result = reconstruct(Fields { + header: None, + length: None, + slot: None, + offset: None, + body: None, + raw_bytes: None, + payload: None, + }); + assert!(matches!(result, Err(StoreError::MaterialCollected))); + } +} diff --git a/platform/event-log/src/canonical/recipe.rs b/platform/event-log/src/canonical/recipe.rs new file mode 100644 index 0000000000..67f8594061 --- /dev/null +++ b/platform/event-log/src/canonical/recipe.rs @@ -0,0 +1,441 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Lossless canonical event recipes and byte-range mapping. +use crate::StoreError; +use serde::Deserialize; +use serde_json::value::RawValue; +use std::{borrow::Cow, ops::Range}; + +fn invalid(reason: &'static str) -> StoreError { + StoreError::Sqlite(rusqlite::Error::InvalidParameterName(reason.into())) +} + +/// A projection token borrowed from the original complete canonical event. +pub(crate) struct Projection<'a> { + pub offset: usize, + pub json: &'a str, +} +pub(crate) fn projection<'a>( + kind: &str, + json: &'a str, +) -> Result>, StoreError> { + if kind != "tool_settled" { + return Ok(None); + } + #[derive(Deserialize)] + struct Envelope<'a> { + #[serde(borrow)] + fact: Fact<'a>, + } + #[derive(Deserialize)] + struct Fact<'a> { + #[serde(borrow)] + outcome: Outcome<'a>, + } + #[derive(Deserialize)] + struct Outcome<'a> { + kind: String, + #[serde(borrow)] + model_projection: Option<&'a RawValue>, + } + let envelope: Envelope<'_> = serde_json::from_str(json)?; + if envelope.fact.outcome.kind != "succeeded" { + return Ok(None); + } + let token = envelope + .fact + .outcome + .model_projection + .ok_or_else(|| invalid("successful tool result lacks its projection"))? + .get(); + let offset = (token.as_ptr() as usize) + .checked_sub(json.as_ptr() as usize) + .ok_or_else(|| invalid("projection does not borrow its event"))?; + offset + .checked_add(token.len()) + .filter(|end| *end <= json.len()) + .ok_or_else(|| invalid("projection lies outside its event"))?; + Ok(Some(Projection { + offset, + json: token, + })) +} +impl Projection<'_> { + /// SQLite json() preserves standard JSON tokens and removes only the four + /// insignificant ASCII whitespace bytes. This token was validated by Serde; + /// counting does not validate JSON or normalize numbers/Unicode escapes. + pub fn sqlite_json_bytes(&self) -> usize { + let mut quoted = false; + let mut escaped = false; + let mut bytes = 0; + for byte in self.json.bytes() { + if quoted { + bytes += 1; + if escaped { + escaped = false; + } else if byte == b'\\' { + escaped = true; + } else if byte == b'"' { + quoted = false; + } + } else if !matches!(byte, b' ' | b'\t' | b'\n' | b'\r') { + bytes += 1; + quoted = byte == b'"'; + } + } + bytes + } + + pub fn header(&self, original: &str) -> Result { + let end = self + .offset + .checked_add(self.json.len()) + .ok_or_else(|| invalid("projection range overflows"))?; + if original.get(self.offset..end) != Some(self.json) { + return Err(invalid("projection differs from its event")); + } + let mut header = String::with_capacity(original.len() - self.json.len() + 4); + header.push_str(&original[..self.offset]); + header.push_str("null"); + header.push_str(&original[end..]); + Ok(header) + } +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Slot { + pub offset: u64, + pub bytes: u64, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Segment { + Header { offset: u64, bytes: u64 }, + Body { offset: u64, bytes: u64 }, +} +/// Logical positions map to physical spans without loading the header body. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Layout { + bytes: u64, + slot: Option, +} +impl Layout { + pub fn new(bytes: u64, slot: Option) -> Result { + if let Some(slot) = slot { + if slot.bytes == 0 + || slot + .offset + .checked_add(slot.bytes) + .is_none_or(|end| end > bytes) + { + return Err(invalid("invalid canonical body slot range")); + } + bytes + .checked_sub(slot.bytes) + .and_then(|len| len.checked_add(4)) + .ok_or_else(|| invalid("canonical header length overflows"))?; + } + Ok(Self { bytes, slot }) + } + pub fn header_bytes(self) -> u64 { + match self.slot { + Some(slot) => self.bytes - slot.bytes + 4, + None => self.bytes, + } + } + /// A logical byte range touches at most the prefix, body, and suffix. + pub fn segments(&self, range: Range) -> Result<[Option; 3], StoreError> { + if range.start > range.end || range.end > self.bytes { + return Err(invalid("canonical byte range is outside its event")); + } + let mut segments = [None; 3]; + if range.is_empty() { + return Ok(segments); + } + let Some(slot) = self.slot else { + segments[0] = Some(Segment::Header { + offset: range.start, + bytes: range.end - range.start, + }); + return Ok(segments); + }; + let body_end = slot + .offset + .checked_add(slot.bytes) + .ok_or_else(|| invalid("canonical slot range overflows"))?; + let prefix_end = range.end.min(slot.offset); + if range.start < prefix_end { + segments[0] = Some(Segment::Header { + offset: range.start, + bytes: prefix_end - range.start, + }); + } + let body_start = range.start.max(slot.offset); + let body_limit = range.end.min(body_end); + if body_start < body_limit { + segments[1] = Some(Segment::Body { + offset: body_start - slot.offset, + bytes: body_limit - body_start, + }); + } + let suffix_start = range.start.max(body_end); + if suffix_start < range.end { + segments[2] = Some(Segment::Header { + offset: suffix_start - body_end + slot.offset + 4, + bytes: range.end - suffix_start, + }); + } + Ok(segments) + } +} + +/// The header and optional typed body slot describe exactly one event. +pub(crate) struct Recipe<'a> { + header: &'a str, + layout: Layout, +} +impl<'a> Recipe<'a> { + pub fn new(header: &'a str, bytes: u64, slot: Option) -> Result { + let layout = Layout::new(bytes, slot)?; + if header.len() as u64 != layout.header_bytes() { + return Err(invalid("canonical event length differs from its recipe")); + } + if let Some(slot) = slot { + let position = usize::try_from(slot.offset) + .map_err(|_| invalid("slot offset exceeds memory range"))?; + let end = position + .checked_add(4) + .ok_or_else(|| invalid("slot offset overflows"))?; + if header.as_bytes().get(position..end) != Some(b"null") { + return Err(invalid("invalid canonical body slot")); + } + } + Ok(Self { header, layout }) + } + pub fn reconstruct(&self, body: Option<&[u8]>) -> Result, StoreError> { + let Some(slot) = self.layout.slot else { + if body.is_some() { + return Err(invalid("canonical body has no slot")); + } + return Ok(Cow::Borrowed(self.header)); + }; + let body = body.ok_or_else(|| invalid("canonical slot lacks its body"))?; + if body.len() as u64 != slot.bytes { + return Err(invalid("canonical body length changed")); + } + self.reconstruct_with(|bytes| { + bytes.extend_from_slice(body); + Ok(()) + }) + } + pub fn reconstruct_with( + &self, + append_body: impl FnOnce(&mut Vec) -> Result<(), StoreError>, + ) -> Result, StoreError> { + let Some(slot) = self.layout.slot else { + return Ok(Cow::Borrowed(self.header)); + }; + let position = usize::try_from(slot.offset) + .map_err(|_| invalid("slot offset exceeds memory range"))?; + let capacity = usize::try_from(self.layout.bytes) + .map_err(|_| invalid("canonical event exceeds memory range"))?; + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(capacity) + .map_err(std::io::Error::other)?; + bytes.extend_from_slice(&self.header.as_bytes()[..position]); + append_body(&mut bytes)?; + if bytes.len() as u64 != slot.offset + slot.bytes { + return Err(invalid("canonical body length changed")); + } + bytes.extend_from_slice(&self.header.as_bytes()[position + 4..]); + Ok(Cow::Owned( + String::from_utf8(bytes).map_err(|_| invalid("canonical body is not UTF-8"))?, + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + const ORIGINAL: &str = r#"{ "outside":null, "fact": {"kind":"tool_settled", "outcome":{"kind":"succeeded", "model_projection": { "kind":"text", "text":"汉字\n\u0061 null" }}}, "after":null }"#; + #[test] + fn impossible_declared_capacity_fails_before_consuming_body() { + let recipe = Recipe::new( + "null", + u64::MAX, + Some(Slot { + offset: 0, + bytes: u64::MAX, + }), + ) + .unwrap(); + assert!( + recipe + .reconstruct_with(|_| panic!("invalid capacity must fail first")) + .is_err() + ); + } + #[test] + fn projection_measure_matches_sqlite_without_binding_a_whole_event() { + let db = rusqlite::Connection::open_in_memory().unwrap(); + let numbers = [ + "1e+00", + "1E-009", + "-0", + "-0.00", + "1e9999", + "123456789012345678901234567890", + ]; + let strings = [ + r#""汉🦀""#, + r#""\u0061\uD83E\uDD80""#, + r#""\\\" \\ \t\r\n""#, + ]; + for number in numbers { + for text in strings { + let token = format!( + "{{ \"kind\" : \"json\", \"value\" : {{ \"n\" : {number}, \"text\" : {text}, \"n\": [ true, null, false ] }} }}" + ); + let event = format!( + "{{\"fact\":{{\"outcome\":{{\"kind\":\"succeeded\",\"model_projection\":{token}}}}}}}" + ); + let source = projection("tool_settled", &event).unwrap().unwrap(); + let expected: i64 = db + .query_row( + "SELECT length(CAST(json(?1) AS BLOB))", + [source.json], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(source.sqlite_json_bytes(), expected as usize); + } + } + } + #[test] + fn extraction_and_every_byte_range_preserve_original_bytes() { + let projection = projection("tool_settled", ORIGINAL).unwrap().unwrap(); + let header = projection.header(ORIGINAL).unwrap(); + let recipe = Recipe::new( + &header, + ORIGINAL.len() as u64, + Some(Slot { + offset: projection.offset as u64, + bytes: projection.json.len() as u64, + }), + ) + .unwrap(); + assert_eq!( + recipe + .reconstruct(Some(projection.json.as_bytes())) + .unwrap(), + ORIGINAL + ); + for start in 0..=ORIGINAL.len() { + for end in start..=ORIGINAL.len() { + let mut result = Vec::new(); + for segment in recipe + .layout + .segments(start as u64..end as u64) + .unwrap() + .into_iter() + .flatten() + { + let (source, offset, bytes) = match segment { + Segment::Header { offset, bytes } => (header.as_bytes(), offset, bytes), + Segment::Body { offset, bytes } => { + (projection.json.as_bytes(), offset, bytes) + } + }; + result.extend_from_slice(&source[offset as usize..(offset + bytes) as usize]); + } + assert_eq!( + result, + &ORIGINAL.as_bytes()[start..end], + "range {start}..{end}" + ); + } + } + } + #[test] + fn escaped_success_tag_preserves_its_original_encoding() { + let original = ORIGINAL.replace("succeeded", r"succe\u0065ded"); + let projection = projection("tool_settled", &original).unwrap().unwrap(); + let header = projection.header(&original).unwrap(); + let recipe = Recipe::new( + &header, + original.len() as u64, + Some(Slot { + offset: projection.offset as u64, + bytes: projection.json.len() as u64, + }), + ) + .unwrap(); + assert_eq!( + recipe + .reconstruct(Some(projection.json.as_bytes())) + .unwrap(), + original + ); + } + #[test] + fn inline_and_failed_events_have_no_hidden_body() { + assert!( + projection("invocation_ended", r#"{"outcome":"completed"}"#) + .unwrap() + .is_none() + ); + assert!( + projection("tool_settled", r#"{"fact":{"outcome":{"kind":"failed"}}}"#) + .unwrap() + .is_none() + ); + let recipe = Recipe::new("null", 4, None).unwrap(); + assert_eq!(recipe.reconstruct(None).unwrap(), "null"); + assert!(recipe.reconstruct(Some(b"null")).is_err()); + assert!(recipe.layout.segments(0..5).is_err()); + } + #[test] + fn invalid_recipes_do_not_silently_become_inline_events() { + assert!(Recipe::new("null", 8, None).is_err()); + assert!( + Recipe::new( + "true", + 4, + Some(Slot { + offset: 0, + bytes: 4 + }) + ) + .is_err() + ); + let recipe = Recipe::new( + "null", + 8, + Some(Slot { + offset: 0, + bytes: 8, + }), + ) + .unwrap(); + assert!(recipe.reconstruct(None).is_err()); + assert!(recipe.reconstruct(Some(b"null")).is_err()); + assert!(recipe.reconstruct(Some(&[255; 8])).is_err()); + } +} diff --git a/platform/event-log/src/canonical/source.rs b/platform/event-log/src/canonical/source.rs new file mode 100644 index 0000000000..9d7195ffd8 --- /dev/null +++ b/platform/event-log/src/canonical/source.rs @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Same-handle immutable body readers; no SQLite handle escapes the callback. +use crate::StoreError; +use std::io::Read; +#[derive(Debug, thiserror::Error)] +#[error("{read}; closing canonical body also failed: {close}")] +struct ReadAndClose { + #[source] + read: StoreError, + close: rusqlite::Error, +} +pub(super) fn with_material( + raw: &rusqlite::Connection, + id: i64, + bytes: u64, + use_reader: impl FnOnce(&mut dyn Read) -> Result, +) -> Result { + let mut blob = raw.blob_open("main", "event_bodies", "payload", id, true)?; + let result = (|| { + if blob.len() as u64 != bytes { + return Err(invalid("canonical body length changed")); + } + use_reader(&mut blob) + })(); + settle(result, blob.close()) +} +fn settle(result: Result, closed: rusqlite::Result<()>) -> Result { + match (result, closed) { + (result, Ok(())) => result, + (Ok(_), Err(error)) => Err(error.into()), + (Err(read), Err(close)) => Err(std::io::Error::other(ReadAndClose { read, close }).into()), + } +} + +use super::recipe::{Layout, Segment, Slot}; +use std::io::{self, Seek, SeekFrom, Take}; +fn invalid(reason: &'static str) -> StoreError { + io::Error::new(io::ErrorKind::InvalidData, reason).into() +} +struct Parts<'a> { + parts: [Option>>; 3], + current: usize, +} +impl Read for Parts<'_> { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if output.is_empty() { + return Ok(0); + } + while self.current < self.parts.len() { + let read = match self.parts[self.current].as_mut() { + Some(reader) => reader.read(output)?, + None => 0, + }; + if read > 0 { + return Ok(read); + } + self.current += 1; + } + Ok(0) + } +} +#[derive(Clone, Copy)] +pub(crate) struct SpanMetadata { + live: bool, + length: Option, + slot_sequence: Option, + slot_offset: Option, + body_id: Option, + raw_bytes: Option, +} +pub(crate) const SPAN_COLUMNS: &str = "e.event_header IS NOT NULL AS canonical_live, + e.canonical_bytes AS canonical_length,canonical_slot.event_sequence AS canonical_slot_sequence, + canonical_slot.header_offset AS canonical_offset,canonical_body.id AS canonical_body_id, + canonical_body.raw_bytes AS canonical_raw_bytes"; + +impl SpanMetadata { + pub(crate) fn from_row(row: &sqlx::sqlite::SqliteRow) -> Result { + use sqlx::Row; + Ok(Self { + live: row.try_get("canonical_live")?, + length: row.try_get("canonical_length")?, + slot_sequence: row.try_get("canonical_slot_sequence")?, + slot_offset: row.try_get("canonical_offset")?, + body_id: row.try_get("canonical_body_id")?, + raw_bytes: row.try_get("canonical_raw_bytes")?, + }) + } +} + +/// Map canonical coordinates to immutable material without reading a full header. +pub(crate) fn with_span( + raw: &rusqlite::Connection, + sequence: i64, + offset: u64, + bytes: u64, + consume: impl FnOnce(&mut dyn Read) -> Result, +) -> Result { + let query = format!( + "SELECT {SPAN_COLUMNS} FROM event_log e {} WHERE e.sequence=?", + super::read::JOINS + ); + raw.query_row(&query, [sequence], |row| { + Ok(with_span_on_row(raw, row, sequence, offset, bytes, consume)) + })? +} + +/// Consume metadata borrowed from a caller's existing same-snapshot query. +pub(crate) fn with_span_on_row( + raw: &rusqlite::Connection, + row: &rusqlite::Row<'_>, + sequence: i64, + offset: u64, + bytes: u64, + consume: impl FnOnce(&mut dyn Read) -> Result, +) -> Result { + // SPAN_COLUMNS is one contiguous projection even when prefixed by a + // caller's fields. Resolve its base once rather than rescan all names. + let base = row.as_ref().column_index("canonical_live")?; + let metadata = SpanMetadata { + live: row.get(base)?, + length: row.get(base + 1)?, + slot_sequence: row.get(base + 2)?, + slot_offset: row.get(base + 3)?, + body_id: row.get(base + 4)?, + raw_bytes: row.get(base + 5)?, + }; + with_span_metadata(raw, sequence, metadata, offset, bytes, consume) +} + +pub(crate) fn with_span_metadata( + raw: &rusqlite::Connection, + sequence: i64, + metadata: SpanMetadata, + offset: u64, + bytes: u64, + consume: impl FnOnce(&mut dyn Read) -> Result, +) -> Result { + let SpanMetadata { + live, + length, + slot_sequence, + slot_offset, + body_id, + raw_bytes, + } = metadata; + if !live { + if length.is_some() || slot_sequence.is_some() || body_id.is_some() { + return Err(invalid("collected canonical event retains live material")); + } + return Err(StoreError::MaterialCollected); + } + let length = + u64::try_from(length.ok_or_else(|| invalid("canonical span lacks logical length"))?) + .map_err(|_| invalid("negative canonical byte length"))?; + let (slot, material) = match (slot_sequence, slot_offset, body_id, raw_bytes) { + (None, None, None, None) => (None, None), + (Some(source), Some(position), Some(id), Some(length)) + if source == sequence && position >= 0 && id > 0 && length > 0 => + { + ( + Some(Slot { + offset: position as u64, + bytes: length as u64, + }), + Some((id, length as u64)), + ) + } + _ => return Err(invalid("canonical span has an incomplete body slot")), + }; + let layout = Layout::new(length, slot)?; + let end = offset + .checked_add(bytes) + .ok_or_else(|| invalid("canonical span length overflows"))?; + let segments = layout.segments(offset..end)?; + let mut parts = Parts { + parts: std::array::from_fn(|_| None), + current: 0, + }; + let result = (|| { + for (index, segment) in segments.into_iter().enumerate() { + let Some(segment) = segment else { continue }; + let (table, column, rowid, expected, offset, bytes) = match segment { + Segment::Header { offset, bytes } => ( + "event_log", + "event_header", + sequence, + layout.header_bytes(), + offset, + bytes, + ), + Segment::Body { offset, bytes } => { + let (id, raw_bytes) = material + .ok_or_else(|| invalid("canonical range requires missing body material"))?; + ("event_bodies", "payload", id, raw_bytes, offset, bytes) + } + }; + let blob = raw.blob_open("main", table, column, rowid, true)?; + // Install ownership before any validation or seek can fail. + parts.parts[index] = Some(blob.take(bytes)); + let reader = parts.parts[index] + .as_mut() + .expect("installed canonical reader"); + if reader.get_ref().len() as u64 != expected { + return Err(invalid("canonical material length changed")); + } + reader.get_mut().seek(SeekFrom::Start(offset))?; + } + consume(&mut parts) + })(); + let mut closing = Ok(()); + for part in parts.parts.into_iter().flatten() { + if let Err(error) = part.into_inner().close() { + closing = match closing { + Ok(()) => Err(error), + Err(previous) => Err(rusqlite::Error::ToSqlConversionFailure(Box::new( + io::Error::other(format!("{previous}; {error}")), + ))), + }; + } + } + settle(result, closing) +} diff --git a/platform/event-log/src/canonical/write.rs b/platform/event-log/src/canonical/write.rs new file mode 100644 index 0000000000..1b28063fb7 --- /dev/null +++ b/platform/event-log/src/canonical/write.rs @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Immutable material and its event slot are published in the original transaction. +use super::{recipe::Projection, source}; +use crate::StoreError; +use sha2::{Digest, Sha256}; +use sqlx::sqlite::SqliteWorkerConnection; +pub(crate) async fn publish( + worker: &mut SqliteWorkerConnection<'_>, + sequence: u64, + projection: &Projection<'_>, +) -> Result<(), StoreError> { + let original = projection.json.as_bytes(); + let digest = Sha256::digest(original); + let inserted=sqlx::query("INSERT INTO event_bodies(digest,raw_bytes,sqlite_json_bytes,payload) VALUES(?1,?2,?3,?4) ON CONFLICT(digest) DO NOTHING") + .bind(digest.as_slice()).bind(original.len() as i64).bind(projection.sqlite_json_bytes() as i64).bind(original) + .execute(&mut *worker).await?; + let (id, bytes): (i64, i64) = + sqlx::query_as("SELECT id,raw_bytes FROM event_bodies WHERE digest=?") + .bind(digest.as_slice()) + .fetch_one(&mut *worker) + .await?; + if bytes != original.len() as i64 { + return Err(StoreError::EventConflict); + } + if inserted.rows_affected() == 0 { + // SAFETY: the worker callback exclusively owns this original handle. + // The borrowed wrapper and its BLOB readers close before returning. + let raw = unsafe { rusqlite::Connection::from_handle(worker.as_raw_handle().as_ptr()) }?; + source::with_material(&raw, id, bytes as u64, |reader| { + let mut position = 0; + let mut buffer = [0; 32 * 1024]; + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + if original.get(position..position + count) != Some(&buffer[..count]) { + return Err(StoreError::EventConflict); + } + position += count; + } + if position != original.len() { + return Err(StoreError::EventConflict); + } + Ok(()) + })?; + drop(raw); + } + sqlx::query("INSERT INTO event_body_slots(event_sequence,header_offset,body_id) VALUES(?,?,?)") + .bind(sequence as i64) + .bind(projection.offset as i64) + .bind(id) + .execute(&mut *worker) + .await?; + Ok(()) +} diff --git a/platform/event-log/src/canonical_insert.rs b/platform/event-log/src/canonical_insert.rs index 89c532e840..59bb7fac2e 100644 --- a/platform/event-log/src/canonical_insert.rs +++ b/platform/event-log/src/canonical_insert.rs @@ -31,10 +31,18 @@ pub(crate) struct Record<'json> { } pub(crate) fn insert(record: Record<'_>) -> Query<'_, Sqlite, SqliteArguments> { + let bytes = record.json.len(); + insert_header(record, bytes) +} + +pub(crate) fn insert_header( + record: Record<'_>, + canonical_bytes: usize, +) -> Query<'_, Sqlite, SqliteArguments> { sqlx::query( - "INSERT INTO event_log(sequence,event_id,invocation_id,kind,operation_id,event_json, + "INSERT INTO event_log(sequence,event_id,invocation_id,kind,operation_id,event_header,canonical_bytes, accounting_at,accounting_usage,invocation_model,accounting_tool,event_session) - VALUES (?1,?2,?3,?4,?5,?6, + VALUES (?1,?2,?3,?4,?5,?6,?7, CASE WHEN ?4 IN ('model_requested', 'model_completed', 'model_interrupted', 'invocation_ended', 'auxiliary_model_started', 'auxiliary_model_settled', 'tool_dispatched', 'tool_settled', 'tool_rejected') @@ -69,4 +77,5 @@ pub(crate) fn insert(record: Record<'_>) -> Query<'_, Sqlite, SqliteArguments> { .bind(record.kind) .bind(record.operation_id) .bind(record.json) + .bind(canonical_bytes as i64) } diff --git a/platform/event-log/src/catalog_changes.rs b/platform/event-log/src/catalog_changes.rs index 879225a914..cbbb51e960 100644 --- a/platform/event-log/src/catalog_changes.rs +++ b/platform/event-log/src/catalog_changes.rs @@ -36,12 +36,12 @@ impl EventLog { } self.connection.run(move |connection| Box::pin(async move { let rows: Vec<(i64, String)> = sqlx::query_as( - "SELECT sequence, json_extract(event_json, '$.invocation.session_id') + "SELECT sequence, json_extract(event_header, '$.invocation.session_id') FROM local_runtime_events WHERE sequence > ?1 AND sequence <= ?2 AND kind IN ('invocation_opened', 'model_completed', 'model_interrupted', 'invocation_ended') AND EXISTS (SELECT 1 FROM session_control - WHERE id = json_extract(event_json, '$.invocation.session_id')) + WHERE id = json_extract(event_header, '$.invocation.session_id')) ORDER BY sequence LIMIT ?3", ).bind(after as i64).bind(through as i64).bind(limit as i64) .fetch_all(connection).await?; diff --git a/platform/event-log/src/composition.rs b/platform/event-log/src/composition.rs index 1358b8268e..536b2f73d9 100644 --- a/platform/event-log/src/composition.rs +++ b/platform/event-log/src/composition.rs @@ -42,9 +42,9 @@ pub(crate) async fn insert( let first: Option> = sqlx::query_scalar( "SELECT c.digest FROM runtime_events r LEFT JOIN model_request_compositions c ON c.event_id=r.event_id WHERE r.invocation_id=?1 AND r.kind='model_requested' AND r.event_id!=?2 - AND json_extract(r.event_json,'$.fact.purpose')='summary' - AND json_extract(r.event_json,'$.fact.source_scope')=json(?3) - AND json_extract(r.event_json,'$.fact.source_high_water')=?4 + AND json_extract(r.event_header,'$.fact.purpose')='summary' + AND json_extract(r.event_header,'$.fact.source_scope')=json(?3) + AND json_extract(r.event_header,'$.fact.source_high_water')=?4 ORDER BY r.sequence LIMIT 1", ).bind(&write.event().invocation.invocation_id).bind(&write.event().id) .bind(serde_json::to_string(source_scope)?).bind(*source_high_water as i64) @@ -95,7 +95,7 @@ pub(crate) async fn validate_retry( let prior: Option> = sqlx::query_scalar( "SELECT c.digest FROM runtime_events r LEFT JOIN model_request_compositions c ON c.event_id=r.event_id WHERE r.invocation_id=?1 AND r.kind='model_requested' AND r.event_id!=?2 AND r.sequence>?3 - AND json_remove(json_extract(r.event_json,'$.fact'),'$.step_id')=json_remove(json(?4),'$.step_id') + AND json_remove(json_extract(r.event_header,'$.fact'),'$.step_id')=json_remove(json(?4),'$.step_id') ORDER BY r.sequence LIMIT 1" ).bind(&event.invocation.invocation_id).bind(&event.id).bind(*source_high_water as i64) .bind(serde_json::to_string(&event.fact)?).fetch_optional(connection).await?; @@ -151,7 +151,7 @@ async fn validate_reference( "SELECT c.digest FROM model_request_compositions c JOIN runtime_events r ON r.event_id=c.event_id WHERE c.event_id=?1 AND r.invocation_id=?2 - AND json_extract(r.event_json,'$.invocation.session_id')=?3", + AND json_extract(r.event_header,'$.invocation.session_id')=?3", ) .bind(event_id) .bind(&event.invocation.invocation_id) @@ -184,7 +184,7 @@ impl EventLog { "SELECT r.event_id FROM runtime_events c JOIN runtime_events r ON r.invocation_id=c.invocation_id AND r.operation_id=c.operation_id WHERE c.sequence=?1 AND c.kind='model_completed' AND r.kind='model_requested' - AND json_extract(c.event_json,'$.invocation.session_id')=?2", + AND json_extract(c.event_header,'$.invocation.session_id')=?2", ) .bind(sequence as i64) .bind(owned_session) @@ -213,7 +213,7 @@ impl EventLog { "SELECT composition.surface FROM request_compositions AS composition JOIN model_request_compositions AS request ON request.digest = composition.digest JOIN runtime_events AS event ON event.event_id = request.event_id - WHERE request.event_id = ? AND json_extract(event.event_json, '$.invocation.session_id') = ?" + WHERE request.event_id = ? AND json_extract(event.event_header, '$.invocation.session_id') = ?" ).bind(event_id).bind(session).fetch_optional(connection).await?; row.map(|bytes| serde_json::from_slice(&bytes).map_err(StoreError::from)).transpose() })).await diff --git a/platform/event-log/src/context/boundary.rs b/platform/event-log/src/context/boundary.rs index 9303469353..53797d63c3 100644 --- a/platform/event-log/src/context/boundary.rs +++ b/platform/event-log/src/context/boundary.rs @@ -101,11 +101,11 @@ pub(super) async fn summary_start( ) -> Result, StoreError> { let first: Option = sqlx::query_scalar( "SELECT sequence FROM event_log INDEXED BY context_summary_round - WHERE invocation_id = ?1 AND event_json IS NOT NULL + WHERE invocation_id = ?1 AND event_header IS NOT NULL AND kind = 'model_requested' AND sequence < ?2 - AND json_extract(event_json, '$.fact.purpose') = 'summary' - AND json_extract(event_json, '$.fact.source_high_water') = ?3 - AND json_extract(event_json, '$.fact.source_scope') = json(?4) + AND json_extract(event_header, '$.fact.purpose') = 'summary' + AND json_extract(event_header, '$.fact.source_high_water') = ?3 + AND json_extract(event_header, '$.fact.source_scope') = json(?4) ORDER BY sequence LIMIT 1", ) .bind(invocation) @@ -118,26 +118,26 @@ pub(super) async fn summary_start( } pub(super) const SUMMARY_ADMISSION: &str = "WITH covered AS ( - SELECT json_extract(event_json,'$.fact.source_high_water') AS high + SELECT json_extract(event_header,'$.fact.source_high_water') AS high FROM event_log INDEXED BY context_summary_round - WHERE invocation_id=?1 AND event_json IS NOT NULL AND kind='model_requested' - AND json_extract(event_json,'$.fact.purpose')='summary' - ORDER BY json_extract(event_json,'$.fact.source_high_water') DESC LIMIT 1) + WHERE invocation_id=?1 AND event_header IS NOT NULL AND kind='model_requested' + AND json_extract(event_header,'$.fact.purpose')='summary' + ORDER BY json_extract(event_header,'$.fact.source_high_water') DESC LIMIT 1) SELECT EXISTS(SELECT 1 FROM context_execution_work w INDEXED BY context_execution_live JOIN context_execution_steps s ON s.invocation_id=w.invocation_id AND s.operation_id=w.operation_id WHERE w.invocation_id=?1 AND w.closed_at IS NULL AND w.kind='model' AND s.purpose='summary'), (SELECT COUNT(*) FROM (SELECT 1 FROM event_log INDEXED BY context_summary_sequence - WHERE invocation_id=?1 AND event_json IS NOT NULL AND kind='model_requested' - AND json_extract(event_json,'$.fact.purpose')='summary' AND sequence>=?2 LIMIT 3)), + WHERE invocation_id=?1 AND event_header IS NOT NULL AND kind='model_requested' + AND json_extract(event_header,'$.fact.purpose')='summary' AND sequence>=?2 LIMIT 3)), NOT EXISTS(SELECT 1 FROM covered) OR EXISTS( SELECT 1 FROM runtime_events c JOIN runtime_events r ON r.invocation_id=c.invocation_id AND r.kind='model_requested' AND r.operation_id=CASE WHEN c.kind='model_observed' - THEN CAST(json_extract(c.event_json,'$.fact.step_id') AS TEXT) ELSE c.operation_id END + THEN CAST(json_extract(c.event_header,'$.fact.step_id') AS TEXT) ELSE c.operation_id END WHERE c.invocation_id=?1 AND (c.kind='model_completed' OR - (c.kind='model_observed' AND json_extract(r.event_json,'$.fact.item_acceptance')=1 - AND json_extract(c.event_json,'$.fact.event.kind')='part')) - AND json_extract(r.event_json, '$.fact.purpose')='main' + (c.kind='model_observed' AND json_extract(r.event_header,'$.fact.item_acceptance')=1 + AND json_extract(c.event_header,'$.fact.event.kind')='part')) + AND json_extract(r.event_header, '$.fact.purpose')='main' AND c.sequence <= ?3 AND c.sequence > (SELECT high FROM covered))"; /// Renewal must be Main progress covered by the new source. A fast Main can diff --git a/platform/event-log/src/context/evidence/roots.rs b/platform/event-log/src/context/evidence/roots.rs index 2f7474ee27..260d381120 100644 --- a/platform/event-log/src/context/evidence/roots.rs +++ b/platform/event-log/src/context/evidence/roots.rs @@ -461,8 +461,8 @@ async fn record_archive( ) -> Result<(), StoreError> { let event = &stored.event; let (target, invocation, identity, operation, is_error): (i64, String, String, String, bool) = - sqlx::query_as("SELECT sequence,invocation_id,json_extract(event_json,'$.invocation'),operation_id, - json_extract(event_json,'$.fact.outcome.kind')='failed' FROM runtime_events WHERE event_id=?") + sqlx::query_as("SELECT sequence,invocation_id,json_extract(event_header,'$.invocation'),operation_id, + json_extract(event_header,'$.fact.outcome.kind')='failed' FROM runtime_events WHERE event_id=?") .bind(&placeholder.identity.runtime_event_id).fetch_one(&mut *db).await?; let encoded = serde_json::to_string(placeholder)?; sqlx::query("INSERT INTO context_archive_records VALUES(?,?,?,?,?,?,?,?,?,?,?)") diff --git a/platform/event-log/src/context/history.rs b/platform/event-log/src/context/history.rs index da88adbe81..83f7087e62 100644 --- a/platform/event-log/src/context/history.rs +++ b/platform/event-log/src/context/history.rs @@ -136,13 +136,13 @@ pub(crate) async fn resolve_cut( let (first, last): (Option, Option) = sqlx::query_as( "SELECT MIN(sequence),MAX(sequence) FROM session_history_events WHERE owner_session_id = ?1 - AND json_extract(event_json,'$.invocation.turn_id') = ?2 + AND json_extract(event_header,'$.invocation.turn_id') = ?2 AND sequence <= ?3 AND EXISTS(SELECT 1 FROM session_history_events opening WHERE (opening.kind = 'message_imported' OR (opening.kind = 'invocation_opened' - AND json_extract(opening.event_json,'$.fact.input.kind') IN ('message','continuation','handoff'))) + AND json_extract(opening.event_header,'$.fact.input.kind') IN ('message','continuation','handoff'))) AND opening.owner_session_id = ?1 - AND json_extract(opening.event_json,'$.invocation.turn_id') = ?2 + AND json_extract(opening.event_header,'$.invocation.turn_id') = ?2 AND opening.sequence <= ?3)", ) .bind(session) @@ -161,7 +161,7 @@ pub(crate) async fn resolve_cut( let crossing: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM session_history_events WHERE owner_session_id = ?1 AND sequence <= ?2 - GROUP BY json_extract(event_json,'$.invocation.turn_id') + GROUP BY json_extract(event_header,'$.invocation.turn_id') HAVING MIN(sequence) <= ?3 AND MAX(sequence) > ?3)", ) .bind(session) diff --git a/platform/event-log/src/context/latest_main.rs b/platform/event-log/src/context/latest_main.rs index 8ffc7745d6..e166ef7869 100644 --- a/platform/event-log/src/context/latest_main.rs +++ b/platform/event-log/src/context/latest_main.rs @@ -87,36 +87,36 @@ pub(super) async fn read_selected_at( // summary can be skipped; missing trace must not reveal an older main. let mut rows = sqlx::query(sqlx::AssertSqlSafe(format!( "WITH runtime_events AS NOT MATERIALIZED (SELECT e.* FROM main.runtime_events e WHERE e.sequence <= ?2 AND {filter}) - SELECT c.sequence, json_extract(c.event_json, '$.invocation') AS ci, - json_extract(r.event_json, '$.invocation') AS ri, json_extract(o.event_json, '$.invocation') AS oi, - json_extract(o.event_json, '$.fact.input.kind') AS opening_kind, - json_extract(r.event_json, '$.fact.purpose') AS purpose, - (SELECT json_extract(f.event_json, '$.recorded_at') FROM runtime_events f + SELECT c.sequence, json_extract(c.event_header, '$.invocation') AS ci, + json_extract(r.event_header, '$.invocation') AS ri, json_extract(o.event_header, '$.invocation') AS oi, + json_extract(o.event_header, '$.fact.input.kind') AS opening_kind, + json_extract(r.event_header, '$.fact.purpose') AS purpose, + (SELECT json_extract(f.event_header, '$.recorded_at') FROM runtime_events f WHERE f.invocation_id=c.invocation_id AND f.kind='model_observed' - AND json_extract(f.event_json, '$.invocation.session_id')=?1 - AND CAST(json_extract(f.event_json, '$.fact.step_id') AS TEXT)=json_extract(c.event_json, '$.fact.step_id') - AND json_extract(f.event_json, '$.fact.event.kind')='finished' + AND json_extract(f.event_header, '$.invocation.session_id')=?1 + AND CAST(json_extract(f.event_header, '$.fact.step_id') AS TEXT)=json_extract(c.event_header, '$.fact.step_id') + AND json_extract(f.event_header, '$.fact.event.kind')='finished' AND f.sequence>r.sequence AND f.sequence r.sequence AND (target.sequence IS NULL OR target.sequence <= json_extract(r.event_json,'$.fact.source_high_water'))) AS projection_current + AND json_extract(a.event_header,'$.invocation.session_id')=json_extract(c.event_header,'$.invocation.session_id') + AND a.sequence > r.sequence AND (target.sequence IS NULL OR target.sequence <= json_extract(r.event_header,'$.fact.source_high_water'))) AS projection_current FROM runtime_events c LEFT JOIN runtime_events r ON r.invocation_id = c.invocation_id AND r.operation_id = c.operation_id AND r.kind = 'model_requested' LEFT JOIN runtime_events o ON o.invocation_id = c.invocation_id AND o.kind = 'invocation_opened' - WHERE c.kind = 'model_completed' AND json_extract(c.event_json, '$.invocation.session_id') = ?1 + WHERE c.kind = 'model_completed' AND json_extract(c.event_header, '$.invocation.session_id') = ?1 ORDER BY c.sequence DESC" ))).bind(session).bind(through as i64).bind(&selection.lineage) .bind(archives_before.min(i64::MAX as u64) as i64).fetch(connection); diff --git a/platform/event-log/src/context/projection.rs b/platform/event-log/src/context/projection.rs index 686aaf5bb7..84f8a4a1dd 100644 --- a/platform/event-log/src/context/projection.rs +++ b/platform/event-log/src/context/projection.rs @@ -126,7 +126,7 @@ async fn obligations( SELECT e.event_session, e.invocation_id, 'tool', e.operation_id, e.sequence, EXISTS(SELECT 1 FROM runtime_events terminal WHERE terminal.invocation_id=e.invocation_id AND terminal.kind='invocation_ended' - AND json_extract(terminal.event_json,'$.fact.outcome.kind') IN ('failed','cancelled')) + AND json_extract(terminal.event_header,'$.fact.outcome.kind') IN ('failed','cancelled')) FROM runtime_events e WHERE e.sequence=?1 AND e.kind='tool_dispatched' AND NOT EXISTS(SELECT 1 FROM runtime_events t WHERE t.invocation_id=e.invocation_id AND t.operation_id=e.operation_id AND t.kind='tool_settled')"#).bind(sequence).execute(&mut *db).await?; @@ -341,10 +341,10 @@ FROM runtime_events e WHERE e.kind='tool_dispatched' AND e.sequence=?1"#).bind(s sqlx::query("INSERT OR IGNORE INTO context_execution_work(invocation_id,kind,operation_id,opened_at,closed_at) SELECT ?1,'executor_tool',?2,?3, (SELECT MIN(e.sequence) FROM event_log e INDEXED BY executor_tool_observations - WHERE e.invocation_id=?1 AND e.event_json IS NOT NULL + WHERE e.invocation_id=?1 AND e.event_header IS NOT NULL AND e.kind IN ('executor_observed','executor_completed') - AND json_extract(e.event_json,'$.fact.output.toolCallId')=?2 - AND json_extract(e.event_json,'$.fact.output.type')='tool_result') + AND json_extract(e.event_header,'$.fact.output.toolCallId')=?2 + AND json_extract(e.event_header,'$.fact.output.type')='tool_result') WHERE NOT EXISTS(SELECT 1 FROM context_execution_work WHERE invocation_id=?1 AND kind='executor_tool' AND operation_id=?2 AND call_id='' AND call_name='')") diff --git a/platform/event-log/src/context/proof.rs b/platform/event-log/src/context/proof.rs index 32885b8e62..43282230b0 100644 --- a/platform/event-log/src/context/proof.rs +++ b/platform/event-log/src/context/proof.rs @@ -25,14 +25,21 @@ use maka_agent_facts::context::TextSummary; use maka_agent_facts::context::resolve_model_purpose; use maka_agent_facts::event::Fact; use maka_agent_facts::event::RuntimeEvent; -use sqlx::SqliteConnection; +use sqlx::{Row, SqliteConnection}; pub(super) async fn by_id( connection: &mut SqliteConnection, id: &str, ) -> Result<(u64, RuntimeEvent), StoreError> { - decode(sqlx::query_as("SELECT sequence, CASE WHEN length(CAST(event_json AS BLOB)) <= 8388608 THEN event_json END - FROM runtime_events WHERE event_id = ?").bind(id).fetch_optional(connection).await?) + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.sequence,e.event_header,{} FROM runtime_events e {} WHERE e.event_id=?", + crate::canonical::COLUMNS, + crate::canonical::JOINS + ))) + .bind(id) + .fetch_optional(connection) + .await?; + decode(row) } pub(super) async fn by_kind( @@ -41,22 +48,35 @@ pub(super) async fn by_kind( kind: &str, operation: Option<&str>, ) -> Result<(u64, RuntimeEvent), StoreError> { - let row = if let Some(operation) = operation { - sqlx::query_as("SELECT sequence, CASE WHEN length(CAST(event_json AS BLOB)) <= 8388608 THEN event_json END - FROM runtime_events WHERE invocation_id = ? AND kind = ? AND operation_id = ?") - .bind(invocation).bind(kind).bind(operation).fetch_optional(connection).await? + let operation_filter = if operation.is_some() { + " AND e.operation_id=?3" + } else { + "" + }; + let query = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.sequence,e.event_header,{} FROM runtime_events e {} + WHERE e.invocation_id=?1 AND e.kind=?2{operation_filter}", + crate::canonical::COLUMNS, + crate::canonical::JOINS + ))) + .bind(invocation) + .bind(kind); + let query = if let Some(operation) = operation { + query.bind(operation) } else { - sqlx::query_as("SELECT sequence, CASE WHEN length(CAST(event_json AS BLOB)) <= 8388608 THEN event_json END - FROM runtime_events WHERE invocation_id = ? AND kind = ?") - .bind(invocation).bind(kind).fetch_optional(connection).await? + query }; + let row = query.fetch_optional(connection).await?; decode(row) } -fn decode(row: Option<(i64, Option)>) -> Result<(u64, RuntimeEvent), StoreError> { - let (sequence, json) = row.ok_or_else(|| invalid("missing proof event"))?; - let json = json.ok_or_else(|| invalid("proof event exceeds bounded reader capacity"))?; - Ok((sequence_number(sequence)?, serde_json::from_str(&json)?)) +fn decode(row: Option) -> Result<(u64, RuntimeEvent), StoreError> { + let row = row.ok_or_else(|| invalid("missing proof event"))?; + let json = crate::canonical::json(&row)?; + Ok(( + sequence_number(row.try_get("sequence")?)?, + serde_json::from_str(&json)?, + )) } /// Validate the proof and return its exact source selection. The chain reader @@ -167,14 +187,14 @@ pub(super) async fn validate( } let changed_route: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM event_log INDEXED BY context_summary_round - WHERE invocation_id = ?1 AND event_json IS NOT NULL AND kind = 'model_requested' AND sequence >= ?9 AND sequence < ?2 - AND json_extract(event_json, '$.fact.purpose') = 'summary' - AND json_extract(event_json, '$.fact.source_high_water') = ?5 - AND json_extract(event_json, '$.fact.source_scope') = json(?10) - AND (json_extract(event_json, '$.fact.model_id') != ?3 OR json_extract(event_json, '$.fact.route_identity') != ?4 - OR json_extract(event_json, '$.fact.source_high_water') != ?5 OR json_extract(event_json, '$.fact.source_digest') != ?6 - OR json_extract(event_json, '$.fact.checkpoint_event_id') IS NOT ?7 - OR json_extract(event_json, '$.fact.effective_source_digest') IS NOT ?8))", + WHERE invocation_id = ?1 AND event_header IS NOT NULL AND kind = 'model_requested' AND sequence >= ?9 AND sequence < ?2 + AND json_extract(event_header, '$.fact.purpose') = 'summary' + AND json_extract(event_header, '$.fact.source_high_water') = ?5 + AND json_extract(event_header, '$.fact.source_scope') = json(?10) + AND (json_extract(event_header, '$.fact.model_id') != ?3 OR json_extract(event_header, '$.fact.route_identity') != ?4 + OR json_extract(event_header, '$.fact.source_high_water') != ?5 OR json_extract(event_header, '$.fact.source_digest') != ?6 + OR json_extract(event_header, '$.fact.checkpoint_event_id') IS NOT ?7 + OR json_extract(event_header, '$.fact.effective_source_digest') IS NOT ?8))", ).bind(&event.invocation.invocation_id).bind(sequence as i64) .bind(model_id).bind(route_identity).bind(checkpoint.covered_through as i64).bind(&checkpoint.source_digest) .bind(&checkpoint.previous_checkpoint_id).bind(effective_source_digest).bind(first_summary as i64).bind(serde_json::to_string(source_scope)?).fetch_one(&mut *connection).await?; diff --git a/platform/event-log/src/context/read.rs b/platform/event-log/src/context/read.rs index c579fac227..825adedfd1 100644 --- a/platform/event-log/src/context/read.rs +++ b/platform/event-log/src/context/read.rs @@ -314,7 +314,7 @@ async fn unknown_tools( .bind(&selection.lineage) .fetch_one(&mut *connection) .await?; - let rows=sqlx::query_as(sqlx::AssertSqlSafe(format!("SELECT json_extract(d.event_json,'$.fact.name'),o.operation_id {from} ORDER BY o.opened_at LIMIT 64"))) + let rows=sqlx::query_as(sqlx::AssertSqlSafe(format!("SELECT json_extract(d.event_header,'$.fact.name'),o.operation_id {from} ORDER BY o.opened_at LIMIT 64"))) .bind(&selection.session).bind(&selection.lineage).fetch_all(connection).await?; Ok((rows, crate::sequence_number(count)?)) } @@ -392,19 +392,19 @@ pub(crate) async fn latest_record( let eligible = format!( "c.kind='context_checkpoint_recorded' AND c.sequence <= ?1 AND {filter} AND NOT EXISTS(SELECT 1 FROM runtime_events r WHERE r.invocation_id=c.invocation_id - AND r.kind='model_requested' AND r.operation_id=json_extract(c.event_json,'$.fact.checkpoint.summary_step_id') - AND json_extract(r.event_json,'$.fact.source_scope.kind')='lineage' + AND r.kind='model_requested' AND r.operation_id=json_extract(c.event_header,'$.fact.checkpoint.summary_step_id') + AND json_extract(r.event_header,'$.fact.source_scope.kind')='lineage' AND (?3 IS NULL OR NOT {contains}))" ); // Limit each indexed branch before combining: a wide UNION view would sort // every old body even when the newest native checkpoint is already known. let row: Option<(i64, String)> = sqlx::query_as(sqlx::AssertSqlSafe(format!( "SELECT * FROM ( - SELECT c.sequence,c.event_json FROM runtime_events c - WHERE json_extract(c.event_json,'$.invocation.session_id')=?2 AND {eligible} + SELECT c.sequence,c.event_header FROM runtime_events c + WHERE json_extract(c.event_header,'$.invocation.session_id')=?2 AND {eligible} ORDER BY c.sequence DESC LIMIT 1) UNION ALL SELECT * FROM ( - SELECT c.sequence,c.event_json FROM session_history_members h + SELECT c.sequence,c.event_header FROM session_history_members h JOIN runtime_events c ON c.sequence=h.sequence WHERE h.session_id=?2 AND {eligible} ORDER BY h.sequence DESC LIMIT 1) ORDER BY sequence DESC LIMIT 1" diff --git a/platform/event-log/src/context/report.rs b/platform/event-log/src/context/report.rs index 3b97f811ea..486011fd12 100644 --- a/platform/event-log/src/context/report.rs +++ b/platform/event-log/src/context/report.rs @@ -81,10 +81,10 @@ impl EventLog { return Err(invalid("fence exceeds committed history")); } let row = sqlx::query( - "SELECT sequence, event_json FROM runtime_events - WHERE json_extract(event_json,'$.invocation.session_id')=?1 + "SELECT sequence, event_header FROM runtime_events + WHERE json_extract(event_header,'$.invocation.session_id')=?1 AND kind='model_requested' AND sequence<=?2 - AND json_extract(event_json,'$.fact.purpose')='main' + AND json_extract(event_header,'$.fact.purpose')='main' ORDER BY sequence DESC LIMIT 1", ) .bind(&session) @@ -93,7 +93,7 @@ impl EventLog { .await?; let request = if let Some(row) = row { let sequence = safe_number(sequence_number(row.try_get("sequence")?)?)?; - let event: RuntimeEvent = serde_json::from_str(row.try_get("event_json")?)?; + let event: RuntimeEvent = serde_json::from_str(row.try_get("event_header")?)?; let Fact::ModelRequested { step_id, model_id, @@ -112,11 +112,11 @@ impl EventLog { breakdown.validate().map_err(invalid)?; } let terminal = sqlx::query( - "SELECT kind, json_extract(event_json,'$.fact.output.usage.input_tokens') AS input_tokens + "SELECT kind, json_extract(event_header,'$.fact.output.usage.input_tokens') AS input_tokens FROM runtime_events WHERE invocation_id=?1 AND operation_id=?2 AND kind IN ('model_completed','model_interrupted') AND sequence>?3 AND sequence<=?4 - AND json_extract(event_json,'$.invocation.session_id')=?5 + AND json_extract(event_header,'$.invocation.session_id')=?5 ORDER BY sequence LIMIT 1" ) .bind(&event.invocation.invocation_id) diff --git a/platform/event-log/src/context/request.rs b/platform/event-log/src/context/request.rs index ca2a36a21f..585f48ada0 100644 --- a/platform/event-log/src/context/request.rs +++ b/platform/event-log/src/context/request.rs @@ -160,7 +160,7 @@ pub(super) async fn validate( ) .await?; let previous_effective: Option<(i64, Option)> = sqlx::query_as( - "SELECT sequence,json_extract(event_json,'$.fact.effective_source_digest') FROM runtime_events + "SELECT sequence,json_extract(event_header,'$.fact.effective_source_digest') FROM runtime_events WHERE sequence = ?", ).bind(first_summary).fetch_optional(&mut *connection).await?; if let Some((sequence, expected)) = previous_effective { @@ -178,9 +178,9 @@ pub(super) async fn validate( .await?; } let first: Option<(String, String, i64, String, Option)> = sqlx::query_as( - "SELECT json_extract(event_json, '$.fact.model_id'), json_extract(event_json, '$.fact.route_identity'), - json_extract(event_json, '$.fact.source_high_water'), json_extract(event_json, '$.fact.source_digest'), - json_extract(event_json, '$.fact.checkpoint_event_id') FROM runtime_events + "SELECT json_extract(event_header, '$.fact.model_id'), json_extract(event_header, '$.fact.route_identity'), + json_extract(event_header, '$.fact.source_high_water'), json_extract(event_header, '$.fact.source_digest'), + json_extract(event_header, '$.fact.checkpoint_event_id') FROM runtime_events WHERE sequence = ?", ).bind(first_summary).fetch_optional(&mut *connection).await?; if first.is_some_and(|(m, r, h, d, p)| { diff --git a/platform/event-log/src/context/safety.rs b/platform/event-log/src/context/safety.rs index 9a48e7b0dc..9e5c5405cb 100644 --- a/platform/event-log/src/context/safety.rs +++ b/platform/event-log/src/context/safety.rs @@ -32,8 +32,8 @@ pub(crate) async fn current_opening( return Ok(None); }; let row: Option<(i64, String)> = sqlx::query_as( - "SELECT sequence, event_json FROM runtime_events e WHERE invocation_id = ? - AND kind = 'invocation_opened' AND json_extract(event_json, '$.invocation.session_id') = ? + "SELECT sequence, event_header FROM runtime_events e WHERE invocation_id = ? + AND kind = 'invocation_opened' AND json_extract(event_header, '$.invocation.session_id') = ? AND NOT EXISTS(SELECT 1 FROM runtime_events t WHERE t.invocation_id = e.invocation_id AND t.kind = 'invocation_ended')", ).bind(id).bind(session).fetch_optional(connection).await?; let (sequence, json) = @@ -129,7 +129,7 @@ async fn require_safe_through_policy( "WITH runtime_events AS NOT MATERIALIZED (SELECT * FROM main.runtime_events WHERE sequence <= ?3), ", crate::model_items::accepted_calls!(), " SELECT EXISTS(SELECT 1 FROM runtime_events e - WHERE json_extract(e.event_json, '$.invocation.session_id') = ?1 + WHERE json_extract(e.event_header, '$.invocation.session_id') = ?1 AND (e.kind = 'invocation_opened' OR e.kind = 'tool_dispatched' OR e.kind = 'model_requested' OR e.kind = 'model_completed') AND (?2 IS NULL OR e.invocation_id != ?2) AND ( @@ -139,7 +139,7 @@ async fn require_safe_through_policy( WHERE t.invocation_id = e.invocation_id AND t.operation_id = e.operation_id AND t.kind = 'tool_settled') AND (?4 = 0 OR NOT EXISTS(SELECT 1 FROM runtime_events terminal WHERE terminal.invocation_id = e.invocation_id AND terminal.kind = 'invocation_ended' - AND json_extract(terminal.event_json, '$.fact.outcome.kind') IN ('failed','cancelled')))) + AND json_extract(terminal.event_header, '$.fact.outcome.kind') IN ('failed','cancelled')))) OR (e.kind = 'model_requested' AND NOT EXISTS(SELECT 1 FROM runtime_events t WHERE t.invocation_id = e.invocation_id AND t.operation_id = e.operation_id AND t.kind IN ('model_completed','model_interrupted'))) OR (e.kind = 'model_requested' AND EXISTS(SELECT 1 FROM accepted_calls p @@ -257,18 +257,18 @@ pub(crate) async fn model_source_unchanged( let changed: bool = sqlx::query_scalar(sqlx::AssertSqlSafe(format!( "SELECT EXISTS(SELECT 1 FROM runtime_events e LEFT JOIN context_execution_steps s ON s.invocation_id=e.invocation_id - AND s.operation_id=CAST(json_extract(e.event_json,'$.fact.step_id') AS TEXT) + AND s.operation_id=CAST(json_extract(e.event_header,'$.fact.step_id') AS TEXT) WHERE e.sequence > ?2 - AND (e.event_json IS NOT NULL AND e.event_session=?3) AND {filter} + AND (e.event_header IS NOT NULL AND e.event_session=?3) AND {filter} AND NOT (e.invocation_id=?1 AND e.kind IN ('tool_dispatched','tool_settled','tool_rejected','tool_notified') AND EXISTS(SELECT 1 FROM context_independent_tools i WHERE i.invocation_id=e.invocation_id AND i.operation_id=CASE WHEN e.kind='tool_notified' - THEN json_extract(e.event_json,'$.fact.operation_id') ELSE e.operation_id END)) + THEN json_extract(e.event_header,'$.fact.operation_id') ELSE e.operation_id END)) AND NOT (e.invocation_id=?1 AND e.kind IN ('model_requested','model_observed','model_interrupted') AND EXISTS(SELECT 1 FROM runtime_events r WHERE r.sequence=s.requested_at AND r.sequence>?2 AND s.terminal_kind='model_interrupted' AND s.terminal_status='retryable_failure' AND (s.item_acceptance!=1 OR s.accepted_at IS NULL) AND s.barrier_at IS NULL - AND json_remove(json_extract(r.event_json,'$.fact'),'$.step_id')=json_remove(json(?5),'$.step_id'))) + AND json_remove(json_extract(r.event_header,'$.fact'),'$.step_id')=json_remove(json(?5),'$.step_id'))) AND NOT (e.invocation_id=?1 AND e.kind IN ('model_requested','model_observed','model_completed','model_interrupted') AND COALESCE(s.purpose='summary' AND s.tool_observed_at IS NULL,0)))" ))) diff --git a/platform/event-log/src/context/selection.rs b/platform/event-log/src/context/selection.rs index 961dd56a76..27946c4c70 100644 --- a/platform/event-log/src/context/selection.rs +++ b/platform/event-log/src/context/selection.rs @@ -138,7 +138,7 @@ impl Selection { let filter = Self::predicate(archive, lineage); format!( "(({target}.inherited = 1 AND {archive}.sequence = {target}.archive_sequence) - OR (json_extract({archive}.event_json,'$.invocation.session_id') = {target}.owner_session_id + OR (json_extract({archive}.event_header,'$.invocation.session_id') = {target}.owner_session_id AND {archive}.sequence < {before} AND {filter}))" ) } @@ -157,7 +157,7 @@ impl Selection { sqlx::query_scalar(sqlx::AssertSqlSafe(format!( "SELECT MAX( (SELECT COALESCE(MAX(e.sequence),0) FROM runtime_events e WHERE e.sequence <= ?1 - AND (e.event_json IS NOT NULL AND e.event_session=?2) AND {filter}), + AND (e.event_header IS NOT NULL AND e.event_session=?2) AND {filter}), (SELECT COALESCE(MAX(h.sequence),0) FROM session_history_members h JOIN runtime_events e ON e.sequence=h.sequence WHERE h.session_id=?2 AND h.sequence <= ?1 AND {filter}))" diff --git a/platform/event-log/src/context/semantic/captured.rs b/platform/event-log/src/context/semantic/captured.rs index 6408178ef5..4f6ef31f22 100644 --- a/platform/event-log/src/context/semantic/captured.rs +++ b/platform/event-log/src/context/semantic/captured.rs @@ -19,9 +19,11 @@ //! Frozen semantic sources retain the original execution guards, never a new admission. mod artifact_identity; +mod canonical; mod locator; use super::{SemanticCursor, SemanticSelection, available_at, invalid, rows_query}; use crate::{EventLog, StoreError}; +use canonical::CanonicalRead; mod directory; pub use directory::{CapturedDirectoryRow, ChildPage, DirectoryPage}; use maka_agent_facts::history::RowPosition; @@ -631,8 +633,16 @@ impl CapturedSource { let mut tx = db.begin().await?; let result = crate::connection::worker_query(&mut tx, move |worker| { Box::pin(async move { + let canonical = CanonicalRead::load(worker, &backing).await?; with_raw(worker, |raw| { - read_chunk(raw, &backing, position, buffer, None) + read_chunk( + raw, + &backing, + position, + buffer, + None, + canonical.as_ref(), + ) }) }) }) @@ -659,6 +669,7 @@ impl CapturedSource { let mut tx = db.begin().await?; let result = crate::connection::worker_query(&mut tx, move |worker| { Box::pin(async move { + let canonical = CanonicalRead::load(worker, &backing).await?; with_raw(worker, |raw| match backing { Backing::Row { rowid, bytes } => with_blob( raw, @@ -669,18 +680,12 @@ impl CapturedSource { bytes, consume, ), - Backing::Canonical { key, offset, bytes } => { - check_canonical(raw, key, offset, bytes)?; - with_blob( - raw, - "event_log", - "event_json", - key.source_sequence as i64, - offset, - bytes, - consume, - ) - } + Backing::Canonical { bytes, .. } => canonical + .as_ref() + .ok_or_else(|| { + invalid("canonical source metadata was not resolved") + })? + .with_range(raw, 0, bytes, consume), Backing::Pieces { key, .. } => { check_pieces(raw, key)?; let mut reader = PieceReader { @@ -767,19 +772,19 @@ pub(super) fn with_fragment_reader( consume: impl FnOnce(&mut dyn Read) -> Result, ) -> Result { match source { - super::source::SemanticPayloadSource::CanonicalToolProjection(span) => with_blob( - raw, - "event_log", - "event_json", - key.source_sequence as i64, - span.offset() as u64, - span.len() as u64, - consume, - ), + super::source::SemanticPayloadSource::CanonicalToolProjection(span) => { + crate::canonical::with_span( + raw, + key.source_sequence as i64, + span.offset() as u64, + span.len() as u64, + consume, + ) + } super::source::SemanticPayloadSource::CanonicalModelPart(span) => with_blob( raw, "event_log", - "event_json", + "event_header", key.source_sequence as i64, span.offset() as u64, span.len() as u64, @@ -815,15 +820,25 @@ fn fragment_backing(raw: &RawConnection, key: RowPosition) -> Result= 0 - && bytes > 0 - && offset - .checked_add(bytes) - .is_some_and(|end| end <= i64::from(i32::MAX)) => - { + Some((kind, offset, bytes)) => { + let source = super::source::SemanticPayloadSource::from_metadata( + key.source_sequence as i64, + key.item_index as i64, + &kind, + offset, + bytes, + )?; + let (offset, bytes) = match source { + super::source::SemanticPayloadSource::CanonicalToolProjection(span) => { + (span.offset(), span.len()) + } + super::source::SemanticPayloadSource::CanonicalModelPart(span) => { + (span.offset(), span.len()) + } + super::source::SemanticPayloadSource::Pieces => { + return Err(invalid("invalid captured pieces")); + } + }; Ok(Backing::Canonical { key, offset: offset as u64, @@ -833,20 +848,78 @@ fn fragment_backing(raw: &RawConnection, key: RowPosition) -> Result Err(invalid("captured semantic fragment is missing or invalid")), } } -fn check_canonical( +fn with_canonical_range( raw: &RawConnection, key: RowPosition, offset: u64, bytes: u64, -) -> Result<(), StoreError> { - match fragment_backing(raw, key)? { - Backing::Canonical { - offset: actual, - bytes: length, - .. - } if actual == offset && length == bytes => Ok(()), - _ => Err(invalid("captured canonical source changed")), - } + start: u64, + count: u64, + consume: impl FnOnce(&mut dyn Read) -> Result, +) -> Result { + let query = format!("SELECT f.storage_kind,f.byte_offset,f.byte_len,{} FROM context_semantic_fragments f + LEFT JOIN event_log e ON e.sequence=f.source_sequence AND f.storage_kind='canonical_tool_projection' {} + WHERE f.source_sequence=? AND f.source_index=?", crate::canonical::SPAN_COLUMNS, crate::canonical::JOINS); + raw.query_row( + &query, + params![key.source_sequence as i64, key.item_index as i64], + |row| { + Ok((|| { + let kind: String = row.get("storage_kind")?; + let source = super::source::SemanticPayloadSource::from_metadata( + key.source_sequence as i64, + key.item_index as i64, + &kind, + row.get("byte_offset")?, + row.get("byte_len")?, + )?; + let (actual, length) = match source { + super::source::SemanticPayloadSource::CanonicalToolProjection(span) => { + (span.offset(), span.len()) + } + super::source::SemanticPayloadSource::CanonicalModelPart(span) => { + (span.offset(), span.len()) + } + super::source::SemanticPayloadSource::Pieces => { + return Err(invalid("captured canonical source changed")); + } + }; + if actual as u64 != offset || length as u64 != bytes { + return Err(invalid("captured canonical source changed")); + } + if start.checked_add(count).is_none_or(|end| end > bytes) { + return Err(invalid("canonical range exceeds captured span")); + } + let position = offset + .checked_add(start) + .ok_or_else(|| invalid("canonical span offset overflows"))?; + match source { + super::source::SemanticPayloadSource::CanonicalModelPart(_) => with_blob( + raw, + "event_log", + "event_header", + key.source_sequence as i64, + position, + count, + consume, + ), + super::source::SemanticPayloadSource::CanonicalToolProjection(_) => { + crate::canonical::with_span_on_row( + raw, + row, + key.source_sequence as i64, + position, + count, + consume, + ) + } + super::source::SemanticPayloadSource::Pieces => { + unreachable!("validated canonical source") + } + } + })()) + }, + )? } fn check_pieces(raw: &RawConnection, key: RowPosition) -> Result<(), StoreError> { let valid:bool=raw.query_row("SELECT EXISTS(SELECT 1 FROM context_semantic_fragments WHERE source_sequence=? AND source_index=? AND storage_kind='pieces')",params![key.source_sequence as i64,key.item_index as i64],|r|r.get(0))?; @@ -953,6 +1026,7 @@ fn read_chunk( position: SourcePosition, mut buffer: B, limit: Option, + canonical: Option<&CanonicalRead>, ) -> Result, StoreError> { let output = buffer.as_mut(); let output_len = output.len().min(limit.unwrap_or(output.len())); @@ -988,16 +1062,18 @@ fn read_chunk( }), ) } - Backing::Canonical { key, offset, bytes } => { - check_canonical(raw, key, offset, bytes)?; - let (count, end) = read_blob( - raw, - ("event_log", "event_json", key.source_sequence as i64), - offset, - bytes, - position.offset, - output, - )?; + Backing::Canonical { key, bytes, .. } => { + if position.offset > bytes { + return Err(invalid("captured source position exceeds its span")); + } + let count = (bytes - position.offset).min(output.len() as u64) as usize; + canonical + .ok_or_else(|| invalid("canonical source metadata was not resolved"))? + .with_range(raw, position.offset, count as u64, |reader| { + reader.read_exact(&mut output[..count])?; + Ok(()) + })?; + let end = position.offset + count as u64 == bytes; ( count, Some(key), @@ -1236,6 +1312,7 @@ impl ByteCursor for CapturedByteCursor { let mut tx = db.begin().await?; let result = crate::connection::worker_query(&mut tx, move |worker| { Box::pin(async move { + let canonical = CanonicalRead::load(worker, &backing).await?; with_raw(worker, |raw| { read_chunk( raw, @@ -1243,6 +1320,7 @@ impl ByteCursor for CapturedByteCursor { cursor, buffer, Some(usize::try_from(remaining).unwrap_or(usize::MAX)), + canonical.as_ref(), ) }) }) diff --git a/platform/event-log/src/context/semantic/captured/artifact_identity.rs b/platform/event-log/src/context/semantic/captured/artifact_identity.rs index 689798f12c..49bb73dd14 100644 --- a/platform/event-log/src/context/semantic/captured/artifact_identity.rs +++ b/platform/event-log/src/context/semantic/captured/artifact_identity.rs @@ -55,18 +55,7 @@ fn with_range( consume, ), Backing::Canonical { key, offset, bytes } => { - check_canonical(raw, key, offset, bytes)?; - with_blob( - raw, - "event_log", - "event_json", - key.source_sequence as i64, - offset - .checked_add(range.start) - .ok_or_else(|| invalid("identity range overflow"))?, - length, - consume, - ) + with_canonical_range(raw, key, offset, bytes, range.start, length, consume) } Backing::Pieces { key, .. } => { check_pieces(raw, key)?; diff --git a/platform/event-log/src/context/semantic/captured/canonical.rs b/platform/event-log/src/context/semantic/captured/canonical.rs new file mode 100644 index 0000000000..8197e67fac --- /dev/null +++ b/platform/event-log/src/context/semantic/captured/canonical.rs @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Callback-local source metadata uses the original worker's statement cache. +use super::{Backing, RawConnection, RowPosition, StoreError, invalid, with_blob}; +use crate::canonical::SpanMetadata; +use sqlx::{Row, sqlite::SqliteWorkerConnection}; +use std::io::Read; + +pub(super) struct CanonicalRead { + key: RowPosition, + offset: u64, + bytes: u64, + material: Option, +} +impl CanonicalRead { + pub(super) async fn load( + worker: &mut SqliteWorkerConnection<'_>, + backing: &Backing, + ) -> Result, StoreError> { + let Backing::Canonical { key, offset, bytes } = *backing else { + return Ok(None); + }; + let query = format!("SELECT f.storage_kind,f.byte_offset,f.byte_len,{} FROM context_semantic_fragments f + LEFT JOIN event_log e ON e.sequence=f.source_sequence AND f.storage_kind='canonical_tool_projection' {} + WHERE f.source_sequence=? AND f.source_index=?", crate::canonical::SPAN_COLUMNS, crate::canonical::JOINS); + let row = sqlx::query(sqlx::AssertSqlSafe(query)) + .bind(key.source_sequence as i64) + .bind(key.item_index as i64) + .fetch_one(&mut *worker) + .await?; + let source = super::super::source::SemanticPayloadSource::from_metadata( + key.source_sequence as i64, + key.item_index as i64, + row.try_get("storage_kind")?, + row.try_get("byte_offset")?, + row.try_get("byte_len")?, + )?; + let (actual, length, model) = match source { + super::super::source::SemanticPayloadSource::CanonicalToolProjection(span) => { + (span.offset(), span.len(), false) + } + super::super::source::SemanticPayloadSource::CanonicalModelPart(span) => { + (span.offset(), span.len(), true) + } + super::super::source::SemanticPayloadSource::Pieces => { + return Err(invalid("captured canonical source changed")); + } + }; + if actual as u64 != offset || length as u64 != bytes { + return Err(invalid("captured canonical source changed")); + } + Ok(Some(Self { + key, + offset, + bytes, + material: if model { + None + } else { + Some(SpanMetadata::from_row(&row)?) + }, + })) + } + pub(super) fn with_range( + &self, + raw: &RawConnection, + start: u64, + count: u64, + consume: impl FnOnce(&mut dyn Read) -> Result, + ) -> Result { + if start.checked_add(count).is_none_or(|end| end > self.bytes) { + return Err(invalid("canonical range exceeds captured span")); + } + let position = self + .offset + .checked_add(start) + .ok_or_else(|| invalid("canonical span offset overflows"))?; + match self.material { + Some(material) => crate::canonical::with_span_metadata( + raw, + self.key.source_sequence as i64, + material, + position, + count, + consume, + ), + None => with_blob( + raw, + "event_log", + "event_header", + self.key.source_sequence as i64, + position, + count, + consume, + ), + } + } +} diff --git a/platform/event-log/src/context/semantic/source.rs b/platform/event-log/src/context/semantic/source.rs index 1e759f8f3d..77287533b7 100644 --- a/platform/event-log/src/context/semantic/source.rs +++ b/platform/event-log/src/context/semantic/source.rs @@ -139,14 +139,14 @@ pub(super) fn locate_model_parts( .ok_or_else(|| invalid("canonical model part is outside its event"))?; i32::try_from(end) .map_err(|_| invalid("canonical model part exceeds SQLite Blob range"))?; - let span = CanonicalModelPartSpan(CanonicalToolProjectionSpan::from_metadata( + let span = CanonicalModelPartSpan::from_metadata( i64::try_from(source_sequence) .map_err(|_| invalid("canonical model sequence exceeds SQLite range"))?, i64::try_from(offset) .map_err(|_| invalid("canonical model offset exceeds SQLite range"))?, i64::try_from(raw.len()) .map_err(|_| invalid("canonical model length exceeds SQLite range"))?, - )?); + )?; if spans.insert(ordinal, span).is_some() { return Err(invalid("duplicate canonical model ordinal")); } @@ -154,6 +154,17 @@ pub(super) fn locate_model_parts( Ok(spans) } impl CanonicalModelPartSpan { + fn from_metadata(sequence: i64, offset: i64, len: i64) -> Result { + let span = CanonicalToolProjectionSpan::from_metadata(sequence, offset, len)?; + if offset + .checked_add(len) + .is_none_or(|end| end > i64::from(i32::MAX)) + { + return Err(invalid("canonical model part exceeds SQLite Blob range")); + } + Ok(Self(span)) + } + pub(super) fn offset(self) -> i64 { self.0.offset() } @@ -164,7 +175,9 @@ impl CanonicalModelPartSpan { self, worker: &mut SqliteWorkerConnection<'_>, ) -> Result { - self.0.read_value::(worker).map(Payload::Part) + self.0 + .read_inline_value::(worker) + .map(Payload::Part) } } @@ -190,12 +203,10 @@ impl CanonicalToolProjectionSpan { let offset = (projection.as_ptr() as usize) .checked_sub(json.as_ptr() as usize) .ok_or_else(|| invalid("canonical projection does not borrow its event"))?; - let end = offset + offset .checked_add(projection.len()) .filter(|end| *end <= json.len()) .ok_or_else(|| invalid("canonical projection is outside its event"))?; - i32::try_from(end) - .map_err(|_| invalid("canonical projection exceeds SQLite Blob range"))?; Self::from_metadata( i64::try_from(source_sequence) .map_err(|_| invalid("canonical source sequence exceeds SQLite range"))?, @@ -207,13 +218,7 @@ impl CanonicalToolProjectionSpan { } fn from_metadata(sequence: i64, offset: i64, len: i64) -> Result { - if sequence <= 0 - || offset < 0 - || len <= 0 - || offset - .checked_add(len) - .is_none_or(|end| end > i64::from(i32::MAX)) - { + if sequence <= 0 || offset < 0 || len <= 0 || offset.checked_add(len).is_none() { return Err(invalid("invalid canonical projection span")); } Ok(Self { @@ -234,9 +239,13 @@ impl CanonicalToolProjectionSpan { json: &str, projection: &DurableToolProjection, ) -> Result { - let start = self.byte_offset as usize; + let start = usize::try_from(self.byte_offset) + .map_err(|_| invalid("canonical projection offset exceeds address range"))?; let end = start - .checked_add(self.byte_len as usize) + .checked_add( + usize::try_from(self.byte_len) + .map_err(|_| invalid("canonical projection length exceeds address range"))?, + ) .ok_or_else(|| invalid("canonical projection span overflows"))?; let expected = json .as_bytes() @@ -249,10 +258,23 @@ impl CanonicalToolProjectionSpan { self, worker: &mut SqliteWorkerConnection<'_>, ) -> Result { - self.read_value::(worker) - .map(Payload::Tool) + super::captured::with_raw(worker, |raw| { + crate::canonical::with_span( + raw, + self.source_sequence, + self.byte_offset, + self.byte_len, + |reader| { + let reader = BufReader::with_capacity(READ_BUFFER_BYTES, reader); + Ok(Payload::Tool(serde_json::from_reader::< + _, + DurableToolProjection, + >(reader)?)) + }, + ) + }) } - fn read_value( + fn read_inline_value( self, worker: &mut SqliteWorkerConnection<'_>, ) -> Result { @@ -263,7 +285,7 @@ impl CanonicalToolProjectionSpan { let mut blob = borrowed.blob_open( "main", "event_log", - "event_json", + "event_header", self.source_sequence, true, )?; @@ -337,9 +359,9 @@ impl SemanticPayloadSource { )) } ("canonical_model_part", Some(offset), Some(len)) if index >= 0 => { - Ok(Self::CanonicalModelPart(CanonicalModelPartSpan( - CanonicalToolProjectionSpan::from_metadata(sequence, offset, len)?, - ))) + Ok(Self::CanonicalModelPart( + CanonicalModelPartSpan::from_metadata(sequence, offset, len)?, + )) } _ => Err(invalid("invalid semantic payload source")), } @@ -365,3 +387,34 @@ impl std::error::Error for ReadAndCloseError { Some(&self.read) } } + +#[cfg(test)] +mod logical_span_tests { + use super::*; + #[test] + fn tools_use_logical_positions_while_model_parts_keep_physical_blob_bounds() { + let offset = i64::from(i32::MAX) + 1; + assert!( + SemanticPayloadSource::from_metadata( + 1, + 0, + "canonical_tool_projection", + Some(offset), + Some(32) + ) + .is_ok() + ); + assert!( + SemanticPayloadSource::from_metadata( + 1, + 0, + "canonical_model_part", + Some(offset), + Some(32) + ) + .is_err() + ); + assert!(CanonicalToolProjectionSpan::from_metadata(1, i64::MAX - 32, 32).is_ok()); + assert!(CanonicalToolProjectionSpan::from_metadata(1, i64::MAX - 31, 32).is_err()); + } +} diff --git a/platform/event-log/src/context/tail.rs b/platform/event-log/src/context/tail.rs index 92ad02f3a5..ad09035ab5 100644 --- a/platform/event-log/src/context/tail.rs +++ b/platform/event-log/src/context/tail.rs @@ -39,31 +39,37 @@ pub(super) async fn read( let archive_filter = Selection::archive_predicate("t", "a", "?4", "?5"); let items = "t.kind IN ('model_completed','model_interrupted') AND EXISTS(SELECT 1 FROM runtime_events r WHERE r.invocation_id=t.invocation_id AND r.operation_id=t.operation_id AND r.kind='model_requested' - AND json_extract(r.event_json,'$.fact.item_acceptance')=1)"; + AND json_extract(r.event_header,'$.fact.item_acceptance')=1)"; macro_rules! selected { ($prefix:literal, $suffix:literal) => { concat!($prefix, " FROM session_history_events t LEFT JOIN runtime_events a ON a.kind='tool_result_archived' - AND CAST(json_extract(a.event_json,'$.fact.placeholder.identity.runtime_event_id') AS TEXT)=t.event_id AND {archive_filter} + AND CAST(json_extract(a.event_header,'$.fact.placeholder.identity.runtime_event_id') AS TEXT)=t.event_id AND {archive_filter} + {material_join} WHERE t.sequence > ?1 AND t.sequence <= ?2 AND t.owner_session_id=?3 AND {filter} AND t.kind NOT IN ('model_observed','context_checkpoint_recorded','tool_result_archived') AND NOT EXISTS(SELECT 1 FROM runtime_events o WHERE o.invocation_id=t.invocation_id - AND o.kind='invocation_opened' AND json_extract(o.event_json,'$.fact.input.kind')='context_compact') + AND o.kind='invocation_opened' AND json_extract(o.event_header,'$.fact.input.kind')='context_compact') AND NOT EXISTS(SELECT 1 FROM runtime_events r WHERE r.invocation_id=t.invocation_id - AND r.kind='model_requested' AND r.operation_id=t.operation_id AND json_extract(r.event_json,'$.fact.purpose')='summary')", + AND r.kind='model_requested' AND r.operation_id=t.operation_id AND json_extract(r.event_header,'$.fact.purpose')='summary')", $suffix) }; } let (count, bytes): (i64,i64) = sqlx::query_as(sqlx::AssertSqlSafe(format!(selected!( - "SELECT COUNT(*),COALESCE(SUM(CASE WHEN a.sequence IS NULL AND NOT ({items}) THEN length(CAST(t.event_json AS BLOB)) ELSE 0 END),0)", "" - ), filter=filter, archive_filter=archive_filter, items=items))).bind(after as i64).bind(through as i64).bind(session).bind(before as i64).bind(&selection.lineage).fetch_one(&mut *connection).await?; + "SELECT COUNT(*),COALESCE(SUM(CASE WHEN a.sequence IS NULL AND NOT ({items}) THEN t.canonical_bytes ELSE 0 END),0)", "" + ), filter=filter, archive_filter=archive_filter, items=items, material_join=""))).bind(after as i64).bind(through as i64).bind(session).bind(before as i64).bind(&selection.lineage).fetch_one(&mut *connection).await?; if sequence_number(count)? > max_events as u64 || sequence_number(bytes)? > max_bytes as u64 { return Err(StoreError::PrefixTooLarge); } // Archived base bodies are NULL here, never retained with the returned tail. + let material_join = format!( + "LEFT JOIN event_log e ON e.sequence=t.sequence AND a.sequence IS NULL AND NOT ({items}) {}", + crate::canonical::JOINS + ); let rows = sqlx::query(sqlx::AssertSqlSafe(format!(selected!( - "SELECT t.sequence,t.event_id,json_extract(a.event_json,'$.invocation.session_id') AS archive_writer, + "SELECT t.sequence,t.event_id,json_extract(a.event_header,'$.invocation.session_id') AS archive_writer, a.sequence + 1 AS archives_before, - CASE WHEN a.sequence IS NULL AND NOT ({items}) THEN t.event_json END AS canonical, + e.event_header,{columns}, CASE WHEN {items} THEN t.operation_id END AS model_step, - CASE WHEN {items} THEN json_extract(t.event_json,'$.invocation') END AS model_invocation", " ORDER BY t.sequence" - ), filter=filter, archive_filter=archive_filter, items=items))).bind(after as i64).bind(through as i64).bind(session).bind(before as i64).bind(&selection.lineage).fetch_all(&mut *connection).await?; + CASE WHEN {items} THEN json_extract(t.event_header,'$.invocation') END AS model_invocation", " ORDER BY t.sequence" + ), filter=filter, archive_filter=archive_filter, items=items, columns=crate::canonical::COLUMNS, + material_join=material_join))).bind(after as i64).bind(through as i64).bind(session).bind(before as i64).bind(&selection.lineage).fetch_all(&mut *connection).await?; let mut used = bytes as usize; let mut result = Vec::with_capacity(count as usize); for row in rows { @@ -84,8 +90,8 @@ pub(super) async fn read( return Err(StoreError::PrefixTooLarge); } result.push(ContextEvent::ModelItems(Box::new(projected))); - } else if let Some(json) = row.try_get::, _>("canonical")? { - let event = serde_json::from_str(json)?; + } else if row.try_get::, _>("event_header")?.is_some() { + let event = serde_json::from_str(&crate::canonical::json(&row)?)?; result.push(ContextEvent::Canonical(Box::new(StoredEvent { sequence: sequence_number(row.try_get("sequence")?)?, event, diff --git a/platform/event-log/src/context/usage.rs b/platform/event-log/src/context/usage.rs index 83b3fa8349..500aabc9ba 100644 --- a/platform/event-log/src/context/usage.rs +++ b/platform/event-log/src/context/usage.rs @@ -29,7 +29,7 @@ use maka_agent_facts::model::ToolInput; use maka_agent_facts::model::{ModelPart, recorded::ModelObservation}; use maka_agent_facts::tool_output::DurableToolProjection; use maka_agent_facts::tool_output::ProjectionPart; -use sqlx::{Connection, Row, SqliteConnection}; +use sqlx::{Connection, SqliteConnection}; #[derive(Debug, PartialEq, Eq)] pub struct ContextUsage { @@ -85,7 +85,7 @@ async fn last_known( // a meaningful reading of the active context, even as a last-known value. let reset: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM runtime_events WHERE sequence>?1 - AND json_extract(event_json,'$.invocation.session_id')=?2 + AND json_extract(event_header,'$.invocation.session_id')=?2 AND kind='context_checkpoint_recorded')", ) .bind(basis.sequence as i64) @@ -118,18 +118,18 @@ async fn additions( "SELECT EXISTS(SELECT 1 FROM runtime_events c JOIN runtime_events r ON r.invocation_id=c.invocation_id AND r.operation_id=c.operation_id AND r.kind='model_requested' JOIN runtime_events n ON n.invocation_id=c.invocation_id - WHERE c.sequence=?1 AND n.sequence>json_extract(r.event_json,'$.fact.source_high_water') AND n.sequencejson_extract(r.event_header,'$.fact.source_high_water') AND n.sequence?1 AND json_extract(i.event_json,'$.invocation.session_id')=?2 - AND json_extract(o.event_json,'$.fact.event.kind')='part')", + AND json_extract(o.event_header,'$.fact.step_id')=i.operation_id AND o.sequence?1 AND json_extract(i.event_header,'$.invocation.session_id')=?2 + AND json_extract(o.event_header,'$.fact.event.kind')='part')", ).bind(basis.sequence as i64).bind(session).fetch_one(&mut *connection).await?; if unobserved_progress { return Ok(None); @@ -143,12 +143,12 @@ async fn additions( AND base.operation_id=c.operation_id AND base.kind='model_requested' LEFT JOIN model_request_compositions bc ON bc.event_id=base.event_id JOIN runtime_events r ON r.sequence>c.sequence AND r.kind='model_requested' - AND json_extract(r.event_json,'$.invocation.session_id')=?2 - AND json_extract(r.event_json,'$.fact.purpose')='main' + AND json_extract(r.event_header,'$.invocation.session_id')=?2 + AND json_extract(r.event_header,'$.fact.purpose')='main' LEFT JOIN model_request_compositions rc ON rc.event_id=r.event_id WHERE c.sequence=?1 AND (rc.digest IS NULL OR bc.digest IS NULL - OR rc.digest!=bc.digest OR json_extract(r.event_json,'$.fact.route_identity') - !=json_extract(base.event_json,'$.fact.route_identity')))", + OR rc.digest!=bc.digest OR json_extract(r.event_header,'$.fact.route_identity') + !=json_extract(base.event_header,'$.fact.route_identity')))", ) .bind(basis.sequence as i64) .bind(session) @@ -161,31 +161,37 @@ async fn additions( // Legacy interrupted fragments are discarded; marked retained progress // was handled above. Completed Main output is already in newest usage. const FILTER: &str = " - e.sequence > ?1 AND (e.event_json IS NOT NULL AND e.event_session=?2) + e.sequence > ?1 AND (e.event_header IS NOT NULL AND e.event_session=?2) AND e.kind IN ('invocation_opened','message_steered','model_requested', 'model_observed','executor_completed','tool_settled','tool_rejected','tool_notified', 'context_checkpoint_recorded','tool_result_archived') AND (e.kind NOT IN ('tool_settled','tool_rejected') OR - json_extract(e.event_json,'$.fact.call.origin.kind')='provider' OR + json_extract(e.event_header,'$.fact.call.origin.kind')='provider' OR EXISTS(SELECT 1 FROM runtime_events d WHERE d.invocation_id=e.invocation_id AND d.operation_id=e.operation_id AND d.kind='tool_dispatched' - AND json_extract(d.event_json,'$.fact.call.origin.kind')='provider')) + AND json_extract(d.event_header,'$.fact.call.origin.kind')='provider')) AND (e.kind!='model_observed' OR EXISTS( SELECT 1 FROM runtime_events r WHERE r.invocation_id=e.invocation_id AND r.kind='model_requested' - AND r.operation_id=json_extract(e.event_json,'$.fact.step_id') - AND json_extract(r.event_json,'$.fact.purpose')='main' + AND r.operation_id=json_extract(e.event_header,'$.fact.step_id') + AND json_extract(r.event_header,'$.fact.purpose')='main' AND NOT EXISTS(SELECT 1 FROM runtime_events c WHERE c.invocation_id=r.invocation_id AND c.operation_id=r.operation_id AND c.kind IN ('model_completed','model_interrupted'))))"; let (count, bytes): (i64, i64) = sqlx::query_as(sqlx::AssertSqlSafe(format!( - "SELECT COUNT(*),COALESCE(SUM(length(CAST(e.event_json AS BLOB))),0) FROM runtime_events e WHERE {FILTER}" - ))).bind(basis.sequence as i64).bind(session).fetch_one(&mut *connection).await?; + "SELECT COUNT(*),COALESCE(SUM(e.canonical_bytes),0) FROM runtime_events e WHERE {FILTER}" + ))) + .bind(basis.sequence as i64) + .bind(session) + .fetch_one(&mut *connection) + .await?; if count > 4096 || bytes > 1024 * 1024 { return Ok(None); } let rows = sqlx::query(sqlx::AssertSqlSafe(format!( - "SELECT e.event_json FROM runtime_events e WHERE {FILTER} ORDER BY e.sequence" + "SELECT e.event_header,{} FROM runtime_events e {} WHERE {FILTER} ORDER BY e.sequence", + crate::canonical::COLUMNS, + crate::canonical::JOINS ))) .bind(basis.sequence as i64) .bind(session) @@ -193,7 +199,7 @@ async fn additions( .await?; let mut units = 0_u64; for row in rows { - let event: RuntimeEvent = serde_json::from_str(row.try_get("event_json")?)?; + let event: RuntimeEvent = serde_json::from_str(&crate::canonical::json(&row)?)?; let Some(added) = increment(&event.fact, basis) else { return Ok(None); }; diff --git a/platform/event-log/src/context/validate.rs b/platform/event-log/src/context/validate.rs index ddb8359cb5..53c9c413da 100644 --- a/platform/event-log/src/context/validate.rs +++ b/platform/event-log/src/context/validate.rs @@ -66,7 +66,7 @@ pub(crate) async fn validate_append( } Fact::InvocationEnded { outcome } => { let compact: bool = sqlx::query_scalar( - "SELECT json_extract(event_json, '$.fact.input.kind') = 'context_compact' + "SELECT json_extract(event_header, '$.fact.input.kind') = 'context_compact' FROM runtime_events WHERE invocation_id = ? AND kind = 'invocation_opened'", ) .bind(&event.invocation.invocation_id) diff --git a/platform/event-log/src/continuation.rs b/platform/event-log/src/continuation.rs index 4701d28e45..75f81a8266 100644 --- a/platform/event-log/src/continuation.rs +++ b/platform/event-log/src/continuation.rs @@ -39,13 +39,13 @@ impl EventLog { let session = session.to_owned(); self.connection.run(move |connection| Box::pin(async move { Ok(sqlx::query_scalar( - "SELECT json_extract(opening.event_json, '$.invocation.run_id') + "SELECT json_extract(opening.event_header, '$.invocation.run_id') FROM runtime_events opening JOIN runtime_events terminal ON terminal.invocation_id = opening.invocation_id AND terminal.kind = 'invocation_ended' WHERE opening.kind = 'invocation_opened' - AND json_extract(opening.event_json, '$.invocation.session_id') = ? - AND json_extract(opening.event_json, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') - AND json_extract(terminal.event_json, '$.fact.outcome.kind') IN ('failed', 'cancelled') + AND json_extract(opening.event_header, '$.invocation.session_id') = ? + AND json_extract(opening.event_header, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') + AND json_extract(terminal.event_header, '$.fact.outcome.kind') IN ('failed', 'cancelled') ORDER BY opening.sequence DESC LIMIT 1" ).bind(session).fetch_optional(connection).await?) })).await @@ -67,12 +67,12 @@ impl EventLog { self.connection.run(move |connection| Box::pin(async move { let mut tx = connection.begin().await?; let rows: Vec<(i64, Option)> = sqlx::query_as( - "SELECT sequence, CASE WHEN length(CAST(event_json AS BLOB)) <= 1048576 THEN event_json END + "SELECT sequence, CASE WHEN length(CAST(event_header AS BLOB)) <= 1048576 THEN event_header END FROM runtime_events WHERE kind='invocation_opened' - AND json_extract(event_json,'$.fact.input.kind') IN ('continuation','handoff') - AND json_extract(event_json,'$.invocation.session_id')=? - AND json_extract(event_json,'$.fact.input.claim.source.invocation.run_id')=? - AND json_extract(event_json,'$.fact.input.claim.source.high_water')=? LIMIT 2" + AND json_extract(event_header,'$.fact.input.kind') IN ('continuation','handoff') + AND json_extract(event_header,'$.invocation.session_id')=? + AND json_extract(event_header,'$.fact.input.claim.source.invocation.run_id')=? + AND json_extract(event_header,'$.fact.input.claim.source.high_water')=? LIMIT 2" ).bind(&source.invocation.session_id).bind(&source.invocation.run_id) .bind(source.high_water as i64).fetch_all(&mut *tx).await?; if rows.len() > 1 { return Err(invalid("ambiguous continuation claim")); } @@ -115,8 +115,8 @@ pub(crate) async fn validate( .ok_or_else(|| invalid("continuation has no observed workspace identity"))?; let reused: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM runtime_events WHERE kind='invocation_opened' - AND json_extract(event_json,'$.invocation.session_id')=?1 - AND ((?4 AND json_extract(event_json,'$.invocation.turn_id')=?2) OR json_extract(event_json,'$.invocation.run_id')=?3))" + AND json_extract(event_header,'$.invocation.session_id')=?1 + AND ((?4 AND json_extract(event_header,'$.invocation.turn_id')=?2) OR json_extract(event_header,'$.invocation.run_id')=?3))" ).bind(&event.invocation.session_id).bind(&event.invocation.turn_id).bind(&event.invocation.run_id) .bind(matches!(input, InvocationInput::Continuation { .. })).fetch_one(&mut *connection).await?; if reused { diff --git a/platform/event-log/src/diagnostics.rs b/platform/event-log/src/diagnostics.rs index 1f56f6cf38..68f5c7db95 100644 --- a/platform/event-log/src/diagnostics.rs +++ b/platform/event-log/src/diagnostics.rs @@ -149,20 +149,20 @@ fn invalid(message: &str) -> StoreError { } const RECORDS: &str = r#"SELECT json_object('sequence',sequence,'event_id',event_id, - 'recorded_at',json_extract(event_json,'$.recorded_at'), + 'recorded_at',json_extract(event_header,'$.recorded_at'), 'detail',json(CASE kind - WHEN 'title_failure' THEN json_object('kind','session_title','invocation',json_extract(event_json,'$.opening'),'cause',json_extract(event_json,'$.cause')) - WHEN 'model_interrupted' THEN json_object('kind','model_step','invocation',json_extract(event_json,'$.invocation'), - 'stepId',json_extract(event_json,'$.fact.step_id'),'status',json_extract(event_json,'$.fact.status'), - 'modelFailure',json_extract(event_json,'$.fact.model_failure')) - ELSE json_object('kind','execution','invocation',json_extract(event_json,'$.invocation'), - 'reason',CASE WHEN json_extract(event_json,'$.fact.outcome.class') IN ('session_busy','continuation_changed','reconciliation_required','invalid_input','cancelled','model_incomplete','event_commit','event_store','model_timeout','model_http','model_adapter','model_provider','context_overflow','tool_execution','runtime_internal','cleanup_unconfirmed','executor','message_preparation_failed','host_interrupted','outcome_unknown') THEN json_extract(event_json,'$.fact.outcome.class') ELSE 'execution_failed' END, - 'modelFailure',json_extract(event_json,'$.fact.outcome.model_failure')) END)) + WHEN 'title_failure' THEN json_object('kind','session_title','invocation',json_extract(event_header,'$.opening'),'cause',json_extract(event_header,'$.cause')) + WHEN 'model_interrupted' THEN json_object('kind','model_step','invocation',json_extract(event_header,'$.invocation'), + 'stepId',json_extract(event_header,'$.fact.step_id'),'status',json_extract(event_header,'$.fact.status'), + 'modelFailure',json_extract(event_header,'$.fact.model_failure')) + ELSE json_object('kind','execution','invocation',json_extract(event_header,'$.invocation'), + 'reason',CASE WHEN json_extract(event_header,'$.fact.outcome.class') IN ('session_busy','continuation_changed','reconciliation_required','invalid_input','cancelled','model_incomplete','event_commit','event_store','model_timeout','model_http','model_adapter','model_provider','context_overflow','tool_execution','runtime_internal','cleanup_unconfirmed','executor','message_preparation_failed','host_interrupted','outcome_unknown') THEN json_extract(event_header,'$.fact.outcome.class') ELSE 'execution_failed' END, + 'modelFailure',json_extract(event_header,'$.fact.outcome.model_failure')) END)) FROM event_log INDEXED BY diagnostic_session_sequence -WHERE (kind='title_failure' OR (kind='invocation_ended' AND json_extract(event_json,'$.fact.outcome.kind')='failed') OR (kind='model_interrupted' AND json_extract(event_json,'$.fact.status')!='cancelled')) AND (CASE WHEN kind='title_failure' THEN json_extract(event_json,'$.opening.session_id') ELSE json_extract(event_json,'$.invocation.session_id') END)=?1 AND sequence<=min(?2,coalesce(?3-1,?2)) +WHERE (kind='title_failure' OR (kind='invocation_ended' AND json_extract(event_header,'$.fact.outcome.kind')='failed') OR (kind='model_interrupted' AND json_extract(event_header,'$.fact.status')!='cancelled')) AND (CASE WHEN kind='title_failure' THEN json_extract(event_header,'$.opening.session_id') ELSE json_extract(event_header,'$.invocation.session_id') END)=?1 AND sequence<=min(?2,coalesce(?3-1,?2)) ORDER BY sequence DESC LIMIT ?4"#; const LATEST: &str = r#"SELECT sequence FROM event_log INDEXED BY diagnostic_session_sequence -WHERE (kind='title_failure' OR (kind='invocation_ended' AND json_extract(event_json,'$.fact.outcome.kind')='failed') OR (kind='model_interrupted' AND json_extract(event_json,'$.fact.status')!='cancelled')) AND (CASE WHEN kind='title_failure' THEN json_extract(event_json,'$.opening.session_id') ELSE json_extract(event_json,'$.invocation.session_id') END)=?1 AND sequence<=?2 +WHERE (kind='title_failure' OR (kind='invocation_ended' AND json_extract(event_header,'$.fact.outcome.kind')='failed') OR (kind='model_interrupted' AND json_extract(event_header,'$.fact.status')!='cancelled')) AND (CASE WHEN kind='title_failure' THEN json_extract(event_header,'$.opening.session_id') ELSE json_extract(event_header,'$.invocation.session_id') END)=?1 AND sequence<=?2 ORDER BY sequence DESC LIMIT 1"#; #[cfg(test)] diff --git a/platform/event-log/src/executor.rs b/platform/event-log/src/executor.rs index cb9d16f353..b738238bbd 100644 --- a/platform/event-log/src/executor.rs +++ b/platform/event-log/src/executor.rs @@ -52,7 +52,7 @@ pub(crate) async fn validate( return Err(invalid("executor already started")); } let opening: String = sqlx::query_scalar( - "SELECT event_json FROM runtime_events WHERE invocation_id = ? AND kind = 'invocation_opened'" + "SELECT event_header FROM runtime_events WHERE invocation_id = ? AND kind = 'invocation_opened'" ).bind(id).fetch_one(&mut *connection).await?; let opening: RuntimeEvent = serde_json::from_str(&opening)?; if !matches!(opening.fact, Fact::InvocationOpened { diff --git a/platform/event-log/src/executor/tests.rs b/platform/event-log/src/executor/tests.rs index e76760145f..8de32452a5 100644 --- a/platform/event-log/src/executor/tests.rs +++ b/platform/event-log/src/executor/tests.rs @@ -55,7 +55,7 @@ async fn check_after_deltas(deltas: usize) -> [usize; 3] { .unwrap(); sqlx::query( "WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x [usize; 3] { let (native, native_steps) = counted(&mut db, &event).await; native.unwrap(); sqlx::query( - "INSERT INTO event_log(event_id,invocation_id,kind,event_json) + "INSERT INTO event_log(event_id,invocation_id,kind,event_header) VALUES('started','invocation','executor_started','{}')", ) .execute(&mut db) @@ -91,7 +91,7 @@ async fn check_after_deltas(deltas: usize) -> [usize; 3] { if reason == "executor completion requires a durable final result") ); sqlx::query( - "INSERT INTO event_log(event_id,invocation_id,kind,event_json) + "INSERT INTO event_log(event_id,invocation_id,kind,event_header) VALUES('completed','invocation','executor_completed','{}')", ) .execute(&mut db) diff --git a/platform/event-log/src/executor_identity_tests.rs b/platform/event-log/src/executor_identity_tests.rs index 04936b2b39..bb9ebea75f 100644 --- a/platform/event-log/src/executor_identity_tests.rs +++ b/platform/event-log/src/executor_identity_tests.rs @@ -103,7 +103,7 @@ fn claim(invocation: &Invocation, message_id: &str) -> EventWrite { }) } fn events(db: &Connection) -> Vec<(i64, Vec)> { - db.prepare("SELECT sequence,CAST(event_json AS BLOB) FROM event_log ORDER BY sequence") + db.prepare("SELECT sequence,CAST(event_header AS BLOB) FROM event_log ORDER BY sequence") .unwrap() .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) .unwrap() diff --git a/platform/event-log/src/handoff.rs b/platform/event-log/src/handoff.rs index e854c66df7..d17419daf9 100644 --- a/platform/event-log/src/handoff.rs +++ b/platform/event-log/src/handoff.rs @@ -45,16 +45,16 @@ impl EventLog { .run(move |tx| { Box::pin(async move { let rows: Vec<(i64, String, String)> = sqlx::query_as( - "SELECT pause.sequence, json_extract(pause.event_json,'$.invocation'), - json_extract(pause.event_json,'$.fact.outcome.pause.intent.host_epoch') + "SELECT pause.sequence, json_extract(pause.event_header,'$.invocation'), + json_extract(pause.event_header,'$.fact.outcome.pause.intent.host_epoch') FROM local_runtime_events pause WHERE pause.sequence > ? AND pause.kind='invocation_ended' AND NOT EXISTS (SELECT 1 FROM session_retirements r - WHERE r.session_id=json_extract(pause.event_json,'$.invocation.session_id')) - AND json_extract(pause.event_json,'$.fact.outcome.kind')='handoff_paused' + WHERE r.session_id=json_extract(pause.event_header,'$.invocation.session_id')) + AND json_extract(pause.event_header,'$.fact.outcome.kind')='handoff_paused' AND NOT EXISTS (SELECT 1 FROM runtime_events successor WHERE successor.kind='invocation_opened' - AND successor.invocation_id=json_extract(pause.event_json, + AND successor.invocation_id=json_extract(pause.event_header, '$.fact.outcome.pause.intent.successor_invocation_id')) ORDER BY pause.sequence LIMIT 64", ) @@ -182,9 +182,9 @@ async fn reservation( ) -> Result, StoreError> { Ok(sqlx::query_scalar( "SELECT pause.invocation_id FROM local_runtime_events pause - WHERE pause.kind='invocation_ended' AND json_extract(pause.event_json,'$.fact.outcome.kind')='handoff_paused' + WHERE pause.kind='invocation_ended' AND json_extract(pause.event_header,'$.fact.outcome.kind')='handoff_paused' AND pause.invocation_id=(SELECT invocation_id FROM local_runtime_events WHERE kind='invocation_opened' - AND json_extract(event_json,'$.invocation.session_id')=? ORDER BY sequence DESC LIMIT 1)" + AND json_extract(event_header,'$.invocation.session_id')=? ORDER BY sequence DESC LIMIT 1)" ).bind(session).fetch_optional(tx).await?) } @@ -196,12 +196,12 @@ pub(crate) async fn reservation_conflict( source: Option<&str>, ) -> Result { Ok(sqlx::query_scalar( - "WITH reservations AS NOT MATERIALIZED (SELECT event_json FROM local_runtime_events - WHERE kind='invocation_ended' AND json_extract(event_json,'$.fact.outcome.kind')='handoff_paused' + "WITH reservations AS NOT MATERIALIZED (SELECT event_header FROM local_runtime_events + WHERE kind='invocation_ended' AND json_extract(event_header,'$.fact.outcome.kind')='handoff_paused' AND (?4 IS NULL OR invocation_id != ?4)) - SELECT EXISTS(SELECT 1 FROM reservations WHERE json_extract(event_json,'$.fact.outcome.pause.intent.successor_run_id')=?1 - UNION ALL SELECT 1 FROM reservations WHERE json_extract(event_json,'$.fact.outcome.pause.intent.successor_invocation_id')=?2 - UNION ALL SELECT 1 FROM reservations WHERE json_extract(event_json,'$.fact.outcome.pause.intent.claim_id')=?3)", + SELECT EXISTS(SELECT 1 FROM reservations WHERE json_extract(event_header,'$.fact.outcome.pause.intent.successor_run_id')=?1 + UNION ALL SELECT 1 FROM reservations WHERE json_extract(event_header,'$.fact.outcome.pause.intent.successor_invocation_id')=?2 + UNION ALL SELECT 1 FROM reservations WHERE json_extract(event_header,'$.fact.outcome.pause.intent.claim_id')=?3)", ).bind(run).bind(invocation).bind(claim).bind(source).fetch_one(tx).await?) } diff --git a/platform/event-log/src/handoff/evidence.rs b/platform/event-log/src/handoff/evidence.rs index b738a4e77d..31e35b1c19 100644 --- a/platform/event-log/src/handoff/evidence.rs +++ b/platform/event-log/src/handoff/evidence.rs @@ -39,7 +39,7 @@ pub(crate) async fn validate( if let InvocationInput::Handoff { claim, pause } = input { let source = &claim.source.invocation; let (opening, terminal): (String, String) = sqlx::query_as( - "SELECT o.event_json, t.event_json FROM runtime_events o JOIN runtime_events t + "SELECT o.event_header, t.event_header FROM runtime_events o JOIN runtime_events t ON t.invocation_id=o.invocation_id AND t.kind='invocation_ended' WHERE o.invocation_id=? AND o.kind='invocation_opened'", ) @@ -72,7 +72,7 @@ pub(crate) async fn validate( return Err(invalid("handoff source is already sealed")); } let opening: String = sqlx::query_scalar( - "SELECT event_json FROM runtime_events WHERE invocation_id=? AND kind='invocation_opened'", + "SELECT event_header FROM runtime_events WHERE invocation_id=? AND kind='invocation_opened'", ) .bind(&event.invocation.invocation_id) .fetch_one(&mut *tx) @@ -153,8 +153,8 @@ pub(crate) async fn validate( crate::context::read::validate_handoff_context(tx, &opening).await?; let occupied: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM runtime_events WHERE kind='invocation_opened' - AND (invocation_id=?1 OR json_extract(event_json,'$.invocation.run_id')=?2 - OR json_extract(event_json,'$.fact.input.claim.id')=?3))", + AND (invocation_id=?1 OR json_extract(event_header,'$.invocation.run_id')=?2 + OR json_extract(event_header,'$.fact.input.claim.id')=?3))", ) .bind(&pause.intent.successor_invocation_id) .bind(&pause.intent.successor_run_id) diff --git a/platform/event-log/src/handoff/owner.rs b/platform/event-log/src/handoff/owner.rs index 761ca46f5d..82b473018b 100644 --- a/platform/event-log/src/handoff/owner.rs +++ b/platform/event-log/src/handoff/owner.rs @@ -30,7 +30,7 @@ use std::collections::HashSet; async fn opening(tx: &mut SqliteConnection, id: &str) -> Result, StoreError> { let json: Option = sqlx::query_scalar( - "SELECT event_json + "SELECT event_header FROM runtime_events WHERE invocation_id=? AND kind='invocation_opened'", ) .bind(id) diff --git a/platform/event-log/src/lib.rs b/platform/event-log/src/lib.rs index 1ce908d790..7c5cc86a5f 100644 --- a/platform/event-log/src/lib.rs +++ b/platform/event-log/src/lib.rs @@ -25,6 +25,7 @@ pub mod applications; pub mod archive; pub mod artifacts; pub mod bundle; +mod canonical; mod canonical_insert; mod catalog_changes; pub mod checkpoints; diff --git a/platform/event-log/src/message_admissions/insert.rs b/platform/event-log/src/message_admissions/insert.rs index eec8e7c00b..66e990db52 100644 --- a/platform/event-log/src/message_admissions/insert.rs +++ b/platform/event-log/src/message_admissions/insert.rs @@ -75,11 +75,11 @@ pub(crate) async fn insert( return Err(StoreError::SessionBusy); } let active: Option = sqlx::query_scalar( - "SELECT json_extract(o.event_json, '$.invocation') FROM ( - SELECT invocation_id, event_json FROM event_log INDEXED BY catalog_message_facts - WHERE invocation_id IS NOT NULL AND event_json IS NOT NULL + "SELECT json_extract(o.event_header, '$.invocation') FROM ( + SELECT invocation_id, event_header FROM event_log INDEXED BY catalog_message_facts + WHERE invocation_id IS NOT NULL AND event_header IS NOT NULL AND kind IN ('invocation_opened', 'model_completed') AND kind = 'invocation_opened' - AND json_extract(event_json, '$.invocation.session_id') = ?1 + AND json_extract(event_header, '$.invocation.session_id') = ?1 ORDER BY sequence DESC LIMIT 1 ) AS o WHERE NOT EXISTS(SELECT 1 FROM imported_invocations i diff --git a/platform/event-log/src/message_identity.rs b/platform/event-log/src/message_identity.rs index 2f8588dfbd..a1a7781b04 100644 --- a/platform/event-log/src/message_identity.rs +++ b/platform/event-log/src/message_identity.rs @@ -159,8 +159,8 @@ pub(crate) async fn validate_derived_tool_id( SELECT 1 FROM session_history_events WHERE owner_session_id=?1 AND invocation_id=?2 AND kind IN ('executor_observed','executor_completed') - AND json_extract(event_json,'$.fact.output.toolCallId')=?3 - AND json_extract(event_json,'$.fact.output.type')='tool_start')", + AND json_extract(event_header,'$.fact.output.toolCallId')=?3 + AND json_extract(event_header,'$.fact.output.type')='tool_start')", ) .bind(session) .bind(invocation) @@ -217,8 +217,8 @@ pub(crate) async fn validate_source_id( AND owner_session_id = ?1 AND maka_tool_use_id(invocation_id, operation_id) = ?2 UNION ALL SELECT 1 FROM session_history_events WHERE kind = 'executor_observed' - AND owner_session_id = ?1 AND json_extract(event_json, '$.fact.output.type') = 'tool_start' - AND maka_tool_use_id(invocation_id, json_extract(event_json, '$.fact.output.toolCallId')) = ?2 + AND owner_session_id = ?1 AND json_extract(event_header, '$.fact.output.type') = 'tool_start' + AND maka_tool_use_id(invocation_id, json_extract(event_header, '$.fact.output.toolCallId')) = ?2 UNION ALL SELECT 1 FROM accepted_calls c JOIN session_history_events e ON e.event_id=c.event_id WHERE e.owner_session_id=?1 AND maka_tool_use_id(c.invocation_id, c.step_id || ':' || json_extract(c.call,'$.id'))=?2)" @@ -256,19 +256,19 @@ async fn executor_pending( // Keep the partial-index kind predicate for executor_tool_observations. // Original JSON equality matches the pending tool ID, not a cast spelling. let count: i64 = sqlx::query_scalar( - "SELECT COUNT(DISTINCT json_extract(start.event_json, '$.fact.output.toolCallId')) + "SELECT COUNT(DISTINCT json_extract(start.event_header, '$.fact.output.toolCallId')) FROM runtime_events AS start WHERE start.invocation_id = ?1 AND start.kind IN ('executor_observed', 'executor_completed') AND start.sequence > ?2 AND start.sequence < ?3 - AND json_extract(start.event_json, '$.fact.output.type') = 'tool_start' + AND json_extract(start.event_header, '$.fact.output.type') = 'tool_start' AND NOT EXISTS(SELECT 1 FROM runtime_events AS result WHERE result.invocation_id = start.invocation_id AND result.kind IN ('executor_observed', 'executor_completed') - AND json_extract(result.event_json, '$.fact.output.toolCallId') = - json_extract(start.event_json, '$.fact.output.toolCallId') + AND json_extract(result.event_header, '$.fact.output.toolCallId') = + json_extract(start.event_header, '$.fact.output.toolCallId') AND result.sequence > start.sequence AND result.sequence < ?3 - AND json_extract(result.event_json, '$.fact.output.type') = 'tool_result')", + AND json_extract(result.event_header, '$.fact.output.type') = 'tool_result')", ) .bind(invocation) .bind(started) diff --git a/platform/event-log/src/message_interrupts.rs b/platform/event-log/src/message_interrupts.rs index 93794a63f1..072c6420b4 100644 --- a/platform/event-log/src/message_interrupts.rs +++ b/platform/event-log/src/message_interrupts.rs @@ -99,9 +99,9 @@ impl EventLog { let active = if let Some(owner) = owner.as_ref().filter(|_| matches_owner) { sqlx::query_scalar::<_, bool>( "SELECT EXISTS(SELECT 1 FROM local_runtime_events o WHERE o.kind = 'invocation_opened' - AND o.invocation_id = ? AND json_extract(o.event_json, '$.invocation') = json(?) + AND o.invocation_id = ? AND json_extract(o.event_header, '$.invocation') = json(?) AND o.sequence = (SELECT MAX(n.sequence) FROM local_runtime_events n - WHERE n.kind = 'invocation_opened' AND json_extract(n.event_json, '$.invocation.session_id') = ?))" + WHERE n.kind = 'invocation_opened' AND json_extract(n.event_header, '$.invocation.session_id') = ?))" ).bind(&owner.invocation_id).bind(serde_json::to_string(owner)?) .bind(&owner.session_id).fetch_one(&mut *tx).await? } else { false }; diff --git a/platform/event-log/src/message_queue/edit.rs b/platform/event-log/src/message_queue/edit.rs index 9ecb298b66..25d1306c1e 100644 --- a/platform/event-log/src/message_queue/edit.rs +++ b/platform/event-log/src/message_queue/edit.rs @@ -71,7 +71,7 @@ pub(crate) async fn apply( } let active: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM local_runtime_events o WHERE o.kind = 'invocation_opened' - AND o.invocation_id = ? AND json_extract(o.event_json, '$.invocation') = json(?) + AND o.invocation_id = ? AND json_extract(o.event_header, '$.invocation') = json(?) AND NOT EXISTS(SELECT 1 FROM runtime_events t WHERE t.invocation_id = o.invocation_id AND t.kind = 'invocation_ended'))" ).bind(&invocation.invocation_id).bind(serde_json::to_string(&invocation)?) .fetch_one(&mut *tx).await?; diff --git a/platform/event-log/src/message_resolution.rs b/platform/event-log/src/message_resolution.rs index 7ac52600a3..4f4961f53a 100644 --- a/platform/event-log/src/message_resolution.rs +++ b/platform/event-log/src/message_resolution.rs @@ -71,7 +71,7 @@ impl EventLog { if !exists { return Ok(resolutions); } for message_id in messages { let owner: Option = sqlx::query_scalar( - "SELECT json_extract(e.event_json, '$.invocation') + "SELECT json_extract(e.event_header, '$.invocation') FROM message_sources s JOIN runtime_events e ON e.event_id = s.event_id WHERE s.session_id = ? AND s.message_id = ?" ).bind(&session).bind(&message_id).fetch_optional(&mut *tx).await?; diff --git a/platform/event-log/src/message_resolution/owner.rs b/platform/event-log/src/message_resolution/owner.rs index d654cbc950..5d9ab729f2 100644 --- a/platform/event-log/src/message_resolution/owner.rs +++ b/platform/event-log/src/message_resolution/owner.rs @@ -99,7 +99,7 @@ async fn read( message: &str, ) -> Result, StoreError> { let json: Option = sqlx::query_scalar( - "SELECT e.event_json + "SELECT e.event_header FROM message_sources s JOIN runtime_events e ON e.event_id = s.event_id WHERE s.session_id = ? AND s.message_id = ?", ) @@ -159,7 +159,7 @@ pub(crate) async fn lineage( return Err(invalid("Message owner continuation cycle")); } let opening: Option = sqlx::query_scalar( - "SELECT event_json + "SELECT event_header FROM runtime_events WHERE invocation_id = ? AND kind = 'invocation_opened'", ) .bind(¤t.invocation_id) @@ -171,13 +171,13 @@ pub(crate) async fn lineage( return Err(invalid("Message owner identity changed")); } let successors: Vec = sqlx::query_scalar( - "SELECT event_json + "SELECT event_header FROM event_log INDEXED BY continuation_source_boundary - WHERE invocation_id IS NOT NULL AND event_json IS NOT NULL + WHERE invocation_id IS NOT NULL AND event_header IS NOT NULL AND kind = 'invocation_opened' - AND json_extract(event_json, '$.fact.input.kind') IN ('continuation', 'handoff') - AND json_extract(event_json, '$.invocation.session_id') = ? - AND json_extract(event_json, '$.fact.input.claim.source.invocation.run_id') = ? + AND json_extract(event_header, '$.fact.input.kind') IN ('continuation', 'handoff') + AND json_extract(event_header, '$.invocation.session_id') = ? + AND json_extract(event_header, '$.fact.input.claim.source.invocation.run_id') = ? LIMIT 2", ) .bind(¤t.session_id) diff --git a/platform/event-log/src/message_sources.rs b/platform/event-log/src/message_sources.rs index 654940b04e..aa1866eede 100644 --- a/platform/event-log/src/message_sources.rs +++ b/platform/event-log/src/message_sources.rs @@ -53,17 +53,17 @@ pub(crate) async fn initialize(connection: &mut SqliteConnection) -> Result<(), let mut tx = connection.begin().await?; sqlx::raw_sql( "INSERT INTO message_sources - SELECT json_extract(event_json, '$.invocation.session_id'), - json_extract(event_json, '$.fact.message.message_id'), event_id + SELECT json_extract(event_header, '$.invocation.session_id'), + json_extract(event_header, '$.fact.message.message_id'), event_id -- Match runtime_events while using the sparse steering identity index. FROM event_log INDEXED BY message_steering_identity - WHERE kind = 'message_steered' AND invocation_id IS NOT NULL AND event_json IS NOT NULL; + WHERE kind = 'message_steered' AND invocation_id IS NOT NULL AND event_header IS NOT NULL; INSERT INTO message_sources - SELECT json_extract(e.event_json, '$.invocation.session_id'), + SELECT json_extract(e.event_header, '$.invocation.session_id'), json_extract(s.value, '$.message_id'), e.event_id - FROM runtime_events e, json_each(e.event_json, '$.fact.input.source_messages') s - WHERE e.kind = 'invocation_opened' AND json_extract(e.event_json, '$.fact.input.kind') = 'message' - AND json_array_length(e.event_json, '$.fact.input.source_messages') > 0;" + FROM runtime_events e, json_each(e.event_header, '$.fact.input.source_messages') s + WHERE e.kind = 'invocation_opened' AND json_extract(e.event_header, '$.fact.input.kind') = 'message' + AND json_array_length(e.event_header, '$.fact.input.source_messages') > 0;" ).execute(&mut *tx).await?; tx.commit().await.map_err(StoreError::CommitUnknown)?; Ok(()) @@ -111,21 +111,21 @@ impl EventLog { "WITH candidates AS ( SELECT sequence FROM ( SELECT e.sequence FROM runtime_events e - WHERE json_extract(e.event_json,'$.invocation.session_id')=?1 AND e.kind='invocation_opened' AND json_extract(e.event_json,'$.fact.input.kind')='message' + WHERE json_extract(e.event_header,'$.invocation.session_id')=?1 AND e.kind='invocation_opened' AND json_extract(e.event_header,'$.fact.input.kind')='message' ORDER BY e.sequence DESC LIMIT 1) UNION ALL SELECT sequence FROM ( SELECT h.sequence FROM session_history_members h JOIN runtime_events e ON e.sequence=h.sequence - WHERE h.session_id=?1 AND e.kind='invocation_opened' AND json_extract(e.event_json,'$.fact.input.kind')='message' + WHERE h.session_id=?1 AND e.kind='invocation_opened' AND json_extract(e.event_header,'$.fact.input.kind')='message' ORDER BY h.sequence DESC LIMIT 1) UNION ALL SELECT sequence FROM ( SELECT r.sequence FROM session_revision_sources r JOIN runtime_events e ON e.sequence=r.sequence - WHERE r.session_id=?1 AND e.kind='invocation_opened' AND json_extract(e.event_json,'$.fact.input.kind')='message' + WHERE r.session_id=?1 AND e.kind='invocation_opened' AND json_extract(e.event_header,'$.fact.input.kind')='message' ORDER BY r.sequence DESC LIMIT 1)) - SELECT e.sequence,e.event_json FROM runtime_events e + SELECT e.sequence,e.event_header FROM runtime_events e WHERE e.sequence=(SELECT MAX(sequence) FROM candidates)" ).bind(session).fetch_optional(connection).await?; row.map(|(sequence, json)| decode_delivery(sequence, json)).transpose() @@ -169,11 +169,11 @@ impl EventLog { Box::pin(async move { let mut tx = connection.begin().await?; let row: Option<(i64, String)> = sqlx::query_as( - "SELECT e.sequence, e.event_json + "SELECT e.sequence, e.event_header FROM session_message_sources s JOIN runtime_events e ON e.event_id=s.event_id WHERE s.owner_session_id=?1 AND s.message_id=?2 AND (e.kind='invocation_opened' OR e.kind='message_steered') - AND json_extract(e.event_json,'$.invocation.turn_id')=?3", + AND json_extract(e.event_header,'$.invocation.turn_id')=?3", ) .bind(&session) .bind(&message) @@ -211,7 +211,7 @@ impl EventLog { .run(move |connection| { Box::pin(async move { let row: Option<(i64, String)> = sqlx::query_as( - "SELECT e.sequence, e.event_json + "SELECT e.sequence, e.event_header FROM message_sources s JOIN runtime_events e ON e.event_id = s.event_id WHERE s.session_id = ? AND s.message_id = ? AND (e.kind='invocation_opened' OR e.kind='message_steered')", @@ -326,21 +326,21 @@ pub(crate) async fn first_message_opening( "WITH candidates AS ( SELECT sequence FROM ( SELECT e.sequence FROM runtime_events e - WHERE e.event_json IS NOT NULL AND e.event_session=?1 AND ((e.kind='invocation_opened' AND json_extract(e.event_json,'$.fact.input.kind')='message') OR (e.kind='message_imported' AND json_extract(e.event_json,'$.fact.record.content.kind')='user')) + WHERE e.event_header IS NOT NULL AND e.event_session=?1 AND ((e.kind='invocation_opened' AND json_extract(e.event_header,'$.fact.input.kind')='message') OR (e.kind='message_imported' AND json_extract(e.event_header,'$.fact.record.content.kind')='user')) ORDER BY e.sequence ASC LIMIT 1) UNION ALL SELECT sequence FROM ( SELECT h.sequence FROM session_history_members h JOIN runtime_events e ON e.sequence=h.sequence - WHERE h.session_id=?1 AND ((e.kind='invocation_opened' AND json_extract(e.event_json,'$.fact.input.kind')='message') OR (e.kind='message_imported' AND json_extract(e.event_json,'$.fact.record.content.kind')='user')) + WHERE h.session_id=?1 AND ((e.kind='invocation_opened' AND json_extract(e.event_header,'$.fact.input.kind')='message') OR (e.kind='message_imported' AND json_extract(e.event_header,'$.fact.record.content.kind')='user')) ORDER BY h.sequence ASC LIMIT 1) UNION ALL SELECT sequence FROM ( SELECT r.sequence FROM session_revision_sources r JOIN runtime_events e ON e.sequence=r.sequence - WHERE r.session_id=?1 AND ((e.kind='invocation_opened' AND json_extract(e.event_json,'$.fact.input.kind')='message') OR (e.kind='message_imported' AND json_extract(e.event_json,'$.fact.record.content.kind')='user')) + WHERE r.session_id=?1 AND ((e.kind='invocation_opened' AND json_extract(e.event_header,'$.fact.input.kind')='message') OR (e.kind='message_imported' AND json_extract(e.event_header,'$.fact.record.content.kind')='user')) ORDER BY r.sequence ASC LIMIT 1)) - SELECT e.sequence,e.event_json,e.kind FROM runtime_events e + SELECT e.sequence,e.event_header,e.kind FROM runtime_events e WHERE e.sequence=(SELECT MIN(sequence) FROM candidates)" ).bind(session).fetch_optional(connection).await?; match row { diff --git a/platform/event-log/src/model_items.rs b/platform/event-log/src/model_items.rs index 712d6a1f29..04cda11602 100644 --- a/platform/event-log/src/model_items.rs +++ b/platform/event-log/src/model_items.rs @@ -38,13 +38,13 @@ macro_rules! accepted_calls { FROM model_selected_parts s JOIN runtime_events m ON m.sequence=s.sequence JOIN model_part_bodies b ON b.event_id=s.body_event_id AND b.ordinal=s.ordinal JOIN runtime_events r ON r.invocation_id=s.invocation_id AND r.operation_id=s.step_id AND r.kind='model_requested' - WHERE s.terminal_kind='model_completed' AND COALESCE(json_extract(r.event_json,'$.fact.item_acceptance'),0)=0 + WHERE s.terminal_kind='model_completed' AND COALESCE(json_extract(r.event_header,'$.fact.item_acceptance'),0)=0 AND json_extract(b.part,'$.kind')='tool_call' UNION ALL SELECT b.invocation_id,b.step_id,b.source_sequence,b.event_id,json_extract(b.part,'$.call') FROM model_part_bodies b JOIN runtime_events o ON o.sequence=b.source_sequence JOIN runtime_events r ON r.invocation_id=b.invocation_id AND r.operation_id=b.step_id AND r.kind='model_requested' - WHERE b.source_kind='model_observed' AND json_extract(r.event_json,'$.fact.item_acceptance')=1 + WHERE b.source_kind='model_observed' AND json_extract(r.event_header,'$.fact.item_acceptance')=1 AND json_extract(b.part,'$.kind')='tool_call' )" }; } @@ -68,8 +68,8 @@ pub(crate) async fn mode( step: &str, ) -> Result<(bool, Option), StoreError> { let (enabled, purpose): (bool, Option) = sqlx::query_as( - "SELECT COALESCE(json_extract(event_json,'$.fact.item_acceptance'),0), - json_extract(event_json,'$.fact.purpose') FROM runtime_events + "SELECT COALESCE(json_extract(event_header,'$.fact.item_acceptance'),0), + json_extract(event_header,'$.fact.purpose') FROM runtime_events WHERE invocation_id=? AND operation_id=? AND kind='model_requested'", ) .bind(invocation) @@ -121,9 +121,9 @@ pub(crate) async fn validate( Fact::ModelCompleted { step_id, output } => { let (enabled, _) = mode(connection, &event.invocation.invocation_id, step_id).await?; let witness: Option<(String, String)> = sqlx::query_as( - "SELECT json_extract(event_json,'$.fact.event.data.reason'),json_extract(event_json,'$.fact.event.data.usage') + "SELECT json_extract(event_header,'$.fact.event.data.reason'),json_extract(event_header,'$.fact.event.data.usage') FROM runtime_events WHERE invocation_id=? AND kind='model_observed' - AND json_extract(event_json,'$.fact.step_id')=? AND json_extract(event_json,'$.fact.event.kind')='finished' + AND json_extract(event_header,'$.fact.step_id')=? AND json_extract(event_header,'$.fact.event.kind')='finished' ORDER BY sequence DESC LIMIT 1", ).bind(&event.invocation.invocation_id).bind(step_id).fetch_optional(&mut *connection).await?; if let Some((reason, usage)) = witness { diff --git a/platform/event-log/src/model_retries.rs b/platform/event-log/src/model_retries.rs index 504ae1d456..58c9d9046d 100644 --- a/platform/event-log/src/model_retries.rs +++ b/platform/event-log/src/model_retries.rs @@ -66,7 +66,7 @@ pub(crate) async fn frontier( { return Ok(false); } - let row: Option<(i64,String)> = sqlx::query_as("SELECT sequence,event_json FROM runtime_events WHERE event_id=? AND invocation_id=? AND kind='model_interrupted'") + let row: Option<(i64,String)> = sqlx::query_as("SELECT sequence,event_header FROM runtime_events WHERE event_id=? AND invocation_id=? AND kind='model_interrupted'") .bind(&plan.failure_event_id).bind(&invocation.invocation_id).fetch_optional(&mut *tx).await?; let Some((sequence, json)) = row else { return Ok(false); @@ -103,7 +103,7 @@ pub(crate) async fn frontier( { return Ok(false); } - let row: Option<(i64,String)> = sqlx::query_as("SELECT sequence,event_json FROM runtime_events WHERE invocation_id=? AND operation_id=? AND kind='model_requested'") + let row: Option<(i64,String)> = sqlx::query_as("SELECT sequence,event_header FROM runtime_events WHERE invocation_id=? AND operation_id=? AND kind='model_requested'") .bind(&invocation.invocation_id).bind(step_id).fetch_optional(&mut *tx).await?; let Some((requested, json)) = row else { return Ok(false); @@ -120,7 +120,7 @@ pub(crate) async fn frontier( if request.invocation != *invocation || requested >= sequence { return Ok(false); } - let advanced: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM runtime_events e WHERE e.invocation_id=? AND e.sequence>? AND (e.kind IN ('executor_started','invocation_ended','tool_dispatched','tool_rejected','tool_notified','message_steered') OR (e.kind='model_requested' AND json_extract(e.event_json,'$.fact.purpose')='main') OR (e.kind='model_completed' AND EXISTS(SELECT 1 FROM runtime_events r WHERE r.invocation_id=e.invocation_id AND r.kind='model_requested' AND r.operation_id=e.operation_id AND json_extract(r.event_json,'$.fact.purpose')='main'))))") + let advanced: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM runtime_events e WHERE e.invocation_id=? AND e.sequence>? AND (e.kind IN ('executor_started','invocation_ended','tool_dispatched','tool_rejected','tool_notified','message_steered') OR (e.kind='model_requested' AND json_extract(e.event_header,'$.fact.purpose')='main') OR (e.kind='model_completed' AND EXISTS(SELECT 1 FROM runtime_events r WHERE r.invocation_id=e.invocation_id AND r.kind='model_requested' AND r.operation_id=e.operation_id AND json_extract(r.event_header,'$.fact.purpose')='main'))))") .bind(&invocation.invocation_id).bind(requested).fetch_one(&mut *tx).await?; let obligations: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM context_obligations WHERE invocation_id=? AND kind IN ('model','tool','call','completed_provider','interrupted_provider'))") .bind(&invocation.invocation_id).fetch_one(&mut *tx).await?; @@ -130,7 +130,7 @@ pub(crate) async fn frontier( return Ok(false); } if *item_acceptance { - let accepted: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM runtime_events WHERE invocation_id=? AND operation_id=? AND kind='model_observed' AND json_extract(event_json,'$.fact.event.kind')='part')") + let accepted: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM runtime_events WHERE invocation_id=? AND operation_id=? AND kind='model_observed' AND json_extract(event_header,'$.fact.event.kind')='part')") .bind(&invocation.invocation_id).bind(step_id).fetch_one(&mut *tx).await?; if accepted { return Ok(false); @@ -178,7 +178,7 @@ impl EventLog { Box::pin(async move { let mut tx = db.begin().await?; let stored: Option<(i64, String)> = sqlx::query_as( - "SELECT sequence,event_json FROM runtime_events WHERE event_id=?", + "SELECT sequence,event_header FROM runtime_events WHERE event_id=?", ) .bind(&write.event().id) .fetch_optional(&mut *tx) @@ -381,7 +381,7 @@ async fn cancellation_confirmed( tx: &mut SqliteConnection, receipt: &ParkedActivationReceipt, ) -> Result { - let row: Option = sqlx::query_scalar("SELECT event_json FROM runtime_events WHERE event_id=? AND invocation_id=? AND kind='invocation_ended'") + let row: Option = sqlx::query_scalar("SELECT event_header FROM runtime_events WHERE event_id=? AND invocation_id=? AND kind='invocation_ended'") .bind(format!("model-retry-cancel:{}", receipt.checkpoint_id())).bind(&receipt.invocation().invocation_id).fetch_optional(tx).await?; row.map(|json| -> Result { let event: RuntimeEvent = serde_json::from_str(&json)?; diff --git a/platform/event-log/src/native_approvals.rs b/platform/event-log/src/native_approvals.rs index 1bef3d7599..52d961fa6f 100644 --- a/platform/event-log/src/native_approvals.rs +++ b/platform/event-log/src/native_approvals.rs @@ -114,7 +114,7 @@ pub(crate) async fn permission_current( tx: &mut SqliteConnection, invocation: &Invocation, ) -> Result { - let (binding,boundary):(Option,Option)=sqlx::query_as("SELECT json_extract(event_json,'$.fact.configuration.workspace_binding'),json_extract(event_json,'$.fact.configuration.boundary_revision') FROM runtime_events WHERE invocation_id=? AND kind='invocation_opened'").bind(&invocation.invocation_id).fetch_one(&mut *tx).await?; + let (binding,boundary):(Option,Option)=sqlx::query_as("SELECT json_extract(event_header,'$.fact.configuration.workspace_binding'),json_extract(event_header,'$.fact.configuration.boundary_revision') FROM runtime_events WHERE invocation_id=? AND kind='invocation_opened'").bind(&invocation.invocation_id).fetch_one(&mut *tx).await?; let expected = binding .map(|json| -> Result<_, StoreError> { Ok(maka_agent_facts::workspace::PermissionBasis { @@ -167,7 +167,7 @@ async fn frontier( { return Ok(false); } - let row:Option<(i64,String)>=sqlx::query_as("SELECT sequence,event_json FROM runtime_events WHERE event_id=? AND invocation_id=? AND operation_id=? AND kind='tool_dispatched'") + let row:Option<(i64,String)>=sqlx::query_as("SELECT sequence,event_header FROM runtime_events WHERE event_id=? AND invocation_id=? AND operation_id=? AND kind='tool_dispatched'") .bind(&item.dispatch.event_id).bind(&invocation.invocation_id).bind(&item.operation_id).fetch_optional(&mut *tx).await?; let Some((sequence, json)) = row else { return Ok(false); @@ -526,7 +526,7 @@ impl EventLog { self.check_bound_session(&pending.invocation.session_id)?; let pending = pending.clone(); self.connection.read(move |db|Box::pin(async move { - let (sequence,json):(i64,String)=sqlx::query_as("SELECT sequence,event_json FROM runtime_events WHERE event_id=? AND invocation_id=? AND operation_id=? AND kind='tool_dispatched'").bind(&pending.dispatch.event_id).bind(&pending.invocation.invocation_id).bind(&pending.operation_id).fetch_one(db).await?; + let (sequence,json):(i64,String)=sqlx::query_as("SELECT sequence,event_header FROM runtime_events WHERE event_id=? AND invocation_id=? AND operation_id=? AND kind='tool_dispatched'").bind(&pending.dispatch.event_id).bind(&pending.invocation.invocation_id).bind(&pending.operation_id).fetch_one(db).await?; let event:RuntimeEvent=serde_json::from_str(&json)?; if event.invocation!=pending.invocation || crate::sequence_number(sequence)?!=pending.dispatch.sequence {return Err(StoreError::EventConflict)} Ok(StoredEvent{sequence:pending.dispatch.sequence,event}) diff --git a/platform/event-log/src/native_computer.rs b/platform/event-log/src/native_computer.rs index aae0a6ecf7..902f7d4218 100644 --- a/platform/event-log/src/native_computer.rs +++ b/platform/event-log/src/native_computer.rs @@ -93,7 +93,7 @@ impl EventLog { return Ok(InterruptionCommit::Recorded { sequence }); } let result = sqlx::query( - "INSERT INTO event_log(event_id,kind,event_json) VALUES (?,'native_turn_interrupted',?)" + "INSERT INTO event_log(event_id,kind,event_header,canonical_bytes) VALUES (?1,'native_turn_interrupted',?2,length(CAST(?2 AS BLOB)))" ).bind(uuid::Uuid::new_v4().to_string()) .bind(serde_json::to_string(&observed)?).execute(&mut *tx).await?; let sequence = crate::sequence_number(result.last_insert_rowid())?; @@ -111,8 +111,8 @@ async fn source( // Compacted T1 retains its real invocation/call identity. Never require or // reconstruct the collected input body merely to observe a later Esc. let encoded: Option<(String, String)> = sqlx::query_as( - "SELECT json_extract(COALESCE(event_json,retained_json),'$.invocation'), - json_extract(COALESCE(event_json,retained_json),'$.fact.call') + "SELECT json_extract(COALESCE(event_header,retained_json),'$.invocation'), + json_extract(COALESCE(event_header,retained_json),'$.fact.call') FROM event_log e WHERE invocation_id=? AND operation_id=? AND kind='tool_dispatched' AND NOT EXISTS (SELECT 1 FROM imported_invocations i WHERE i.invocation_id=e.invocation_id)" ).bind(&observed.opening.invocation_id).bind(&observed.operation_id) @@ -139,10 +139,10 @@ async fn interruption( turn: &str, ) -> Result, StoreError> { let row: Option<(i64, String)> = sqlx::query_as( - "SELECT sequence,event_json FROM event_log INDEXED BY native_turn_interruption_lookup + "SELECT sequence,event_header FROM event_log INDEXED BY native_turn_interruption_lookup WHERE kind='native_turn_interrupted' - AND json_extract(event_json,'$.opening.session_id')=? - AND json_extract(event_json,'$.opening.turn_id')=?", + AND json_extract(event_header,'$.opening.session_id')=? + AND json_extract(event_header,'$.opening.turn_id')=?", ) .bind(session) .bind(turn) diff --git a/platform/event-log/src/observation.rs b/platform/event-log/src/observation.rs index 0338e88faa..903096a596 100644 --- a/platform/event-log/src/observation.rs +++ b/platform/event-log/src/observation.rs @@ -86,15 +86,14 @@ impl EventLog { let invocation = invocation.clone(); let event_id = event_id.to_owned(); self.connection.read(move |connection| Box::pin(async move { - let row: Option<(i64, String)> = sqlx::query_as( - "SELECT sequence, event_json FROM runtime_events WHERE event_id = ?1 AND sequence <= ?2 - AND (event_json IS NOT NULL AND event_session=?3) - AND json_extract(event_json, '$.invocation.turn_id') = ?4" - ).bind(event_id).bind(through as i64).bind(invocation.session_id).bind(invocation.turn_id) + let row=sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.sequence,e.event_header,{columns} FROM runtime_events e {joins} WHERE e.event_id=?1 AND e.sequence<=?2 + AND (e.event_header IS NOT NULL AND e.event_session=?3) + AND json_extract(e.event_header,'$.invocation.turn_id')=?4", + columns=crate::canonical::COLUMNS,joins=crate::canonical::JOINS, + ))).bind(event_id).bind(through as i64).bind(invocation.session_id).bind(invocation.turn_id) .fetch_optional(connection).await?; - row.map(|(sequence, json)| Ok(StoredEvent { - sequence: sequence_number(sequence)?, event: serde_json::from_str(&json)?, - })).transpose() + row.map(|row|Ok(StoredEvent {sequence:sequence_number(row.try_get(0)?)?,event:serde_json::from_str(&crate::canonical::json(&row)?)?})).transpose() })).await } @@ -113,7 +112,7 @@ impl EventLog { COALESCE((SELECT revision FROM message_queue_state WHERE session_id = session.id), 0), COALESCE((SELECT sequence FROM event_log INDEXED BY event_material_owner WHERE invocation_id IS NOT NULL - AND (event_json IS NOT NULL AND event_session=session.id) + AND (event_header IS NOT NULL AND event_session=session.id) ORDER BY sequence DESC LIMIT 1), 0), control.revision FROM session_control session CROSS JOIN root_app_revision control @@ -281,9 +280,10 @@ impl EventLog { let mut next_after = None; { let mut rows = sqlx::query( - "SELECT sequence, length(CAST(event_json AS BLOB)), event_json FROM runtime_events - WHERE (event_json IS NOT NULL AND event_session=?1) - AND sequence > ?2 AND sequence <= ?3 ORDER BY sequence LIMIT ?4", + sqlx::AssertSqlSafe(format!("SELECT e.sequence,e.canonical_bytes,e.event_header,{columns} FROM runtime_events e {joins} + WHERE (e.event_header IS NOT NULL AND e.event_session=?1) + AND e.sequence>?2 AND e.sequence<=?3 ORDER BY e.sequence LIMIT ?4", + columns=crate::canonical::COLUMNS,joins=crate::canonical::JOINS)), ) .bind(&session_id) .bind(after_sql) @@ -300,7 +300,7 @@ impl EventLog { } let sequence = sequence_number(row.try_get(0)?)?; let event: RuntimeEvent = - serde_json::from_str(row.try_get::<&str, _>(2)?)?; + serde_json::from_str(&crate::canonical::json(&row)?)?; if event.invocation.session_id != session_id { return Err(invalid("stored observation scope changed")); } diff --git a/platform/event-log/src/observation/delivery.rs b/platform/event-log/src/observation/delivery.rs index 139e4d1873..6c2f98e09d 100644 --- a/platform/event-log/src/observation/delivery.rs +++ b/platform/event-log/src/observation/delivery.rs @@ -236,104 +236,104 @@ const STREAM_EVENTS: &str = " WITH events AS MATERIALIZED ( SELECT json_object( 'sequence', sequence, 'id', event_id, - 'recorded_at', json_extract(event_json, '$.recorded_at'), + 'recorded_at', json_extract(event_header, '$.recorded_at'), 'invocation', json_object( - 'session_id', json_extract(event_json, '$.invocation.session_id'), - 'turn_id', json_extract(event_json, '$.invocation.turn_id'), - 'run_id', json_extract(event_json, '$.invocation.run_id'), + 'session_id', json_extract(event_header, '$.invocation.session_id'), + 'turn_id', json_extract(event_header, '$.invocation.turn_id'), + 'run_id', json_extract(event_header, '$.invocation.run_id'), 'invocation_id', invocation_id), 'fact', json(CASE WHEN kind IN ('model_requested', 'context_checkpoint_recorded', 'context_compaction_waiting') OR (kind IN ('model_completed', 'model_interrupted') AND EXISTS ( SELECT 1 FROM runtime_events request WHERE request.invocation_id = runtime_events.invocation_id AND request.kind = 'model_requested' AND request.operation_id = runtime_events.operation_id - AND json_extract(request.event_json, '$.fact.purpose') = 'summary')) + AND json_extract(request.event_header, '$.fact.purpose') = 'summary')) THEN json_object('kind', 'context_changed') WHEN kind IN ('executor_started', 'executor_completed') THEN json_object('kind', kind) - WHEN kind = 'executor_observed' AND json_extract(event_json, '$.fact.output.type') IN ('output_delta','thinking_delta') THEN + WHEN kind = 'executor_observed' AND json_extract(event_header, '$.fact.output.type') IN ('output_delta','thinking_delta') THEN json_object('kind', 'executor_delta', - 'text_kind', CASE WHEN json_extract(event_json, '$.fact.output.type') = 'thinking_delta' THEN 'thinking' ELSE 'text' END, - 'text', json_extract(event_json, '$.fact.output.text')) - WHEN kind = 'executor_observed' AND json_extract(event_json, '$.fact.output.type') = 'tool_start' THEN - json_object('kind', 'executor_tool_start', 'tool_call_id', json_extract(event_json, '$.fact.output.toolCallId'), 'name', json_extract(event_json, '$.fact.output.name')) - WHEN kind = 'executor_observed' AND json_extract(event_json, '$.fact.output.type') = 'tool_progress' THEN - json_object('kind', 'executor_tool_progress', 'tool_call_id', json_extract(event_json, '$.fact.output.toolCallId'), 'text', substr(json_extract(event_json, '$.fact.output.text'), -2048)) + 'text_kind', CASE WHEN json_extract(event_header, '$.fact.output.type') = 'thinking_delta' THEN 'thinking' ELSE 'text' END, + 'text', json_extract(event_header, '$.fact.output.text')) + WHEN kind = 'executor_observed' AND json_extract(event_header, '$.fact.output.type') = 'tool_start' THEN + json_object('kind', 'executor_tool_start', 'tool_call_id', json_extract(event_header, '$.fact.output.toolCallId'), 'name', json_extract(event_header, '$.fact.output.name')) + WHEN kind = 'executor_observed' AND json_extract(event_header, '$.fact.output.type') = 'tool_progress' THEN + json_object('kind', 'executor_tool_progress', 'tool_call_id', json_extract(event_header, '$.fact.output.toolCallId'), 'text', substr(json_extract(event_header, '$.fact.output.text'), -2048)) WHEN kind = 'executor_observed' THEN - json_object('kind', 'executor_tool_result', 'tool_call_id', json_extract(event_json, '$.fact.output.toolCallId'), - 'is_error', json(CASE WHEN json_extract(event_json, '$.fact.output.isError') THEN 'true' ELSE 'false' END)) + json_object('kind', 'executor_tool_result', 'tool_call_id', json_extract(event_header, '$.fact.output.toolCallId'), + 'is_error', json(CASE WHEN json_extract(event_header, '$.fact.output.isError') THEN 'true' ELSE 'false' END)) WHEN kind = 'tool_dispatched' THEN json_object('kind', kind, 'operation_id', operation_id, - 'name', json_extract(event_json, '$.fact.name'), - 'title', json_extract(event_json, '$.fact.title')) + 'name', json_extract(event_header, '$.fact.name'), + 'title', json_extract(event_header, '$.fact.title')) WHEN kind = 'tool_rejected' THEN json_object('kind', kind, 'operation_id', operation_id, - 'name', json_extract(event_json, '$.fact.name')) + 'name', json_extract(event_header, '$.fact.name')) WHEN kind = 'tool_settled' THEN json_object('kind', kind, 'operation_id', operation_id, - 'outcome', json_extract(event_json, '$.fact.outcome.kind')) + 'outcome', json_extract(event_header, '$.fact.outcome.kind')) WHEN kind IN ('invocation_opened', 'message_steered') THEN json_object('kind', kind) WHEN kind = 'invocation_ended' THEN json_object('kind', kind, - 'failed', json(CASE WHEN json_extract(event_json, '$.fact.outcome.kind') = 'failed' - OR (json_extract(event_json, '$.fact.outcome.kind') = 'context_compact_finished' - AND json_extract(event_json, '$.fact.outcome.outcome.kind') = 'failed') - OR (json_extract(event_json, '$.fact.outcome.kind') = 'cancelled' AND EXISTS ( + 'failed', json(CASE WHEN json_extract(event_header, '$.fact.outcome.kind') = 'failed' + OR (json_extract(event_header, '$.fact.outcome.kind') = 'context_compact_finished' + AND json_extract(event_header, '$.fact.outcome.outcome.kind') = 'failed') + OR (json_extract(event_header, '$.fact.outcome.kind') = 'cancelled' AND EXISTS ( SELECT 1 FROM runtime_events started WHERE started.invocation_id = runtime_events.invocation_id AND started.kind = 'executor_started')) THEN 'true' ELSE 'false' END)) WHEN kind IN ('model_completed', 'model_interrupted') THEN json_object('kind', 'step_ended', - 'step_id', json_extract(event_json, '$.fact.step_id'), + 'step_id', json_extract(event_header, '$.fact.step_id'), 'completed', json(CASE WHEN kind='model_completed' THEN 'true' ELSE 'false' END), - 'failed', json(CASE WHEN kind = 'model_interrupted' AND json_extract(event_json, '$.fact.status') != 'cancelled' THEN 'true' ELSE 'false' END)) - WHEN json_extract(event_json, '$.fact.event.data.part.kind') = 'tool_call' THEN + 'failed', json(CASE WHEN kind = 'model_interrupted' AND json_extract(event_header, '$.fact.status') != 'cancelled' THEN 'true' ELSE 'false' END)) + WHEN json_extract(event_header, '$.fact.event.data.part.kind') = 'tool_call' THEN json_object('kind', 'tool_input_ready', - 'step_id', json_extract(event_json, '$.fact.step_id'), - 'tool_call_id', json_extract(event_json, '$.fact.event.data.part.call.id'), - 'name', json_extract(event_json, '$.fact.event.data.part.call.name'), - 'provider_executed', json(CASE WHEN json_extract(event_json, '$.fact.event.data.part.call.provider_executed') THEN 'true' ELSE 'false' END)) - WHEN json_extract(event_json, '$.fact.event.data.part.kind') = 'tool_result' THEN + 'step_id', json_extract(event_header, '$.fact.step_id'), + 'tool_call_id', json_extract(event_header, '$.fact.event.data.part.call.id'), + 'name', json_extract(event_header, '$.fact.event.data.part.call.name'), + 'provider_executed', json(CASE WHEN json_extract(event_header, '$.fact.event.data.part.call.provider_executed') THEN 'true' ELSE 'false' END)) + WHEN json_extract(event_header, '$.fact.event.data.part.kind') = 'tool_result' THEN json_object('kind', 'provider_tool_result', - 'step_id', json_extract(event_json, '$.fact.step_id'), - 'tool_call_id', json_extract(event_json, '$.fact.event.data.part.id'), - 'is_error', json(CASE WHEN json_extract(event_json, '$.fact.event.data.part.is_error') THEN 'true' ELSE 'false' END)) + 'step_id', json_extract(event_header, '$.fact.step_id'), + 'tool_call_id', json_extract(event_header, '$.fact.event.data.part.id'), + 'is_error', json(CASE WHEN json_extract(event_header, '$.fact.event.data.part.is_error') THEN 'true' ELSE 'false' END)) ELSE json_object('kind', 'model_part_committed', - 'step_id', json_extract(event_json, '$.fact.step_id'), - 'message_id', json_extract(event_json, '$.fact.event.data.message_id'), - 'ordinal', json_extract(event_json, '$.fact.event.data.ordinal')) + 'step_id', json_extract(event_header, '$.fact.step_id'), + 'message_id', json_extract(event_header, '$.fact.event.data.message_id'), + 'ordinal', json_extract(event_header, '$.fact.event.data.ordinal')) END)) AS projected, invocation_id AS owner, sequence FROM runtime_events WHERE sequence > ?2 AND sequence <= ?3 -AND (event_json IS NOT NULL AND event_session=?1) +AND (event_header IS NOT NULL AND event_session=?1) AND (kind IN ('invocation_ended', 'model_requested', 'model_completed', 'model_interrupted', 'context_checkpoint_recorded', 'context_compaction_waiting') OR NOT EXISTS ( SELECT 1 FROM runtime_events opening WHERE opening.invocation_id = runtime_events.invocation_id AND opening.kind = 'invocation_opened' - AND json_extract(opening.event_json, '$.fact.input.kind') = 'context_compact')) + AND json_extract(opening.event_header, '$.fact.input.kind') = 'context_compact')) AND (kind != 'model_observed' OR EXISTS (SELECT 1 FROM runtime_events request JOIN runtime_events opening ON opening.invocation_id = request.invocation_id AND opening.kind = 'invocation_opened' WHERE request.kind = 'model_requested' AND request.invocation_id = runtime_events.invocation_id - AND request.operation_id = json_extract(runtime_events.event_json, '$.fact.step_id') - AND json_extract(opening.event_json, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') - AND json_extract(request.event_json, '$.fact.purpose') = 'main')) -AND (kind != 'model_observed' OR json_extract(event_json, '$.fact.event.data.part.kind') != 'tool_result' + AND request.operation_id = json_extract(runtime_events.event_header, '$.fact.step_id') + AND json_extract(opening.event_header, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') + AND json_extract(request.event_header, '$.fact.purpose') = 'main')) +AND (kind != 'model_observed' OR json_extract(event_header, '$.fact.event.data.part.kind') != 'tool_result' OR EXISTS (SELECT 1 FROM runtime_events call WHERE call.invocation_id=runtime_events.invocation_id AND call.kind='model_observed' AND call.sequence { let opening: Option = sqlx::query_scalar( - "SELECT event_json + "SELECT event_header FROM runtime_events WHERE invocation_id = ?1 AND kind = 'invocation_opened' - AND json_extract(event_json, '$.invocation.session_id') = ?2", + AND json_extract(event_header, '$.invocation.session_id') = ?2", ) .bind(invocation) .bind(&session) @@ -140,7 +140,7 @@ async fn observe( // Blob slicing preserves NUL and bounds bytes, not characters. let page: Option<(Vec, i64)> = sqlx::query_as( "SELECT substr(CAST(text AS BLOB), ?2, 16388), length(CAST(text AS BLOB)) FROM ( - SELECT sequence, json_extract(event_json, '$.fact.text') AS text + SELECT sequence, json_extract(event_header, '$.fact.text') AS text FROM runtime_events WHERE invocation_id = ?1 AND kind = 'executor_completed' UNION ALL SELECT event.sequence, ( @@ -148,17 +148,17 @@ async fn observe( SELECT CASE json_extract(part.value, '$.kind') WHEN 'own' THEN json_extract(part.value, '$.data.part') WHEN 'ref' THEN ( - SELECT json_extract(original.event_json, '$.fact.event.data.part') + SELECT json_extract(original.event_header, '$.fact.event.data.part') FROM runtime_events original WHERE original.event_id = json_extract(part.value, '$.data.event_id') AND original.invocation_id = event.invocation_id AND original.sequence < event.sequence AND original.kind = 'model_observed' - AND json_extract(original.event_json, '$.fact.step_id') = event.operation_id - AND json_extract(original.event_json, '$.fact.event.kind') = 'part' - AND json_extract(original.event_json, '$.fact.event.data.ordinal') = json_extract(part.value, '$.data.ordinal') + AND json_extract(original.event_header, '$.fact.step_id') = event.operation_id + AND json_extract(original.event_header, '$.fact.event.kind') = 'part' + AND json_extract(original.event_header, '$.fact.event.data.ordinal') = json_extract(part.value, '$.data.ordinal') ) END AS body - FROM json_each(event.event_json, '$.fact.output.parts') part + FROM json_each(event.event_header, '$.fact.output.parts') part ORDER BY CAST(part.key AS INTEGER) ) WHERE json_extract(body, '$.kind') = 'text' AND json_extract(body, '$.text_kind') = 'text' diff --git a/platform/event-log/src/observation/streams.rs b/platform/event-log/src/observation/streams.rs index db41194a09..3df99b3e31 100644 --- a/platform/event-log/src/observation/streams.rs +++ b/platform/event-log/src/observation/streams.rs @@ -96,7 +96,7 @@ pub(super) async fn compacting( maka_agent_facts::input::InvocationInput::ContextCompact { .. } ) { sqlx::query_as( - "SELECT opening.event_id, catalog_time(json_extract(opening.event_json, '$.recorded_at')) + "SELECT opening.event_id, catalog_time(json_extract(opening.event_header, '$.recorded_at')) FROM runtime_events opening WHERE opening.event_id = ? AND NOT EXISTS (SELECT 1 FROM runtime_events checkpoint WHERE checkpoint.invocation_id = opening.invocation_id @@ -110,27 +110,27 @@ pub(super) async fn compacting( "WITH waiting AS MATERIALIZED ( SELECT sequence FROM event_log INDEXED BY context_compaction_waiting WHERE invocation_id = ?1 AND kind = 'context_compaction_waiting' - AND event_json IS NOT NULL + AND event_header IS NOT NULL ORDER BY sequence DESC LIMIT 1 ), latest AS MATERIALIZED ( - SELECT sequence, event_json FROM event_log INDEXED BY context_summary_sequence + SELECT sequence, event_header FROM event_log INDEXED BY context_summary_sequence WHERE invocation_id = ?1 AND kind = 'model_requested' - AND json_extract(event_json, '$.fact.purpose') = 'summary' + AND json_extract(event_header, '$.fact.purpose') = 'summary' ORDER BY sequence DESC LIMIT 1 ) - SELECT first.event_id, catalog_time(json_extract(first.event_json, '$.recorded_at')) + SELECT first.event_id, catalog_time(json_extract(first.event_header, '$.recorded_at')) FROM waiting, latest JOIN event_log first INDEXED BY context_summary_round ON first.invocation_id = ?1 AND first.kind = 'model_requested' - AND json_extract(first.event_json, '$.fact.purpose') = 'summary' - AND json_extract(first.event_json, '$.fact.source_high_water') = json_extract(latest.event_json, '$.fact.source_high_water') - AND json_extract(first.event_json, '$.fact.source_digest') = json_extract(latest.event_json, '$.fact.source_digest') + AND json_extract(first.event_header, '$.fact.purpose') = 'summary' + AND json_extract(first.event_header, '$.fact.source_high_water') = json_extract(latest.event_header, '$.fact.source_high_water') + AND json_extract(first.event_header, '$.fact.source_digest') = json_extract(latest.event_header, '$.fact.source_digest') AND first.sequence <= latest.sequence - WHERE first.event_json IS NOT NULL + WHERE first.event_header IS NOT NULL AND NOT EXISTS (SELECT 1 FROM runtime_events boundary WHERE boundary.invocation_id = ?1 AND boundary.sequence > waiting.sequence AND (boundary.kind = 'context_checkpoint_recorded' - OR (boundary.kind = 'model_requested' AND json_extract(boundary.event_json, '$.fact.purpose') = 'main') - OR (boundary.kind = 'model_interrupted' AND json_extract(boundary.event_json, '$.fact.status') = 'cancelled'))) + OR (boundary.kind = 'model_requested' AND json_extract(boundary.event_header, '$.fact.purpose') = 'main') + OR (boundary.kind = 'model_interrupted' AND json_extract(boundary.event_header, '$.fact.status') = 'cancelled'))) AND NOT EXISTS (SELECT 1 FROM runtime_events boundary WHERE boundary.invocation_id = ?1 AND boundary.sequence > first.sequence AND boundary.kind = 'context_checkpoint_recorded') diff --git a/platform/event-log/src/observation/tools.rs b/platform/event-log/src/observation/tools.rs index 8328b86e52..10ff73264c 100644 --- a/platform/event-log/src/observation/tools.rs +++ b/platform/event-log/src/observation/tools.rs @@ -112,7 +112,7 @@ LEFT JOIN context_execution_work call ON call.invocation_id=w.invocation_id AND call.call_id='' AND call.call_name='' LEFT JOIN runtime_events source ON w.kind='call' AND source.sequence=w.opened_at LEFT JOIN context_execution_steps step ON step.invocation_id=w.invocation_id - AND step.operation_id=json_extract(source.event_json,'$.fact.step_id') + AND step.operation_id=json_extract(source.event_header,'$.fact.step_id') WHERE w.invocation_id=?1 AND w.closed_at IS NULL AND w.kind IN ('call','tool','executor_tool') AND (w.kind!='call' OR (step.purpose='main' @@ -121,10 +121,10 @@ WHERE w.invocation_id=?1 AND w.closed_at IS NULL const PROVIDER_PARTS: &str = r#" SELECT p.source_sequence,p.step_id, - json_extract(e.event_json,'$.fact.event.data.part.kind')='tool_result' AS settled, - CASE json_extract(e.event_json,'$.fact.event.data.part.kind') - WHEN 'tool_call' THEN json_extract(e.event_json,'$.fact.event.data.part.call.id') - ELSE json_extract(e.event_json,'$.fact.event.data.part.id') END AS call_id + json_extract(e.event_header,'$.fact.event.data.part.kind')='tool_result' AS settled, + CASE json_extract(e.event_header,'$.fact.event.data.part.kind') + WHEN 'tool_call' THEN json_extract(e.event_header,'$.fact.event.data.part.call.id') + ELSE json_extract(e.event_header,'$.fact.event.data.part.id') END AS call_id FROM context_execution_work w INDEXED BY context_execution_live CROSS JOIN context_execution_steps step ON step.invocation_id=w.invocation_id AND step.operation_id=w.operation_id AND step.purpose='main' @@ -132,9 +132,9 @@ CROSS JOIN model_part_records p ON p.invocation_id=w.invocation_id AND p.step_id AND p.record_kind='part' CROSS JOIN runtime_events e ON e.sequence=p.source_sequence AND e.kind='model_observed' WHERE w.invocation_id=?1 AND w.kind='model' AND w.closed_at IS NULL - AND (json_extract(e.event_json,'$.fact.event.data.part.kind')='tool_result' - OR (json_extract(e.event_json,'$.fact.event.data.part.kind')='tool_call' - AND json_extract(e.event_json,'$.fact.event.data.part.call.provider_executed')=1)) + AND (json_extract(e.event_header,'$.fact.event.data.part.kind')='tool_result' + OR (json_extract(e.event_header,'$.fact.event.data.part.kind')='tool_call' + AND json_extract(e.event_header,'$.fact.event.data.part.call.provider_executed')=1)) "#; #[cfg(test)] diff --git a/platform/event-log/src/prefix.rs b/platform/event-log/src/prefix.rs index 248d93c9f2..9328b85ef1 100644 --- a/platform/event-log/src/prefix.rs +++ b/platform/event-log/src/prefix.rs @@ -17,11 +17,10 @@ * under the License. */ -use futures_util::TryStreamExt; use maka_agent_facts::event::LogPrefix; use maka_agent_facts::event::LogScope; use maka_agent_facts::event::StoredEvent; -use sqlx::{Connection, Row}; +use sqlx::Connection; use crate::{EventLog, StoreError, context::selection::Selection, sequence_number}; @@ -54,15 +53,15 @@ impl EventLog { Some(session) => sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM context_source_prefixes p JOIN event_log e ON e.sequence=p.sequence - WHERE p.kind='session' AND p.owner=? AND e.event_json IS NULL)", + WHERE p.kind='session' AND p.owner=? AND e.event_header IS NULL)", ).bind(session).fetch_one(&mut *transaction).await?, None => sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM event_log INDEXED BY invocation_sequence - WHERE invocation_id IS NOT NULL AND event_json IS NULL)", + WHERE invocation_id IS NOT NULL AND event_header IS NULL)", ).fetch_one(&mut *transaction).await?, }; if collected { return Err(StoreError::MaterialCollected); } - let lineage = Selection::predicate("runtime_events", "?2"); + let lineage = Selection::predicate("e", "?2"); let source = if selection.session.is_some() { "session_history_events" } else { "runtime_events" }; let session_filter = if selection.session.is_some() { "owner_session_id=?1" @@ -70,34 +69,39 @@ impl EventLog { let filter = format!("{session_filter} AND {lineage}"); // Only internal SQL clauses are formatted; all identities remain bound. let budget = sqlx::query_as::<_, (i64, i64, i64)>(sqlx::AssertSqlSafe(format!( - "SELECT COALESCE(MAX(sequence), 0), COUNT(*), COALESCE(SUM(length(CAST(event_json AS BLOB))), 0) FROM {source} runtime_events WHERE {filter}" + "SELECT COALESCE(MAX(sequence), 0), COUNT(*), COALESCE(SUM(canonical_bytes), 0) FROM {source} e WHERE {filter}" ))).bind(&selection.session).bind(&selection.lineage); let (high_water, count, bytes) = budget.fetch_one(&mut *transaction).await?; if sequence_number(count)? > max_events as u64 || sequence_number(bytes)? > max_bytes as u64 { return Err(StoreError::PrefixTooLarge); } - let mut root_digest = selection.session.is_none().then(crate::context::evidence::RootDigest::new); - let mut events = Vec::new(); - { - let statement = sqlx::query(sqlx::AssertSqlSafe(format!( - "SELECT sequence, event_json FROM {source} runtime_events WHERE {filter} ORDER BY sequence" - ))).bind(&selection.session).bind(&selection.lineage); - let mut rows = statement.fetch(&mut *transaction); - while let Some(row) = rows.try_next().await? { - let sequence: i64 = row.try_get(0)?; - let json: &str = row.try_get(1)?; - if let Some(digest) = &mut root_digest { - digest.record(sequence, json); + let query = format!( + "SELECT e.sequence,e.event_header,{columns} FROM {source} e {joins} WHERE {filter} ORDER BY e.sequence", + columns=crate::canonical::COLUMNS,joins=crate::canonical::JOINS + ); + let session = selection.session.clone(); + let selected_lineage = selection.lineage.clone(); + let (events, root_digest) = crate::connection::worker_query(&mut transaction, move |worker| Box::pin(async move { + // SAFETY: this callback exclusively owns the original SQLite worker. + // The wrapper and all borrowed rows/statements close before return. + let raw = unsafe { rusqlite::Connection::from_handle(worker.as_raw_handle().as_ptr()) }?; + let result = (|| { + let mut digest = session.is_none().then(crate::context::evidence::RootDigest::new); + let mut events = Vec::new(); + let mut statement = raw.prepare(&query)?; + let mut rows = statement.query(rusqlite::params![session, selected_lineage])?; + while let Some(row) = rows.next()? { + let sequence: i64 = row.get("sequence")?; + let json = crate::canonical::json_on_handle(row)?; + if let Some(digest) = &mut digest { digest.record(sequence, &json); } + events.push(StoredEvent { sequence: sequence_number(sequence)?, event: serde_json::from_str(&json)? }); } - let sequence = sequence_number(sequence)?; - let event = serde_json::from_str(json)?; - events.push(StoredEvent { - sequence, - event, - }); - } - } + Ok((events, digest)) + })(); + drop(raw); + result + })).await?; let digest = match root_digest { Some(digest) => digest.finish(high_water)?, None => crate::context::evidence::selected( diff --git a/platform/event-log/src/read_resources.rs b/platform/event-log/src/read_resources.rs index ab9711fe24..99266a3d8a 100644 --- a/platform/event-log/src/read_resources.rs +++ b/platform/event-log/src/read_resources.rs @@ -51,11 +51,11 @@ impl EventLog { AND (t.event_session=?1 OR EXISTS( SELECT 1 FROM session_history_members h WHERE h.session_id=?1 AND h.sequence=t.sequence)) - AND json_extract(t.event_json,'$.fact.outcome.kind') IN ('succeeded','failed') + AND json_extract(t.event_header,'$.fact.outcome.kind') IN ('succeeded','failed') AND EXISTS(SELECT 1 FROM runtime_events d WHERE d.invocation_id=t.invocation_id AND d.operation_id=t.operation_id AND d.kind='tool_dispatched' - AND json_extract(d.event_json,'$.fact.call.origin.kind')='provider')", + AND json_extract(d.event_header,'$.fact.call.origin.kind')='provider')", "t.event_id", ), }; diff --git a/platform/event-log/src/recovery.rs b/platform/event-log/src/recovery.rs index 8f091ede07..c19ccbdcf8 100644 --- a/platform/event-log/src/recovery.rs +++ b/platform/event-log/src/recovery.rs @@ -54,7 +54,7 @@ macro_rules! unresolved { WHERE done.invocation_id = started.invocation_id AND (done.kind = 'executor_completed' OR (done.kind = 'invocation_ended' AND - COALESCE(json_extract(done.event_json, '$.fact.outcome.class'), '') != 'outcome_unknown'))) + COALESCE(json_extract(done.event_header, '$.fact.outcome.class'), '') != 'outcome_unknown'))) )", $tail ) }; @@ -107,9 +107,9 @@ const UNFINISHED_INVOCATIONS: &str = "WITH pending AS MATERIALIZED ( WHERE i.invocation_id=context_obligations.invocation_id) ORDER BY opened_at LIMIT ?2 ) - SELECT json_extract(opening.event_json, '$.invocation.session_id'), - json_extract(opening.event_json, '$.invocation.turn_id'), - json_extract(opening.event_json, '$.invocation.run_id'), + SELECT json_extract(opening.event_header, '$.invocation.session_id'), + json_extract(opening.event_header, '$.invocation.turn_id'), + json_extract(opening.event_header, '$.invocation.run_id'), opening.invocation_id, pending.opened_at FROM pending CROSS JOIN runtime_events AS opening WHERE opening.sequence = pending.opened_at @@ -123,9 +123,9 @@ const SESSION_UNFINISHED_INVOCATIONS: &str = "WITH pending AS MATERIALIZED ( WHERE i.invocation_id=context_obligations.invocation_id) ORDER BY opened_at LIMIT ?2 ) - SELECT json_extract(opening.event_json, '$.invocation.session_id'), - json_extract(opening.event_json, '$.invocation.turn_id'), - json_extract(opening.event_json, '$.invocation.run_id'), + SELECT json_extract(opening.event_header, '$.invocation.session_id'), + json_extract(opening.event_header, '$.invocation.turn_id'), + json_extract(opening.event_header, '$.invocation.run_id'), opening.invocation_id, pending.opened_at FROM pending CROSS JOIN runtime_events AS opening WHERE opening.sequence = pending.opened_at @@ -146,7 +146,7 @@ impl EventLog { let mut transaction = connection.begin().await?; require_local(&mut transaction, &invocation.invocation_id).await?; let identity: String = sqlx::query_scalar( - "SELECT json_extract(event_json, '$.invocation') FROM runtime_events + "SELECT json_extract(event_header, '$.invocation') FROM runtime_events WHERE invocation_id = ? AND kind = 'invocation_opened'", ) .bind(&invocation.invocation_id) diff --git a/platform/event-log/src/root_store/catalog.rs b/platform/event-log/src/root_store/catalog.rs index 544a433a27..2fc5d02d74 100644 --- a/platform/event-log/src/root_store/catalog.rs +++ b/platform/event-log/src/root_store/catalog.rs @@ -619,7 +619,7 @@ async fn workspace_page( use sqlx::{Connection,Row}; let mut tx=db.begin().await?; require_snapshot(&mut tx,expected).await?; let rows=sqlx::query("SELECT s.id,s.revision,b.metadata_epoch,json_extract(s.configuration,'$.workspace.metadataEpochAtAttach') AS attached_epoch, - COALESCE((SELECT MAX(e.sequence) FROM runtime_events e WHERE e.event_json IS NOT NULL AND e.event_session=s.id),0) AS sequence, + COALESCE((SELECT MAX(e.sequence) FROM runtime_events e WHERE e.event_header IS NOT NULL AND e.event_session=s.id),0) AS sequence, EXISTS(SELECT 1 FROM shell_runs r WHERE r.session_id=s.id AND json_extract(r.record_json,'$.state.outcome.kind')='orphaned') AS orphaned FROM workspace_bindings b JOIN session_control s INDEXED BY session_workspace_binding ON json_extract(s.configuration,'$.workspace.id')=b.id diff --git a/platform/event-log/src/root_store/material/tests.rs b/platform/event-log/src/root_store/material/tests.rs index 5e9321e423..45c12e0bde 100644 --- a/platform/event-log/src/root_store/material/tests.rs +++ b/platform/event-log/src/root_store/material/tests.rs @@ -154,7 +154,7 @@ async fn material_state(log: &EventLog) -> (i64, i64, i64) { .read(|db| { Box::pin(async move { let bodies = sqlx::query_scalar( - "SELECT COUNT(*) FROM event_log WHERE event_json IS NOT NULL", + "SELECT COUNT(*) FROM event_log WHERE event_header IS NOT NULL", ) .fetch_one(&mut *db) .await?; @@ -228,7 +228,7 @@ async fn retirement_automatically_collects_after_the_last_archived_fork_is_remov "automatic vacuum must shrink the database" ); let proof: (i64, i64) = log.connection.read(|db| Box::pin(async move { - Ok(sqlx::query_as("SELECT COUNT(*), SUM(event_json IS NULL AND body_digest IS NOT NULL AND retained_json NOT LIKE '%private input%') FROM event_log") + Ok(sqlx::query_as("SELECT COUNT(*), SUM(event_header IS NULL AND body_digest IS NOT NULL AND retained_json NOT LIKE '%private input%') FROM event_log") .fetch_one(db).await?) })).await.unwrap(); assert_eq!( @@ -390,7 +390,7 @@ async fn shutdown_drains_the_accepted_batch_and_reopen_resumes_without_new_write .unwrap(); let remaining: i64 = inspect .query_row( - "SELECT COUNT(*) FROM event_log WHERE event_json IS NOT NULL", + "SELECT COUNT(*) FROM event_log WHERE event_header IS NOT NULL", [], |row| row.get(0), ) diff --git a/platform/event-log/src/root_store/usage_meter.rs b/platform/event-log/src/root_store/usage_meter.rs index 8ec89b0528..dfa74923f8 100644 --- a/platform/event-log/src/root_store/usage_meter.rs +++ b/platform/event-log/src/root_store/usage_meter.rs @@ -115,8 +115,8 @@ async fn observed( step: &str, ) -> Result { let usage: Option = sqlx::query_scalar("SELECT accounting_usage FROM event_log INDEXED BY model_usage_observation - WHERE kind='model_observed' AND json_extract(COALESCE(event_json,retained_json),'$.fact.event.kind')='finished' - AND invocation_id=? AND json_extract(COALESCE(event_json,retained_json),'$.fact.step_id')=? + WHERE kind='model_observed' AND json_extract(COALESCE(event_header,retained_json),'$.fact.event.kind')='finished' + AND invocation_id=? AND json_extract(COALESCE(event_header,retained_json),'$.fact.step_id')=? ORDER BY sequence DESC LIMIT 1") .bind(invocation).bind(step).fetch_optional(db).await?; Ok(usage @@ -175,12 +175,12 @@ pub(crate) async fn settle_auxiliary( db: &mut SqliteConnection, request: uuid::Uuid, ) -> Result<(), StoreError> { - let session: Option = sqlx::query_scalar("SELECT json_extract(event_json,'$.session_id') FROM event_log WHERE event_id=? AND kind='auxiliary_model_started'") + let session: Option = sqlx::query_scalar("SELECT json_extract(event_header,'$.session_id') FROM event_log WHERE event_id=? AND kind='auxiliary_model_started'") .bind(request.to_string()).fetch_one(&mut *db).await?; let Some(session) = session else { return Ok(()); }; - let usage: Option = sqlx::query_scalar("SELECT accounting_usage FROM event_log INDEXED BY auxiliary_usage_meter_observation WHERE kind='auxiliary_model_usage' AND json_extract(COALESCE(event_json,retained_json),'$.request_id')=?") + let usage: Option = sqlx::query_scalar("SELECT accounting_usage FROM event_log INDEXED BY auxiliary_usage_meter_observation WHERE kind='auxiliary_model_usage' AND json_extract(COALESCE(event_header,retained_json),'$.request_id')=?") .bind(request.to_string()).fetch_optional(&mut *db).await?; let usage = usage .map(|json| serde_json::from_str(&json)) diff --git a/platform/event-log/src/root_store/usage_meter/tests.rs b/platform/event-log/src/root_store/usage_meter/tests.rs index d395f7871a..906ce3fbba 100644 --- a/platform/event-log/src/root_store/usage_meter/tests.rs +++ b/platform/event-log/src/root_store/usage_meter/tests.rs @@ -285,7 +285,7 @@ async fn lifetime_read_and_terminal_pending_lookup_remain_indexed_after_large_hi .unwrap(); let log = session.owner().unwrap(); log.connection.run(|db|Box::pin(async move { - sqlx::query("WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<20000) INSERT INTO event_log(event_id,kind,event_json) SELECT 'large-history-'||x,'unrelated','{}' FROM n").execute(db).await?;Ok(()) + sqlx::query("WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<20000) INSERT INTO event_log(event_id,kind,event_header,canonical_bytes) SELECT 'large-history-'||x,'unrelated','{}',2 FROM n").execute(db).await?;Ok(()) })).await.unwrap(); log.connection.read(|db|Box::pin(async move { let plans=sqlx::query("EXPLAIN QUERY PLAN SELECT input_known,input_missing,output_known,output_missing FROM session_usage_totals WHERE session_id='source'").fetch_all(&mut *db).await?; diff --git a/platform/event-log/src/run_source.rs b/platform/event-log/src/run_source.rs index 6a51051ea7..6672e5a234 100644 --- a/platform/event-log/src/run_source.rs +++ b/platform/event-log/src/run_source.rs @@ -114,19 +114,13 @@ pub(crate) async fn read( )); } let (opened,json):(i64,String)=sqlx::query_as( - "SELECT sequence,event_json FROM runtime_events WHERE invocation_id=? ORDER BY sequence LIMIT 1") + "SELECT sequence,event_header FROM runtime_events WHERE invocation_id=? ORDER BY sequence LIMIT 1") .bind(invocation_id).fetch_one(&mut *connection).await?; let opening = StoredEvent { sequence: sequence_number(opened)?, event: serde_json::from_str::(&json)?, }; - let json: String = sqlx::query_scalar( - "SELECT event_json FROM runtime_events WHERE invocation_id=? AND sequence=?", - ) - .bind(invocation_id) - .bind(through) - .fetch_one(connection) - .await?; + let json = crate::canonical::one(connection, sequence_number(through)?).await?; let last = StoredEvent { sequence: sequence_number(through)?, event: serde_json::from_str::(&json)?, diff --git a/platform/event-log/src/schema/fixtures.rs b/platform/event-log/src/schema/fixtures.rs index 0d87ffbcd3..f25a3684b6 100644 --- a/platform/event-log/src/schema/fixtures.rs +++ b/platform/event-log/src/schema/fixtures.rs @@ -23,10 +23,12 @@ use rusqlite::Connection; pub(crate) fn schema(include: impl Fn(&str, &str) -> bool) -> String { let db = Connection::open_in_memory().unwrap(); crate::context::semantic::migration::register(&db).unwrap(); + crate::canonical::migration::register(&db).unwrap(); for migration in super::MIGRATIONS.iter() { db.execute_batch(migration.sql.as_ref()).unwrap(); } crate::context::semantic::migration::unregister(&db).unwrap(); + crate::canonical::migration::unregister(&db).unwrap(); let mut query = db.prepare( "SELECT type,name,sql FROM sqlite_schema WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%' AND tbl_name NOT IN (SELECT name FROM pragma_table_list WHERE schema='main' AND type='shadow') ORDER BY rowid", ).unwrap(); diff --git a/platform/event-log/src/sessions/activity.rs b/platform/event-log/src/sessions/activity.rs index 4a4c7a191e..e0eb3c2e46 100644 --- a/platform/event-log/src/sessions/activity.rs +++ b/platform/event-log/src/sessions/activity.rs @@ -104,7 +104,7 @@ pub(crate) async fn project_execution( let sequence = sequence as i64; let (session, invocation, step, kind): (String, String, Option, String) = sqlx::query_as( - "SELECT json_extract(event_json, '$.invocation.session_id'), invocation_id, + "SELECT json_extract(event_header, '$.invocation.session_id'), invocation_id, operation_id, kind FROM runtime_events WHERE sequence = ?", ) .bind(sequence) @@ -115,7 +115,7 @@ pub(crate) async fn project_execution( let compact: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM runtime_events WHERE invocation_id = ? AND kind = 'invocation_opened' - AND json_extract(event_json, '$.fact.input.kind') = 'context_compact')", + AND json_extract(event_header, '$.fact.input.kind') = 'context_compact')", ) .bind(&invocation) .fetch_one(&mut *connection) @@ -127,8 +127,8 @@ pub(crate) async fn project_execution( AND opening.kind = 'invocation_opened' WHERE request.kind = 'model_requested' AND request.invocation_id = ?1 AND request.operation_id = ?2 - AND json_extract(opening.event_json, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') - AND json_extract(request.event_json, '$.fact.purpose') = 'main')", + AND json_extract(opening.event_header, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') + AND json_extract(request.event_header, '$.fact.purpose') = 'main')", ) .bind(&invocation) .bind(&step) @@ -157,10 +157,10 @@ pub(crate) async fn project_execution( if kind == "message_imported" { sqlx::query( "INSERT OR IGNORE INTO catalog_messages - SELECT sequence,0,?2,catalog_time(json_extract(event_json,'$.recorded_at')), - catalog_preview(json_extract(event_json,'$.fact.record.content.text')),event_id + SELECT sequence,0,?2,catalog_time(json_extract(event_header,'$.recorded_at')), + catalog_preview(json_extract(event_header,'$.fact.record.content.text')),event_id FROM runtime_events WHERE sequence=?1 - AND json_extract(event_json,'$.fact.record.content.kind') IN ('user','assistant')", + AND json_extract(event_header,'$.fact.record.content.kind') IN ('user','assistant')", ) .bind(sequence) .bind(&session) @@ -169,14 +169,14 @@ pub(crate) async fn project_execution( } else if kind == "invocation_opened" { sqlx::query( "INSERT OR IGNORE INTO catalog_messages - SELECT sequence, 0, ?2, catalog_time(json_extract(event_json, '$.recorded_at')), + SELECT sequence, 0, ?2, catalog_time(json_extract(event_header, '$.recorded_at')), catalog_preview(COALESCE( - json_extract(event_json, '$.fact.input.content.display_text'), - json_extract(event_json, '$.fact.input.content.text'))), - CASE WHEN json_array_length(event_json, '$.fact.input.source_messages') = 1 - THEN json_extract(event_json, '$.fact.input.source_messages[0].message_id') ELSE event_id END + json_extract(event_header, '$.fact.input.content.display_text'), + json_extract(event_header, '$.fact.input.content.text'))), + CASE WHEN json_array_length(event_header, '$.fact.input.source_messages') = 1 + THEN json_extract(event_header, '$.fact.input.source_messages[0].message_id') ELSE event_id END FROM runtime_events WHERE sequence = ?1 - AND json_extract(event_json, '$.fact.input.kind') = 'message'", + AND json_extract(event_header, '$.fact.input.kind') = 'message'", ) .bind(sequence) .bind(&session) diff --git a/platform/event-log/src/sessions/activity/partial.rs b/platform/event-log/src/sessions/activity/partial.rs index d5690cd438..84b4dd6145 100644 --- a/platform/event-log/src/sessions/activity/partial.rs +++ b/platform/event-log/src/sessions/activity/partial.rs @@ -104,7 +104,7 @@ pub(super) async fn project( AND result.invocation_id = ?1 AND result.sequence < ?2 AND result.kind IN ('model_completed', 'model_interrupted')) AND EXISTS (SELECT 1 FROM runtime_events WHERE sequence = ?2 - AND json_extract(event_json, '$.fact.outcome.kind') IN ('failed', 'cancelled')) + AND json_extract(event_header, '$.fact.outcome.kind') IN ('failed', 'cancelled')) LIMIT 1", ).bind(invocation).bind(boundary).fetch_optional(&mut *connection).await? }; @@ -117,8 +117,8 @@ pub(super) async fn project( AND opening.kind = 'invocation_opened' WHERE request.kind = 'model_requested' AND request.invocation_id = ?1 AND request.operation_id = ?2 - AND json_extract(opening.event_json, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') - AND json_extract(request.event_json, '$.fact.purpose') = 'main')", + AND json_extract(opening.event_header, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') + AND json_extract(request.event_header, '$.fact.purpose') = 'main')", ) .bind(invocation) .bind(&step) diff --git a/platform/event-log/src/sessions/copy.rs b/platform/event-log/src/sessions/copy.rs index e81e1e6d2b..0ae9059182 100644 --- a/platform/event-log/src/sessions/copy.rs +++ b/platform/event-log/src/sessions/copy.rs @@ -196,8 +196,8 @@ impl EventLog { "INSERT INTO session_history_members SELECT ?1, e.sequence, COALESCE((SELECT a.sequence FROM runtime_events a WHERE a.kind = 'tool_result_archived' - AND CAST(json_extract(a.event_json,'$.fact.placeholder.identity.runtime_event_id') AS TEXT) = e.event_id - AND json_extract(a.event_json,'$.invocation.session_id') = e.owner_session_id + AND CAST(json_extract(a.event_header,'$.fact.placeholder.identity.runtime_event_id') AS TEXT) = e.event_id + AND json_extract(a.event_header,'$.invocation.session_id') = e.owner_session_id AND a.sequence < ?2), e.archive_sequence) FROM session_history_events e WHERE e.owner_session_id = ?3 AND e.sequence <= ?4", ).bind(&target).bind(observed.saturating_add(1)) @@ -224,10 +224,10 @@ impl EventLog { "INSERT INTO session_revision_sources SELECT ?1, sequence FROM session_history_events WHERE owner_session_id = ?2 AND sequence <= ?3 - AND json_extract(event_json, '$.invocation.turn_id') = ?4 + AND json_extract(event_header, '$.invocation.turn_id') = ?4 AND ((kind = 'invocation_opened' - AND json_extract(event_json, '$.fact.input.kind') = 'message' - AND json_array_length(event_json, '$.fact.input.source_messages') > 0) + AND json_extract(event_header, '$.fact.input.kind') = 'message' + AND json_array_length(event_header, '$.fact.input.source_messages') > 0) OR kind = 'message_steered')" ).bind(&request.target_session_id).bind(&request.source_session_id).bind(observed) .bind(turn_id).execute(&mut *tx).await?; diff --git a/platform/event-log/src/sessions/execution.rs b/platform/event-log/src/sessions/execution.rs index 8b3f4badf1..97721b11e1 100644 --- a/platform/event-log/src/sessions/execution.rs +++ b/platform/event-log/src/sessions/execution.rs @@ -55,9 +55,9 @@ pub(crate) async fn has_active_invocation( Ok(sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM ( SELECT invocation_id FROM event_log INDEXED BY catalog_message_facts - WHERE invocation_id IS NOT NULL AND event_json IS NOT NULL + WHERE invocation_id IS NOT NULL AND event_header IS NOT NULL AND kind IN ('invocation_opened', 'model_completed') AND kind = 'invocation_opened' - AND json_extract(event_json, '$.invocation.session_id') = ? + AND json_extract(event_header, '$.invocation.session_id') = ? ORDER BY sequence DESC LIMIT 1 ) AS opening WHERE NOT EXISTS(SELECT 1 FROM imported_invocations i @@ -76,12 +76,12 @@ pub(super) async fn read( id: &str, ) -> Result, StoreError> { let opening = sqlx::query( - "SELECT invocation_id, json_extract(event_json, '$.invocation.turn_id'), - catalog_time(json_extract(event_json, '$.recorded_at')) + "SELECT invocation_id, json_extract(event_header, '$.invocation.turn_id'), + catalog_time(json_extract(event_header, '$.recorded_at')) FROM event_log INDEXED BY catalog_message_facts WHERE invocation_id IS NOT NULL AND kind IN ('invocation_opened', 'model_completed') AND kind = 'invocation_opened' - AND json_extract(event_json, '$.invocation.session_id') = ? + AND json_extract(event_header, '$.invocation.session_id') = ? AND NOT EXISTS(SELECT 1 FROM imported_invocations i WHERE i.invocation_id=event_log.invocation_id) ORDER BY sequence DESC LIMIT 1", ) @@ -95,9 +95,9 @@ pub(super) async fn read( let turn_id = opening.try_get(1)?; let opened_at = number(&opening, 2)?; let terminal = sqlx::query( - "SELECT json_extract(event_json, '$.fact.outcome.kind'), - catalog_time(json_extract(event_json, '$.recorded_at')), - json_extract(event_json, '$.fact.outcome.outcome.kind') FROM event_log + "SELECT json_extract(event_header, '$.fact.outcome.kind'), + catalog_time(json_extract(event_header, '$.recorded_at')), + json_extract(event_header, '$.fact.outcome.outcome.kind') FROM event_log INDEXED BY invocation_boundary WHERE invocation_id = ? AND kind IN ('invocation_opened', 'invocation_ended') AND kind = 'invocation_ended' ORDER BY sequence DESC LIMIT 1", diff --git a/platform/event-log/src/sessions/import.rs b/platform/event-log/src/sessions/import.rs index 7b0256955a..4b028f8f38 100644 --- a/platform/event-log/src/sessions/import.rs +++ b/platform/event-log/src/sessions/import.rs @@ -188,7 +188,7 @@ async fn append( let json = serde_json::to_string(&event)?; if replay { let prior: (Option, Option) = - sqlx::query_as("SELECT event_json,body_digest FROM event_log WHERE event_id=?") + sqlx::query_as("SELECT event_header,body_digest FROM event_log WHERE event_id=?") .bind(event_id) .fetch_one(&mut *connection) .await?; diff --git a/platform/event-log/src/sessions/material.rs b/platform/event-log/src/sessions/material.rs index 4a813906e4..d3ef9d7180 100644 --- a/platform/event-log/src/sessions/material.rs +++ b/platform/event-log/src/sessions/material.rs @@ -72,7 +72,7 @@ impl EventLog { let mut tx = connection.begin_with("BEGIN IMMEDIATE").await?; let session: Option = sqlx::query_scalar( "SELECT DISTINCT e.event_session FROM event_log e INDEXED BY event_material_owner - WHERE e.event_json IS NOT NULL AND e.event_session IS NOT NULL + WHERE e.event_header IS NOT NULL AND e.event_session IS NOT NULL AND (?1 IS NULL OR e.event_session>?1) AND ( EXISTS(SELECT 1 FROM session_retirements r WHERE r.session_id=e.event_session AND r.remove_session=1 AND r.completed=1) @@ -94,7 +94,7 @@ impl EventLog { let released: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM context_source_prefixes p JOIN event_log e ON e.sequence=p.sequence - WHERE p.kind='session' AND p.owner=? AND e.event_json IS NULL)" + WHERE p.kind='session' AND p.owner=? AND e.event_header IS NULL)" ).bind(&session).fetch_one(&mut *tx).await?; if !released && sqlx::query_scalar::<_, bool>(RETAINED) .bind(&session).fetch_one(&mut *tx).await? @@ -105,11 +105,14 @@ impl EventLog { // remain the material-collection authority; descendants were checked above. // Also covers imported owners that never had a local retirement. crate::context::semantic::gc::retire_owner(&mut tx,&session).await?; - let rows: Vec<(i64, String, String)> = sqlx::query_as( - "SELECT sequence,event_id,event_json FROM event_log WHERE event_session=? - AND event_json IS NOT NULL ORDER BY sequence LIMIT 8" - ).bind(&session).fetch_all(&mut *tx).await?; - for (sequence,id,body) in rows { + let rows=sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.sequence,e.event_id,e.event_header,{columns} FROM event_log e {joins} WHERE e.event_session=? + AND e.event_header IS NOT NULL ORDER BY e.sequence LIMIT 8",columns=crate::canonical::COLUMNS,joins=crate::canonical::JOINS, + ))).bind(&session).fetch_all(&mut *tx).await?; + for row in rows { + use sqlx::Row; + let sequence:i64=row.try_get(0)?;let id:String=row.try_get(1)?; + let body=crate::canonical::json(&row)?; let event: RuntimeEvent = serde_json::from_str(&body)?; let proof = retained(&event); crate::context::semantic::gc::collect_event(&mut tx,crate::sequence_number(sequence)?).await?; @@ -125,7 +128,8 @@ impl EventLog { AND NOT EXISTS (SELECT 1 FROM model_request_compositions WHERE digest=?1)") .bind(surface).execute(&mut *tx).await?; } - sqlx::query("UPDATE event_log SET retained_json=?, body_digest=?, event_json=NULL WHERE event_id=?") + sqlx::query("DELETE FROM event_body_slots WHERE event_sequence=?").bind(sequence).execute(&mut *tx).await?; + sqlx::query("UPDATE event_log SET retained_json=?, body_digest=?, event_header=NULL,canonical_bytes=NULL WHERE event_id=?") .bind(serde_json::to_string(&proof)?).bind(content_digest(body.as_bytes())) .bind(&id).execute(&mut *tx).await?; } @@ -288,7 +292,7 @@ mod cursor_tests { let proof = serde_json::to_string(&retained(&ended)).unwrap(); let digest = content_digest(serde_json::to_string(&ended).unwrap().as_bytes()); log.connection.run(move |db| Box::pin(async move { - sqlx::query("UPDATE event_log SET retained_json=?,body_digest=?,event_json=NULL WHERE sequence=2") + sqlx::query("UPDATE event_log SET retained_json=?,body_digest=?,event_header=NULL,canonical_bytes=NULL WHERE sequence=2") .bind(proof).bind(digest).execute(&mut *db).await?; let publication: (f64, String) = sqlx::query_as("SELECT accounting_at,event_session FROM event_log WHERE sequence=2") .fetch_one(&mut *db).await?; @@ -379,7 +383,8 @@ mod cursor_tests { .into_iter() .filter_map(|statement| statement.sql) .filter(|sql| { - (sql.contains("context_source_prefixes p") && sql.contains("e.event_json IS NULL")) + (sql.contains("context_source_prefixes p") + && sql.contains("e.event_header IS NULL")) || sql.contains("event_log INDEXED BY invocation_sequence") }) .collect(); diff --git a/platform/event-log/src/sessions/read_state.rs b/platform/event-log/src/sessions/read_state.rs index 684585178e..38e0f2dd30 100644 --- a/platform/event-log/src/sessions/read_state.rs +++ b/platform/event-log/src/sessions/read_state.rs @@ -60,7 +60,7 @@ pub(crate) async fn mark_unread( SELECT id, 1, NULL FROM session_control WHERE id = ?1 AND NOT EXISTS (SELECT 1 FROM runtime_events WHERE invocation_id = ?2 AND kind = 'invocation_opened' - AND json_extract(event_json, '$.fact.input.kind') = 'context_compact') + AND json_extract(event_header, '$.fact.input.kind') = 'context_compact') ON CONFLICT(session_id) DO UPDATE SET has_unread = 1", ) .bind(id) diff --git a/platform/event-log/src/sessions/removal.rs b/platform/event-log/src/sessions/removal.rs index 976c6f37a1..a374e0f44d 100644 --- a/platform/event-log/src/sessions/removal.rs +++ b/platform/event-log/src/sessions/removal.rs @@ -335,10 +335,10 @@ async fn settled(connection: &mut SqliteConnection, session: &str) -> Result Result<(), St pub(crate) async fn register_migrations( connection: &mut SqliteConnection, ) -> Result<(), StoreError> { - on_handle(connection, crate::context::semantic::migration::register).await + on_handle(connection, |db| { + crate::context::semantic::migration::register(db)?; + crate::canonical::migration::register(db) + }) + .await } pub(crate) async fn unregister_migrations( connection: &mut SqliteConnection, ) -> Result<(), StoreError> { - on_handle(connection, crate::context::semantic::migration::unregister).await + on_handle(connection, |db| { + crate::context::semantic::migration::unregister(db)?; + crate::canonical::migration::unregister(db) + }) + .await } async fn on_handle( diff --git a/platform/event-log/src/steering.rs b/platform/event-log/src/steering.rs index 6bc767b492..a7cb9a0eb0 100644 --- a/platform/event-log/src/steering.rs +++ b/platform/event-log/src/steering.rs @@ -35,7 +35,7 @@ pub(crate) async fn validate_append( .map_err(|error| StoreError::InvalidTransition(error.into()))?; let eligible: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM runtime_events WHERE invocation_id = ? - AND kind = 'invocation_opened' AND json_extract(event_json, '$.fact.input.kind') IN ('message', 'continuation', 'handoff'))" + AND kind = 'invocation_opened' AND json_extract(event_header, '$.fact.input.kind') IN ('message', 'continuation', 'handoff'))" ).bind(&event.invocation.invocation_id).fetch_one(&mut *tx).await?; if !eligible { return Err(StoreError::InvalidTransition( @@ -77,7 +77,7 @@ impl EventLog { .run(move |tx| { Box::pin(async move { let row: Option<(i64, String)> = sqlx::query_as( - "SELECT e.sequence, e.event_json FROM message_sources s JOIN runtime_events e ON e.event_id = s.event_id + "SELECT e.sequence, e.event_header FROM message_sources s JOIN runtime_events e ON e.event_id = s.event_id WHERE e.kind = 'message_steered' AND s.session_id = ? AND s.message_id = ?", ) .bind(&session_id) @@ -121,11 +121,11 @@ pub(crate) async fn project_catalog( ) -> Result<(), StoreError> { sqlx::query( "INSERT OR IGNORE INTO catalog_messages - SELECT sequence, 0, ?2, catalog_time(json_extract(event_json, '$.recorded_at')), + SELECT sequence, 0, ?2, catalog_time(json_extract(event_header, '$.recorded_at')), catalog_preview(COALESCE( - json_extract(event_json, '$.fact.message.content.display_text'), - json_extract(event_json, '$.fact.message.content.text'))), - json_extract(event_json, '$.fact.message.message_id') + json_extract(event_header, '$.fact.message.content.display_text'), + json_extract(event_header, '$.fact.message.content.text'))), + json_extract(event_header, '$.fact.message.message_id') FROM runtime_events WHERE sequence = ?1 AND kind = 'message_steered'", ) .bind(sequence) diff --git a/platform/event-log/src/tool_calls.rs b/platform/event-log/src/tool_calls.rs index 89bcca42ba..1869876731 100644 --- a/platform/event-log/src/tool_calls.rs +++ b/platform/event-log/src/tool_calls.rs @@ -50,7 +50,7 @@ impl crate::EventLog { let name = name.to_owned(); let input = input.clone(); self.connection.read(move |db|Box::pin(async move { - let (sequence,json):(i64,String)=sqlx::query_as("SELECT sequence,event_json FROM runtime_events WHERE invocation_id=? AND operation_id=? AND kind='tool_dispatched'").bind(&context.invocation.invocation_id).bind(&context.operation_id).fetch_optional(db).await?.ok_or_else(||invalid("service confirmation requires durable dispatch"))?; + let (sequence,json):(i64,String)=sqlx::query_as("SELECT sequence,event_header FROM runtime_events WHERE invocation_id=? AND operation_id=? AND kind='tool_dispatched'").bind(&context.invocation.invocation_id).bind(&context.operation_id).fetch_optional(db).await?.ok_or_else(||invalid("service confirmation requires durable dispatch"))?; let event:RuntimeEvent=serde_json::from_str(&json)?; if event.invocation!=context.invocation || !matches!(&event.fact,Fact::ToolDispatched{name:original,input:original_input,..} if *original==name && *original_input==input) {return Err(invalid("service confirmation differs from durable dispatch"))} Ok(maka_agent_facts::native_approval::DispatchReceipt{event_id:event.id,sequence:crate::sequence_number(sequence)?}) @@ -70,7 +70,7 @@ impl crate::EventLog { .run(move |connection| { Box::pin(async move { let (sequence, encoded): (i64, String) = sqlx::query_as( - "SELECT sequence, event_json FROM runtime_events + "SELECT sequence, event_header FROM runtime_events WHERE invocation_id = ? AND operation_id = ? AND kind = 'tool_dispatched'", ) .bind(&context.invocation.invocation_id) @@ -127,7 +127,7 @@ pub(crate) async fn validate( } let live: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM runtime_events d WHERE d.invocation_id=? AND d.operation_id=? AND d.kind='tool_dispatched' - AND json_extract(d.event_json,'$.fact.call.origin.kind')='code_cell' + AND json_extract(d.event_header,'$.fact.call.origin.kind')='code_cell' AND NOT EXISTS(SELECT 1 FROM runtime_events s WHERE s.invocation_id=d.invocation_id AND s.operation_id=d.operation_id AND s.kind='tool_settled'))") .bind(invocation).bind(operation_id).fetch_one(&mut *tx).await?; @@ -164,14 +164,14 @@ pub(crate) async fn validate( WHERE operation_id=?1 AND kind IN ('tool_dispatched','tool_rejected')) OR EXISTS(SELECT 1 FROM runtime_events WHERE invocation_id=?2 AND kind IN ('tool_dispatched','tool_rejected') - AND json_extract(event_json,'$.fact.call.tool_call_id')=?3 - AND CASE WHEN json_extract(event_json,'$.fact.call.origin.kind')='provider' - THEN json_extract(event_json,'$.fact.call.origin.step_id') END IS NULL) + AND json_extract(event_header,'$.fact.call.tool_call_id')=?3 + AND CASE WHEN json_extract(event_header,'$.fact.call.origin.kind')='provider' + THEN json_extract(event_header,'$.fact.call.origin.step_id') END IS NULL) OR EXISTS(SELECT 1 FROM runtime_events WHERE invocation_id=?2 AND kind IN ('tool_dispatched','tool_rejected') - AND json_extract(event_json,'$.fact.call.tool_call_id')=?3 - AND CASE WHEN json_extract(event_json,'$.fact.call.origin.kind')='provider' - THEN json_extract(event_json,'$.fact.call.origin.step_id') END=?4)", + AND json_extract(event_header,'$.fact.call.tool_call_id')=?3 + AND CASE WHEN json_extract(event_header,'$.fact.call.origin.kind')='provider' + THEN json_extract(event_header,'$.fact.call.origin.step_id') END=?4)", ) .bind(operation_id) .bind(invocation) @@ -185,7 +185,7 @@ pub(crate) async fn validate( WHERE operation_id=?1 AND kind IN ('tool_dispatched','tool_rejected')) OR EXISTS(SELECT 1 FROM runtime_events WHERE invocation_id=?2 AND kind IN ('tool_dispatched','tool_rejected') - AND json_extract(event_json,'$.fact.call.tool_call_id')=?3)", + AND json_extract(event_header,'$.fact.call.tool_call_id')=?3)", ) .bind(operation_id) .bind(invocation) @@ -247,7 +247,7 @@ pub(crate) async fn validate( "SELECT EXISTS(SELECT 1 FROM runtime_events AS parent WHERE parent.invocation_id = ? AND parent.operation_id = ? AND parent.kind = 'tool_dispatched' - AND json_extract(parent.event_json, '$.fact.call.tool_call_id') = ? + AND json_extract(parent.event_header, '$.fact.call.tool_call_id') = ? AND NOT EXISTS(SELECT 1 FROM runtime_events AS settled WHERE settled.invocation_id = parent.invocation_id AND settled.operation_id = parent.operation_id @@ -300,11 +300,11 @@ pub(crate) async fn validate( "SELECT EXISTS(SELECT 1 FROM context_obligations live CROSS JOIN runtime_events child ON child.sequence=live.opened_at WHERE live.invocation_id = ?1 AND live.kind='tool' - AND ((json_extract(child.event_json, '$.fact.call.origin.kind') = 'code_mode' - AND json_extract(child.event_json, '$.fact.call.origin.parent_operation_id') = ?2) - OR (json_extract(child.event_json, '$.fact.call.origin.kind') = 'host_sdk' - AND json_extract(child.event_json, '$.fact.call.origin.parent_relationship.kind') IS NOT 'evaluation_observer' - AND json_extract(child.event_json, '$.fact.call.origin.parent_operation_id') = ?2)))", + AND ((json_extract(child.event_header, '$.fact.call.origin.kind') = 'code_mode' + AND json_extract(child.event_header, '$.fact.call.origin.parent_operation_id') = ?2) + OR (json_extract(child.event_header, '$.fact.call.origin.kind') = 'host_sdk' + AND json_extract(child.event_header, '$.fact.call.origin.parent_relationship.kind') IS NOT 'evaluation_observer' + AND json_extract(child.event_header, '$.fact.call.origin.parent_operation_id') = ?2)))", ).bind(invocation).bind(operation_id).fetch_one(&mut *tx).await?; if pending { return Err(invalid("parent tool cannot settle before its children")); diff --git a/platform/event-log/src/tool_payloads.rs b/platform/event-log/src/tool_payloads.rs index d66aa7e99e..93bdb0ad9e 100644 --- a/platform/event-log/src/tool_payloads.rs +++ b/platform/event-log/src/tool_payloads.rs @@ -138,8 +138,14 @@ pub(crate) async fn verify_replay( Ok(()) } -const RAW_META: &str = r#"SELECT e.event_json, length(p.payload) FROM runtime_events e - LEFT JOIN tool_result_payloads p ON p.event_id = e.event_id WHERE e.event_id = ?"#; +fn raw_meta() -> String { + format!( + "SELECT e.event_header, length(p.payload) AS payload_bytes,{} FROM runtime_events e {} + LEFT JOIN tool_result_payloads p ON p.event_id=e.event_id WHERE e.event_id=?", + crate::canonical::COLUMNS, + crate::canonical::JOINS + ) +} const RAW_BYTES: &str = r#"SELECT payload FROM tool_result_payloads WHERE event_id = ? AND length(payload) = ?"#; @@ -155,14 +161,19 @@ pub(crate) fn resolve_in_transaction( else { return Err(invalid()); }; - let row: Option<(String, Option)> = db - .prepare_cached(RAW_META)? - .query_row([&event.id], |row| Ok((row.get(0)?, row.get(1)?))) + let row = db + .prepare_cached(&raw_meta())? + .query_row([&event.id], |row| { + Ok(( + crate::canonical::json_on_handle(row).map(|json| json.into_owned()), + row.get::<_, Option>("payload_bytes")?, + )) + }) .optional()?; let Some((canonical, bytes)) = row else { return Err(invalid()); }; - if serde_json::from_str::(&canonical)? != *event { + if serde_json::from_str::(&canonical?)? != *event { return Err(invalid()); } verify_binding(event, bytes)?; @@ -186,12 +197,13 @@ impl EventLog { .run(move |connection| { Box::pin(async move { let mut tx = connection.begin().await?; - let canonical: Option = sqlx::query_scalar( - "SELECT event_json FROM runtime_events WHERE event_id = ? AND kind = 'tool_settled' - AND json_extract(event_json, '$.invocation.session_id') = ?", - ).bind(event_id).bind(session_id).fetch_optional(&mut *tx).await?; - let event = - serde_json::from_str::(&canonical.ok_or_else(invalid)?)?; + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.event_header,{} FROM runtime_events e {} WHERE e.event_id=? AND e.kind='tool_settled' + AND json_extract(e.event_header,'$.invocation.session_id')=?", + crate::canonical::COLUMNS, crate::canonical::JOINS + ))).bind(event_id).bind(session_id).fetch_optional(&mut *tx).await? + .ok_or_else(invalid)?; + let event = serde_json::from_str::(&crate::canonical::json(&row)?)?; let output = tx .run_on_worker(move |handle| { // SAFETY: exclusive callback-local non-owning connection; diff --git a/platform/event-log/src/transcript/active.rs b/platform/event-log/src/transcript/active.rs index 71e6508015..3ac49a8baf 100644 --- a/platform/event-log/src/transcript/active.rs +++ b/platform/event-log/src/transcript/active.rs @@ -74,7 +74,7 @@ impl EventLog { "SELECT EXISTS(SELECT 1 FROM runtime_events WHERE invocation_id = ?1 AND sequence <= ?2 AND kind = 'invocation_opened' - AND json_extract(event_json, '$.fact.input.kind') = 'context_compact')", + AND json_extract(event_header, '$.fact.input.kind') = 'context_compact')", ) .bind(id) .bind(fence) diff --git a/platform/event-log/src/transcript/compaction.rs b/platform/event-log/src/transcript/compaction.rs index 9c7f46f255..837bffd635 100644 --- a/platform/event-log/src/transcript/compaction.rs +++ b/platform/event-log/src/transcript/compaction.rs @@ -21,24 +21,24 @@ use crate::{StoreError, sequence_number}; use maka_presentation::{Content, Message, Row}; use rusqlite::Connection as RawConnection; -const COMPACTION: &str = r#"SELECT c.event_id, json_extract(c.event_json, '$.invocation.turn_id') AS turn_id, - catalog_time(json_extract(c.event_json, '$.recorded_at')) AS adopted_at, - CASE WHEN json_extract(opening.event_json, '$.fact.input.kind') = 'context_compact' - THEN catalog_time(json_extract(opening.event_json, '$.recorded_at')) - ELSE (SELECT catalog_time(json_extract(r.event_json, '$.recorded_at')) +const COMPACTION: &str = r#"SELECT c.event_id, json_extract(c.event_header, '$.invocation.turn_id') AS turn_id, + catalog_time(json_extract(c.event_header, '$.recorded_at')) AS adopted_at, + CASE WHEN json_extract(opening.event_header, '$.fact.input.kind') = 'context_compact' + THEN catalog_time(json_extract(opening.event_header, '$.recorded_at')) + ELSE (SELECT catalog_time(json_extract(r.event_header, '$.recorded_at')) FROM event_log r INDEXED BY context_summary_round - WHERE r.event_json IS NOT NULL AND r.invocation_id = c.invocation_id AND r.kind = 'model_requested' - AND json_extract(r.event_json, '$.fact.purpose') = 'summary' + WHERE r.event_header IS NOT NULL AND r.invocation_id = c.invocation_id AND r.kind = 'model_requested' + AND json_extract(r.event_header, '$.fact.purpose') = 'summary' AND r.sequence <= adopted.sequence - AND json_extract(r.event_json, '$.fact.source_high_water') = json_extract(adopted.event_json, '$.fact.source_high_water') - AND json_extract(r.event_json, '$.fact.source_digest') = json_extract(adopted.event_json, '$.fact.source_digest') + AND json_extract(r.event_header, '$.fact.source_high_water') = json_extract(adopted.event_header, '$.fact.source_high_water') + AND json_extract(r.event_header, '$.fact.source_digest') = json_extract(adopted.event_header, '$.fact.source_digest') AND NOT EXISTS (SELECT 1 FROM runtime_events boundary WHERE boundary.invocation_id = c.invocation_id AND boundary.sequence > r.sequence AND boundary.sequence < c.sequence AND boundary.kind = 'context_checkpoint_recorded') ORDER BY r.sequence LIMIT 1) END AS started_at FROM runtime_events c JOIN runtime_events adopted ON adopted.invocation_id = c.invocation_id AND adopted.kind = 'model_requested' - AND adopted.operation_id = json_extract(c.event_json, '$.fact.checkpoint.summary_step_id') + AND adopted.operation_id = json_extract(c.event_header, '$.fact.checkpoint.summary_step_id') LEFT JOIN runtime_events opening ON opening.invocation_id = c.invocation_id AND opening.kind = 'invocation_opened' WHERE c.sequence = ? AND c.kind = 'context_checkpoint_recorded'"#; diff --git a/platform/event-log/src/transcript/directory.rs b/platform/event-log/src/transcript/directory.rs index 02cff8586f..5b83beb317 100644 --- a/platform/event-log/src/transcript/directory.rs +++ b/platform/event-log/src/transcript/directory.rs @@ -107,7 +107,7 @@ fn ensure_for( } // Canonical membership retains copied material. A bare imported owner can // lose its own material without a session retirement row. - let unavailable: bool = db.prepare_cached("SELECT EXISTS(SELECT 1 FROM event_log INDEXED BY event_retired_material_owner WHERE event_session=?1 AND sequence<=?2 AND event_json IS NULL AND invocation_id IS NOT NULL)")? + let unavailable: bool = db.prepare_cached("SELECT EXISTS(SELECT 1 FROM event_log INDEXED BY event_retired_material_owner WHERE event_session=?1 AND sequence<=?2 AND event_header IS NULL AND invocation_id IS NOT NULL)")? .query_row(params![session,through], |row| row.get(0))?; if unavailable { return Err(StoreError::InvalidTransition( diff --git a/platform/event-log/src/transcript/emit.rs b/platform/event-log/src/transcript/emit.rs index 90767b5b6f..a95348ed90 100644 --- a/platform/event-log/src/transcript/emit.rs +++ b/platform/event-log/src/transcript/emit.rs @@ -28,31 +28,31 @@ use sqlx::sqlite::SqliteWorkerConnection; pub(super) const ELIGIBLE: &str = r#"(e.kind = 'context_checkpoint_recorded' OR NOT EXISTS (SELECT 1 FROM runtime_events opening WHERE opening.invocation_id = e.invocation_id AND opening.kind = 'invocation_opened' - AND json_extract(opening.event_json, '$.fact.input.kind') = 'context_compact')) + AND json_extract(opening.event_header, '$.fact.input.kind') = 'context_compact')) AND (e.kind NOT IN ('model_completed', 'model_interrupted') OR EXISTS (SELECT 1 FROM runtime_events request JOIN runtime_events opening ON opening.invocation_id = request.invocation_id AND opening.kind = 'invocation_opened' WHERE request.kind = 'model_requested' AND request.invocation_id = e.invocation_id AND request.operation_id = e.operation_id - AND json_extract(opening.event_json, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') - AND json_extract(request.event_json, '$.fact.purpose') = 'main')) + AND json_extract(opening.event_header, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') + AND json_extract(request.event_header, '$.fact.purpose') = 'main')) AND (e.kind IN ('message_imported', 'invocation_opened', 'message_steered', 'model_completed', 'model_interrupted', 'invocation_ended', 'context_checkpoint_recorded', 'tool_dispatched', 'tool_rejected', 'tool_settled', 'executor_completed') OR (e.kind = 'model_observed' - AND json_extract(e.event_json, '$.fact.event.kind') = 'part' - AND json_type(e.event_json, '$.fact.step_id') = 'text' + AND json_extract(e.event_header, '$.fact.event.kind') = 'part' + AND json_type(e.event_header, '$.fact.step_id') = 'text' AND EXISTS (SELECT 1 FROM runtime_events request JOIN runtime_events opening ON opening.invocation_id = request.invocation_id AND opening.kind = 'invocation_opened' WHERE request.kind = 'model_requested' AND request.invocation_id = e.invocation_id AND request.sequence < e.sequence - AND request.operation_id = CAST(json_extract(e.event_json, '$.fact.step_id') AS TEXT) - AND json_extract(e.event_json, '$.fact.step_id') = request.operation_id - AND json_extract(request.event_json, '$.fact.purpose') = 'main' - AND json_extract(request.event_json, '$.fact.item_acceptance') = 1 - AND json_extract(opening.event_json, '$.fact.input.kind') IN ('message','continuation','handoff'))) - OR (e.kind = 'executor_observed' AND json_extract(e.event_json, '$.fact.output.type') IN ('tool_start','tool_result')))"#; + AND request.operation_id = CAST(json_extract(e.event_header, '$.fact.step_id') AS TEXT) + AND json_extract(e.event_header, '$.fact.step_id') = request.operation_id + AND json_extract(request.event_header, '$.fact.purpose') = 'main' + AND json_extract(request.event_header, '$.fact.item_acceptance') = 1 + AND json_extract(opening.event_header, '$.fact.input.kind') IN ('message','continuation','handoff'))) + OR (e.kind = 'executor_observed' AND json_extract(e.event_header, '$.fact.output.type') IN ('tool_start','tool_result')))"#; pub(super) async fn source_on_worker( connection: &mut SqliteWorkerConnection<'_>, @@ -94,7 +94,7 @@ pub(super) async fn source_on_worker( fn candidate_sql() -> String { format!( - "SELECT e.invocation_id, CASE WHEN e.kind = 'model_observed' THEN json_extract(e.event_json, '$.fact.step_id') ELSE e.operation_id END, e.kind FROM runtime_events e WHERE e.sequence = ?1 AND (e.event_session = ?2 OR EXISTS(SELECT 1 FROM session_history_members h WHERE h.session_id = ?2 AND h.sequence = e.sequence)) AND {ELIGIBLE}" + "SELECT e.invocation_id, CASE WHEN e.kind = 'model_observed' THEN json_extract(e.event_header, '$.fact.step_id') ELSE e.operation_id END, e.kind FROM runtime_events e WHERE e.sequence = ?1 AND (e.event_session = ?2 OR EXISTS(SELECT 1 FROM session_history_members h WHERE h.session_id = ?2 AND h.sequence = e.sequence)) AND {ELIGIBLE}" ) } diff --git a/platform/event-log/src/transcript/evidence.rs b/platform/event-log/src/transcript/evidence.rs index 18218ca68b..6f76cc9506 100644 --- a/platform/event-log/src/transcript/evidence.rs +++ b/platform/event-log/src/transcript/evidence.rs @@ -37,8 +37,8 @@ const EXEC_BOUNDARY: &str = r#"SELECT sequence FROM runtime_events SELECT sequence FROM runtime_events WHERE invocation_id = ?1 AND sequence <= ?2 AND kind IN ('executor_observed','executor_completed') - AND json_extract(event_json, '$.fact.output.toolCallId') = - (SELECT json_extract(event_json, '$.fact.output.toolCallId') FROM runtime_events WHERE sequence = ?2) + AND json_extract(event_header, '$.fact.output.toolCallId') = + (SELECT json_extract(event_header, '$.fact.output.toolCallId') FROM runtime_events WHERE sequence = ?2) ORDER BY sequence"#; const EXEC_REST: &str = r#"SELECT sequence FROM runtime_events WHERE invocation_id = ?1 AND sequence <= ?2 AND ( @@ -46,8 +46,8 @@ const EXEC_REST: &str = r#"SELECT sequence FROM runtime_events OR (?3 AND sequence = ?2) OR (kind IN ('executor_observed','executor_completed') AND ( NOT ?3 OR NOT EXISTS(SELECT 1 FROM runtime_events boundary WHERE boundary.sequence = ?2 AND boundary.kind = 'executor_observed') - OR json_extract(event_json, '$.fact.output.toolCallId') = - (SELECT json_extract(event_json, '$.fact.output.toolCallId') FROM runtime_events WHERE sequence = ?2)))) + OR json_extract(event_header, '$.fact.output.toolCallId') = + (SELECT json_extract(event_header, '$.fact.output.toolCallId') FROM runtime_events WHERE sequence = ?2)))) ORDER BY sequence"#; const STEP: &str = r#"SELECT sequence FROM runtime_events WHERE invocation_id = ?1 AND sequence <= ?3 AND kind = 'invocation_opened' @@ -67,16 +67,16 @@ UNION SELECT sequence FROM runtime_events WHERE invocation_id = ?1 AND sequence <= ?3 AND ?2 IS NOT NULL AND kind = 'model_observed' - AND CAST(json_extract(event_json, '$.fact.step_id') AS TEXT) = ?2 - AND json_extract(event_json, '$.fact.step_id') = ?2 + AND CAST(json_extract(event_header, '$.fact.step_id') AS TEXT) = ?2 + AND json_extract(event_header, '$.fact.step_id') = ?2 ORDER BY sequence"#; -const FACT: &str = r#"SELECT event_json FROM runtime_events WHERE sequence = ?"#; + const PENDING: &str = r#"SELECT operation_id FROM runtime_events AS request WHERE invocation_id = ?1 AND sequence <= ?2 AND kind = 'model_requested' AND EXISTS(SELECT 1 FROM runtime_events opening WHERE opening.invocation_id = request.invocation_id AND opening.kind = 'invocation_opened' - AND json_extract(opening.event_json, '$.fact.input.kind') IN ('message', 'continuation', 'handoff')) - AND json_extract(request.event_json, '$.fact.purpose') = 'main' + AND json_extract(opening.event_header, '$.fact.input.kind') IN ('message', 'continuation', 'handoff')) + AND json_extract(request.event_header, '$.fact.purpose') = 'main' AND NOT EXISTS(SELECT 1 FROM runtime_events AS boundary WHERE boundary.invocation_id = request.invocation_id AND boundary.operation_id = request.operation_id @@ -188,18 +188,27 @@ pub(super) async fn fact( tx: &mut SqliteConnection, sequence: i64, ) -> Result { - let json: String = sqlx::query_scalar(FACT) - .bind(sequence) - .fetch_one(tx) - .await?; - decoded_fact(sequence, &json) + let row = sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT e.event_header,{columns} FROM runtime_events e {joins} WHERE e.sequence=?", + columns = crate::canonical::COLUMNS, + joins = crate::canonical::JOINS + ))) + .bind(sequence) + .fetch_one(tx) + .await?; + decoded_fact(sequence, &crate::canonical::json(&row)?) } pub(super) fn fact_on_handle(db: &RawConnection, sequence: i64) -> Result { - let json: String = db - .prepare_cached(FACT)? - .query_row([sequence], |row| row.get(0))?; - decoded_fact(sequence, &json) + let query = format!( + "SELECT e.event_header,{columns} FROM runtime_events e {joins} WHERE e.sequence=?", + columns = crate::canonical::COLUMNS, + joins = crate::canonical::JOINS + ); + db.prepare_cached(&query)?.query_row([sequence], |row| { + // Decode failures keep their storage cause after the raw SQLite callback. + Ok(crate::canonical::json_on_handle(row).and_then(|json| decoded_fact(sequence, &json))) + })? } fn decoded_fact(sequence: i64, json: &str) -> Result { diff --git a/platform/event-log/src/transcript/final_output.rs b/platform/event-log/src/transcript/final_output.rs index ea3615cc78..f2ceed7b58 100644 --- a/platform/event-log/src/transcript/final_output.rs +++ b/platform/event-log/src/transcript/final_output.rs @@ -32,18 +32,18 @@ const TERMINAL: &str = "WITH latest AS ( ) SELECT e.invocation_id, e.sequence FROM latest JOIN runtime_events e ON e.sequence = latest.source_sequence WHERE latest.state_status = 'completed' AND e.kind = 'invocation_ended' - AND json_extract(e.event_json, '$.fact.outcome.kind') = 'completed' + AND json_extract(e.event_header, '$.fact.outcome.kind') = 'completed' AND (e.event_session = ?1 OR EXISTS( SELECT 1 FROM session_history_members h WHERE h.session_id = ?1 AND h.sequence = e.sequence))"; const REQUESTED: &str = "SELECT e.operation_id FROM runtime_events e WHERE e.invocation_id = ?2 AND e.kind = 'model_requested' AND e.sequence <= ?3 - AND json_extract(e.event_json, '$.fact.purpose') = 'main' + AND json_extract(e.event_header, '$.fact.purpose') = 'main' AND (e.event_session = ?1 OR EXISTS( SELECT 1 FROM session_history_members h WHERE h.session_id = ?1 AND h.sequence = e.sequence)) ORDER BY e.sequence DESC LIMIT 1"; const COMPLETED: &str = "SELECT EXISTS(SELECT 1 FROM runtime_events e WHERE e.operation_id = ?3 AND e.kind = 'model_completed' AND e.invocation_id = ?2 - AND e.sequence <= ?4 AND json_extract(e.event_json, '$.fact.step_id') = ?3 + AND e.sequence <= ?4 AND json_extract(e.event_header, '$.fact.step_id') = ?3 AND (e.event_session = ?1 OR EXISTS( SELECT 1 FROM session_history_members h WHERE h.session_id = ?1 AND h.sequence = e.sequence)))"; const EXECUTOR: &str = "SELECT e.sequence FROM runtime_events e diff --git a/platform/event-log/src/transcript/model.rs b/platform/event-log/src/transcript/model.rs index 45493aca31..f5462052cd 100644 --- a/platform/event-log/src/transcript/model.rs +++ b/platform/event-log/src/transcript/model.rs @@ -41,27 +41,27 @@ const PRODUCER: &str = "SELECT e.invocation_id, e.kind, e.operation_id FROM runtime_events e WHERE e.sequence = ?2 AND e.sequence <= ?3 AND (e.event_session = ?1 OR EXISTS( SELECT 1 FROM session_history_members h WHERE h.session_id = ?1 AND h.sequence = e.sequence))"; -const REQUESTED: &str = "SELECT json_extract(e.event_json, '$.fact.model_id') +const REQUESTED: &str = "SELECT json_extract(e.event_header, '$.fact.model_id') FROM runtime_events e WHERE e.operation_id = ?3 AND e.kind = 'model_requested' AND e.invocation_id = ?2 AND e.sequence <= ?4 - AND json_extract(e.event_json, '$.fact.step_id') = ?3 + AND json_extract(e.event_header, '$.fact.step_id') = ?3 AND (e.event_session = ?1 OR EXISTS( SELECT 1 FROM session_history_members h WHERE h.session_id = ?1 AND h.sequence = e.sequence))"; -const OBSERVED: &str = "SELECT e.sequence, json_extract(e.event_json, '$.fact.event.data.model') +const OBSERVED: &str = "SELECT e.sequence, json_extract(e.event_header, '$.fact.event.data.model') FROM runtime_events e WHERE e.invocation_id = ?2 - AND CAST(json_extract(e.event_json, '$.fact.step_id') AS TEXT) = ?3 - AND json_extract(e.event_json, '$.fact.step_id') = ?3 - AND e.sequence <= ?4 AND e.kind = 'model_observed' AND e.event_json IS NOT NULL - AND json_extract(e.event_json, '$.fact.event.kind') = 'response_metadata' - AND json_type(e.event_json, '$.fact.event.data.model') = 'text' + AND CAST(json_extract(e.event_header, '$.fact.step_id') AS TEXT) = ?3 + AND json_extract(e.event_header, '$.fact.step_id') = ?3 + AND e.sequence <= ?4 AND e.kind = 'model_observed' AND e.event_header IS NOT NULL + AND json_extract(e.event_header, '$.fact.event.kind') = 'response_metadata' + AND json_type(e.event_header, '$.fact.event.data.model') = 'text' AND (e.event_session = ?1 OR EXISTS( SELECT 1 FROM session_history_members h WHERE h.session_id = ?1 AND h.sequence = e.sequence)) ORDER BY e.sequence DESC LIMIT 1"; -const COMPLETED: &str = "SELECT e.sequence, json_extract(e.event_json, '$.fact.output.model') +const COMPLETED: &str = "SELECT e.sequence, json_extract(e.event_header, '$.fact.output.model') FROM runtime_events e WHERE e.operation_id = ?3 AND e.kind = 'model_completed' AND e.invocation_id = ?2 AND e.sequence <= ?4 - AND json_extract(e.event_json, '$.fact.step_id') = ?3 - AND json_type(e.event_json, '$.fact.output.model') = 'text' + AND json_extract(e.event_header, '$.fact.step_id') = ?3 + AND json_type(e.event_header, '$.fact.output.model') = 'text' AND (e.event_session = ?1 OR EXISTS( SELECT 1 FROM session_history_members h WHERE h.session_id = ?1 AND h.sequence = e.sequence))"; const EXECUTOR: &str = "SELECT EXISTS(SELECT 1 FROM runtime_events e @@ -100,8 +100,8 @@ pub(super) async fn resolve_on_worker( let kind: String = producer.try_get(1)?; if kind == "message_imported" { let (content, model): (String, Option) = sqlx::query_as( - "SELECT json_extract(event_json, '$.fact.record.content.kind'), - json_extract(event_json, '$.fact.record.content.model') + "SELECT json_extract(event_header, '$.fact.record.content.kind'), + json_extract(event_header, '$.fact.record.content.model') FROM runtime_events WHERE sequence = ?", ) .bind(producer_sequence) @@ -138,7 +138,7 @@ pub(super) async fn resolve_on_worker( // later public fence that may contain unrelated model steps. super::evidence::pending_step_on_worker(tx, &invocation, producer_sequence).await? } else if kind == "model_observed" { - sqlx::query_scalar("SELECT json_extract(event_json, '$.fact.step_id') FROM runtime_events WHERE sequence = ?") + sqlx::query_scalar("SELECT json_extract(event_header, '$.fact.step_id') FROM runtime_events WHERE sequence = ?") .bind(producer_sequence).fetch_one(&mut *tx).await? } else if matches!(kind.as_str(), "model_completed" | "model_interrupted") { producer.try_get(2)? diff --git a/platform/event-log/src/transcript/navigation/queries.rs b/platform/event-log/src/transcript/navigation/queries.rs index 9c6b166fbb..0b7fa5cd8a 100644 --- a/platform/event-log/src/transcript/navigation/queries.rs +++ b/platform/event-log/src/transcript/navigation/queries.rs @@ -21,14 +21,14 @@ pub(super) const FENCE: &str = " SELECT MAX(sequence) FROM ( SELECT MAX(event.sequence) AS sequence FROM runtime_events event JOIN runtime_events opening ON opening.invocation_id = event.invocation_id AND opening.kind = 'invocation_opened' - WHERE (event.event_json IS NOT NULL AND event.event_session=?1) - AND json_extract(opening.event_json, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') + WHERE (event.event_header IS NOT NULL AND event.event_session=?1) + AND json_extract(opening.event_header, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') UNION ALL SELECT MAX(h.sequence) FROM session_history_members h JOIN runtime_events event ON event.sequence=h.sequence JOIN runtime_events opening ON opening.invocation_id = event.invocation_id AND opening.kind = 'invocation_opened' WHERE h.session_id = ?1 - AND json_extract(opening.event_json, '$.fact.input.kind') IN ('message', 'continuation', 'handoff'))"; + AND json_extract(opening.event_header, '$.fact.input.kind') IN ('message', 'continuation', 'handoff'))"; pub(super) const ROWS: &str = " SELECT row.source_sequence,row.item_index,row.turn_id,row.role,source.kind @@ -39,16 +39,16 @@ ORDER BY row.source_sequence,row.item_index LIMIT 257"; pub(super) const LANDMARKS: &str = " WITH sources AS ( - SELECT opening.invocation_id,opening.sequence,json_extract(opening.event_json,'$.invocation.turn_id') AS turn_id + SELECT opening.invocation_id,opening.sequence,json_extract(opening.event_header,'$.invocation.turn_id') AS turn_id FROM runtime_events opening - WHERE json_extract(opening.event_json,'$.invocation.session_id')=?1 + WHERE json_extract(opening.event_header,'$.invocation.session_id')=?1 AND opening.kind='invocation_opened' AND opening.sequence<=?2 - AND json_extract(opening.event_json,'$.fact.input.kind') IN ('message','continuation') + AND json_extract(opening.event_header,'$.fact.input.kind') IN ('message','continuation') UNION ALL - SELECT opening.invocation_id,opening.sequence,json_extract(opening.event_json,'$.invocation.turn_id') + SELECT opening.invocation_id,opening.sequence,json_extract(opening.event_header,'$.invocation.turn_id') FROM session_history_members h JOIN runtime_events opening ON opening.sequence=h.sequence WHERE h.session_id=?1 AND h.sequence<=?2 AND opening.kind='invocation_opened' - AND json_extract(opening.event_json,'$.fact.input.kind') IN ('message','continuation') + AND json_extract(opening.event_header,'$.fact.input.kind') IN ('message','continuation') ), openings AS ( SELECT invocation_id,sequence,ROW_NUMBER() OVER (PARTITION BY turn_id ORDER BY sequence) AS first FROM sources diff --git a/platform/event-log/src/transcript/tools.rs b/platform/event-log/src/transcript/tools.rs index 05bf70771d..ddf84d7088 100644 --- a/platform/event-log/src/transcript/tools.rs +++ b/platform/event-log/src/transcript/tools.rs @@ -35,8 +35,8 @@ const TOOL_MAIN: &str = r#"SELECT EXISTS(SELECT 1 FROM runtime_events request AND opening.kind = 'invocation_opened' WHERE request.invocation_id = ?1 AND request.operation_id = ?2 AND request.kind = 'model_requested' - AND json_extract(opening.event_json, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') - AND json_extract(request.event_json, '$.fact.purpose') = 'main')"#; + AND json_extract(opening.event_header, '$.fact.input.kind') IN ('message', 'continuation', 'handoff') + AND json_extract(request.event_header, '$.fact.purpose') = 'main')"#; /// Reconstruct one tool boundary, its actual dispatch ancestry and accepted /// provider step. Exclude sibling outcomes and unrelated raw payloads. diff --git a/platform/event-log/src/turns.rs b/platform/event-log/src/turns.rs index b052f04eec..d82ff24493 100644 --- a/platform/event-log/src/turns.rs +++ b/platform/event-log/src/turns.rs @@ -102,9 +102,9 @@ impl EventLog { .read(move |connection| { Box::pin(async move { let row: Option<(String, Option, Option)> = sqlx::query_as( - "SELECT json_extract(event_json, '$.invocation'), - json_extract(event_json, '$.fact.input.claim.source'), - json_extract(event_json, '$.fact.input.request_fingerprint') + "SELECT json_extract(event_header, '$.invocation'), + json_extract(event_header, '$.fact.input.claim.source'), + json_extract(event_header, '$.fact.input.request_fingerprint') FROM runtime_events WHERE invocation_id = ? AND kind = 'invocation_opened' AND (? IS NULL OR event_session = ?)", ) @@ -142,9 +142,9 @@ impl EventLog { Box::pin(async move { let mut tx = connection.begin().await?; let opening: Option = sqlx::query_scalar( - "SELECT event_json FROM runtime_events WHERE invocation_id = ?1 + "SELECT event_header FROM runtime_events WHERE invocation_id = ?1 AND kind = 'invocation_opened' - AND json_extract(event_json, '$.invocation.session_id') = ?2", + AND json_extract(event_header, '$.invocation.session_id') = ?2", ) .bind(invocation) .bind(session) @@ -196,11 +196,11 @@ impl EventLog { let invocation = invocation.clone(); self.connection.read(move |connection| Box::pin(async move { let configuration: Option = sqlx::query_scalar( - "SELECT json_extract(event_json, '$.fact.configuration') FROM runtime_events + "SELECT json_extract(event_header, '$.fact.configuration') FROM runtime_events WHERE invocation_id = ?1 AND kind = 'invocation_opened' - AND json_extract(event_json, '$.invocation.session_id') = ?2 - AND json_extract(event_json, '$.invocation.run_id') = ?3 - AND json_extract(event_json, '$.invocation.turn_id') = ?4" + AND json_extract(event_header, '$.invocation.session_id') = ?2 + AND json_extract(event_header, '$.invocation.run_id') = ?3 + AND json_extract(event_header, '$.invocation.turn_id') = ?4" ).bind(&invocation.invocation_id).bind(&invocation.session_id).bind(&invocation.run_id).bind(&invocation.turn_id) .fetch_optional(connection).await?.flatten(); configuration.map(|value| serde_json::from_str(&value).map_err(StoreError::from)).transpose() @@ -225,7 +225,7 @@ impl EventLog { return Ok(None); }; let opening: String = sqlx::query_scalar( - "SELECT event_json FROM runtime_events WHERE invocation_id = ? AND kind = 'invocation_opened'", + "SELECT event_header FROM runtime_events WHERE invocation_id = ? AND kind = 'invocation_opened'", ).bind(invocation).fetch_one(&mut *tx).await?; let result = project(&mut tx, serde_json::from_str(&opening)?).await?; tx.commit().await?; @@ -267,10 +267,10 @@ pub(crate) async fn read( turn_id: Option<&str>, ) -> Result, StoreError> { let opening: Option = sqlx::query_scalar( - "SELECT event_json FROM runtime_events + "SELECT event_header FROM runtime_events WHERE kind = 'invocation_opened' - AND json_extract(event_json, '$.invocation.session_id') = ?1 - AND (?2 IS NULL OR json_extract(event_json, '$.invocation.turn_id') = ?2) + AND json_extract(event_header, '$.invocation.session_id') = ?1 + AND (?2 IS NULL OR json_extract(event_header, '$.invocation.turn_id') = ?2) ORDER BY sequence DESC LIMIT 1", ) .bind(session_id) @@ -293,8 +293,8 @@ pub(crate) async fn run_invocation( ) -> Result, StoreError> { let ids: Vec = sqlx::query_scalar( "SELECT invocation_id FROM runtime_events WHERE kind = 'invocation_opened' - AND json_extract(event_json, '$.invocation.session_id') = ? - AND json_extract(event_json, '$.invocation.run_id') = ? LIMIT 2", + AND json_extract(event_header, '$.invocation.session_id') = ? + AND json_extract(event_header, '$.invocation.run_id') = ? LIMIT 2", ) .bind(session) .bind(run) @@ -319,7 +319,7 @@ pub(crate) async fn project( }; let root = root::read(connection, &opening.invocation, &input).await?; let terminal: Option = sqlx::query_scalar( - "SELECT event_json FROM runtime_events WHERE invocation_id = ? AND kind = 'invocation_ended'", + "SELECT event_header FROM runtime_events WHERE invocation_id = ? AND kind = 'invocation_ended'", ).bind(&opening.invocation.invocation_id).fetch_optional(&mut *connection).await?; let state = if let Some(terminal) = terminal { let terminal: RuntimeEvent = serde_json::from_str(&terminal)?; diff --git a/platform/event-log/src/turns/root.rs b/platform/event-log/src/turns/root.rs index 32044759dc..cb8b4a2094 100644 --- a/platform/event-log/src/turns/root.rs +++ b/platform/event-log/src/turns/root.rs @@ -42,12 +42,12 @@ pub(super) async fn read( // The canonical seal already authenticated this root across every handoff. // Resolve provenance once, without replaying model/tool bodies or guessing latest. let rows: Vec = sqlx::query_scalar( - "SELECT event_json FROM event_log INDEXED BY turn_opening_lookup - WHERE invocation_id IS NOT NULL AND event_json IS NOT NULL + "SELECT event_header FROM event_log INDEXED BY turn_opening_lookup + WHERE invocation_id IS NOT NULL AND event_header IS NOT NULL AND kind='invocation_opened' - AND json_extract(event_json,'$.invocation.session_id')=?1 - AND json_extract(event_json,'$.invocation.turn_id')=?2 - AND json_extract(event_json,'$.invocation.run_id')=?3 LIMIT 2", + AND json_extract(event_header,'$.invocation.session_id')=?1 + AND json_extract(event_header,'$.invocation.turn_id')=?2 + AND json_extract(event_header,'$.invocation.run_id')=?3 LIMIT 2", ) .bind(¤t.session_id) .bind(¤t.turn_id) diff --git a/platform/event-log/src/usage/auxiliary.rs b/platform/event-log/src/usage/auxiliary.rs index a14dd2f0b3..e086e516da 100644 --- a/platform/event-log/src/usage/auxiliary.rs +++ b/platform/event-log/src/usage/auxiliary.rs @@ -145,7 +145,7 @@ impl EventLog { if local_session { crate::sessions::removal::require_accepting(&mut tx, &session).await?; } let used: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM event_log WHERE kind='auxiliary_model_started' - AND json_extract(event_json, '$.source') = json(?))" + AND json_extract(event_header, '$.source') = json(?))" ).bind(serde_json::to_string(&source)?).fetch_one(&mut *tx).await?; if used { return Err(invalid("Application operation already attempted its auxiliary model")); } (binding, Some(session)) @@ -248,7 +248,7 @@ impl EventLog { "SELECT event_id FROM event_log started WHERE kind = 'auxiliary_model_started' AND NOT EXISTS ( SELECT 1 FROM event_log settled WHERE settled.kind = 'auxiliary_model_settled' - AND json_extract(settled.event_json, '$.request_id') = started.event_id) + AND json_extract(settled.event_header, '$.request_id') = started.event_id) ORDER BY sequence LIMIT 128" ).fetch_all(&mut *tx).await?; let mut last = None; @@ -289,8 +289,8 @@ async fn binding( } let used: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM event_log WHERE kind='auxiliary_model_started' - AND json_extract(event_json,'$.source.kind')='session_title' - AND json_extract(event_json,'$.source.invocation.session_id')=?)", + AND json_extract(event_header,'$.source.kind')='session_title' + AND json_extract(event_header,'$.source.invocation.session_id')=?)", ) .bind(&invocation.session_id) .fetch_one(&mut *connection) @@ -308,12 +308,12 @@ async fn binding( operation_id, } => { let raw: Option = sqlx::query_scalar( - "SELECT opening.event_json FROM runtime_events opening + "SELECT opening.event_header FROM runtime_events opening JOIN runtime_events dispatch ON dispatch.invocation_id = opening.invocation_id WHERE opening.kind = 'invocation_opened' AND opening.invocation_id = ?1 AND dispatch.kind = 'tool_dispatched' AND dispatch.operation_id = ?2 - AND json_extract(dispatch.event_json, '$.fact.name') = 'llm.generate' - AND json_extract(dispatch.event_json, '$.fact.call.origin.kind') = 'host_sdk' + AND json_extract(dispatch.event_header, '$.fact.name') = 'llm.generate' + AND json_extract(dispatch.event_header, '$.fact.call.origin.kind') = 'host_sdk' AND NOT EXISTS (SELECT 1 FROM runtime_events settled WHERE (settled.kind = 'tool_settled' AND settled.operation_id = ?2) OR (settled.kind = 'invocation_ended' AND settled.invocation_id = ?1))", @@ -371,11 +371,11 @@ async fn require_pending( // Membership and pending status are checked together, in the same transaction // that observes/settles this private source admission. let session: Option> = sqlx::query_scalar( - "SELECT json_extract(event_json, '$.session_id') FROM event_log + "SELECT json_extract(event_header, '$.session_id') FROM event_log WHERE event_id = ?1 AND kind = 'auxiliary_model_started' - AND (?2 IS NULL OR json_extract(event_json, '$.session_id') = ?2) + AND (?2 IS NULL OR json_extract(event_header, '$.session_id') = ?2) AND NOT EXISTS(SELECT 1 FROM event_log WHERE kind = 'auxiliary_model_settled' - AND json_extract(event_json, '$.request_id') = ?1)", + AND json_extract(event_header, '$.request_id') = ?1)", ) .bind(id.to_string()) .bind(expected_session) diff --git a/platform/event-log/src/workspace.rs b/platform/event-log/src/workspace.rs index a458501f74..82fb9131f7 100644 --- a/platform/event-log/src/workspace.rs +++ b/platform/event-log/src/workspace.rs @@ -71,7 +71,7 @@ impl EventLog { "SELECT s.id, s.revision, b.metadata_epoch,json_extract(s.configuration,'$.workspace.metadataEpochAtAttach') AS attached_epoch, COALESCE((SELECT MAX(e.sequence) FROM runtime_events e WHERE - (e.event_json IS NOT NULL AND e.event_session=s.id)), 0) AS sequence, + (e.event_header IS NOT NULL AND e.event_session=s.id)), 0) AS sequence, EXISTS(SELECT 1 FROM shell_runs r WHERE r.session_id = s.id AND json_extract(r.record_json, '$.state.outcome.kind') = 'orphaned') AS orphaned FROM workspace_bindings b diff --git a/platform/event-log/tests/integration/archive/mod.rs b/platform/event-log/tests/integration/archive/mod.rs index df42b7da22..306902e2be 100644 --- a/platform/event-log/tests/integration/archive/mod.rs +++ b/platform/event-log/tests/integration/archive/mod.rs @@ -321,7 +321,13 @@ async fn archive_publication_is_atomic_and_preserves_original_source_integrity() .unwrap(), expected ); - inspect.execute("UPDATE event_log SET event_json=replace(event_json,'xxxxxxxx','yyyyyyyy') WHERE event_id=?", [&target.event().id]).unwrap(); + // Deliberately corrupt immutable physical material outside its publisher. + inspect + .execute_batch("DROP TRIGGER event_body_immutable") + .unwrap(); + inspect.execute("UPDATE event_bodies SET payload=CAST(replace(CAST(payload AS TEXT),'xxxxxxxx','yyyyyyyy') AS BLOB) WHERE id=( + SELECT s.body_id FROM event_body_slots s JOIN event_log e ON e.sequence=s.event_sequence WHERE e.event_id=?)", + [&target.event().id]).unwrap(); assert!( log.read_tool_result("session", &target.event().id) .await diff --git a/platform/event-log/tests/integration/bundle/import.rs b/platform/event-log/tests/integration/bundle/import.rs index f440eef340..4cbbfcfd70 100644 --- a/platform/event-log/tests/integration/bundle/import.rs +++ b/platform/event-log/tests/integration/bundle/import.rs @@ -256,7 +256,7 @@ pub(crate) async fn roundtrip(bytes: &[u8]) { let inspect = rusqlite::Connection::open(&collision_path).unwrap(); inspect .execute_batch( - "UPDATE event_log SET retained_json=event_json,body_digest='fixture',event_json=NULL", + "DELETE FROM event_body_slots; UPDATE event_log SET retained_json=event_header,body_digest='fixture',event_header=NULL,canonical_bytes=NULL", ) .unwrap(); let collision = EventLog::open(&collision_path).await.unwrap(); diff --git a/platform/event-log/tests/integration/context_compaction/active/latest.rs b/platform/event-log/tests/integration/context_compaction/active/latest.rs index 21e2fcdddc..3326b48ff7 100644 --- a/platform/event-log/tests/integration/context_compaction/active/latest.rs +++ b/platform/event-log/tests/integration/context_compaction/active/latest.rs @@ -137,7 +137,7 @@ async fn preturn_keeps_anchor_and_latest_main_never_falls_back_a_broken_newer_tr Some(write.event().id.as_str()) ); let inspect = rusqlite::Connection::open(&path).unwrap(); - inspect.execute("UPDATE event_log SET event_json=json_remove(event_json, '$.fact.context') WHERE operation_id='new-main' AND kind='model_requested'", []).unwrap(); + inspect.execute("UPDATE event_log SET event_header=json_remove(event_header, '$.fact.context') WHERE operation_id='new-main' AND kind='model_requested'", []).unwrap(); assert!( matches!(log.latest_main_context("session").await.unwrap(), LatestMainContext::Selected(value) if value.context.is_none() && value.finished_at == Some(finished_at)) ); diff --git a/platform/event-log/tests/integration/continuation/handoff_cancel.rs b/platform/event-log/tests/integration/continuation/handoff_cancel.rs index 4f9824f7da..2d569c079e 100644 --- a/platform/event-log/tests/integration/continuation/handoff_cancel.rs +++ b/platform/event-log/tests/integration/continuation/handoff_cancel.rs @@ -139,7 +139,7 @@ async fn paused_cancellation_claim_is_atomic_idempotent_and_never_cancels_an_exi let database = rusqlite::Connection::open(&path).unwrap(); database.execute_batch( "CREATE TRIGGER reject_cancel BEFORE INSERT ON event_log - WHEN NEW.kind='invocation_ended' AND json_extract(NEW.event_json,'$.fact.outcome.kind')='cancelled' + WHEN NEW.kind='invocation_ended' AND json_extract(NEW.event_header,'$.fact.outcome.kind')='cancelled' BEGIN SELECT RAISE(ABORT, 'cancel fault'); END;" ).unwrap(); let before = log.prefix(100, 8 * 1024 * 1024).await.unwrap(); diff --git a/platform/event-log/tests/integration/continuation/mod.rs b/platform/event-log/tests/integration/continuation/mod.rs index 88a599cc79..9ff93f47d0 100644 --- a/platform/event-log/tests/integration/continuation/mod.rs +++ b/platform/event-log/tests/integration/continuation/mod.rs @@ -153,7 +153,7 @@ async fn canonical_claim_is_atomic_unique_and_preserves_the_entire_lineage_after let db = rusqlite::Connection::open(&path).unwrap(); db.execute_batch( "CREATE TRIGGER reject_continuation BEFORE INSERT ON event_log - WHEN json_extract(NEW.event_json,'$.fact.input.kind')='continuation' + WHEN json_extract(NEW.event_header,'$.fact.input.kind')='continuation' BEGIN SELECT RAISE(ABORT,'injected opening failure'); END;", ) .unwrap(); diff --git a/platform/event-log/tests/integration/import_atomic.rs b/platform/event-log/tests/integration/import_atomic.rs index 1e40a43d7a..5a77d0a19c 100644 --- a/platform/event-log/tests/integration/import_atomic.rs +++ b/platform/event-log/tests/integration/import_atomic.rs @@ -222,9 +222,9 @@ async fn second_record_sql_failure_rolls_back_the_complete_import_and_allows_ret .execute_batch( "CREATE TRIGGER fail_second_import_record BEFORE INSERT ON event_log WHEN NEW.kind='message_imported' - AND json_extract(NEW.event_json,'$.fact.record.sourceMessageId')='source-1' + AND json_extract(NEW.event_header,'$.fact.record.sourceMessageId')='source-1' AND EXISTS(SELECT 1 FROM event_log WHERE kind='message_imported' - AND json_extract(event_json,'$.fact.record.sourceMessageId')='source-0') + AND json_extract(event_header,'$.fact.record.sourceMessageId')='source-0') BEGIN SELECT RAISE(ABORT,'atomic second record fault'); END;", ) .unwrap(); @@ -356,7 +356,7 @@ fn duplicate_calls() -> Vec { fn import_state(db: &rusqlite::Connection) -> (Vec<(i64, Vec)>, String) { let events = db - .prepare("SELECT sequence,CAST(event_json AS BLOB) FROM event_log ORDER BY sequence") + .prepare("SELECT sequence,CAST(event_header AS BLOB) FROM event_log ORDER BY sequence") .unwrap() .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) .unwrap() @@ -489,7 +489,7 @@ async fn retained_orphan_physical_proof_still_blocks_complete_import() { let inspect = rusqlite::Connection::open(&path).unwrap(); inspect .execute_batch( - "UPDATE event_log SET retained_json=event_json,body_digest='fixture',event_json=NULL", + "DELETE FROM event_body_slots; UPDATE event_log SET retained_json=event_header,body_digest='fixture',event_header=NULL,canonical_bytes=NULL", ) .unwrap(); let log = EventLog::open(&path).await.unwrap(); diff --git a/platform/event-log/tests/integration/session_material.rs b/platform/event-log/tests/integration/session_material.rs index 0baf7f6e38..05796d106b 100644 --- a/platform/event-log/tests/integration/session_material.rs +++ b/platform/event-log/tests/integration/session_material.rs @@ -220,7 +220,7 @@ async fn collection_waits_for_the_last_history_owner_and_preserves_frozen_accoun let inspect = rusqlite::Connection::open(&path).unwrap(); let (bodies, leaked, digest): (i64, i64, String) = inspect .query_row( - "SELECT (SELECT COUNT(*) FROM event_log WHERE event_json IS NOT NULL), + "SELECT (SELECT COUNT(*) FROM event_log WHERE event_header IS NOT NULL), (SELECT COUNT(*) FROM event_log WHERE retained_json LIKE '%private-%'), (SELECT body_digest FROM event_log WHERE event_id=?1)", [&opening.event().id], @@ -270,3 +270,114 @@ async fn retire(log: &EventLog, session: &str) { SessionRetirement::Removed ); } + +#[tokio::test] +async fn shared_tool_projection_survives_collection_of_one_canonical_owner() { + use maka_agent_facts::{ + event::{EventWrite, Invocation, RuntimeEvent}, + tool_call::ToolCallIdentity, + tool_output::{DurableToolProjection, ToolOutput, ToolSuccess}, + }; + use maka_event_log::sessions::MaterialCollection; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("shared.sqlite"); + let log = EventLog::open(&path).await.unwrap(); + let text = "shared projection 汉字\\n".repeat(4000); + for id in ["first", "second"] { + log.create_session(id, &format!("create-{id}"), &json!({}), 1) + .await + .unwrap(); + let invocation = Invocation { + session_id: id.into(), + turn_id: id.into(), + run_id: id.into(), + invocation_id: id.into(), + }; + let plain = |fact| EventWrite::plain(RuntimeEvent::new(invocation.clone(), fact)).unwrap(); + log.append(&plain(Fact::InvocationOpened { + configuration: None, + input: InvocationInput::Code { + source: "fixture".into(), + }, + })) + .await + .unwrap(); + let operation = format!("{id}-operation"); + log.append(&plain(Fact::ToolDispatched { + presentation: None, + title: None, + operation_id: operation.clone(), + call: ToolCallIdentity::standalone(format!("{id}-call")), + name: "Read".into(), + input: json!({}), + })) + .await + .unwrap(); + let result = EventWrite::tool_success( + format!("{id}-result"), + std::time::UNIX_EPOCH, + invocation.clone(), + operation, + ToolSuccess::projected( + ToolOutput::Text(text.clone()), + DurableToolProjection::Text { text: text.clone() }, + ), + ) + .unwrap() + .0; + log.append(&result).await.unwrap(); + log.append(&result).await.unwrap(); + log.append(&plain(Fact::InvocationEnded { + outcome: InvocationOutcome::Completed, + })) + .await + .unwrap(); + } + let inspect = rusqlite::Connection::open(&path).unwrap(); + let counts = || { + inspect.query_row("SELECT (SELECT COUNT(*) FROM event_bodies),(SELECT COUNT(*) FROM event_body_slots)", [], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?))).unwrap() + }; + assert_eq!(counts(), (1, 2)); + let stored_bytes: i64 = inspect + .query_row("SELECT SUM(length(payload)) FROM event_bodies", [], |row| { + row.get(0) + }) + .unwrap(); + let projection_bytes = serde_json::to_vec(&DurableToolProjection::Text { text: text.clone() }) + .unwrap() + .len(); + assert_eq!( + stored_bytes as usize, projection_bytes, + "two canonical owners store one exact raw projection token" + ); + let scope = LogScope::Session { + id: "second".into(), + }; + let before = log + .scoped_prefix(scope.clone(), 100, 4 * 1024 * 1024) + .await + .unwrap(); + retire(&log, "first").await; + while log.collect_session_material(None).await.unwrap() == MaterialCollection::Collected {} + assert_eq!(counts(), (1, 1)); + assert_eq!( + log.resolve_tool_result("second", "second-result") + .await + .unwrap(), + ToolOutput::Text(text.clone()) + ); + log.close().await.unwrap(); + let log = EventLog::open(&path).await.unwrap(); + let after = log + .scoped_prefix(scope, 100, 4 * 1024 * 1024) + .await + .unwrap(); + assert_eq!( + serde_json::to_vec(&before).unwrap(), + serde_json::to_vec(&after).unwrap() + ); + retire(&log, "second").await; + while log.collect_session_material(None).await.unwrap() == MaterialCollection::Collected {} + assert_eq!(counts(), (0, 0)); + log.close().await.unwrap(); +} diff --git a/platform/event-log/tests/integration/startup.rs b/platform/event-log/tests/integration/startup.rs index a51d9ea9fe..a7111adcb8 100644 --- a/platform/event-log/tests/integration/startup.rs +++ b/platform/event-log/tests/integration/startup.rs @@ -298,7 +298,7 @@ async fn initialization_and_high_water_failures_close_the_actual_sql_owner_befor // Deliberately corrupt canonical storage outside RuntimeEvent ingress. // Current projection ownership is application-level; no append UDF // trigger must be disabled or restored around this fault fixture. - raw.execute_batch("PRAGMA ignore_check_constraints=ON; INSERT INTO event_log(sequence,event_id,kind,event_json) VALUES(-1,'invalid','diagnostic','{}');").unwrap(); + raw.execute_batch("PRAGMA ignore_check_constraints=ON; INSERT INTO event_log(sequence,event_id,kind,event_header,canonical_bytes) VALUES(-1,'invalid','diagnostic','{}',2);").unwrap(); } else { raw.execute_batch("PRAGMA application_id=123;").unwrap(); } diff --git a/plugins/agent/native/src/owner/tests.rs b/plugins/agent/native/src/owner/tests.rs index e149629ad0..fe06f06129 100644 --- a/plugins/agent/native/src/owner/tests.rs +++ b/plugins/agent/native/src/owner/tests.rs @@ -434,7 +434,7 @@ impl Fixture { ) .unwrap(); let (count, bytes): (i64, i64) = database.query_row( - "SELECT COUNT(*), COALESCE(SUM(length(CAST(event_json AS BLOB))),0) FROM runtime_events WHERE event_session=?", + "SELECT COUNT(*), COALESCE(SUM(canonical_bytes),0) FROM runtime_events WHERE event_session=?", [session], |row| Ok((row.get(0)?, row.get(1)?)), ).unwrap(); assert!( @@ -443,7 +443,7 @@ impl Fixture { ); let mut statement = database .prepare( - "SELECT event_json FROM runtime_events WHERE event_session=? ORDER BY sequence", + "SELECT event_header FROM runtime_events WHERE event_session=? ORDER BY sequence", ) .unwrap(); statement diff --git a/plugins/agent/native/src/owner/tests/prompt_language.rs b/plugins/agent/native/src/owner/tests/prompt_language.rs index bf0864dc14..66985bfc15 100644 --- a/plugins/agent/native/src/owner/tests/prompt_language.rs +++ b/plugins/agent/native/src/owner/tests/prompt_language.rs @@ -108,7 +108,7 @@ fn stored_configuration(fixture: &Fixture, invocation: &str) -> Value { ) .unwrap(); let raw: String = connection.query_row( - "SELECT json_extract(event_json, '$.fact.configuration') FROM runtime_events WHERE invocation_id=?1 AND kind='invocation_opened'", + "SELECT json_extract(event_header, '$.fact.configuration') FROM runtime_events WHERE invocation_id=?1 AND kind='invocation_opened'", [invocation], |row| row.get(0), ).unwrap(); serde_json::from_str(&raw).unwrap() diff --git a/plugins/agent/native/src/owner/tests/title_tests.rs b/plugins/agent/native/src/owner/tests/title_tests.rs index 5370083bd7..40966b127d 100644 --- a/plugins/agent/native/src/owner/tests/title_tests.rs +++ b/plugins/agent/native/src/owner/tests/title_tests.rs @@ -260,7 +260,7 @@ async fn independently_admitted_title_uses_current_language_after_english_main_c rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, ).unwrap(); let recorded_language: String = raw.query_row( - "SELECT json_extract(event_json, '$.language') FROM event_log WHERE kind='auxiliary_model_started' AND json_extract(event_json, '$.source.kind')='session_title' AND json_extract(event_json, '$.source.invocation.session_id')=?1", + "SELECT json_extract(event_header, '$.language') FROM event_log WHERE kind='auxiliary_model_started' AND json_extract(event_header, '$.source.kind')='session_title' AND json_extract(event_header, '$.source.invocation.session_id')=?1", [&session], |row| row.get(0), ).unwrap(); drop(raw); diff --git a/scripts/bench-event-storage.py b/scripts/bench-event-storage.py index 9269854530..fffcc276e4 100644 --- a/scripts/bench-event-storage.py +++ b/scripts/bench-event-storage.py @@ -60,14 +60,15 @@ def storage_stats(directory): page_size=db.execute("PRAGMA page_size").fetchone()[0], pages=db.execute("PRAGMA page_count").fetchone()[0], free_pages=db.execute("PRAGMA freelist_count").fetchone()[0], - directories=directories)) + directories=directories, + bodies=(dict(zip(("records","raw_bytes","encoded_bytes"), db.execute("SELECT COUNT(*),COALESCE(SUM(raw_bytes),0),COALESCE(SUM(length(payload)),0) FROM event_bodies").fetchone())) if "event_bodies" in tables else None))) return result -def run(binary, directory, mode, events, body_bytes, iterations): +def run(binary, directory, mode, events, body_bytes, iterations, body_pattern): command = [str(binary), "--database", str(directory), "--mode", mode, "--events", str(events), "--body-bytes", str(body_bytes), - "--iterations", str(iterations)] + "--iterations", str(iterations), "--body-pattern", body_pattern] before = {"load": os.getloadavg(), "free_bytes": shutil.disk_usage(directory.parent).free} start = time.monotonic_ns() result = subprocess.run(command, check=True, text=True, capture_output=True) @@ -87,6 +88,7 @@ def main(): parser.add_argument("--output", type=Path, required=True) parser.add_argument("--runs", type=int, default=5) parser.add_argument("--iterations", type=int, default=20) + parser.add_argument("--body-pattern", choices=("repeated", "entropy"), default="repeated") parser.add_argument("--case", action="append", metavar="NAME:EVENTS:BODY_BYTES") args = parser.parse_args() args.output = args.output.resolve() @@ -105,7 +107,7 @@ def main(): binaries[label] = dict(path=str(binary), sha256=hashlib.file_digest(stream, "sha256").hexdigest()) (args.output / "manifest.json").write_text(json.dumps(dict( binaries=binaries, platform=platform.platform(), cpu_count=os.cpu_count(), - runs=args.runs, iterations=args.iterations, cases=cases, + runs=args.runs, iterations=args.iterations, cases=cases, body_pattern=args.body_pattern, cache_state="one validation/warmup pass; steady state and OS-warm reopen"), indent=2) + "\n") summaries = {} fixtures = {} @@ -117,7 +119,7 @@ def main(): directory = args.output / f"{case}-{repeat}-{label}" for mode in ("create", "read"): report = run(binary, directory, mode, int(event_count), - int(body_size), args.iterations) + int(body_size), args.iterations, args.body_pattern) fixture = report["fixture"] previous = fixtures.setdefault(case, fixture) if fixture != previous: From 541c7eb6fc1542dea2e8a64a9f8bd18cccddf367 Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Sat, 31 Oct 2026 18:41:19 +0800 Subject: [PATCH 3/8] refactor(storage): publish body references with canonical events Keep one body reference on each event row, enforce atomic collection, and measure prefix API time separately from complete result validation. Generated-by: Codex --- platform/event-log/README.md | 6 +- platform/event-log/README.zh-CN.md | 6 +- .../event-log/examples/event_storage_bench.rs | 86 ++++++++++++------- ...es.sql => 0014_canonical_event_bodies.sql} | 77 +++++++++-------- platform/event-log/src/append/ingress.rs | 12 ++- platform/event-log/src/archive/candidates.rs | 3 +- platform/event-log/src/archive/target.rs | 4 +- platform/event-log/src/canonical.rs | 2 +- platform/event-log/src/canonical/migration.rs | 44 ++++++++-- platform/event-log/src/canonical/read.rs | 45 +++++----- platform/event-log/src/canonical/source.rs | 25 +++--- platform/event-log/src/canonical/write.rs | 20 ++--- platform/event-log/src/canonical_insert.rs | 15 +++- platform/event-log/src/sessions/material.rs | 5 +- .../tests/integration/archive/mod.rs | 2 +- .../tests/integration/bundle/import.rs | 2 +- .../context_compaction/active/latest.rs | 3 +- .../tests/integration/import_atomic.rs | 2 +- .../tests/integration/session_material.rs | 2 +- .../event-log/tests/integration/startup.rs | 2 +- scripts/bench-event-storage.py | 10 ++- 21 files changed, 226 insertions(+), 147 deletions(-) rename platform/event-log/migrations/{0013_canonical_event_bodies.sql => 0014_canonical_event_bodies.sql} (81%) diff --git a/platform/event-log/README.md b/platform/event-log/README.md index dba66b3ca3..96c3222e31 100644 --- a/platform/event-log/README.md +++ b/platform/event-log/README.md @@ -69,16 +69,18 @@ New runtime events use UUIDv7 identities to keep event and derived message index Canonical runtime-event ingress inserts each event, captures its sequence, then maintains safety, execution, source and semantic projections as separate statements on the same SQLite worker and writer transaction. The shared insert statement derives indexed accounting and owner fields from those same JSON bytes; collection preserves their published values when removing retired bodies. Typed facts select the safety and execution statements to run. The next event cannot begin before those projections finish. Projection storage failures roll back the entire batch, including imported events; notifications remain tied to successful commit. Copied-prefix projections are maintained by their current copy paths. -Canonical databases use UTF-8. Successful tool results store their original `model_projection` token separately: `event_header` preserves every surrounding byte with a literal `null` placeholder, and an immutable slot records the original byte offset and body identity. Reconstructing the header and original token reproduces the exact canonical event, including imported whitespace, number spelling and Unicode escapes. `canonical_bytes` supplies logical byte budgets; encoded size never substitutes for admitted canonical size. Other event kinds remain inline. +Canonical databases use UTF-8. Successful tool results store their original `model_projection` token separately: `event_header` preserves every surrounding byte with a literal `null` placeholder, and the event row records the original byte offset and immutable body reference. Reconstructing the header and original token reproduces the exact canonical event, including imported whitespace, number spelling and Unicode escapes. `canonical_bytes` supplies logical byte budgets; encoded size never substitutes for admitted canonical size. Other event kinds remain inline. Bodies are directly seekable raw bytes, shared by SHA-256 of their original token with exact byte comparison on a repeated identity. Reads validate logical lengths and selected ranges. Framed compression was evaluated against raw storage and rejected because it repeatedly slowed supported window and sequential reads; it is not a persisted format. Space savings come from compact directories and sharing identical projection tokens, not from compressing unique bodies. -Complete-event queries join the slot and body into the selected SQL row, then reconstruct in Rust within the original snapshot. Metadata predicates use the header and retain their indexes. Semantic spans retain original canonical coordinates; tool spans map those logical coordinates to header/body ranges, while inline model parts retain physical SQLite BLOB bounds. Resolution stays on the original SQLite worker. Hot range reads use its existing prepared-statement cache; bounded prefix decoding borrows rows on that same worker without copying every nullable column across the callback. Slots follow the existing history-owner collection proof; deleting the final slot removes its body in the same transaction. Neither a body digest nor a derived directory grants access or extends lifetime. Derived archive placeholders and other fragment bodies keep their chunked storage. +Complete-event queries join the referenced body into the selected SQL row, then reconstruct in Rust within the original snapshot. Metadata predicates use the header and retain their indexes. Semantic spans retain original canonical coordinates; tool spans map those logical coordinates to header/body ranges, while inline model parts retain physical SQLite BLOB bounds. Resolution stays on the original SQLite worker. Hot range reads use its existing prepared-statement cache; bounded prefix decoding borrows rows on that same worker without copying every nullable column across the callback. Body references follow the existing history-owner collection proof; clearing or deleting the final reference removes its body in the same transaction. The event row publishes the header, logical length, offset and reference atomically; live references cannot be redirected. Neither a body digest nor a derived directory grants access or extends lifetime. Derived archive placeholders and other fragment bodies keep their chunked storage. Semantic row, payload and child directories are finite derived descriptors encoded with Postcard. Writers encode them once in the publication transaction. Pages copy their stored bytes alongside fixed little-endian locator fields into the caller's charged allocation; consumers decode the original typed descriptors without navigating whole bodies. Original byte positions and leaf commitments remain unchanged. Directory encoding never supplies a canonical fact or source identity. The SQL migration converts former JSON descriptors atomically; its conversion functions exist only during migration. `event_storage_bench` exercises real Root writes, captured directories, child pages, first/tail and repeated 64-byte windows, sequential bodies, event prefixes and bundle export. Build the same example in release against both revisions and retain each binary, then run `python3 scripts/bench-event-storage.py --output /tmp/storage-comparison original=/path/to/original candidate=/path/to/candidate`. It alternates at least five runs and validates event, descriptor, body and export digests. The first read pass validates and warms caches; reopened measurements are OS-cache-warm. Raw distributions, process CPU/RSS/I/O availability, SQLite cache counters and separate main/WAL/SHM file sizes are recorded. Live prepared-statement snapshots do not represent total SQL work, and JSON/text/content leaf reads and settlement writes are also reported separately. Use `--body-pattern entropy` to include deterministic low-compressibility printable input. +`prefix_read` stops when the prefix API returns; `prefix_all` also includes reserialization and full result digest verification. Keep both when interpreting a difference. `--operation prefix` restricts measured iterations to consecutive prefix reads while retaining the first complete validation/warmup pass and every prefix oracle; use the same option in both binaries. This is an OS-warm focused workload, distinct from the default mixed-operation loop. + A statement failure rolls back the entire group unless it is a domain rejection isolated by its own savepoint. Former ordinary writer requests can continue recovery on the same owner after acknowledged rollback and fresh authority validation; pure observation batches retain their stricter retirement policy. Failed commits always return unknown outcomes without execution claims. Reusing a former ordinary writer group after a failed commit additionally requires acknowledged cleanup and zero transaction depth. Savepoint, authority and unconfirmed cleanup failures retire the writer. Single-key and paged plugin-data reads use the independent reader owner and see committed snapshots; dependent reads must still await their preceding write receipt. Saved application inspection uses `RootOwner::open_existing` and `EventLog::for_application_inspection`. It requires existing private coordination state and the original exclusive Root/database leases, reads committed application facts including retained WAL content, and closes without changing stored members or bytes. It performs no migration or derived initialization; missing state fails without repair. Only application queries on the owned connection belong to this opening. CLI application read and default-update preview share it; applying an update opens a fresh ordinary writer and recomputes the change. diff --git a/platform/event-log/README.zh-CN.md b/platform/event-log/README.zh-CN.md index c0768b2c00..d79fdf5b37 100644 --- a/platform/event-log/README.zh-CN.md +++ b/platform/event-log/README.zh-CN.md @@ -63,16 +63,18 @@ Control 中的调用意图、项目登记、workspace attachment、Session 路 规范运行时事件入口逐条插入事件并取得序号，再在同一 SQLite worker、同一写事务内，用独立语句维护安全、执行、来源和语义投影。用于索引的记账和所有者字段由统一插入语句从同一 JSON 计算，正文回收后保留其发布值。安全与执行维护由类型化事实选择相应语句。当前事件投影完成后才开始下一条。投影存储错误使整个批次回滚，导入事件同样如此；通知仍只在提交成功后发布。复制前缀投影由当前复制路径维护。 -规范数据库使用 UTF-8。成功工具结果的原始 `model_projection` token 单独保存：`event_header` 用字面量 `null` 占位，保留周围全部原始字节；不可变 slot 记录原字节偏移和正文身份。两者重建出精确的规范事件，包括导入内容的空白、数字拼写和 Unicode 转义。`canonical_bytes` 用于逻辑字节预算，压缩大小不替代已接受的规范大小。其他事件类型仍内联保存。 +规范数据库使用 UTF-8。成功工具结果的原始 `model_projection` token 单独保存：`event_header` 用字面量 `null` 占位，保留周围全部原始字节；事件行记录原字节偏移和不可变正文引用。两者重建出精确的规范事件，包括导入内容的空白、数字拼写和 Unicode 转义。`canonical_bytes` 用于逻辑字节预算，压缩大小不替代已接受的规范大小。其他事件类型仍内联保存。 正文保存为可直接 seek 的原始字节，按原始 token 的 SHA-256 共享，重复身份还需逐字节核对。读取校验逻辑长度与选中区间。对照实验中，分帧压缩稳定拖慢了受支持的窗口和顺序读取，因此未作为持久化格式保留。空间收益来自紧凑目录与相同投影 token 的共享，不能把压缩实验中独有正文的压缩率套到最终实现。 -完整事件查询在同一 SQL 行 JOIN slot 和正文，在原快照内由 Rust 重建；元数据谓词继续读取 header 并使用原索引。语义区间保留原规范坐标，工具区间映射至实际 header／body，内联 model part 则保留 SQLite BLOB 的物理边界。读取仍由原 SQLite worker 执行。热区间读取复用其已有 statement cache；有界前缀在同一 worker 上借用行并解码，避免把每个可空列复制出回调。Slot 存活由既有历史所有者回收证明决定，最后一个 slot 删除时，在同一事务删除正文；正文摘要和派生目录都不授予访问权，也不延长存活期。归档占位内容和其他片段正文继续使用分块存储。 +完整事件查询在同一 SQL 行 JOIN 引用的正文，在原快照内由 Rust 重建；元数据谓词继续读取 header 并使用原索引。语义区间保留原规范坐标，工具区间映射至实际 header／body，内联 model part 则保留 SQLite BLOB 的物理边界。读取仍由原 SQLite worker 执行。热区间读取复用其已有 statement cache；有界前缀在同一 worker 上借用行并解码，避免把每个可空列复制出回调。正文引用的存活由既有历史所有者回收证明决定，最后一个引用清除或删除时，在同一事务删除正文。事件行原子发布 header、逻辑长度、偏移和引用，存活引用不可重定向；正文摘要和派生目录都不授予访问权，也不延长存活期。归档占位内容和其他片段正文继续使用分块存储。 语义行、payload 和子项目录是有限的派生描述符，采用 Postcard 编码。writer 在发布事务内编码一次；分页把已存字节和固定小端序定位字段直接复制到调用者计费的缓冲区，消费端恢复原有类型，无需读取整个正文。原始字节位置与叶子摘要保持不变，目录编码不参与规范事实或来源身份。SQL 迁移在同一事务内转换原 JSON 描述符；转换函数仅在迁移期间存在。 `event_storage_bench` 测量真实 Root 写入、冻结目录、子项分页、首尾及重复 64 字节窗口、顺序正文、事件前缀和 bundle 导出。在两个版本上用 release 构建同一 example 并各自保存二进制后，运行 `python3 scripts/bench-event-storage.py --output /tmp/storage-comparison original=/path/to/original candidate=/path/to/candidate`。脚本交错执行至少五轮，校验事件、目录、正文与导出摘要。首次读取用于校验和预热；重开测量保留操作系统缓存。记录保留原始分布、进程 CPU/RSS/I/O 可用性、SQLite 缓存计数，以及分列的主库、WAL、SHM 大小。存活 prepared statement 的快照不能代表全部 SQL 工作；另外分别记录 JSON／文本／多段内容的叶子读取和结算写入耗时。 +`prefix_read` 在前缀 API 返回时停止计时；`prefix_all` 还包含重新序列化及完整结果摘要验证，解释差异时保留两者。`--operation prefix` 仅将计量迭代切换为连续前缀读取，首次完整校验／预热及每次前缀校验仍然执行；两个二进制必须使用相同选项。这是保留操作系统缓存的定向工作负载，与默认混合操作循环分别报告。 + 语句失败会回滚整组，只有已由自身保存点隔离的领域拒绝例外。原先使用普通写入路径的请求，在确认回滚并重新校验所有权后，可以继续通过同一 owner 恢复；纯 observation 批次保留更严格的退出策略。提交失败始终返回结果未知，不产生执行凭据；此后复用原普通写入组，还要求确认清理完成且事务深度为零。保存点、所有权或无法确认的清理失败都会让写入器退出。单键与分页插件数据读取使用独立 reader，读取已提交的快照；依赖写入的读取仍需先等待相应回执。 已保存应用的检查使用 `RootOwner::open_existing` 和 `EventLog::for_application_inspection`，要求已有的私有协调状态，并持有原有的独占 Root／数据库租约。检查读取已提交的应用事实，包括保留在 WAL 中的内容，关闭后不改变存储成员或字节；不执行迁移或派生初始化，缺失状态直接报错，不做修复。该打开方式仅供原连接上的应用查询使用。CLI 的 application read 与默认更新预览共用此入口；实际应用更新时重新打开普通写入所有者，并重新计算变更。 diff --git a/platform/event-log/examples/event_storage_bench.rs b/platform/event-log/examples/event_storage_bench.rs index cd3a459a59..e50fd77693 100644 --- a/platform/event-log/examples/event_storage_bench.rs +++ b/platform/event-log/examples/event_storage_bench.rs @@ -285,6 +285,49 @@ impl Samples { output.into() } } +async fn record_prefix( + session: &SessionStore, + manifest: &serde_json::Value, + samples: &mut Samples, + measured: bool, +) -> Result<()> { + let start = Instant::now(); + let prefix = session + .scoped_prefix( + LogScope::Session { id: "bench".into() }, + usize::MAX, + usize::MAX, + ) + .await?; + if measured { + samples.add("prefix_read", start); + } + if prefix.digest + != manifest["prefix_digest"] + .as_str() + .ok_or("missing prefix digest")? + { + return Err("event proof changed".into()); + } + if prefix.events.len() + != manifest["canonical_events"] + .as_u64() + .ok_or("missing event count")? as usize + { + return Err("event count changed".into()); + } + if digest(&serde_json::to_vec(&prefix.events)?) + != manifest["events_digest"] + .as_str() + .ok_or("missing event content digest")? + { + return Err("event content changed".into()); + } + if measured { + samples.add("prefix_all", start); + } + Ok(()) +} fn io_stats() -> serde_json::Value { match std::fs::read_to_string("/proc/self/io") { Ok(value) => json!({"value":value}), @@ -333,6 +376,10 @@ async fn main() -> Result<()> { return Err("body pattern must be repeated or entropy".into()); } let iterations: usize = value("--iterations", "10").parse()?; + let operation = value("--operation", "all"); + if !matches!(operation.as_str(), "all" | "prefix") { + return Err("operation must be all or prefix".into()); + } if events == 0 || iterations == 0 { return Err("events and iterations must be positive".into()); } @@ -469,6 +516,10 @@ async fn main() -> Result<()> { let mut child_pages = 0u64; // The first pass validates descriptors and warms application/SQLite caches. for iteration in 0..=iterations { + if iteration > 0 && operation == "prefix" { + record_prefix(&session, &manifest, &mut samples, true).await?; + continue; + } let start = Instant::now(); let metadata = session.read_model_view("bench", None).await?; std::hint::black_box(metadata.source_evidence.digest); @@ -682,38 +733,7 @@ async fn main() -> Result<()> { if iteration > 0 { samples.add("descriptor_leaves_all", start); } - let start = Instant::now(); - let prefix = session - .scoped_prefix( - LogScope::Session { id: "bench".into() }, - usize::MAX, - usize::MAX, - ) - .await?; - if prefix.digest - != manifest["prefix_digest"] - .as_str() - .ok_or("missing prefix digest")? - { - return Err("event proof changed".into()); - } - if prefix.events.len() - != manifest["canonical_events"] - .as_u64() - .ok_or("missing event count")? as usize - { - return Err("event count changed".into()); - } - if digest(&serde_json::to_vec(&prefix.events)?) - != manifest["events_digest"] - .as_str() - .ok_or("missing event content digest")? - { - return Err("event content changed".into()); - } - if iteration > 0 { - samples.add("prefix_all", start); - } + record_prefix(&session, &manifest, &mut samples, iteration > 0).await?; let start = Instant::now(); let inventory = root.preview_bundle("bench").await?; let (_, export) = root @@ -746,7 +766,7 @@ async fn main() -> Result<()> { } println!( "{}", - json!({"mode":mode,"fixture":manifest,"iterations":iterations,"profile":if cfg!(debug_assertions){"debug"}else{"release"},"samples":samples.report(),"completed_rows":completed_rows,"completed_bytes":completed_bytes,"directory_pages":directory_pages,"child_pages":child_pages,"peak_charged_bytes":PEAK.load(Ordering::Relaxed),"charged_allocations":ALLOCATIONS.load(Ordering::Relaxed),"errors":0,"database_bytes_after_close":database_bytes,"io_before":before_io,"io_after":after_io,"sqlite_resources":resources,"usage_before":before_usage,"usage_after":usage(),"timestamp":SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs()}) + json!({"mode":mode,"fixture":manifest,"operation":operation,"iterations":iterations,"profile":if cfg!(debug_assertions){"debug"}else{"release"},"samples":samples.report(),"completed_rows":completed_rows,"completed_bytes":completed_bytes,"directory_pages":directory_pages,"child_pages":child_pages,"peak_charged_bytes":PEAK.load(Ordering::Relaxed),"charged_allocations":ALLOCATIONS.load(Ordering::Relaxed),"errors":0,"database_bytes_after_close":database_bytes,"io_before":before_io,"io_after":after_io,"sqlite_resources":resources,"usage_before":before_usage,"usage_after":usage(),"timestamp":SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs()}) ); Ok(()) } diff --git a/platform/event-log/migrations/0013_canonical_event_bodies.sql b/platform/event-log/migrations/0014_canonical_event_bodies.sql similarity index 81% rename from platform/event-log/migrations/0013_canonical_event_bodies.sql rename to platform/event-log/migrations/0014_canonical_event_bodies.sql index e393aa1aec..14d5c1ee99 100644 --- a/platform/event-log/migrations/0013_canonical_event_bodies.sql +++ b/platform/event-log/migrations/0014_canonical_event_bodies.sql @@ -32,23 +32,20 @@ DROP VIEW model_part_bodies; DROP VIEW runtime_events; ALTER TABLE event_log RENAME COLUMN event_json TO event_header; -ALTER TABLE event_log ADD COLUMN canonical_bytes INTEGER CHECK(canonical_bytes IS NULL OR canonical_bytes>=0); -UPDATE event_log SET canonical_bytes=length(CAST(event_header AS BLOB)) WHERE event_header IS NOT NULL; - CREATE TABLE event_bodies ( id INTEGER PRIMARY KEY, digest BLOB NOT NULL UNIQUE CHECK(length(digest)=32), raw_bytes INTEGER NOT NULL CHECK(raw_bytes>0), sqlite_json_bytes INTEGER NOT NULL CHECK(sqlite_json_bytes>0), - payload BLOB NOT NULL, - CHECK(length(payload)=raw_bytes) -); -CREATE TABLE event_body_slots ( - event_sequence INTEGER PRIMARY KEY REFERENCES event_log(sequence) ON DELETE CASCADE, - header_offset INTEGER NOT NULL CHECK(header_offset>=0), - body_id INTEGER NOT NULL REFERENCES event_bodies(id) + payload BLOB NOT NULL CHECK(length(payload)=raw_bytes) ); -CREATE INDEX event_body_slot_material ON event_body_slots(body_id); +ALTER TABLE event_log ADD COLUMN canonical_bytes INTEGER CHECK(canonical_bytes IS NULL OR canonical_bytes>=0); +ALTER TABLE event_log ADD COLUMN body_id INTEGER REFERENCES event_bodies(id); +ALTER TABLE event_log ADD COLUMN header_offset INTEGER + CHECK(header_offset IS NULL OR header_offset>=0) + CHECK((body_id IS NULL)=(header_offset IS NULL)); +CREATE INDEX event_body_references ON event_log(body_id) WHERE body_id IS NOT NULL; +UPDATE event_log SET canonical_bytes=length(CAST(event_header AS BLOB)) WHERE event_header IS NOT NULL; CREATE TEMP TABLE canonical_body_stage AS SELECT sequence, @@ -60,40 +57,50 @@ INSERT INTO event_bodies(digest,raw_bytes,sqlite_json_bytes,payload) SELECT DISTINCT maka_body_sha(raw),length(raw), length(CAST(json(CAST(raw AS TEXT)) AS BLOB)),raw FROM canonical_body_stage; -UPDATE event_log SET event_header=maka_event_header(kind,event_header) - WHERE sequence IN (SELECT sequence FROM canonical_body_stage); -INSERT INTO event_body_slots(event_sequence,header_offset,body_id) - SELECT stage.sequence,stage.header_offset,body.id - FROM canonical_body_stage stage JOIN event_bodies body ON body.digest=maka_body_sha(stage.raw); +UPDATE event_log SET event_header=maka_event_header(kind,event_header), + body_id=body.id,header_offset=stage.header_offset + FROM canonical_body_stage stage JOIN event_bodies body ON body.digest=maka_body_sha(stage.raw) + WHERE event_log.sequence=stage.sequence; DROP TABLE canonical_body_stage; CREATE TRIGGER event_body_immutable BEFORE UPDATE ON event_bodies BEGIN SELECT RAISE(ABORT,'canonical body material is immutable'); END; -CREATE TRIGGER event_body_slot_immutable BEFORE UPDATE ON event_body_slots BEGIN - SELECT RAISE(ABORT,'canonical body slots are immutable'); +CREATE TRIGGER event_body_reference_immutable BEFORE UPDATE OF body_id,header_offset ON event_log + WHEN (OLD.body_id IS NOT NEW.body_id OR OLD.header_offset IS NOT NEW.header_offset) + AND NOT (NEW.body_id IS NULL AND NEW.header_offset IS NULL AND NEW.event_header IS NULL) BEGIN + SELECT RAISE(ABORT,'live canonical body references are immutable'); END; -CREATE TRIGGER event_body_released AFTER DELETE ON event_body_slots BEGIN +CREATE TRIGGER event_body_released AFTER UPDATE OF body_id ON event_log + WHEN OLD.body_id IS NOT NULL AND OLD.body_id IS NOT NEW.body_id BEGIN DELETE FROM event_bodies WHERE id=OLD.body_id - AND NOT EXISTS(SELECT 1 FROM event_body_slots WHERE body_id=OLD.body_id); + AND NOT EXISTS(SELECT 1 FROM event_log WHERE body_id=OLD.body_id); END; -CREATE TRIGGER event_live_length_insert BEFORE INSERT ON event_log - WHEN (NEW.event_header IS NULL)!=(NEW.canonical_bytes IS NULL) BEGIN - SELECT RAISE(ABORT,'canonical byte length must follow live material'); +CREATE TRIGGER event_body_deleted AFTER DELETE ON event_log WHEN OLD.body_id IS NOT NULL BEGIN + DELETE FROM event_bodies WHERE id=OLD.body_id + AND NOT EXISTS(SELECT 1 FROM event_log WHERE body_id=OLD.body_id); END; -CREATE TRIGGER event_live_length_update BEFORE UPDATE OF event_header,canonical_bytes ON event_log - WHEN (NEW.event_header IS NULL)!=(NEW.canonical_bytes IS NULL) - OR (NEW.event_header IS NULL AND EXISTS(SELECT 1 FROM event_body_slots WHERE event_sequence=NEW.sequence)) BEGIN - SELECT RAISE(ABORT,'collected events cannot retain live canonical material'); +CREATE TRIGGER event_material_insert BEFORE INSERT ON event_log + WHEN NOT EXISTS(SELECT 1 WHERE (NEW.event_header IS NULL AND NEW.canonical_bytes IS NULL AND NEW.body_id IS NULL AND NEW.header_offset IS NULL) + OR (NEW.event_header IS NOT NULL AND NEW.canonical_bytes IS NOT NULL AND ( + (NEW.body_id IS NULL AND NEW.header_offset IS NULL AND NEW.canonical_bytes=length(CAST(NEW.event_header AS BLOB))) + OR (NEW.body_id IS NOT NULL AND NEW.header_offset IS NOT NULL AND EXISTS( + SELECT 1 FROM event_bodies b WHERE b.id=NEW.body_id + AND NEW.kind='tool_settled' AND json_extract(NEW.event_header,'$.fact.outcome.kind')='succeeded' + AND NEW.canonical_bytes=length(CAST(NEW.event_header AS BLOB))-4+b.raw_bytes + AND substr(CAST(NEW.event_header AS BLOB),NEW.header_offset+1,4)=X'6e756c6c'))))) BEGIN + SELECT RAISE(ABORT,'invalid canonical event material recipe'); END; -CREATE TRIGGER event_body_slot_valid BEFORE INSERT ON event_body_slots - WHEN NOT EXISTS(SELECT 1 FROM event_log e JOIN event_bodies b ON b.id=NEW.body_id - WHERE e.sequence=NEW.event_sequence AND e.event_header IS NOT NULL - AND e.kind='tool_settled' - AND json_extract(e.event_header,'$.fact.outcome.kind')='succeeded' - AND e.canonical_bytes=length(CAST(e.event_header AS BLOB))-4+b.raw_bytes - AND substr(CAST(e.event_header AS BLOB),NEW.header_offset+1,4)=X'6e756c6c') BEGIN - SELECT RAISE(ABORT,'invalid canonical body slot recipe'); +CREATE TRIGGER event_material_update BEFORE UPDATE OF event_header,canonical_bytes,body_id,header_offset ON event_log + WHEN NOT EXISTS(SELECT 1 WHERE (NEW.event_header IS NULL AND NEW.canonical_bytes IS NULL AND NEW.body_id IS NULL AND NEW.header_offset IS NULL) + OR (NEW.event_header IS NOT NULL AND NEW.canonical_bytes IS NOT NULL AND ( + (NEW.body_id IS NULL AND NEW.header_offset IS NULL AND NEW.canonical_bytes=length(CAST(NEW.event_header AS BLOB))) + OR (NEW.body_id IS NOT NULL AND NEW.header_offset IS NOT NULL AND EXISTS( + SELECT 1 FROM event_bodies b WHERE b.id=NEW.body_id + AND NEW.kind='tool_settled' AND json_extract(NEW.event_header,'$.fact.outcome.kind')='succeeded' + AND NEW.canonical_bytes=length(CAST(NEW.event_header AS BLOB))-4+b.raw_bytes + AND substr(CAST(NEW.event_header AS BLOB),NEW.header_offset+1,4)=X'6e756c6c'))))) BEGIN + SELECT RAISE(ABORT,'invalid canonical event material recipe'); END; CREATE VIEW runtime_events AS diff --git a/platform/event-log/src/append/ingress.rs b/platform/event-log/src/append/ingress.rs index 0e436f3a5c..b16a4ff54b 100644 --- a/platform/event-log/src/append/ingress.rs +++ b/platform/event-log/src/append/ingress.rs @@ -65,6 +65,13 @@ pub(crate) async fn insert( }; let projection = crate::canonical::projection(event.fact.kind(), &json)?; let header = projection.as_ref().map(|projection| projection.header(&json)).transpose()?; + let body = match &projection { + Some(projection) => Some(crate::canonical_insert::BodyReference { + id: crate::canonical::prepare(worker, projection).await?, + offset: i64::try_from(projection.offset).map_err(|_| StoreError::PrefixTooLarge)?, + }), + None => None, + }; let inserted = crate::canonical_insert::insert_header(crate::canonical_insert::Record { sequence: requested_sequence, event_id: &event.id, @@ -72,15 +79,12 @@ pub(crate) async fn insert( kind: event.fact.kind(), operation_id: event.fact.operation_id(), json: header.as_deref().unwrap_or(&json), - }, json.len()).execute(&mut *worker).await?; + }, json.len(), body).execute(&mut *worker).await?; if inserted.rows_affected() != 1 { return Err(StoreError::EventConflict); } // Capture before the projections issue INSERTs on this same handle. let sequence = sequence_number(inserted.last_insert_rowid())?; - if let Some(projection) = &projection { - crate::canonical::publish(worker, sequence, projection).await?; - } let stored = StoredEvent { sequence, event }; crate::model_items::record(worker, &stored).await?; crate::context::projection::record(worker, &stored, &json).await?; diff --git a/platform/event-log/src/archive/candidates.rs b/platform/event-log/src/archive/candidates.rs index d946da6485..0b1505d79b 100644 --- a/platform/event-log/src/archive/candidates.rs +++ b/platform/event-log/src/archive/candidates.rs @@ -101,8 +101,7 @@ async fn scan( "SELECT t.sequence AS sequence,t.event_id, COALESCE(b.sqlite_json_bytes, length(CAST(json_extract(t.event_header,'$.fact.outcome.message') AS BLOB)),0) - FROM ({source}) t LEFT JOIN event_body_slots slot ON slot.event_sequence=t.sequence - LEFT JOIN event_bodies b ON b.id=slot.body_id CROSS JOIN runtime_events d + FROM ({source}) t LEFT JOIN event_bodies b ON b.id=t.body_id CROSS JOIN runtime_events d WHERE d.invocation_id=t.invocation_id AND d.operation_id=t.operation_id AND d.kind='tool_dispatched' AND t.sequence>?1 AND t.sequence<=?3 AND t.owner_session_id=?2 AND {filter} AND json_extract(d.event_header,'$.fact.call.origin.kind')='provider' diff --git a/platform/event-log/src/archive/target.rs b/platform/event-log/src/archive/target.rs index 3abb9199f7..c2ca30730d 100644 --- a/platform/event-log/src/archive/target.rs +++ b/platform/event-log/src/archive/target.rs @@ -29,11 +29,11 @@ use sqlx::{Row, SqliteConnection}; // combined history view makes SQLite sort the entire remainder on every page. // The copied branch exposes h.sequence so its membership index supplies order. pub(super) const SOURCES: [&str; 2] = [ - "SELECT e.sequence,e.event_id,e.invocation_id,e.operation_id,e.event_header, + "SELECT e.sequence,e.event_id,e.invocation_id,e.operation_id,e.event_header,e.body_id, json_extract(e.event_header,'$.invocation.session_id') AS owner_session_id, 0 AS inherited,NULL AS archive_sequence FROM runtime_events e WHERE e.kind='tool_settled'", - "SELECT h.sequence,e.event_id,e.invocation_id,e.operation_id,e.event_header, + "SELECT h.sequence,e.event_id,e.invocation_id,e.operation_id,e.event_header,e.body_id, h.session_id AS owner_session_id,1 AS inherited,h.archive_sequence FROM session_history_members h CROSS JOIN runtime_events e WHERE e.sequence=h.sequence AND e.kind='tool_settled'", diff --git a/platform/event-log/src/canonical.rs b/platform/event-log/src/canonical.rs index 57b0f52564..66bce01fbc 100644 --- a/platform/event-log/src/canonical.rs +++ b/platform/event-log/src/canonical.rs @@ -29,4 +29,4 @@ pub(crate) use recipe::projection; pub(crate) use source::{ SPAN_COLUMNS, SpanMetadata, with_span, with_span_metadata, with_span_on_row, }; -pub(crate) use write::publish; +pub(crate) use write::prepare; diff --git a/platform/event-log/src/canonical/migration.rs b/platform/event-log/src/canonical/migration.rs index 1ade8255f9..747019f344 100644 --- a/platform/event-log/src/canonical/migration.rs +++ b/platform/event-log/src/canonical/migration.rs @@ -83,7 +83,7 @@ mod tests { let previous = sqlx::migrate::Migrator { migrations: all .iter() - .filter(|m| m.version < 13) + .filter(|m| m.version < 14) .cloned() .collect::>() .into(), @@ -153,6 +153,41 @@ mod tests { .await .unwrap(); assert_eq!(bodies, 1); + // The parent cannot disappear while referenced, and a valid alternate + // body cannot redirect an already published event. + assert!( + sqlx::query("DELETE FROM event_bodies") + .execute(&mut db) + .await + .is_err() + ); + sqlx::query("INSERT INTO event_bodies(id,digest,raw_bytes,sqlite_json_bytes,payload) SELECT id+100,zeroblob(32),raw_bytes,sqlite_json_bytes,payload FROM event_bodies") + .execute(&mut db).await.unwrap(); + let error = sqlx::query("UPDATE event_log SET body_id=body_id+100 WHERE sequence=1") + .execute(&mut db) + .await + .unwrap_err(); + assert!( + error.to_string().contains("references are immutable"), + "{error}" + ); + sqlx::query("DELETE FROM event_bodies WHERE digest=zeroblob(32)") + .execute(&mut db) + .await + .unwrap(); + { + let mut tx = db.begin().await.unwrap(); + sqlx::query("DELETE FROM event_log WHERE sequence IN (1,2)") + .execute(&mut *tx) + .await + .unwrap(); + let bodies: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM event_bodies") + .fetch_one(&mut *tx) + .await + .unwrap(); + assert_eq!(bodies, 0); + tx.rollback().await.unwrap(); + } let expected: i64 = sqlx::query_scalar( "SELECT length(CAST(json_extract(?,'$.fact.outcome.model_projection') AS BLOB))", ) @@ -216,12 +251,7 @@ mod tests { assert!(sqlx::query("UPDATE event_log SET event_header=NULL,canonical_bytes=NULL,retained_json='{}',body_digest='digest' WHERE sequence=1").execute(&mut db).await.is_err()); for (sequence, remaining) in [(1, 1), (2, 0)] { let mut tx = db.begin().await.unwrap(); - sqlx::query("DELETE FROM event_body_slots WHERE event_sequence=?") - .bind(sequence) - .execute(&mut *tx) - .await - .unwrap(); - sqlx::query("UPDATE event_log SET event_header=NULL,canonical_bytes=NULL,retained_json='{}',body_digest='digest' WHERE sequence=?").bind(sequence).execute(&mut *tx).await.unwrap(); + sqlx::query("UPDATE event_log SET event_header=NULL,canonical_bytes=NULL,body_id=NULL,header_offset=NULL,retained_json='{}',body_digest='digest' WHERE sequence=?").bind(sequence).execute(&mut *tx).await.unwrap(); tx.commit().await.unwrap(); let bodies: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM event_bodies") .fetch_one(&mut db) diff --git a/platform/event-log/src/canonical/read.rs b/platform/event-log/src/canonical/read.rs index cd1495b294..9d5313401f 100644 --- a/platform/event-log/src/canonical/read.rs +++ b/platform/event-log/src/canonical/read.rs @@ -26,14 +26,13 @@ use std::{borrow::Cow, io}; // Full-event queries bind their selected event as e. Metadata-only queries do // not join these tables or fetch body payloads. pub(crate) const COLUMNS: &str = "e.canonical_bytes AS canonical_length, - canonical_slot.event_sequence AS canonical_slot_sequence, - canonical_slot.header_offset AS canonical_offset, + e.body_id AS canonical_body_reference, + e.header_offset AS canonical_offset, canonical_body.id AS canonical_body_id, canonical_body.raw_bytes AS canonical_raw_bytes, canonical_body.payload AS canonical_payload"; pub(crate) const JOINS: &str = - "LEFT JOIN event_body_slots canonical_slot ON canonical_slot.event_sequence=e.sequence - LEFT JOIN event_bodies canonical_body ON canonical_body.id=canonical_slot.body_id"; + "LEFT JOIN event_bodies canonical_body ON canonical_body.id=e.body_id"; fn invalid(reason: &'static str) -> StoreError { io::Error::new(io::ErrorKind::InvalidData, reason).into() } @@ -43,7 +42,7 @@ fn nonnegative(value: i64) -> Result { struct Fields<'a> { header: Option<&'a str>, length: Option, - slot: Option, + reference: Option, offset: Option, body: Option, raw_bytes: Option, @@ -53,7 +52,7 @@ fn reconstruct(fields: Fields<'_>) -> Result, StoreError> { let Fields { header, length, - slot, + reference, offset, body, raw_bytes, @@ -61,7 +60,7 @@ fn reconstruct(fields: Fields<'_>) -> Result, StoreError> { } = fields; let Some(header) = header else { if length.is_some() - || slot.is_some() + || reference.is_some() || offset.is_some() || body.is_some() || raw_bytes.is_some() @@ -73,10 +72,10 @@ fn reconstruct(fields: Fields<'_>) -> Result, StoreError> { }; let length = nonnegative(length.ok_or_else(|| invalid("live event lacks its canonical byte length"))?)?; - match (slot, offset, body, raw_bytes, payload) { + match (reference, offset, body, raw_bytes, payload) { (None, None, None, None, None) => Recipe::new(header, length, None)?.reconstruct(None), - (Some(sequence), Some(offset), Some(id), Some(raw_bytes), Some(payload)) => { - if sequence <= 0 || id <= 0 || raw_bytes <= 0 { + (Some(reference), Some(offset), Some(id), Some(raw_bytes), Some(payload)) => { + if reference <= 0 || reference != id || id <= 0 || raw_bytes <= 0 { return Err(invalid("invalid canonical body identity or length")); } let raw_bytes = nonnegative(raw_bytes)?; @@ -96,14 +95,14 @@ fn reconstruct(fields: Fields<'_>) -> Result, StoreError> { pub(crate) fn json(row: &SqliteRow) -> Result, StoreError> { let header = row.try_get("event_header")?; let length = row.try_get("canonical_length")?; - let slot: Option = row.try_get("canonical_slot_sequence")?; - // The shared LEFT JOIN cannot expose body fields without a slot. A dangling - // body still has slot presence and takes the complete validation path. - if slot.is_none() { + let reference: Option = row.try_get("canonical_body_reference")?; + // Presence comes from the event reference, not its joined body. A dangling + // body reference must take the complete validation path and fail. + if reference.is_none() { return reconstruct(Fields { header, length, - slot, + reference, offset: None, body: None, raw_bytes: None, @@ -113,7 +112,7 @@ pub(crate) fn json(row: &SqliteRow) -> Result, StoreError> { reconstruct(Fields { header, length, - slot, + reference, offset: row.try_get("canonical_offset")?, body: row.try_get("canonical_body_id")?, raw_bytes: row.try_get("canonical_raw_bytes")?, @@ -134,12 +133,12 @@ pub(crate) fn json_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result = row.get("canonical_slot_sequence")?; - if slot.is_none() { + let reference: Option = row.get("canonical_body_reference")?; + if reference.is_none() { return reconstruct(Fields { header, length, - slot, + reference, offset: None, body: None, raw_bytes: None, @@ -161,7 +160,7 @@ pub(crate) fn json_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result { pub(crate) struct SpanMetadata { live: bool, length: Option, - slot_sequence: Option, + body_reference: Option, slot_offset: Option, body_id: Option, raw_bytes: Option, } pub(crate) const SPAN_COLUMNS: &str = "e.event_header IS NOT NULL AS canonical_live, - e.canonical_bytes AS canonical_length,canonical_slot.event_sequence AS canonical_slot_sequence, - canonical_slot.header_offset AS canonical_offset,canonical_body.id AS canonical_body_id, + e.canonical_bytes AS canonical_length,e.body_id AS canonical_body_reference, + e.header_offset AS canonical_offset,canonical_body.id AS canonical_body_id, canonical_body.raw_bytes AS canonical_raw_bytes"; impl SpanMetadata { @@ -97,7 +97,7 @@ impl SpanMetadata { Ok(Self { live: row.try_get("canonical_live")?, length: row.try_get("canonical_length")?, - slot_sequence: row.try_get("canonical_slot_sequence")?, + body_reference: row.try_get("canonical_body_reference")?, slot_offset: row.try_get("canonical_offset")?, body_id: row.try_get("canonical_body_id")?, raw_bytes: row.try_get("canonical_raw_bytes")?, @@ -137,7 +137,7 @@ pub(crate) fn with_span_on_row( let metadata = SpanMetadata { live: row.get(base)?, length: row.get(base + 1)?, - slot_sequence: row.get(base + 2)?, + body_reference: row.get(base + 2)?, slot_offset: row.get(base + 3)?, body_id: row.get(base + 4)?, raw_bytes: row.get(base + 5)?, @@ -156,13 +156,18 @@ pub(crate) fn with_span_metadata( let SpanMetadata { live, length, - slot_sequence, + body_reference, slot_offset, body_id, raw_bytes, } = metadata; if !live { - if length.is_some() || slot_sequence.is_some() || body_id.is_some() { + if length.is_some() + || body_reference.is_some() + || slot_offset.is_some() + || body_id.is_some() + || raw_bytes.is_some() + { return Err(invalid("collected canonical event retains live material")); } return Err(StoreError::MaterialCollected); @@ -170,10 +175,10 @@ pub(crate) fn with_span_metadata( let length = u64::try_from(length.ok_or_else(|| invalid("canonical span lacks logical length"))?) .map_err(|_| invalid("negative canonical byte length"))?; - let (slot, material) = match (slot_sequence, slot_offset, body_id, raw_bytes) { + let (slot, material) = match (body_reference, slot_offset, body_id, raw_bytes) { (None, None, None, None) => (None, None), - (Some(source), Some(position), Some(id), Some(length)) - if source == sequence && position >= 0 && id > 0 && length > 0 => + (Some(reference), Some(position), Some(id), Some(length)) + if reference == id && position >= 0 && id > 0 && length > 0 => { ( Some(Slot { diff --git a/platform/event-log/src/canonical/write.rs b/platform/event-log/src/canonical/write.rs index 1b28063fb7..61ee884137 100644 --- a/platform/event-log/src/canonical/write.rs +++ b/platform/event-log/src/canonical/write.rs @@ -17,21 +17,23 @@ * under the License. */ -//! Immutable material and its event slot are published in the original transaction. +//! Prepare immutable material for an event reference in the original transaction. use super::{recipe::Projection, source}; use crate::StoreError; use sha2::{Digest, Sha256}; use sqlx::sqlite::SqliteWorkerConnection; -pub(crate) async fn publish( +pub(crate) async fn prepare( worker: &mut SqliteWorkerConnection<'_>, - sequence: u64, projection: &Projection<'_>, -) -> Result<(), StoreError> { +) -> Result { let original = projection.json.as_bytes(); let digest = Sha256::digest(original); let inserted=sqlx::query("INSERT INTO event_bodies(digest,raw_bytes,sqlite_json_bytes,payload) VALUES(?1,?2,?3,?4) ON CONFLICT(digest) DO NOTHING") .bind(digest.as_slice()).bind(original.len() as i64).bind(projection.sqlite_json_bytes() as i64).bind(original) .execute(&mut *worker).await?; + if inserted.rows_affected() == 1 { + return Ok(inserted.last_insert_rowid()); + } let (id, bytes): (i64, i64) = sqlx::query_as("SELECT id,raw_bytes FROM event_bodies WHERE digest=?") .bind(digest.as_slice()) @@ -40,7 +42,7 @@ pub(crate) async fn publish( if bytes != original.len() as i64 { return Err(StoreError::EventConflict); } - if inserted.rows_affected() == 0 { + { // SAFETY: the worker callback exclusively owns this original handle. // The borrowed wrapper and its BLOB readers close before returning. let raw = unsafe { rusqlite::Connection::from_handle(worker.as_raw_handle().as_ptr()) }?; @@ -64,11 +66,5 @@ pub(crate) async fn publish( })?; drop(raw); } - sqlx::query("INSERT INTO event_body_slots(event_sequence,header_offset,body_id) VALUES(?,?,?)") - .bind(sequence as i64) - .bind(projection.offset as i64) - .bind(id) - .execute(&mut *worker) - .await?; - Ok(()) + Ok(id) } diff --git a/platform/event-log/src/canonical_insert.rs b/platform/event-log/src/canonical_insert.rs index 59bb7fac2e..4bea7969c4 100644 --- a/platform/event-log/src/canonical_insert.rs +++ b/platform/event-log/src/canonical_insert.rs @@ -30,19 +30,26 @@ pub(crate) struct Record<'json> { pub json: &'json str, } +#[derive(Clone, Copy)] +pub(crate) struct BodyReference { + pub id: i64, + pub offset: i64, +} + pub(crate) fn insert(record: Record<'_>) -> Query<'_, Sqlite, SqliteArguments> { let bytes = record.json.len(); - insert_header(record, bytes) + insert_header(record, bytes, None) } pub(crate) fn insert_header( record: Record<'_>, canonical_bytes: usize, + body: Option, ) -> Query<'_, Sqlite, SqliteArguments> { sqlx::query( - "INSERT INTO event_log(sequence,event_id,invocation_id,kind,operation_id,event_header,canonical_bytes, + "INSERT INTO event_log(sequence,event_id,invocation_id,kind,operation_id,event_header,canonical_bytes,body_id,header_offset, accounting_at,accounting_usage,invocation_model,accounting_tool,event_session) - VALUES (?1,?2,?3,?4,?5,?6,?7, + VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9, CASE WHEN ?4 IN ('model_requested', 'model_completed', 'model_interrupted', 'invocation_ended', 'auxiliary_model_started', 'auxiliary_model_settled', 'tool_dispatched', 'tool_settled', 'tool_rejected') @@ -78,4 +85,6 @@ pub(crate) fn insert_header( .bind(record.operation_id) .bind(record.json) .bind(canonical_bytes as i64) + .bind(body.map(|body| body.id)) + .bind(body.map(|body| body.offset)) } diff --git a/platform/event-log/src/sessions/material.rs b/platform/event-log/src/sessions/material.rs index d3ef9d7180..709f8e19b0 100644 --- a/platform/event-log/src/sessions/material.rs +++ b/platform/event-log/src/sessions/material.rs @@ -128,8 +128,7 @@ impl EventLog { AND NOT EXISTS (SELECT 1 FROM model_request_compositions WHERE digest=?1)") .bind(surface).execute(&mut *tx).await?; } - sqlx::query("DELETE FROM event_body_slots WHERE event_sequence=?").bind(sequence).execute(&mut *tx).await?; - sqlx::query("UPDATE event_log SET retained_json=?, body_digest=?, event_header=NULL,canonical_bytes=NULL WHERE event_id=?") + sqlx::query("UPDATE event_log SET retained_json=?, body_digest=?, event_header=NULL,canonical_bytes=NULL,body_id=NULL,header_offset=NULL WHERE event_id=?") .bind(serde_json::to_string(&proof)?).bind(content_digest(body.as_bytes())) .bind(&id).execute(&mut *tx).await?; } @@ -292,7 +291,7 @@ mod cursor_tests { let proof = serde_json::to_string(&retained(&ended)).unwrap(); let digest = content_digest(serde_json::to_string(&ended).unwrap().as_bytes()); log.connection.run(move |db| Box::pin(async move { - sqlx::query("UPDATE event_log SET retained_json=?,body_digest=?,event_header=NULL,canonical_bytes=NULL WHERE sequence=2") + sqlx::query("UPDATE event_log SET retained_json=?,body_digest=?,event_header=NULL,canonical_bytes=NULL,body_id=NULL,header_offset=NULL WHERE sequence=2") .bind(proof).bind(digest).execute(&mut *db).await?; let publication: (f64, String) = sqlx::query_as("SELECT accounting_at,event_session FROM event_log WHERE sequence=2") .fetch_one(&mut *db).await?; diff --git a/platform/event-log/tests/integration/archive/mod.rs b/platform/event-log/tests/integration/archive/mod.rs index 306902e2be..d5aab5dbc6 100644 --- a/platform/event-log/tests/integration/archive/mod.rs +++ b/platform/event-log/tests/integration/archive/mod.rs @@ -326,7 +326,7 @@ async fn archive_publication_is_atomic_and_preserves_original_source_integrity() .execute_batch("DROP TRIGGER event_body_immutable") .unwrap(); inspect.execute("UPDATE event_bodies SET payload=CAST(replace(CAST(payload AS TEXT),'xxxxxxxx','yyyyyyyy') AS BLOB) WHERE id=( - SELECT s.body_id FROM event_body_slots s JOIN event_log e ON e.sequence=s.event_sequence WHERE e.event_id=?)", + SELECT e.body_id FROM event_log e WHERE e.event_id=?)", [&target.event().id]).unwrap(); assert!( log.read_tool_result("session", &target.event().id) diff --git a/platform/event-log/tests/integration/bundle/import.rs b/platform/event-log/tests/integration/bundle/import.rs index 4cbbfcfd70..6619b24db4 100644 --- a/platform/event-log/tests/integration/bundle/import.rs +++ b/platform/event-log/tests/integration/bundle/import.rs @@ -256,7 +256,7 @@ pub(crate) async fn roundtrip(bytes: &[u8]) { let inspect = rusqlite::Connection::open(&collision_path).unwrap(); inspect .execute_batch( - "DELETE FROM event_body_slots; UPDATE event_log SET retained_json=event_header,body_digest='fixture',event_header=NULL,canonical_bytes=NULL", + "UPDATE event_log SET retained_json=event_header,body_digest='fixture',event_header=NULL,canonical_bytes=NULL,body_id=NULL,header_offset=NULL", ) .unwrap(); let collision = EventLog::open(&collision_path).await.unwrap(); diff --git a/platform/event-log/tests/integration/context_compaction/active/latest.rs b/platform/event-log/tests/integration/context_compaction/active/latest.rs index 3326b48ff7..633037f7fe 100644 --- a/platform/event-log/tests/integration/context_compaction/active/latest.rs +++ b/platform/event-log/tests/integration/context_compaction/active/latest.rs @@ -137,7 +137,8 @@ async fn preturn_keeps_anchor_and_latest_main_never_falls_back_a_broken_newer_tr Some(write.event().id.as_str()) ); let inspect = rusqlite::Connection::open(&path).unwrap(); - inspect.execute("UPDATE event_log SET event_header=json_remove(event_header, '$.fact.context') WHERE operation_id='new-main' AND kind='model_requested'", []).unwrap(); + // Keep the storage recipe valid while removing the trace being tested. + inspect.execute("UPDATE event_log SET event_header=json_remove(event_header, '$.fact.context'),canonical_bytes=length(CAST(json_remove(event_header, '$.fact.context') AS BLOB)) WHERE operation_id='new-main' AND kind='model_requested'", []).unwrap(); assert!( matches!(log.latest_main_context("session").await.unwrap(), LatestMainContext::Selected(value) if value.context.is_none() && value.finished_at == Some(finished_at)) ); diff --git a/platform/event-log/tests/integration/import_atomic.rs b/platform/event-log/tests/integration/import_atomic.rs index 5a77d0a19c..8ccd0e1c37 100644 --- a/platform/event-log/tests/integration/import_atomic.rs +++ b/platform/event-log/tests/integration/import_atomic.rs @@ -489,7 +489,7 @@ async fn retained_orphan_physical_proof_still_blocks_complete_import() { let inspect = rusqlite::Connection::open(&path).unwrap(); inspect .execute_batch( - "DELETE FROM event_body_slots; UPDATE event_log SET retained_json=event_header,body_digest='fixture',event_header=NULL,canonical_bytes=NULL", + "UPDATE event_log SET retained_json=event_header,body_digest='fixture',event_header=NULL,canonical_bytes=NULL,body_id=NULL,header_offset=NULL", ) .unwrap(); let log = EventLog::open(&path).await.unwrap(); diff --git a/platform/event-log/tests/integration/session_material.rs b/platform/event-log/tests/integration/session_material.rs index 05796d106b..6344c35545 100644 --- a/platform/event-log/tests/integration/session_material.rs +++ b/platform/event-log/tests/integration/session_material.rs @@ -335,7 +335,7 @@ async fn shared_tool_projection_survives_collection_of_one_canonical_owner() { } let inspect = rusqlite::Connection::open(&path).unwrap(); let counts = || { - inspect.query_row("SELECT (SELECT COUNT(*) FROM event_bodies),(SELECT COUNT(*) FROM event_body_slots)", [], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?))).unwrap() + inspect.query_row("SELECT (SELECT COUNT(*) FROM event_bodies),(SELECT COUNT(*) FROM event_log WHERE body_id IS NOT NULL)", [], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?))).unwrap() }; assert_eq!(counts(), (1, 2)); let stored_bytes: i64 = inspect diff --git a/platform/event-log/tests/integration/startup.rs b/platform/event-log/tests/integration/startup.rs index a7111adcb8..b3b5a1a8c6 100644 --- a/platform/event-log/tests/integration/startup.rs +++ b/platform/event-log/tests/integration/startup.rs @@ -158,7 +158,7 @@ async fn catalog_migration_preserves_existing_history_and_backfills_cached_activ } let checksums = migration_checksums(&Connection::open(&path).unwrap()); assert_eq!(&checksums[..old_checksums.len()], old_checksums); - assert_eq!(checksums.last().unwrap().0, 13); + assert_eq!(checksums.last().unwrap().0, 14); } #[tokio::test] diff --git a/scripts/bench-event-storage.py b/scripts/bench-event-storage.py index fffcc276e4..e418ca4647 100644 --- a/scripts/bench-event-storage.py +++ b/scripts/bench-event-storage.py @@ -65,10 +65,12 @@ def storage_stats(directory): return result -def run(binary, directory, mode, events, body_bytes, iterations, body_pattern): +def run(binary, directory, mode, events, body_bytes, iterations, body_pattern, operation="all"): command = [str(binary), "--database", str(directory), "--mode", mode, "--events", str(events), "--body-bytes", str(body_bytes), "--iterations", str(iterations), "--body-pattern", body_pattern] + if operation != "all": + command.extend(["--operation", operation]) before = {"load": os.getloadavg(), "free_bytes": shutil.disk_usage(directory.parent).free} start = time.monotonic_ns() result = subprocess.run(command, check=True, text=True, capture_output=True) @@ -76,6 +78,8 @@ def run(binary, directory, mode, events, body_bytes, iterations, body_pattern): if (report["errors"] or report["profile"] != "release" or report["completed_rows"] <= 0 or report["completed_bytes"] <= 0): raise RuntimeError(f"invalid or empty benchmark: {report}") + if operation != "all" and report.get("operation") != operation: + raise RuntimeError("binary does not support the requested operation") report.update(environment=before, wall_ns=time.monotonic_ns() - start, stderr=result.stderr, command=command) report["storage_after_close"] = storage_stats(directory) @@ -89,6 +93,7 @@ def main(): parser.add_argument("--runs", type=int, default=5) parser.add_argument("--iterations", type=int, default=20) parser.add_argument("--body-pattern", choices=("repeated", "entropy"), default="repeated") + parser.add_argument("--operation", choices=("all", "prefix"), default="all") parser.add_argument("--case", action="append", metavar="NAME:EVENTS:BODY_BYTES") args = parser.parse_args() args.output = args.output.resolve() @@ -108,6 +113,7 @@ def main(): (args.output / "manifest.json").write_text(json.dumps(dict( binaries=binaries, platform=platform.platform(), cpu_count=os.cpu_count(), runs=args.runs, iterations=args.iterations, cases=cases, body_pattern=args.body_pattern, + operation=args.operation, cache_state="one validation/warmup pass; steady state and OS-warm reopen"), indent=2) + "\n") summaries = {} fixtures = {} @@ -119,7 +125,7 @@ def main(): directory = args.output / f"{case}-{repeat}-{label}" for mode in ("create", "read"): report = run(binary, directory, mode, int(event_count), - int(body_size), args.iterations, args.body_pattern) + int(body_size), args.iterations, args.body_pattern, args.operation) fixture = report["fixture"] previous = fixtures.setdefault(case, fixture) if fixture != previous: From c7fe1071981b8f63f7404f3946e0f87ee5429ef9 Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Sat, 31 Oct 2026 23:21:28 +0800 Subject: [PATCH 4/8] refactor(storage): overlap prefix reads and decoding Prepare canonical headers and original projection bytes on the SQLite worker, then decode owned events on a joined blocking task. Drain both stages before releasing the original transaction, preserving prefix budgets and raw evidence. Verify checkpoint tool history after source collection and import. Record chronological benchmark samples and isolate RootStore opening cost. Generated-by: Codex --- platform/event-log/README.md | 2 +- platform/event-log/README.zh-CN.md | 2 +- .../event-log/examples/event_storage_bench.rs | 15 +- platform/event-log/src/canonical.rs | 4 +- platform/event-log/src/canonical/event.rs | 262 ++++++++++++++++++ platform/event-log/src/canonical/read.rs | 24 +- platform/event-log/src/canonical/recipe.rs | 28 ++ .../event-log/src/context/evidence/roots.rs | 53 +++- platform/event-log/src/prefix.rs | 22 +- platform/event-log/src/prefix/decode.rs | 177 ++++++++++++ .../src/root_store/checkpoints/tests.rs | 2 + .../src/root_store/checkpoints/tests/body.rs | 175 ++++++++++++ scripts/bench-event-storage.py | 3 +- 13 files changed, 745 insertions(+), 24 deletions(-) create mode 100644 platform/event-log/src/canonical/event.rs create mode 100644 platform/event-log/src/prefix/decode.rs create mode 100644 platform/event-log/src/root_store/checkpoints/tests/body.rs diff --git a/platform/event-log/README.md b/platform/event-log/README.md index 96c3222e31..4ac2128c5a 100644 --- a/platform/event-log/README.md +++ b/platform/event-log/README.md @@ -73,7 +73,7 @@ Canonical databases use UTF-8. Successful tool results store their original `mod Bodies are directly seekable raw bytes, shared by SHA-256 of their original token with exact byte comparison on a repeated identity. Reads validate logical lengths and selected ranges. Framed compression was evaluated against raw storage and rejected because it repeatedly slowed supported window and sequential reads; it is not a persisted format. Space savings come from compact directories and sharing identical projection tokens, not from compressing unique bodies. -Complete-event queries join the referenced body into the selected SQL row, then reconstruct in Rust within the original snapshot. Metadata predicates use the header and retain their indexes. Semantic spans retain original canonical coordinates; tool spans map those logical coordinates to header/body ranges, while inline model parts retain physical SQLite BLOB bounds. Resolution stays on the original SQLite worker. Hot range reads use its existing prepared-statement cache; bounded prefix decoding borrows rows on that same worker without copying every nullable column across the callback. Body references follow the existing history-owner collection proof; clearing or deleting the final reference removes its body in the same transaction. The event row publishes the header, logical length, offset and reference atomically; live references cannot be redirected. Neither a body digest nor a derived directory grants access or extends lifetime. Derived archive placeholders and other fragment bodies keep their chunked storage. +Complete-event queries join the referenced body into the selected SQL row within the original snapshot. JSON readers reconstruct the original bytes; typed prefix readers prepare the header and original projection bytes on the SQLite worker, then decode owned values on a joined blocking task while reading continues. They avoid a full temporary JSON string and keep the original transaction until both stages finish, including error and panic paths. The transfer contains only the prefix admitted by the existing event/byte budgets; no SQLite borrow crosses to the decoder. Typed headers reuse fact field types and defaults, ignore unknown metadata, and verify that the projection placeholder is exactly at its stored offset. Root evidence hashes the three original byte slices as one length-prefixed JSON field. Metadata predicates use the header and retain their indexes. Semantic spans retain original canonical coordinates; tool spans map those logical coordinates to header/body ranges, while inline model parts retain physical SQLite BLOB bounds. Resolution stays on the original SQLite worker. Hot range reads use its existing prepared-statement cache. Body references follow the existing history-owner collection proof; clearing or deleting the final reference removes its body in the same transaction. The event row publishes the header, logical length, offset and reference atomically; live references cannot be redirected. Neither a body digest nor a derived directory grants access or extends lifetime. Derived archive placeholders and other fragment bodies keep their chunked storage. Semantic row, payload and child directories are finite derived descriptors encoded with Postcard. Writers encode them once in the publication transaction. Pages copy their stored bytes alongside fixed little-endian locator fields into the caller's charged allocation; consumers decode the original typed descriptors without navigating whole bodies. Original byte positions and leaf commitments remain unchanged. Directory encoding never supplies a canonical fact or source identity. The SQL migration converts former JSON descriptors atomically; its conversion functions exist only during migration. diff --git a/platform/event-log/README.zh-CN.md b/platform/event-log/README.zh-CN.md index d79fdf5b37..84525625e4 100644 --- a/platform/event-log/README.zh-CN.md +++ b/platform/event-log/README.zh-CN.md @@ -67,7 +67,7 @@ Control 中的调用意图、项目登记、workspace attachment、Session 路 正文保存为可直接 seek 的原始字节，按原始 token 的 SHA-256 共享，重复身份还需逐字节核对。读取校验逻辑长度与选中区间。对照实验中，分帧压缩稳定拖慢了受支持的窗口和顺序读取，因此未作为持久化格式保留。空间收益来自紧凑目录与相同投影 token 的共享，不能把压缩实验中独有正文的压缩率套到最终实现。 -完整事件查询在同一 SQL 行 JOIN 引用的正文，在原快照内由 Rust 重建；元数据谓词继续读取 header 并使用原索引。语义区间保留原规范坐标，工具区间映射至实际 header／body，内联 model part 则保留 SQLite BLOB 的物理边界。读取仍由原 SQLite worker 执行。热区间读取复用其已有 statement cache；有界前缀在同一 worker 上借用行并解码，避免把每个可空列复制出回调。正文引用的存活由既有历史所有者回收证明决定，最后一个引用清除或删除时，在同一事务删除正文。事件行原子发布 header、逻辑长度、偏移和引用，存活引用不可重定向；正文摘要和派生目录都不授予访问权，也不延长存活期。归档占位内容和其他片段正文继续使用分块存储。 +完整事件查询在原快照内的同一 SQL 行 JOIN 引用的正文。JSON reader 重建原始字节；类型化前缀 reader 在原 SQLite worker 上准备 header 和原始投影字节，由等待完成的阻塞任务解码已拥有的数据，同时继续读取。它避免构造完整临时 JSON 字符串；包括错误和 panic 在内，都等待两段结束后才结束原事务。传递范围仅限已通过原有事件／字节预算的前缀，不把 SQLite 借用交给解码任务。Header 复用事实字段的类型和默认值、忽略未知元数据，并验证投影占位符恰好位于已存偏移。Root 证据把原始三段字节作为一个带长度前缀的 JSON 字段计算摘要；元数据谓词继续读取 header 并使用原索引。语义区间保留原规范坐标，工具区间映射至实际 header／body，内联 model part 则保留 SQLite BLOB 的物理边界。读取仍由原 SQLite worker 执行。热区间读取复用其已有 statement cache。正文引用的存活由既有历史所有者回收证明决定，最后一个引用清除或删除时，在同一事务删除正文。事件行原子发布 header、逻辑长度、偏移和引用，存活引用不可重定向；正文摘要和派生目录都不授予访问权，也不延长存活期。归档占位内容和其他片段正文继续使用分块存储。 语义行、payload 和子项目录是有限的派生描述符，采用 Postcard 编码。writer 在发布事务内编码一次；分页把已存字节和固定小端序定位字段直接复制到调用者计费的缓冲区，消费端恢复原有类型，无需读取整个正文。原始字节位置与叶子摘要保持不变，目录编码不参与规范事实或来源身份。SQL 迁移在同一事务内转换原 JSON 描述符；转换函数仅在迁移期间存在。 diff --git a/platform/event-log/examples/event_storage_bench.rs b/platform/event-log/examples/event_storage_bench.rs index e50fd77693..47bf9ddfec 100644 --- a/platform/event-log/examples/event_storage_bench.rs +++ b/platform/event-log/examples/event_storage_bench.rs @@ -277,10 +277,11 @@ impl Samples { } fn report(self) -> serde_json::Value { let mut output = serde_json::Map::new(); - for (operation, mut values) in self.0 { - values.sort_unstable(); - let at = |quantile: usize| values[((values.len() - 1) * quantile) / 100]; - output.insert(operation.into(),json!({"samples":values.len(),"min_ns":values[0],"p50_ns":at(50),"p95_ns":at(95),"p99_ns":at(99),"max_ns":values[values.len()-1],"values_ns":values})); + for (operation, values) in self.0 { + let mut sorted = values.clone(); + sorted.sort_unstable(); + let at = |quantile: usize| sorted[((sorted.len() - 1) * quantile) / 100]; + output.insert(operation.into(),json!({"samples":values.len(),"min_ns":sorted[0],"p50_ns":at(50),"p95_ns":at(95),"p99_ns":at(99),"max_ns":sorted[sorted.len()-1],"values_ns":values})); } output.into() } @@ -345,7 +346,7 @@ fn usage() -> serde_json::Value { let micros = |time: libc::timeval| { i128::from(time.tv_sec) * 1_000_000 + i128::from(time.tv_usec) }; - return json!({"user_us":micros(value.ru_utime),"system_us":micros(value.ru_stime),"max_rss_native_units":value.ru_maxrss,"minor_faults":value.ru_minflt,"major_faults":value.ru_majflt,"block_inputs":value.ru_inblock,"block_outputs":value.ru_oublock}); + return json!({"user_us":micros(value.ru_utime),"system_us":micros(value.ru_stime),"max_rss_native_units":value.ru_maxrss,"minor_faults":value.ru_minflt,"major_faults":value.ru_majflt,"block_inputs":value.ru_inblock,"block_outputs":value.ru_oublock,"voluntary_context_switches":value.ru_nvcsw,"involuntary_context_switches":value.ru_nivcsw}); } } serde_json::Value::Null @@ -393,7 +394,9 @@ async fn main() -> Result<()> { } else { RootOwner::open(&directory.join("root"), &namespaces)? }; + let root_store_open = Instant::now(); let root = RootStore::open(Arc::new(owner)).await?; + let root_store_open_ns = root_store_open.elapsed().as_nanos() as u64; let mut samples = Samples::default(); let before_io = io_stats(); let before_usage = usage(); @@ -766,7 +769,7 @@ async fn main() -> Result<()> { } println!( "{}", - json!({"mode":mode,"fixture":manifest,"operation":operation,"iterations":iterations,"profile":if cfg!(debug_assertions){"debug"}else{"release"},"samples":samples.report(),"completed_rows":completed_rows,"completed_bytes":completed_bytes,"directory_pages":directory_pages,"child_pages":child_pages,"peak_charged_bytes":PEAK.load(Ordering::Relaxed),"charged_allocations":ALLOCATIONS.load(Ordering::Relaxed),"errors":0,"database_bytes_after_close":database_bytes,"io_before":before_io,"io_after":after_io,"sqlite_resources":resources,"usage_before":before_usage,"usage_after":usage(),"timestamp":SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs()}) + json!({"mode":mode,"root_store_open_ns":root_store_open_ns,"fixture":manifest,"operation":operation,"iterations":iterations,"profile":if cfg!(debug_assertions){"debug"}else{"release"},"samples":samples.report(),"completed_rows":completed_rows,"completed_bytes":completed_bytes,"directory_pages":directory_pages,"child_pages":child_pages,"peak_charged_bytes":PEAK.load(Ordering::Relaxed),"charged_allocations":ALLOCATIONS.load(Ordering::Relaxed),"errors":0,"database_bytes_after_close":database_bytes,"io_before":before_io,"io_after":after_io,"sqlite_resources":resources,"usage_before":before_usage,"usage_after":usage(),"timestamp":SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs()}) ); Ok(()) } diff --git a/platform/event-log/src/canonical.rs b/platform/event-log/src/canonical.rs index 66bce01fbc..e91c895a2d 100644 --- a/platform/event-log/src/canonical.rs +++ b/platform/event-log/src/canonical.rs @@ -18,13 +18,15 @@ */ //! Lossless canonical events over immutable headers and typed body slots. +mod event; pub(crate) mod migration; mod read; mod recipe; mod source; mod write; -pub(crate) use read::{COLUMNS, JOINS, json, json_on_handle, one}; +pub(crate) use event::PreparedEvent; +pub(crate) use read::{COLUMNS, JOINS, json, json_on_handle, one, prepare_event_on_handle}; pub(crate) use recipe::projection; pub(crate) use source::{ SPAN_COLUMNS, SpanMetadata, with_span, with_span_metadata, with_span_on_row, diff --git a/platform/event-log/src/canonical/event.rs b/platform/event-log/src/canonical/event.rs new file mode 100644 index 0000000000..233a23f427 --- /dev/null +++ b/platform/event-log/src/canonical/event.rs @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Decode the stored tool header and its original projection without joining bytes. +use crate::StoreError; +use maka_agent_facts::{ + artifact::ArtifactEvidence, + event::{Fact, Invocation, RuntimeEvent, ToolOutcome}, + tool_output::{DurableToolProjection, RawToolResultRef}, +}; +use serde::Deserialize; +use serde_json::value::RawValue; +use std::time::SystemTime; + +#[derive(Deserialize)] +struct Header<'a> { + id: String, + recorded_at: SystemTime, + invocation: Invocation, + #[serde(borrow)] + fact: HeaderFact<'a>, +} +#[derive(Deserialize)] +struct HeaderFact<'a> { + kind: String, + operation_id: String, + #[serde(borrow)] + outcome: HeaderOutcome<'a>, +} +#[derive(Deserialize)] +struct HeaderOutcome<'a> { + kind: String, + raw: RawToolResultRef, + #[serde(borrow)] + model_projection: &'a RawValue, + #[serde(default)] + artifacts: Vec, +} + +pub(crate) struct PreparedEvent(Prepared); + +enum Prepared { + Inline(String), + Tool { + header: Box, + projection: String, + }, +} +struct ToolHeader { + id: String, + recorded_at: SystemTime, + invocation: Invocation, + operation_id: String, + raw: RawToolResultRef, + artifacts: Vec, +} + +impl PreparedEvent { + pub(super) fn inline(json: &str) -> Self { + Self(Prepared::Inline(json.to_owned())) + } + pub(crate) fn decode(self) -> Result { + match self.0 { + Prepared::Inline(json) => Ok(serde_json::from_str(&json)?), + Prepared::Tool { header, projection } => { + let model_projection: DurableToolProjection = serde_json::from_str(&projection)?; + let ToolHeader { + id, + recorded_at, + invocation, + operation_id, + raw, + artifacts, + } = *header; + Ok(RuntimeEvent { + id, + recorded_at, + invocation, + fact: Fact::ToolSettled { + operation_id, + outcome: ToolOutcome::Succeeded { + raw, + model_projection, + artifacts, + }, + }, + }) + } + } + } +} + +pub(super) fn prepare( + header: &str, + offset: usize, + body: &[u8], +) -> Result { + let Header { + id, + recorded_at, + invocation, + fact, + } = serde_json::from_str::>(header)?; + let HeaderFact { + kind, + operation_id, + outcome, + } = fact; + let HeaderOutcome { + kind: outcome_kind, + raw, + model_projection, + artifacts, + } = outcome; + let placeholder = model_projection.get(); + if kind != "tool_settled" + || outcome_kind != "succeeded" + || placeholder != "null" + || (placeholder.as_ptr() as usize).checked_sub(header.as_ptr() as usize) != Some(offset) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "canonical slot does not identify a successful tool projection", + ) + .into()); + } + let body = std::str::from_utf8(body) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + Ok(PreparedEvent(Prepared::Tool { + header: Box::new(ToolHeader { + id, + recorded_at, + invocation, + operation_id, + raw, + artifacts, + }), + projection: body.to_owned(), + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::canonical::recipe::{Recipe, Slot, projection}; + use maka_agent_facts::tool_output::ProjectionPart; + use serde_json::json; + + fn original(value: DurableToolProjection) -> String { + serde_json::to_string_pretty(&json!({ + "extra_null": null, + "id":"settled", "recorded_at":{"secs_since_epoch":1,"nanos_since_epoch":2}, + "invocation":{"session_id":"s","turn_id":"t","run_id":"r","invocation_id":"i"}, + "fact":{"kind":"tool_settled","operation_id":"call","unknown":{"a":[1,true,null]}, + "outcome":{"kind":"succeeded","raw":{"bytes":0,"digest":"sha256:empty"}, + "model_projection":value}} + })) + .unwrap() + } + + #[test] + fn split_events_match_original_typed_facts_and_preserve_the_raw_segments() { + for value in [ + DurableToolProjection::Json { + value: json!({"n":1.0,"text":"汉🦀\n\\\""}), + }, + DurableToolProjection::Text { + text: "text 汉\n\\".repeat(4000), + }, + DurableToolProjection::Content { + parts: vec![ProjectionPart::Text { + text: "part\n汉".into(), + }], + }, + DurableToolProjection::Failure, + ] { + let json = original(value) + .replace("\"n\": 1.0", "\"n\": 1e+00") + .replace("\"tool_settled\"", "\"tool_\\u0073ettled\""); + let expected: RuntimeEvent = serde_json::from_str(&json).unwrap(); + let projection = projection("tool_settled", &json).unwrap().unwrap(); + let header = projection.header(&json).unwrap(); + let recipe = Recipe::new( + &header, + json.len() as u64, + Some(Slot { + offset: projection.offset as u64, + bytes: projection.json.len() as u64, + }), + ) + .unwrap(); + let (prepared, parts) = recipe.prepare(Some(projection.json.as_bytes())).unwrap(); + let event = prepared.decode().unwrap(); + assert_eq!(event, expected); + assert_eq!(parts.concat(), json.as_bytes()); + } + } + + #[test] + fn another_null_cannot_impersonate_the_projection_slot() { + let json = original(DurableToolProjection::Text { + text: "body".into(), + }); + let projection = projection("tool_settled", &json).unwrap().unwrap(); + let header = projection.header(&json).unwrap(); + let wrong = header.find("null").unwrap(); + assert_ne!(wrong, projection.offset); + let recipe = Recipe::new( + &header, + json.len() as u64, + Some(Slot { + offset: wrong as u64, + bytes: projection.json.len() as u64, + }), + ) + .unwrap(); + assert!(recipe.prepare(Some(projection.json.as_bytes())).is_err()); + } + + #[test] + fn known_header_fields_remain_required_and_typed() { + let json = original(DurableToolProjection::Text { + text: "body".into(), + }); + let projection = projection("tool_settled", &json).unwrap().unwrap(); + let header = projection.header(&json).unwrap(); + for changed in [ + header.replace("\"succeeded\"", "\"failed\""), + header.replace("\"tool_settled\"", "\"model_completed\""), + header.replace("\"raw\": {", "\"artifacts\": null, \"raw\": {"), + header.replace("\"id\": \"settled\"", "\"id\": null"), + header.replace( + "\"operation_id\": \"call\"", + "\"operation_id\": \"a\", \"operation_id\": \"b\"", + ), + ] { + let position = changed.find("\"model_projection\": null").unwrap() + + "\"model_projection\": ".len(); + assert!( + prepare(&changed, position, projection.json.as_bytes()) + .and_then(PreparedEvent::decode) + .is_err() + ); + } + } +} diff --git a/platform/event-log/src/canonical/read.rs b/platform/event-log/src/canonical/read.rs index 9d5313401f..3df6cc13e6 100644 --- a/platform/event-log/src/canonical/read.rs +++ b/platform/event-log/src/canonical/read.rs @@ -49,6 +49,10 @@ struct Fields<'a> { payload: Option<&'a [u8]>, } fn reconstruct(fields: Fields<'_>) -> Result, StoreError> { + let (recipe, payload) = material(fields)?; + recipe.reconstruct(payload) +} +fn material<'a>(fields: Fields<'a>) -> Result<(Recipe<'a>, Option<&'a [u8]>), StoreError> { let Fields { header, length, @@ -73,7 +77,7 @@ fn reconstruct(fields: Fields<'_>) -> Result, StoreError> { let length = nonnegative(length.ok_or_else(|| invalid("live event lacks its canonical byte length"))?)?; match (reference, offset, body, raw_bytes, payload) { - (None, None, None, None, None) => Recipe::new(header, length, None)?.reconstruct(None), + (None, None, None, None, None) => Ok((Recipe::new(header, length, None)?, None)), (Some(reference), Some(offset), Some(id), Some(raw_bytes), Some(payload)) => { if reference <= 0 || reference != id || id <= 0 || raw_bytes <= 0 { return Err(invalid("invalid canonical body identity or length")); @@ -87,7 +91,7 @@ fn reconstruct(fields: Fields<'_>) -> Result, StoreError> { bytes: raw_bytes, }), )?; - recipe.reconstruct(Some(payload)) + Ok((recipe, Some(payload))) } _ => Err(invalid("canonical slot or body is missing joined metadata")), } @@ -120,6 +124,18 @@ pub(crate) fn json(row: &SqliteRow) -> Result, StoreError> { }) } pub(crate) fn json_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result, StoreError> { + reconstruct(fields_on_handle(row)?) +} +pub(crate) fn prepare_event_on_handle<'a>( + row: &'a rusqlite::Row<'_>, + visit: impl FnOnce([&'a [u8]; 3]), +) -> Result { + let (recipe, body) = material(fields_on_handle(row)?)?; + let (event, parts) = recipe.prepare(body)?; + visit(parts); + Ok(event) +} +fn fields_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result, StoreError> { let header = row .get_ref("event_header")? .as_str_or_null() @@ -135,7 +151,7 @@ pub(crate) fn json_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result = row.get("canonical_body_reference")?; if reference.is_none() { - return reconstruct(Fields { + return Ok(Fields { header, length, reference, @@ -157,7 +173,7 @@ pub(crate) fn json_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result Recipe<'a> { Ok(()) }) } + pub fn prepare( + &self, + body: Option<&'a [u8]>, + ) -> Result<(super::event::PreparedEvent, [&'a [u8]; 3]), StoreError> { + match (self.layout.slot, body) { + (None, None) => Ok(( + super::event::PreparedEvent::inline(self.header), + [self.header.as_bytes(), &[], &[]], + )), + (Some(slot), Some(body)) => { + if body.len() as u64 != slot.bytes { + return Err(invalid("canonical body length changed")); + } + let position = usize::try_from(slot.offset) + .map_err(|_| invalid("slot offset exceeds memory range"))?; + let event = super::event::prepare(self.header, position, body)?; + Ok(( + event, + [ + &self.header.as_bytes()[..position], + body, + &self.header.as_bytes()[position + 4..], + ], + )) + } + _ => Err(invalid("canonical body does not match its slot")), + } + } pub fn reconstruct_with( &self, append_body: impl FnOnce(&mut Vec) -> Result<(), StoreError>, diff --git a/platform/event-log/src/context/evidence/roots.rs b/platform/event-log/src/context/evidence/roots.rs index 260d381120..e82b9fd7c5 100644 --- a/platform/event-log/src/context/evidence/roots.rs +++ b/platform/event-log/src/context/evidence/roots.rs @@ -63,9 +63,23 @@ impl RootDigest { Self(Prefix::empty().hash) } - pub(crate) fn record(&mut self, sequence: i64, json: &str) { + pub(crate) fn record(&mut self, sequence: i64, json: &[&[u8]]) { let sequence = sequence.to_be_bytes(); - let leaf = hash(b"maka.source.leaf.v1\0", &[&sequence, json.as_bytes()]); + let mut leaf = Sha256::new(); + leaf.update(b"maka.source.leaf.v1\0"); + leaf.update((sequence.len() as u64).to_be_bytes()); + leaf.update(sequence); + // The chunks are one original JSON field, with one length prefix. + leaf.update( + json.iter() + .map(|part| part.len() as u64) + .sum::() + .to_be_bytes(), + ); + for part in json { + leaf.update(part); + } + let leaf = leaf.finalize(); self.0 = hash(b"maka.source.chain.v1\0", &[&self.0, &sequence, &leaf]); } @@ -77,6 +91,41 @@ impl RootDigest { } } +#[cfg(test)] +mod root_digest_tests { + use super::*; + + #[test] + fn segmented_raw_json_keeps_the_original_single_field_framing() { + let mut expected = Prefix::empty().hash; + let mut actual = RootDigest::new(); + for (sequence, json) in [ + (7_i64, "{ \"n\":1e+00,\"t\":\"汉\\u0061\" }"), + (91, "{\"empty\":null}"), + ] { + let bytes = json.as_bytes(); + let seq = sequence.to_be_bytes(); + let leaf = hash(b"maka.source.leaf.v1\0", &[&seq, bytes]); + expected = hash(b"maka.source.chain.v1\0", &[&expected, &seq, &leaf]); + // Include a split within a UTF-8 character: hashing is over bytes. + let split = bytes.iter().position(|b| *b >= 128).map_or(3, |i| i + 1); + actual.record( + sequence, + &[ + &bytes[..split], + &bytes[split..split + 1], + &bytes[split + 1..], + ], + ); + } + assert_eq!(actual.0, expected); + assert_eq!( + hex::encode(actual.0), + "c15a4587be8da9a4aabdb746de55daa2d5913e429e929f47ed2ed677f00b245f" + ); + } +} + fn native(db: &Connection, kind: &str, owner: &str, cut: i64) -> rusqlite::Result { Ok(db .query_row(NATIVE, params![kind, owner, cut], |row| { diff --git a/platform/event-log/src/prefix.rs b/platform/event-log/src/prefix.rs index 9328b85ef1..60f5eb2c25 100644 --- a/platform/event-log/src/prefix.rs +++ b/platform/event-log/src/prefix.rs @@ -19,11 +19,12 @@ use maka_agent_facts::event::LogPrefix; use maka_agent_facts::event::LogScope; -use maka_agent_facts::event::StoredEvent; use sqlx::Connection; use crate::{EventLog, StoreError, context::selection::Selection, sequence_number}; +mod decode; + impl EventLog { /// Read one complete bounded execution prefix, or fail without returning a partial /// history that a caller could accidentally treat as complete. Session-owned @@ -82,26 +83,31 @@ impl EventLog { ); let session = selection.session.clone(); let selected_lineage = selection.lineage.clone(); - let (events, root_digest) = crate::connection::worker_query(&mut transaction, move |worker| Box::pin(async move { + let root_scope = session.is_none(); + let (events, root_digest) = if count == 0 { + (Vec::new(), root_scope.then(crate::context::evidence::RootDigest::new)) + } else { decode::run(count as usize, |sender| crate::connection::worker_query(&mut transaction, move |worker| Box::pin(async move { // SAFETY: this callback exclusively owns the original SQLite worker. // The wrapper and all borrowed rows/statements close before return. let raw = unsafe { rusqlite::Connection::from_handle(worker.as_raw_handle().as_ptr()) }?; let result = (|| { let mut digest = session.is_none().then(crate::context::evidence::RootDigest::new); - let mut events = Vec::new(); let mut statement = raw.prepare(&query)?; let mut rows = statement.query(rusqlite::params![session, selected_lineage])?; while let Some(row) = rows.next()? { let sequence: i64 = row.get("sequence")?; - let json = crate::canonical::json_on_handle(row)?; - if let Some(digest) = &mut digest { digest.record(sequence, &json); } - events.push(StoredEvent { sequence: sequence_number(sequence)?, event: serde_json::from_str(&json)? }); + let prepared = crate::canonical::prepare_event_on_handle(row, |parts| { + if let Some(digest) = &mut digest { digest.record(sequence, &parts); } + })?; + if sender.send((sequence_number(sequence)?, prepared)).is_err() { + return Ok(decode::Produced::DecoderStopped); + } } - Ok((events, digest)) + Ok(decode::Produced::Complete(digest)) })(); drop(raw); result - })).await?; + }))).await? }; let digest = match root_digest { Some(digest) => digest.finish(high_water)?, None => crate::context::evidence::selected( diff --git a/platform/event-log/src/prefix/decode.rs b/platform/event-log/src/prefix/decode.rs new file mode 100644 index 0000000000..672ee809a1 --- /dev/null +++ b/platform/event-log/src/prefix/decode.rs @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +//! Join both stages before the caller commits, rolls back, or fences its owner. +use crate::{StoreError, canonical::PreparedEvent, context::evidence::RootDigest}; +use futures_util::FutureExt; +use maka_agent_facts::event::StoredEvent; +use std::{future::Future, panic::AssertUnwindSafe, sync::mpsc}; +use tokio::task::JoinHandle; + +pub(super) enum Produced { + Complete(Option), + DecoderStopped, +} + +pub(super) async fn run( + count: usize, + produce: P, +) -> Result<(Vec, Option), StoreError> +where + P: FnOnce(mpsc::Sender<(u64, PreparedEvent)>) -> F, + F: Future>, +{ + // The snapshot's preflight already bounds this finite batch by the caller's + // event/byte budgets. No producer data survives this one admitted prefix. + let (sender, receiver) = mpsc::channel::<(u64, PreparedEvent)>(); + let decoder = tokio::task::spawn_blocking(move || { + receiver + .into_iter() + .map(|(sequence, prepared)| { + Ok(StoredEvent { + sequence, + event: prepared.decode()?, + }) + }) + .collect() + }); + finish(async move { produce(sender).await }, decoder, count).await +} + +async fn finish( + produce: F, + decoder: JoinHandle, StoreError>>, + count: usize, +) -> Result<(Vec, Option), StoreError> +where + F: Future>, +{ + // SQLx rethrows worker panics at await. Drain the CPU task before restoring + // that panic so the existing owner fence cannot detach unfinished work. + let produced = AssertUnwindSafe(produce).catch_unwind().await; + let decoded = decoder.await; + let produced = match produced { + Ok(result) => result, + Err(panic) => std::panic::resume_unwind(panic), + }; + let decoded = match decoded { + Ok(result) => result, + Err(error) if error.is_panic() => std::panic::resume_unwind(error.into_panic()), + Err(error) => Err(std::io::Error::other(error).into()), + }; + let produced = produced?; + let events = decoded?; + match produced { + Produced::Complete(digest) if events.len() == count => Ok((events, digest)), + _ => Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "prefix decoder stopped before completing the admitted snapshot", + ) + .into()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }; + + #[tokio::test] + async fn producer_panic_waits_for_decoder_before_restoring_the_owner_panic() { + let (release, blocked) = mpsc::channel(); + let (started, ready) = tokio::sync::oneshot::channel(); + let finished = Arc::new(AtomicBool::new(false)); + let done = finished.clone(); + let decoder = tokio::task::spawn_blocking(move || { + let _ = started.send(()); + let _ = blocked.recv(); + done.store(true, Ordering::Release); + Ok(Vec::new()) + }); + ready.await.unwrap(); + let mut parent = Box::pin( + AssertUnwindSafe(finish(async { panic!("producer panic") }, decoder, 0)).catch_unwind(), + ); + assert!(parent.as_mut().now_or_never().is_none()); + assert!(!finished.load(Ordering::Acquire)); + release.send(()).unwrap(); + let result = parent.await; + let Err(panic) = result else { + panic!("producer panic was hidden") + }; + assert_eq!(panic.downcast_ref::<&str>(), Some(&"producer panic")); + assert!(finished.load(Ordering::Acquire)); + } + + #[tokio::test] + async fn producer_error_also_drains_the_decoder() { + let (release, blocked) = mpsc::channel(); + let (started, ready) = tokio::sync::oneshot::channel(); + let decoder = tokio::task::spawn_blocking(move || { + let _ = started.send(()); + let _ = blocked.recv(); + Ok(Vec::new()) + }); + ready.await.unwrap(); + let mut parent = Box::pin(finish(async { Err(StoreError::EventConflict) }, decoder, 0)); + assert!(parent.as_mut().now_or_never().is_none()); + release.send(()).unwrap(); + assert!(matches!(parent.await, Err(StoreError::EventConflict))); + } + + #[tokio::test] + async fn decoder_failure_is_not_masked_by_a_stopped_producer() { + let decoder = tokio::task::spawn_blocking(|| Err(StoreError::EventConflict)); + assert!(matches!( + finish(async { Ok(Produced::DecoderStopped) }, decoder, 1).await, + Err(StoreError::EventConflict) + )); + let decoder = tokio::task::spawn_blocking(|| Ok(Vec::new())); + assert!( + finish(async { Ok(Produced::Complete(None)) }, decoder, 1) + .await + .is_err() + ); + } + + #[tokio::test] + async fn decoder_panic_is_restored_after_the_producer_finishes() { + let finished = Arc::new(AtomicBool::new(false)); + let done = finished.clone(); + let decoder = tokio::task::spawn_blocking(|| panic!("decoder panic")); + let result = AssertUnwindSafe(finish( + async move { + done.store(true, Ordering::Release); + Ok(Produced::Complete(None)) + }, + decoder, + 0, + )) + .catch_unwind() + .await; + assert!(finished.load(Ordering::Acquire)); + let Err(panic) = result else { + panic!("decoder panic was hidden") + }; + assert_eq!(panic.downcast_ref::<&str>(), Some(&"decoder panic")); + } +} diff --git a/platform/event-log/src/root_store/checkpoints/tests.rs b/platform/event-log/src/root_store/checkpoints/tests.rs index b088124b09..40d94ec1f5 100644 --- a/platform/event-log/src/root_store/checkpoints/tests.rs +++ b/platform/event-log/src/root_store/checkpoints/tests.rs @@ -25,6 +25,8 @@ use std::{ }; use uuid::Uuid; +mod body; + fn namespaces(path: &Path) -> root::RootNamespaces { root::RootNamespaces { ownership: path.join("ownership"), diff --git a/platform/event-log/src/root_store/checkpoints/tests/body.rs b/platform/event-log/src/root_store/checkpoints/tests/body.rs new file mode 100644 index 0000000000..d741f934eb --- /dev/null +++ b/platform/event-log/src/root_store/checkpoints/tests/body.rs @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use super::*; +use crate::sessions::{MaterialCollection, RemovalAuthority, RemoveFamilyResult}; +use maka_agent_facts::{ + event::{ + EventWrite, Fact, Invocation, InvocationInput, InvocationOutcome, LogScope, RuntimeEvent, + }, + tool_call::ToolCallIdentity, + tool_output::{DurableToolProjection, ToolOutput, ToolSuccess}, +}; +use std::collections::BTreeMap; + +#[tokio::test] +async fn checkpoint_keeps_tool_bytes_after_source_collection_and_import() { + let directory = tempfile::tempdir().unwrap(); + let (store, handle, request) = fixture(directory.path()).await; + let log = handle.session.scoped().await.unwrap(); + let invocation = Invocation { + session_id: "source".into(), + turn_id: "turn".into(), + run_id: "run".into(), + invocation_id: "invocation".into(), + }; + let plain = |fact| EventWrite::plain(RuntimeEvent::new(invocation.clone(), fact)).unwrap(); + log.append(&plain(Fact::InvocationOpened { + configuration: None, + input: InvocationInput::Code { + source: "fixture".into(), + }, + })) + .await + .unwrap(); + log.append(&plain(Fact::ToolDispatched { + presentation: None, + title: None, + operation_id: "read".into(), + call: ToolCallIdentity::standalone("call".into()), + name: "Read".into(), + input: json!({}), + })) + .await + .unwrap(); + let text = "checkpoint 汉🦀\n\\\"".repeat(4000); + let result = EventWrite::tool_success( + "result".into(), + std::time::UNIX_EPOCH, + invocation.clone(), + "read".into(), + ToolSuccess::projected( + ToolOutput::Text(text.clone()), + DurableToolProjection::Text { text: text.clone() }, + ), + ) + .unwrap() + .0; + log.append(&result).await.unwrap(); + log.append(&plain(Fact::InvocationEnded { + outcome: InvocationOutcome::Completed, + })) + .await + .unwrap(); + let original = log + .scoped_prefix( + LogScope::Session { + id: "source".into(), + }, + usize::MAX, + usize::MAX, + ) + .await + .unwrap() + .events; + drop(log); + let value = handle.publish(request.clone()).await.unwrap(); + let head = published(&value); + let log = handle.session.owner().unwrap(); + let revision = store + .get_session::("source") + .await + .unwrap() + .unwrap() + .revision; + assert!(matches!( + store + .remove_session_family("source", revision, RemovalAuthority::Unmanaged) + .await + .unwrap(), + RemoveFamilyResult::Accepted(_) + )); + store.finish_session_retirement("source").await.unwrap(); + while log.collect_session_material(None).await.unwrap() == MaterialCollection::Collected {} + let bodies = log + .connection + .run(|db| { + Box::pin(async move { + Ok( + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM event_bodies") + .fetch_one(db) + .await?, + ) + }) + }) + .await + .unwrap(); + assert_eq!( + bodies, 0, + "checkpoint bytes are independent of collected canonical material" + ); + drop(log); + store.close().await.unwrap(); + + let store = RootStore::open(owner(directory.path())).await.unwrap(); + let handle = store.history_checkpoints("source", scope()).await.unwrap(); + assert_eq!(handle.receipt(request).await.unwrap(), Some(value)); + let checkout = handle.checkout_exact(head.revision).await.unwrap(); + let mut staged = StagedBundle::read(tokio::fs::File::from_std(checkout.bundle)) + .await + .unwrap(); + staged.validate_history().await.unwrap(); + let destination_directory = tempfile::tempdir().unwrap(); + let destination = RootStore::open(owner(destination_directory.path())) + .await + .unwrap(); + let receipt = destination + .import_bundle( + staged, + &maka_agent_facts::artifact::content_digest(b"checkpoint destination"), + BTreeMap::from([("source".to_owned(), json!({"title":"restored"}))]), + ) + .await + .unwrap(); + let session = destination.session(&receipt.root_session_id).await.unwrap(); + assert_eq!( + session + .resolve_tool_result(&receipt.root_session_id, "result") + .await + .unwrap(), + ToolOutput::Text(text) + ); + let restored = session + .scoped_prefix( + LogScope::Session { + id: receipt.root_session_id, + }, + usize::MAX, + usize::MAX, + ) + .await + .unwrap() + .events; + assert_eq!( + serde_json::to_value(restored).unwrap(), + serde_json::to_value(original).unwrap() + ); + destination.close().await.unwrap(); + store.close().await.unwrap(); +} diff --git a/scripts/bench-event-storage.py b/scripts/bench-event-storage.py index e418ca4647..4b44a41309 100644 --- a/scripts/bench-event-storage.py +++ b/scripts/bench-event-storage.py @@ -25,6 +25,7 @@ """ import argparse +from contextlib import closing import hashlib import json import os @@ -46,7 +47,7 @@ def storage_stats(directory): for suffix in ("-wal", "-shm"): sidecar = path.with_name(path.name + suffix) sidecars[suffix] = sidecar.stat().st_size if sidecar.exists() else 0 - with sqlite3.connect(f"{path.as_uri()}?mode=ro", uri=True) as db: + with closing(sqlite3.connect(f"{path.as_uri()}?mode=ro", uri=True)) as db: tables = {name for (name,) in db.execute("SELECT name FROM sqlite_schema WHERE type='table'")} directories = {} for table in ("context_semantic_rows", "context_semantic_fragments", "context_semantic_directory_children"): From 3b5dcec4b64097f8a33d108a444ec4ee3efe453b Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Sun, 1 Nov 2026 05:20:36 +0800 Subject: [PATCH 5/8] refactor(storage): decode canonical prefixes on the SQLite worker Decode borrowed headers and original projection slices before advancing each SQLite row. Remove the temporary body string, transfer queue and independent decoder while preserving the original snapshot, budgets and raw evidence. Align the cleanup scheduling test with upstream's readable Root contract, retaining sibling drainage, confirmed outcomes and failed shutdown evidence. Generated-by: Codex --- platform/event-log/README.md | 2 +- platform/event-log/README.zh-CN.md | 2 +- platform/event-log/src/canonical.rs | 3 +- platform/event-log/src/canonical/event.rs | 93 ++--------- platform/event-log/src/canonical/read.rs | 6 +- platform/event-log/src/canonical/recipe.rs | 8 +- platform/event-log/src/prefix.rs | 20 +-- platform/event-log/src/prefix/decode.rs | 177 --------------------- 8 files changed, 38 insertions(+), 273 deletions(-) delete mode 100644 platform/event-log/src/prefix/decode.rs diff --git a/platform/event-log/README.md b/platform/event-log/README.md index 4ac2128c5a..a20870a767 100644 --- a/platform/event-log/README.md +++ b/platform/event-log/README.md @@ -73,7 +73,7 @@ Canonical databases use UTF-8. Successful tool results store their original `mod Bodies are directly seekable raw bytes, shared by SHA-256 of their original token with exact byte comparison on a repeated identity. Reads validate logical lengths and selected ranges. Framed compression was evaluated against raw storage and rejected because it repeatedly slowed supported window and sequential reads; it is not a persisted format. Space savings come from compact directories and sharing identical projection tokens, not from compressing unique bodies. -Complete-event queries join the referenced body into the selected SQL row within the original snapshot. JSON readers reconstruct the original bytes; typed prefix readers prepare the header and original projection bytes on the SQLite worker, then decode owned values on a joined blocking task while reading continues. They avoid a full temporary JSON string and keep the original transaction until both stages finish, including error and panic paths. The transfer contains only the prefix admitted by the existing event/byte budgets; no SQLite borrow crosses to the decoder. Typed headers reuse fact field types and defaults, ignore unknown metadata, and verify that the projection placeholder is exactly at its stored offset. Root evidence hashes the three original byte slices as one length-prefixed JSON field. Metadata predicates use the header and retain their indexes. Semantic spans retain original canonical coordinates; tool spans map those logical coordinates to header/body ranges, while inline model parts retain physical SQLite BLOB bounds. Resolution stays on the original SQLite worker. Hot range reads use its existing prepared-statement cache. Body references follow the existing history-owner collection proof; clearing or deleting the final reference removes its body in the same transaction. The event row publishes the header, logical length, offset and reference atomically; live references cannot be redirected. Neither a body digest nor a derived directory grants access or extends lifetime. Derived archive placeholders and other fragment bodies keep their chunked storage. +Complete-event queries join the referenced body into the selected SQL row within the original snapshot. JSON readers reconstruct the original bytes; typed prefix readers decode the borrowed header and original projection bytes directly on the original SQLite worker. They avoid copying the body into a temporary string and finish decoding before advancing its row. The existing event/byte preflight and transaction cover the complete result; no SQLite borrow or unfinished decoder leaves that worker. Typed headers reuse fact field types and defaults, ignore unknown metadata, and verify that the projection placeholder is exactly at its stored offset. Root evidence hashes the three original byte slices as one length-prefixed JSON field. Metadata predicates use the header and retain their indexes. Semantic spans retain original canonical coordinates; tool spans map those logical coordinates to header/body ranges, while inline model parts retain physical SQLite BLOB bounds. Resolution stays on the original SQLite worker. Hot range reads use its existing prepared-statement cache. Body references follow the existing history-owner collection proof; clearing or deleting the final reference removes its body in the same transaction. The event row publishes the header, logical length, offset and reference atomically; live references cannot be redirected. Neither a body digest nor a derived directory grants access or extends lifetime. Derived archive placeholders and other fragment bodies keep their chunked storage. Semantic row, payload and child directories are finite derived descriptors encoded with Postcard. Writers encode them once in the publication transaction. Pages copy their stored bytes alongside fixed little-endian locator fields into the caller's charged allocation; consumers decode the original typed descriptors without navigating whole bodies. Original byte positions and leaf commitments remain unchanged. Directory encoding never supplies a canonical fact or source identity. The SQL migration converts former JSON descriptors atomically; its conversion functions exist only during migration. diff --git a/platform/event-log/README.zh-CN.md b/platform/event-log/README.zh-CN.md index 84525625e4..468d696592 100644 --- a/platform/event-log/README.zh-CN.md +++ b/platform/event-log/README.zh-CN.md @@ -67,7 +67,7 @@ Control 中的调用意图、项目登记、workspace attachment、Session 路 正文保存为可直接 seek 的原始字节，按原始 token 的 SHA-256 共享，重复身份还需逐字节核对。读取校验逻辑长度与选中区间。对照实验中，分帧压缩稳定拖慢了受支持的窗口和顺序读取，因此未作为持久化格式保留。空间收益来自紧凑目录与相同投影 token 的共享，不能把压缩实验中独有正文的压缩率套到最终实现。 -完整事件查询在原快照内的同一 SQL 行 JOIN 引用的正文。JSON reader 重建原始字节；类型化前缀 reader 在原 SQLite worker 上准备 header 和原始投影字节，由等待完成的阻塞任务解码已拥有的数据，同时继续读取。它避免构造完整临时 JSON 字符串；包括错误和 panic 在内，都等待两段结束后才结束原事务。传递范围仅限已通过原有事件／字节预算的前缀，不把 SQLite 借用交给解码任务。Header 复用事实字段的类型和默认值、忽略未知元数据，并验证投影占位符恰好位于已存偏移。Root 证据把原始三段字节作为一个带长度前缀的 JSON 字段计算摘要；元数据谓词继续读取 header 并使用原索引。语义区间保留原规范坐标，工具区间映射至实际 header／body，内联 model part 则保留 SQLite BLOB 的物理边界。读取仍由原 SQLite worker 执行。热区间读取复用其已有 statement cache。正文引用的存活由既有历史所有者回收证明决定，最后一个引用清除或删除时，在同一事务删除正文。事件行原子发布 header、逻辑长度、偏移和引用，存活引用不可重定向；正文摘要和派生目录都不授予访问权，也不延长存活期。归档占位内容和其他片段正文继续使用分块存储。 +完整事件查询在原快照内的同一 SQL 行 JOIN 引用的正文。JSON reader 重建原始字节；类型化前缀 reader 在原 SQLite worker 上直接解码借用的 header 和原始投影字节，避免把正文复制到临时字符串中，并在推进当前行之前完成解码。原有事件／字节预检和事务覆盖完整结果；SQLite 借用和未完成的解码都不会离开该 worker。Header 复用事实字段的类型和默认值、忽略未知元数据，并验证投影占位符恰好位于已存偏移。Root 证据把原始三段字节作为一个带长度前缀的 JSON 字段计算摘要；元数据谓词继续读取 header 并使用原索引。语义区间保留原规范坐标，工具区间映射至实际 header／body，内联 model part 则保留 SQLite BLOB 的物理边界。读取仍由原 SQLite worker 执行。热区间读取复用其已有 statement cache。正文引用的存活由既有历史所有者回收证明决定，最后一个引用清除或删除时，在同一事务删除正文。事件行原子发布 header、逻辑长度、偏移和引用，存活引用不可重定向；正文摘要和派生目录都不授予访问权，也不延长存活期。归档占位内容和其他片段正文继续使用分块存储。 语义行、payload 和子项目录是有限的派生描述符，采用 Postcard 编码。writer 在发布事务内编码一次；分页把已存字节和固定小端序定位字段直接复制到调用者计费的缓冲区，消费端恢复原有类型，无需读取整个正文。原始字节位置与叶子摘要保持不变，目录编码不参与规范事实或来源身份。SQL 迁移在同一事务内转换原 JSON 描述符；转换函数仅在迁移期间存在。 diff --git a/platform/event-log/src/canonical.rs b/platform/event-log/src/canonical.rs index e91c895a2d..722e5bf082 100644 --- a/platform/event-log/src/canonical.rs +++ b/platform/event-log/src/canonical.rs @@ -25,8 +25,7 @@ mod recipe; mod source; mod write; -pub(crate) use event::PreparedEvent; -pub(crate) use read::{COLUMNS, JOINS, json, json_on_handle, one, prepare_event_on_handle}; +pub(crate) use read::{COLUMNS, JOINS, decode_event_on_handle, json, json_on_handle, one}; pub(crate) use recipe::projection; pub(crate) use source::{ SPAN_COLUMNS, SpanMetadata, with_span, with_span_metadata, with_span_on_row, diff --git a/platform/event-log/src/canonical/event.rs b/platform/event-log/src/canonical/event.rs index 233a23f427..d4c53fad62 100644 --- a/platform/event-log/src/canonical/event.rs +++ b/platform/event-log/src/canonical/event.rs @@ -53,64 +53,7 @@ struct HeaderOutcome<'a> { artifacts: Vec, } -pub(crate) struct PreparedEvent(Prepared); - -enum Prepared { - Inline(String), - Tool { - header: Box, - projection: String, - }, -} -struct ToolHeader { - id: String, - recorded_at: SystemTime, - invocation: Invocation, - operation_id: String, - raw: RawToolResultRef, - artifacts: Vec, -} - -impl PreparedEvent { - pub(super) fn inline(json: &str) -> Self { - Self(Prepared::Inline(json.to_owned())) - } - pub(crate) fn decode(self) -> Result { - match self.0 { - Prepared::Inline(json) => Ok(serde_json::from_str(&json)?), - Prepared::Tool { header, projection } => { - let model_projection: DurableToolProjection = serde_json::from_str(&projection)?; - let ToolHeader { - id, - recorded_at, - invocation, - operation_id, - raw, - artifacts, - } = *header; - Ok(RuntimeEvent { - id, - recorded_at, - invocation, - fact: Fact::ToolSettled { - operation_id, - outcome: ToolOutcome::Succeeded { - raw, - model_projection, - artifacts, - }, - }, - }) - } - } - } -} - -pub(super) fn prepare( - header: &str, - offset: usize, - body: &[u8], -) -> Result { +pub(super) fn decode(header: &str, offset: usize, body: &[u8]) -> Result { let Header { id, recorded_at, @@ -142,17 +85,20 @@ pub(super) fn prepare( } let body = std::str::from_utf8(body) .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; - Ok(PreparedEvent(Prepared::Tool { - header: Box::new(ToolHeader { - id, - recorded_at, - invocation, + let model_projection: DurableToolProjection = serde_json::from_str(body)?; + Ok(RuntimeEvent { + id, + recorded_at, + invocation, + fact: Fact::ToolSettled { operation_id, - raw, - artifacts, - }), - projection: body.to_owned(), - })) + outcome: ToolOutcome::Succeeded { + raw, + model_projection, + artifacts, + }, + }, + }) } #[cfg(test)] @@ -205,8 +151,7 @@ mod tests { }), ) .unwrap(); - let (prepared, parts) = recipe.prepare(Some(projection.json.as_bytes())).unwrap(); - let event = prepared.decode().unwrap(); + let (event, parts) = recipe.decode(Some(projection.json.as_bytes())).unwrap(); assert_eq!(event, expected); assert_eq!(parts.concat(), json.as_bytes()); } @@ -230,7 +175,7 @@ mod tests { }), ) .unwrap(); - assert!(recipe.prepare(Some(projection.json.as_bytes())).is_err()); + assert!(recipe.decode(Some(projection.json.as_bytes())).is_err()); } #[test] @@ -252,11 +197,7 @@ mod tests { ] { let position = changed.find("\"model_projection\": null").unwrap() + "\"model_projection\": ".len(); - assert!( - prepare(&changed, position, projection.json.as_bytes()) - .and_then(PreparedEvent::decode) - .is_err() - ); + assert!(decode(&changed, position, projection.json.as_bytes()).is_err()); } } } diff --git a/platform/event-log/src/canonical/read.rs b/platform/event-log/src/canonical/read.rs index 3df6cc13e6..593d790f3e 100644 --- a/platform/event-log/src/canonical/read.rs +++ b/platform/event-log/src/canonical/read.rs @@ -126,12 +126,12 @@ pub(crate) fn json(row: &SqliteRow) -> Result, StoreError> { pub(crate) fn json_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result, StoreError> { reconstruct(fields_on_handle(row)?) } -pub(crate) fn prepare_event_on_handle<'a>( +pub(crate) fn decode_event_on_handle<'a>( row: &'a rusqlite::Row<'_>, visit: impl FnOnce([&'a [u8]; 3]), -) -> Result { +) -> Result { let (recipe, body) = material(fields_on_handle(row)?)?; - let (event, parts) = recipe.prepare(body)?; + let (event, parts) = recipe.decode(body)?; visit(parts); Ok(event) } diff --git a/platform/event-log/src/canonical/recipe.rs b/platform/event-log/src/canonical/recipe.rs index fcaf62f0d1..d76d6dae32 100644 --- a/platform/event-log/src/canonical/recipe.rs +++ b/platform/event-log/src/canonical/recipe.rs @@ -244,13 +244,13 @@ impl<'a> Recipe<'a> { Ok(()) }) } - pub fn prepare( + pub fn decode( &self, body: Option<&'a [u8]>, - ) -> Result<(super::event::PreparedEvent, [&'a [u8]; 3]), StoreError> { + ) -> Result<(maka_agent_facts::event::RuntimeEvent, [&'a [u8]; 3]), StoreError> { match (self.layout.slot, body) { (None, None) => Ok(( - super::event::PreparedEvent::inline(self.header), + serde_json::from_str(self.header)?, [self.header.as_bytes(), &[], &[]], )), (Some(slot), Some(body)) => { @@ -259,7 +259,7 @@ impl<'a> Recipe<'a> { } let position = usize::try_from(slot.offset) .map_err(|_| invalid("slot offset exceeds memory range"))?; - let event = super::event::prepare(self.header, position, body)?; + let event = super::event::decode(self.header, position, body)?; Ok(( event, [ diff --git a/platform/event-log/src/prefix.rs b/platform/event-log/src/prefix.rs index 60f5eb2c25..0f497e2159 100644 --- a/platform/event-log/src/prefix.rs +++ b/platform/event-log/src/prefix.rs @@ -19,12 +19,11 @@ use maka_agent_facts::event::LogPrefix; use maka_agent_facts::event::LogScope; +use maka_agent_facts::event::StoredEvent; use sqlx::Connection; use crate::{EventLog, StoreError, context::selection::Selection, sequence_number}; -mod decode; - impl EventLog { /// Read one complete bounded execution prefix, or fail without returning a partial /// history that a caller could accidentally treat as complete. Session-owned @@ -86,28 +85,31 @@ impl EventLog { let root_scope = session.is_none(); let (events, root_digest) = if count == 0 { (Vec::new(), root_scope.then(crate::context::evidence::RootDigest::new)) - } else { decode::run(count as usize, |sender| crate::connection::worker_query(&mut transaction, move |worker| Box::pin(async move { + } else { crate::connection::worker_query(&mut transaction, move |worker| Box::pin(async move { // SAFETY: this callback exclusively owns the original SQLite worker. // The wrapper and all borrowed rows/statements close before return. let raw = unsafe { rusqlite::Connection::from_handle(worker.as_raw_handle().as_ptr()) }?; let result = (|| { let mut digest = session.is_none().then(crate::context::evidence::RootDigest::new); + let mut events = Vec::new(); let mut statement = raw.prepare(&query)?; let mut rows = statement.query(rusqlite::params![session, selected_lineage])?; while let Some(row) = rows.next()? { let sequence: i64 = row.get("sequence")?; - let prepared = crate::canonical::prepare_event_on_handle(row, |parts| { + let event = crate::canonical::decode_event_on_handle(row, |parts| { if let Some(digest) = &mut digest { digest.record(sequence, &parts); } })?; - if sender.send((sequence_number(sequence)?, prepared)).is_err() { - return Ok(decode::Produced::DecoderStopped); - } + events.push(StoredEvent { sequence: sequence_number(sequence)?, event }); + } + if events.len() as u64 != sequence_number(count)? { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, + "prefix decoder stopped before completing the admitted snapshot").into()); } - Ok(decode::Produced::Complete(digest)) + Ok((events, digest)) })(); drop(raw); result - }))).await? }; + })).await? }; let digest = match root_digest { Some(digest) => digest.finish(high_water)?, None => crate::context::evidence::selected( diff --git a/platform/event-log/src/prefix/decode.rs b/platform/event-log/src/prefix/decode.rs deleted file mode 100644 index 672ee809a1..0000000000 --- a/platform/event-log/src/prefix/decode.rs +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -//! Join both stages before the caller commits, rolls back, or fences its owner. -use crate::{StoreError, canonical::PreparedEvent, context::evidence::RootDigest}; -use futures_util::FutureExt; -use maka_agent_facts::event::StoredEvent; -use std::{future::Future, panic::AssertUnwindSafe, sync::mpsc}; -use tokio::task::JoinHandle; - -pub(super) enum Produced { - Complete(Option), - DecoderStopped, -} - -pub(super) async fn run( - count: usize, - produce: P, -) -> Result<(Vec, Option), StoreError> -where - P: FnOnce(mpsc::Sender<(u64, PreparedEvent)>) -> F, - F: Future>, -{ - // The snapshot's preflight already bounds this finite batch by the caller's - // event/byte budgets. No producer data survives this one admitted prefix. - let (sender, receiver) = mpsc::channel::<(u64, PreparedEvent)>(); - let decoder = tokio::task::spawn_blocking(move || { - receiver - .into_iter() - .map(|(sequence, prepared)| { - Ok(StoredEvent { - sequence, - event: prepared.decode()?, - }) - }) - .collect() - }); - finish(async move { produce(sender).await }, decoder, count).await -} - -async fn finish( - produce: F, - decoder: JoinHandle, StoreError>>, - count: usize, -) -> Result<(Vec, Option), StoreError> -where - F: Future>, -{ - // SQLx rethrows worker panics at await. Drain the CPU task before restoring - // that panic so the existing owner fence cannot detach unfinished work. - let produced = AssertUnwindSafe(produce).catch_unwind().await; - let decoded = decoder.await; - let produced = match produced { - Ok(result) => result, - Err(panic) => std::panic::resume_unwind(panic), - }; - let decoded = match decoded { - Ok(result) => result, - Err(error) if error.is_panic() => std::panic::resume_unwind(error.into_panic()), - Err(error) => Err(std::io::Error::other(error).into()), - }; - let produced = produced?; - let events = decoded?; - match produced { - Produced::Complete(digest) if events.len() == count => Ok((events, digest)), - _ => Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "prefix decoder stopped before completing the admitted snapshot", - ) - .into()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }; - - #[tokio::test] - async fn producer_panic_waits_for_decoder_before_restoring_the_owner_panic() { - let (release, blocked) = mpsc::channel(); - let (started, ready) = tokio::sync::oneshot::channel(); - let finished = Arc::new(AtomicBool::new(false)); - let done = finished.clone(); - let decoder = tokio::task::spawn_blocking(move || { - let _ = started.send(()); - let _ = blocked.recv(); - done.store(true, Ordering::Release); - Ok(Vec::new()) - }); - ready.await.unwrap(); - let mut parent = Box::pin( - AssertUnwindSafe(finish(async { panic!("producer panic") }, decoder, 0)).catch_unwind(), - ); - assert!(parent.as_mut().now_or_never().is_none()); - assert!(!finished.load(Ordering::Acquire)); - release.send(()).unwrap(); - let result = parent.await; - let Err(panic) = result else { - panic!("producer panic was hidden") - }; - assert_eq!(panic.downcast_ref::<&str>(), Some(&"producer panic")); - assert!(finished.load(Ordering::Acquire)); - } - - #[tokio::test] - async fn producer_error_also_drains_the_decoder() { - let (release, blocked) = mpsc::channel(); - let (started, ready) = tokio::sync::oneshot::channel(); - let decoder = tokio::task::spawn_blocking(move || { - let _ = started.send(()); - let _ = blocked.recv(); - Ok(Vec::new()) - }); - ready.await.unwrap(); - let mut parent = Box::pin(finish(async { Err(StoreError::EventConflict) }, decoder, 0)); - assert!(parent.as_mut().now_or_never().is_none()); - release.send(()).unwrap(); - assert!(matches!(parent.await, Err(StoreError::EventConflict))); - } - - #[tokio::test] - async fn decoder_failure_is_not_masked_by_a_stopped_producer() { - let decoder = tokio::task::spawn_blocking(|| Err(StoreError::EventConflict)); - assert!(matches!( - finish(async { Ok(Produced::DecoderStopped) }, decoder, 1).await, - Err(StoreError::EventConflict) - )); - let decoder = tokio::task::spawn_blocking(|| Ok(Vec::new())); - assert!( - finish(async { Ok(Produced::Complete(None)) }, decoder, 1) - .await - .is_err() - ); - } - - #[tokio::test] - async fn decoder_panic_is_restored_after_the_producer_finishes() { - let finished = Arc::new(AtomicBool::new(false)); - let done = finished.clone(); - let decoder = tokio::task::spawn_blocking(|| panic!("decoder panic")); - let result = AssertUnwindSafe(finish( - async move { - done.store(true, Ordering::Release); - Ok(Produced::Complete(None)) - }, - decoder, - 0, - )) - .catch_unwind() - .await; - assert!(finished.load(Ordering::Acquire)); - let Err(panic) = result else { - panic!("decoder panic was hidden") - }; - assert_eq!(panic.downcast_ref::<&str>(), Some(&"decoder panic")); - } -} From e93efbc9ae3a332068ed5aacedda05672b8028c0 Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Sun, 1 Nov 2026 07:45:56 +0800 Subject: [PATCH 6/8] refactor(storage): keep event bodies out of prefix sort records Order canonical headers and references before resolving bodies by primary key on the same SQLite worker and transaction. Reuse one body statement while decoding borrowed rows and recording original canonical evidence. Reject missing bodies without publishing partial canonical bytes. Generated-by: Codex --- platform/event-log/README.md | 2 +- platform/event-log/README.zh-CN.md | 2 +- platform/event-log/src/canonical/read.rs | 101 ++++++++++++++++------- platform/event-log/src/prefix.rs | 13 ++- 4 files changed, 85 insertions(+), 33 deletions(-) diff --git a/platform/event-log/README.md b/platform/event-log/README.md index a20870a767..75cbe5539d 100644 --- a/platform/event-log/README.md +++ b/platform/event-log/README.md @@ -73,7 +73,7 @@ Canonical databases use UTF-8. Successful tool results store their original `mod Bodies are directly seekable raw bytes, shared by SHA-256 of their original token with exact byte comparison on a repeated identity. Reads validate logical lengths and selected ranges. Framed compression was evaluated against raw storage and rejected because it repeatedly slowed supported window and sequential reads; it is not a persisted format. Space savings come from compact directories and sharing identical projection tokens, not from compressing unique bodies. -Complete-event queries join the referenced body into the selected SQL row within the original snapshot. JSON readers reconstruct the original bytes; typed prefix readers decode the borrowed header and original projection bytes directly on the original SQLite worker. They avoid copying the body into a temporary string and finish decoding before advancing its row. The existing event/byte preflight and transaction cover the complete result; no SQLite borrow or unfinished decoder leaves that worker. Typed headers reuse fact field types and defaults, ignore unknown metadata, and verify that the projection placeholder is exactly at its stored offset. Root evidence hashes the three original byte slices as one length-prefixed JSON field. Metadata predicates use the header and retain their indexes. Semantic spans retain original canonical coordinates; tool spans map those logical coordinates to header/body ranges, while inline model parts retain physical SQLite BLOB bounds. Resolution stays on the original SQLite worker. Hot range reads use its existing prepared-statement cache. Body references follow the existing history-owner collection proof; clearing or deleting the final reference removes its body in the same transaction. The event row publishes the header, logical length, offset and reference atomically; live references cannot be redirected. Neither a body digest nor a derived directory grants access or extends lifetime. Derived archive placeholders and other fragment bodies keep their chunked storage. +Complete-event JSON queries join the referenced body into the selected SQL row within the original snapshot and reconstruct the original bytes. Typed prefix readers order headers and body references, keeping large payloads out of SQLite's sort records. One prepared primary-key lookup resolves each referenced body on the same connection and transaction; inline events require no body lookup. Readers decode the borrowed header and original projection bytes directly on the original SQLite worker. They avoid copying the body into a temporary string and finish decoding before advancing either row. The existing event/byte preflight and transaction cover the complete result; no SQLite borrow or unfinished decoder leaves that worker. Typed headers reuse fact field types and defaults, ignore unknown metadata, and verify that the projection placeholder is exactly at its stored offset. Root evidence hashes the three original byte slices as one length-prefixed JSON field. Metadata predicates use the header and retain their indexes. Semantic spans retain original canonical coordinates; tool spans map those logical coordinates to header/body ranges, while inline model parts retain physical SQLite BLOB bounds. Resolution stays on the original SQLite worker. Hot range reads use its existing prepared-statement cache. Body references follow the existing history-owner collection proof; clearing or deleting the final reference removes its body in the same transaction. The event row publishes the header, logical length, offset and reference atomically; live references cannot be redirected. Neither a body digest nor a derived directory grants access or extends lifetime. Derived archive placeholders and other fragment bodies keep their chunked storage. Semantic row, payload and child directories are finite derived descriptors encoded with Postcard. Writers encode them once in the publication transaction. Pages copy their stored bytes alongside fixed little-endian locator fields into the caller's charged allocation; consumers decode the original typed descriptors without navigating whole bodies. Original byte positions and leaf commitments remain unchanged. Directory encoding never supplies a canonical fact or source identity. The SQL migration converts former JSON descriptors atomically; its conversion functions exist only during migration. diff --git a/platform/event-log/README.zh-CN.md b/platform/event-log/README.zh-CN.md index 468d696592..3696ed3e41 100644 --- a/platform/event-log/README.zh-CN.md +++ b/platform/event-log/README.zh-CN.md @@ -67,7 +67,7 @@ Control 中的调用意图、项目登记、workspace attachment、Session 路 正文保存为可直接 seek 的原始字节，按原始 token 的 SHA-256 共享，重复身份还需逐字节核对。读取校验逻辑长度与选中区间。对照实验中，分帧压缩稳定拖慢了受支持的窗口和顺序读取，因此未作为持久化格式保留。空间收益来自紧凑目录与相同投影 token 的共享，不能把压缩实验中独有正文的压缩率套到最终实现。 -完整事件查询在原快照内的同一 SQL 行 JOIN 引用的正文。JSON reader 重建原始字节；类型化前缀 reader 在原 SQLite worker 上直接解码借用的 header 和原始投影字节，避免把正文复制到临时字符串中，并在推进当前行之前完成解码。原有事件／字节预检和事务覆盖完整结果；SQLite 借用和未完成的解码都不会离开该 worker。Header 复用事实字段的类型和默认值、忽略未知元数据，并验证投影占位符恰好位于已存偏移。Root 证据把原始三段字节作为一个带长度前缀的 JSON 字段计算摘要；元数据谓词继续读取 header 并使用原索引。语义区间保留原规范坐标，工具区间映射至实际 header／body，内联 model part 则保留 SQLite BLOB 的物理边界。读取仍由原 SQLite worker 执行。热区间读取复用其已有 statement cache。正文引用的存活由既有历史所有者回收证明决定，最后一个引用清除或删除时，在同一事务删除正文。事件行原子发布 header、逻辑长度、偏移和引用，存活引用不可重定向；正文摘要和派生目录都不授予访问权，也不延长存活期。归档占位内容和其他片段正文继续使用分块存储。 +完整事件的 JSON 查询在原快照内的同一 SQL 行 JOIN 引用的正文，并重建原始字节。类型化前缀 reader 只排列 header 和正文引用，避免大正文进入 SQLite 排序记录；随后在同一连接、同一事务内复用一条 prepared 主键查询读取引用的正文，内联事件无需查询正文。Reader 在原 SQLite worker 上直接解码借用的 header 和原始投影字节，避免把正文复制到临时字符串中，并在推进任一行之前完成解码。原有事件／字节预检和事务覆盖完整结果；SQLite 借用和未完成的解码都不会离开该 worker。Header 复用事实字段的类型和默认值、忽略未知元数据，并验证投影占位符恰好位于已存偏移。Root 证据把原始三段字节作为一个带长度前缀的 JSON 字段计算摘要；元数据谓词继续读取 header 并使用原索引。语义区间保留原规范坐标，工具区间映射至实际 header／body，内联 model part 则保留 SQLite BLOB 的物理边界。读取仍由原 SQLite worker 执行。热区间读取复用其已有 statement cache。正文引用的存活由既有历史所有者回收证明决定，最后一个引用清除或删除时，在同一事务删除正文。事件行原子发布 header、逻辑长度、偏移和引用，存活引用不可重定向；正文摘要和派生目录都不授予访问权，也不延长存活期。归档占位内容和其他片段正文继续使用分块存储。 语义行、payload 和子项目录是有限的派生描述符，采用 Postcard 编码。writer 在发布事务内编码一次；分页把已存字节和固定小端序定位字段直接复制到调用者计费的缓冲区，消费端恢复原有类型，无需读取整个正文。原始字节位置与叶子摘要保持不变，目录编码不参与规范事实或来源身份。SQL 迁移在同一事务内转换原 JSON 描述符；转换函数仅在迁移期间存在。 diff --git a/platform/event-log/src/canonical/read.rs b/platform/event-log/src/canonical/read.rs index 593d790f3e..d3525aea7a 100644 --- a/platform/event-log/src/canonical/read.rs +++ b/platform/event-log/src/canonical/read.rs @@ -17,7 +17,7 @@ * under the License. */ -//! Complete canonical reads join one immutable body into the same selected row. +//! Canonical reads resolve immutable bodies within the selected event snapshot. use super::recipe::{Recipe, Slot}; use crate::StoreError; use sqlx::{Row, sqlite::SqliteRow}; @@ -126,16 +126,30 @@ pub(crate) fn json(row: &SqliteRow) -> Result, StoreError> { pub(crate) fn json_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result, StoreError> { reconstruct(fields_on_handle(row)?) } -pub(crate) fn decode_event_on_handle<'a>( - row: &'a rusqlite::Row<'_>, - visit: impl FnOnce([&'a [u8]; 3]), +pub(crate) fn decode_event_on_handle( + row: &rusqlite::Row<'_>, + bodies: &mut rusqlite::Statement<'_>, + visit: impl FnOnce([&[u8]; 3]), ) -> Result { - let (recipe, body) = material(fields_on_handle(row)?)?; + let mut fields = header_fields_on_handle(row)?; + let mut body_rows = match fields.reference { + Some(reference) => Some(bodies.query([reference])?), + None => None, + }; + if let Some(rows) = &mut body_rows { + let body = rows + .next()? + .ok_or_else(|| invalid("canonical body is missing"))?; + fields.body = body.get("canonical_body_id")?; + fields.raw_bytes = body.get("canonical_raw_bytes")?; + fields.payload = payload_on_handle(body)?; + } + let (recipe, body) = material(fields)?; let (event, parts) = recipe.decode(body)?; visit(parts); Ok(event) } -fn fields_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result, StoreError> { +fn header_fields_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result, StoreError> { let header = row .get_ref("event_header")? .as_str_or_null() @@ -150,18 +164,22 @@ fn fields_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result, StoreE })?; let length = row.get("canonical_length")?; let reference: Option = row.get("canonical_body_reference")?; - if reference.is_none() { - return Ok(Fields { - header, - length, - reference, - offset: None, - body: None, - raw_bytes: None, - payload: None, - }); - } - let payload = row + Ok(Fields { + header, + length, + reference, + offset: if reference.is_some() { + row.get("canonical_offset")? + } else { + None + }, + body: None, + raw_bytes: None, + payload: None, + }) +} +fn payload_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result, StoreError> { + Ok(row .get_ref("canonical_payload")? .as_blob_or_null() .map_err(|error| { @@ -172,16 +190,16 @@ fn fields_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result, StoreE rusqlite::types::Type::Blob, Box::new(error), ) - })?; - Ok(Fields { - header, - length, - reference, - offset: row.get("canonical_offset")?, - body: row.get("canonical_body_id")?, - raw_bytes: row.get("canonical_raw_bytes")?, - payload, - }) + })?) +} +fn fields_on_handle<'a>(row: &'a rusqlite::Row<'_>) -> Result, StoreError> { + let mut fields = header_fields_on_handle(row)?; + if fields.reference.is_some() { + fields.body = row.get("canonical_body_id")?; + fields.raw_bytes = row.get("canonical_raw_bytes")?; + fields.payload = payload_on_handle(row)?; + } + Ok(fields) } pub(crate) async fn one( connection: &mut sqlx::SqliteConnection, @@ -199,6 +217,33 @@ pub(crate) async fn one( #[cfg(test)] mod tests { use super::*; + #[test] + fn missing_body_lookup_fails_without_visiting_partial_canonical_bytes() { + let db = rusqlite::Connection::open_in_memory().unwrap(); + let mut headers = db + .prepare( + "SELECT 'null' AS event_header,4 AS canonical_length, + 1 AS canonical_body_reference,0 AS canonical_offset", + ) + .unwrap(); + let mut bodies = db + .prepare( + "SELECT 1 AS canonical_body_id,4 AS canonical_raw_bytes, + X'6E756C6C' AS canonical_payload WHERE ?1 IS NULL AND 0", + ) + .unwrap(); + let mut rows = headers.query([]).unwrap(); + let row = rows.next().unwrap().unwrap(); + let error = decode_event_on_handle(row, &mut bodies, |_| { + panic!("missing body must not publish canonical bytes"); + }) + .unwrap_err(); + assert!( + error.to_string().contains("canonical body is missing"), + "{error}" + ); + } + #[test] fn corrupt_large_length_is_rejected_before_reserving_the_event() { let result = reconstruct(Fields { diff --git a/platform/event-log/src/prefix.rs b/platform/event-log/src/prefix.rs index 0f497e2159..962c5ebb04 100644 --- a/platform/event-log/src/prefix.rs +++ b/platform/event-log/src/prefix.rs @@ -77,8 +77,9 @@ impl EventLog { return Err(StoreError::PrefixTooLarge); } let query = format!( - "SELECT e.sequence,e.event_header,{columns} FROM {source} e {joins} WHERE {filter} ORDER BY e.sequence", - columns=crate::canonical::COLUMNS,joins=crate::canonical::JOINS + "SELECT e.sequence,e.event_header,e.canonical_bytes AS canonical_length, + e.body_id AS canonical_body_reference,e.header_offset AS canonical_offset + FROM {source} e WHERE {filter} ORDER BY e.sequence" ); let session = selection.session.clone(); let selected_lineage = selection.lineage.clone(); @@ -93,10 +94,16 @@ impl EventLog { let mut digest = session.is_none().then(crate::context::evidence::RootDigest::new); let mut events = Vec::new(); let mut statement = raw.prepare(&query)?; + // Keep large body payloads out of the ordered event records. + // Both statements borrow the original transaction's snapshot. + let mut bodies = raw.prepare( + "SELECT id AS canonical_body_id,raw_bytes AS canonical_raw_bytes, + payload AS canonical_payload FROM event_bodies WHERE id=?" + )?; let mut rows = statement.query(rusqlite::params![session, selected_lineage])?; while let Some(row) = rows.next()? { let sequence: i64 = row.get("sequence")?; - let event = crate::canonical::decode_event_on_handle(row, |parts| { + let event = crate::canonical::decode_event_on_handle(row, &mut bodies, |parts| { if let Some(digest) = &mut digest { digest.record(sequence, &parts); } })?; events.push(StoredEvent { sequence: sequence_number(sequence)?, event }); From a2b7d44a9a37aa56f5d78ffc1836e8a307d98e89 Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Sun, 1 Nov 2026 10:29:20 +0800 Subject: [PATCH 7/8] refactor(storage): specialize canonical reads to body spans Read tool projection ranges directly from one immutable body and remove unused cross-header streaming. Share captured source validation and range dispatch through CanonicalRead on the original SQLite worker. Cover original and malformed canonical artifact tokens through the real capture path, including continued reads after malformed-token rejection. Generated-by: Codex --- platform/event-log/README.md | 2 +- platform/event-log/README.zh-CN.md | 2 +- platform/event-log/src/canonical.rs | 4 +- platform/event-log/src/canonical/migration.rs | 17 ++- platform/event-log/src/canonical/recipe.rs | 83 +--------- platform/event-log/src/canonical/source.rs | 143 +++++------------- platform/event-log/src/canonical/write.rs | 1 + .../src/context/semantic/captured.rs | 73 --------- .../semantic/captured/artifact_identity.rs | 7 +- .../context/semantic/captured/canonical.rs | 80 ++++++---- .../engine/src/prepared_prompt/media/tests.rs | 81 ++++++++++ 11 files changed, 195 insertions(+), 298 deletions(-) diff --git a/platform/event-log/README.md b/platform/event-log/README.md index 75cbe5539d..8b7e3d7647 100644 --- a/platform/event-log/README.md +++ b/platform/event-log/README.md @@ -73,7 +73,7 @@ Canonical databases use UTF-8. Successful tool results store their original `mod Bodies are directly seekable raw bytes, shared by SHA-256 of their original token with exact byte comparison on a repeated identity. Reads validate logical lengths and selected ranges. Framed compression was evaluated against raw storage and rejected because it repeatedly slowed supported window and sequential reads; it is not a persisted format. Space savings come from compact directories and sharing identical projection tokens, not from compressing unique bodies. -Complete-event JSON queries join the referenced body into the selected SQL row within the original snapshot and reconstruct the original bytes. Typed prefix readers order headers and body references, keeping large payloads out of SQLite's sort records. One prepared primary-key lookup resolves each referenced body on the same connection and transaction; inline events require no body lookup. Readers decode the borrowed header and original projection bytes directly on the original SQLite worker. They avoid copying the body into a temporary string and finish decoding before advancing either row. The existing event/byte preflight and transaction cover the complete result; no SQLite borrow or unfinished decoder leaves that worker. Typed headers reuse fact field types and defaults, ignore unknown metadata, and verify that the projection placeholder is exactly at its stored offset. Root evidence hashes the three original byte slices as one length-prefixed JSON field. Metadata predicates use the header and retain their indexes. Semantic spans retain original canonical coordinates; tool spans map those logical coordinates to header/body ranges, while inline model parts retain physical SQLite BLOB bounds. Resolution stays on the original SQLite worker. Hot range reads use its existing prepared-statement cache. Body references follow the existing history-owner collection proof; clearing or deleting the final reference removes its body in the same transaction. The event row publishes the header, logical length, offset and reference atomically; live references cannot be redirected. Neither a body digest nor a derived directory grants access or extends lifetime. Derived archive placeholders and other fragment bodies keep their chunked storage. +Complete-event JSON queries join the referenced body into the selected SQL row within the original snapshot and reconstruct the original bytes. Typed prefix readers order headers and body references, keeping large payloads out of SQLite's sort records. One prepared primary-key lookup resolves each referenced body on the same connection and transaction; inline events require no body lookup. Readers decode the borrowed header and original projection bytes directly on the original SQLite worker. They avoid copying the body into a temporary string and finish decoding before advancing either row. The existing event/byte preflight and transaction cover the complete result; no SQLite borrow or unfinished decoder leaves that worker. Typed headers reuse fact field types and defaults, ignore unknown metadata, and verify that the projection placeholder is exactly at its stored offset. Root evidence hashes the three original byte slices as one length-prefixed JSON field. Metadata predicates use the header and retain their indexes. Semantic spans retain original canonical coordinates; tool spans translate those logical coordinates into offsets within their immutable body, while inline model parts retain physical SQLite BLOB bounds. Resolution stays on the original SQLite worker. Hot range reads use its existing prepared-statement cache. Body references follow the existing history-owner collection proof; clearing or deleting the final reference removes its body in the same transaction. The event row publishes the header, logical length, offset and reference atomically; live references cannot be redirected. Neither a body digest nor a derived directory grants access or extends lifetime. Derived archive placeholders and other fragment bodies keep their chunked storage. Semantic row, payload and child directories are finite derived descriptors encoded with Postcard. Writers encode them once in the publication transaction. Pages copy their stored bytes alongside fixed little-endian locator fields into the caller's charged allocation; consumers decode the original typed descriptors without navigating whole bodies. Original byte positions and leaf commitments remain unchanged. Directory encoding never supplies a canonical fact or source identity. The SQL migration converts former JSON descriptors atomically; its conversion functions exist only during migration. diff --git a/platform/event-log/README.zh-CN.md b/platform/event-log/README.zh-CN.md index 3696ed3e41..d19c119b45 100644 --- a/platform/event-log/README.zh-CN.md +++ b/platform/event-log/README.zh-CN.md @@ -67,7 +67,7 @@ Control 中的调用意图、项目登记、workspace attachment、Session 路 正文保存为可直接 seek 的原始字节，按原始 token 的 SHA-256 共享，重复身份还需逐字节核对。读取校验逻辑长度与选中区间。对照实验中，分帧压缩稳定拖慢了受支持的窗口和顺序读取，因此未作为持久化格式保留。空间收益来自紧凑目录与相同投影 token 的共享，不能把压缩实验中独有正文的压缩率套到最终实现。 -完整事件的 JSON 查询在原快照内的同一 SQL 行 JOIN 引用的正文，并重建原始字节。类型化前缀 reader 只排列 header 和正文引用，避免大正文进入 SQLite 排序记录；随后在同一连接、同一事务内复用一条 prepared 主键查询读取引用的正文，内联事件无需查询正文。Reader 在原 SQLite worker 上直接解码借用的 header 和原始投影字节，避免把正文复制到临时字符串中，并在推进任一行之前完成解码。原有事件／字节预检和事务覆盖完整结果；SQLite 借用和未完成的解码都不会离开该 worker。Header 复用事实字段的类型和默认值、忽略未知元数据，并验证投影占位符恰好位于已存偏移。Root 证据把原始三段字节作为一个带长度前缀的 JSON 字段计算摘要；元数据谓词继续读取 header 并使用原索引。语义区间保留原规范坐标，工具区间映射至实际 header／body，内联 model part 则保留 SQLite BLOB 的物理边界。读取仍由原 SQLite worker 执行。热区间读取复用其已有 statement cache。正文引用的存活由既有历史所有者回收证明决定，最后一个引用清除或删除时，在同一事务删除正文。事件行原子发布 header、逻辑长度、偏移和引用，存活引用不可重定向；正文摘要和派生目录都不授予访问权，也不延长存活期。归档占位内容和其他片段正文继续使用分块存储。 +完整事件的 JSON 查询在原快照内的同一 SQL 行 JOIN 引用的正文，并重建原始字节。类型化前缀 reader 只排列 header 和正文引用，避免大正文进入 SQLite 排序记录；随后在同一连接、同一事务内复用一条 prepared 主键查询读取引用的正文，内联事件无需查询正文。Reader 在原 SQLite worker 上直接解码借用的 header 和原始投影字节，避免把正文复制到临时字符串中，并在推进任一行之前完成解码。原有事件／字节预检和事务覆盖完整结果；SQLite 借用和未完成的解码都不会离开该 worker。Header 复用事实字段的类型和默认值、忽略未知元数据，并验证投影占位符恰好位于已存偏移。Root 证据把原始三段字节作为一个带长度前缀的 JSON 字段计算摘要；元数据谓词继续读取 header 并使用原索引。语义区间保留原规范坐标，工具区间把原规范坐标换算为不可变正文内部的偏移，内联 model part 则保留 SQLite BLOB 的物理边界。读取仍由原 SQLite worker 执行。热区间读取复用其已有 statement cache。正文引用的存活由既有历史所有者回收证明决定，最后一个引用清除或删除时，在同一事务删除正文。事件行原子发布 header、逻辑长度、偏移和引用，存活引用不可重定向；正文摘要和派生目录都不授予访问权，也不延长存活期。归档占位内容和其他片段正文继续使用分块存储。 语义行、payload 和子项目录是有限的派生描述符，采用 Postcard 编码。writer 在发布事务内编码一次；分页把已存字节和固定小端序定位字段直接复制到调用者计费的缓冲区，消费端恢复原有类型，无需读取整个正文。原始字节位置与叶子摘要保持不变，目录编码不参与规范事实或来源身份。SQL 迁移在同一事务内转换原 JSON 描述符；转换函数仅在迁移期间存在。 diff --git a/platform/event-log/src/canonical.rs b/platform/event-log/src/canonical.rs index 722e5bf082..778ced3609 100644 --- a/platform/event-log/src/canonical.rs +++ b/platform/event-log/src/canonical.rs @@ -27,7 +27,5 @@ mod write; pub(crate) use read::{COLUMNS, JOINS, decode_event_on_handle, json, json_on_handle, one}; pub(crate) use recipe::projection; -pub(crate) use source::{ - SPAN_COLUMNS, SpanMetadata, with_span, with_span_metadata, with_span_on_row, -}; +pub(crate) use source::{SPAN_COLUMNS, SpanMetadata, with_span, with_span_metadata}; pub(crate) use write::prepare; diff --git a/platform/event-log/src/canonical/migration.rs b/platform/event-log/src/canonical/migration.rs index 747019f344..c92d78eab5 100644 --- a/platform/event-log/src/canonical/migration.rs +++ b/platform/event-log/src/canonical/migration.rs @@ -210,20 +210,29 @@ mod tests { .unwrap() .unwrap(); let ranges = [ - 0..64, - (source.offset - 2)..(source.offset + 32), - (source.offset + source.json.len() - 2)..(source.offset + source.json.len() + 4), - (original.len() - 64)..original.len(), + source.offset..source.offset + 64, + source.offset + 32..source.offset + 96, + source.offset + source.json.len() - 64..source.offset + source.json.len(), + source.offset + source.json.len()..source.offset + source.json.len(), ]; let expected_ranges: Vec<_> = ranges .iter() .map(|range| original.as_bytes()[range.clone()].to_vec()) .collect(); + let outside_body = [source.offset - 1, source.offset + source.json.len()]; crate::connection::worker_query(&mut db, move |worker| { Box::pin(async move { // SAFETY: the callback exclusively owns this original SQLite worker. let raw = unsafe { rusqlite::Connection::from_handle(worker.as_raw_handle().as_ptr()) }?; + for offset in outside_body { + assert!( + crate::canonical::with_span::<()>(&raw, 1, offset as u64, 1, |_| { + panic!("tool range must remain inside its canonical body"); + }) + .is_err() + ); + } for (range, expected) in ranges.into_iter().zip(expected_ranges) { let actual = crate::canonical::with_span( &raw, diff --git a/platform/event-log/src/canonical/recipe.rs b/platform/event-log/src/canonical/recipe.rs index d76d6dae32..f4dc73f9a5 100644 --- a/platform/event-log/src/canonical/recipe.rs +++ b/platform/event-log/src/canonical/recipe.rs @@ -17,11 +17,11 @@ * under the License. */ -//! Lossless canonical event recipes and byte-range mapping. +//! Lossless canonical event recipes over a header and its optional body slot. use crate::StoreError; use serde::Deserialize; use serde_json::value::RawValue; -use std::{borrow::Cow, ops::Range}; +use std::borrow::Cow; fn invalid(reason: &'static str) -> StoreError { StoreError::Sqlite(rusqlite::Error::InvalidParameterName(reason.into())) @@ -124,12 +124,7 @@ pub(crate) struct Slot { pub offset: u64, pub bytes: u64, } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum Segment { - Header { offset: u64, bytes: u64 }, - Body { offset: u64, bytes: u64 }, -} -/// Logical positions map to physical spans without loading the header body. +/// The logical event size and optional body slot determine the stored header size. #[derive(Clone, Copy, Debug)] pub(crate) struct Layout { bytes: u64, @@ -159,50 +154,6 @@ impl Layout { None => self.bytes, } } - /// A logical byte range touches at most the prefix, body, and suffix. - pub fn segments(&self, range: Range) -> Result<[Option; 3], StoreError> { - if range.start > range.end || range.end > self.bytes { - return Err(invalid("canonical byte range is outside its event")); - } - let mut segments = [None; 3]; - if range.is_empty() { - return Ok(segments); - } - let Some(slot) = self.slot else { - segments[0] = Some(Segment::Header { - offset: range.start, - bytes: range.end - range.start, - }); - return Ok(segments); - }; - let body_end = slot - .offset - .checked_add(slot.bytes) - .ok_or_else(|| invalid("canonical slot range overflows"))?; - let prefix_end = range.end.min(slot.offset); - if range.start < prefix_end { - segments[0] = Some(Segment::Header { - offset: range.start, - bytes: prefix_end - range.start, - }); - } - let body_start = range.start.max(slot.offset); - let body_limit = range.end.min(body_end); - if body_start < body_limit { - segments[1] = Some(Segment::Body { - offset: body_start - slot.offset, - bytes: body_limit - body_start, - }); - } - let suffix_start = range.start.max(body_end); - if suffix_start < range.end { - segments[2] = Some(Segment::Header { - offset: suffix_start - body_end + slot.offset + 4, - bytes: range.end - suffix_start, - }); - } - Ok(segments) - } } /// The header and optional typed body slot describe exactly one event. @@ -357,7 +308,7 @@ mod tests { } } #[test] - fn extraction_and_every_byte_range_preserve_original_bytes() { + fn extraction_and_reconstruction_preserve_original_bytes() { let projection = projection("tool_settled", ORIGINAL).unwrap().unwrap(); let header = projection.header(ORIGINAL).unwrap(); let recipe = Recipe::new( @@ -375,31 +326,6 @@ mod tests { .unwrap(), ORIGINAL ); - for start in 0..=ORIGINAL.len() { - for end in start..=ORIGINAL.len() { - let mut result = Vec::new(); - for segment in recipe - .layout - .segments(start as u64..end as u64) - .unwrap() - .into_iter() - .flatten() - { - let (source, offset, bytes) = match segment { - Segment::Header { offset, bytes } => (header.as_bytes(), offset, bytes), - Segment::Body { offset, bytes } => { - (projection.json.as_bytes(), offset, bytes) - } - }; - result.extend_from_slice(&source[offset as usize..(offset + bytes) as usize]); - } - assert_eq!( - result, - &ORIGINAL.as_bytes()[start..end], - "range {start}..{end}" - ); - } - } } #[test] fn escaped_success_tag_preserves_its_original_encoding() { @@ -437,7 +363,6 @@ mod tests { let recipe = Recipe::new("null", 4, None).unwrap(); assert_eq!(recipe.reconstruct(None).unwrap(), "null"); assert!(recipe.reconstruct(Some(b"null")).is_err()); - assert!(recipe.layout.segments(0..5).is_err()); } #[test] fn invalid_recipes_do_not_silently_become_inline_events() { diff --git a/platform/event-log/src/canonical/source.rs b/platform/event-log/src/canonical/source.rs index 6abbcecc90..cac94db6f6 100644 --- a/platform/event-log/src/canonical/source.rs +++ b/platform/event-log/src/canonical/source.rs @@ -31,7 +31,7 @@ pub(super) fn with_material( raw: &rusqlite::Connection, id: i64, bytes: u64, - use_reader: impl FnOnce(&mut dyn Read) -> Result, + use_reader: impl FnOnce(&mut rusqlite::blob::Blob<'_>) -> Result, ) -> Result { let mut blob = raw.blob_open("main", "event_bodies", "payload", id, true)?; let result = (|| { @@ -50,33 +50,11 @@ fn settle(result: Result, closed: rusqlite::Result<()>) -> Res } } -use super::recipe::{Layout, Segment, Slot}; -use std::io::{self, Seek, SeekFrom, Take}; +use super::recipe::{Layout, Slot}; +use std::io::{self, Seek, SeekFrom}; fn invalid(reason: &'static str) -> StoreError { io::Error::new(io::ErrorKind::InvalidData, reason).into() } -struct Parts<'a> { - parts: [Option>>; 3], - current: usize, -} -impl Read for Parts<'_> { - fn read(&mut self, output: &mut [u8]) -> io::Result { - if output.is_empty() { - return Ok(0); - } - while self.current < self.parts.len() { - let read = match self.parts[self.current].as_mut() { - Some(reader) => reader.read(output)?, - None => 0, - }; - if read > 0 { - return Ok(read); - } - self.current += 1; - } - Ok(0) - } -} #[derive(Clone, Copy)] pub(crate) struct SpanMetadata { live: bool, @@ -103,6 +81,17 @@ impl SpanMetadata { raw_bytes: row.try_get("canonical_raw_bytes")?, }) } + pub(crate) fn from_handle(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let base = row.as_ref().column_index("canonical_live")?; + Ok(Self { + live: row.get(base)?, + length: row.get(base + 1)?, + body_reference: row.get(base + 2)?, + slot_offset: row.get(base + 3)?, + body_id: row.get(base + 4)?, + raw_bytes: row.get(base + 5)?, + }) + } } /// Map canonical coordinates to immutable material without reading a full header. @@ -118,36 +107,18 @@ pub(crate) fn with_span( super::read::JOINS ); raw.query_row(&query, [sequence], |row| { - Ok(with_span_on_row(raw, row, sequence, offset, bytes, consume)) + Ok(with_span_metadata( + raw, + SpanMetadata::from_handle(row)?, + offset, + bytes, + consume, + )) })? } -/// Consume metadata borrowed from a caller's existing same-snapshot query. -pub(crate) fn with_span_on_row( - raw: &rusqlite::Connection, - row: &rusqlite::Row<'_>, - sequence: i64, - offset: u64, - bytes: u64, - consume: impl FnOnce(&mut dyn Read) -> Result, -) -> Result { - // SPAN_COLUMNS is one contiguous projection even when prefixed by a - // caller's fields. Resolve its base once rather than rescan all names. - let base = row.as_ref().column_index("canonical_live")?; - let metadata = SpanMetadata { - live: row.get(base)?, - length: row.get(base + 1)?, - body_reference: row.get(base + 2)?, - slot_offset: row.get(base + 3)?, - body_id: row.get(base + 4)?, - raw_bytes: row.get(base + 5)?, - }; - with_span_metadata(raw, sequence, metadata, offset, bytes, consume) -} - pub(crate) fn with_span_metadata( raw: &rusqlite::Connection, - sequence: i64, metadata: SpanMetadata, offset: u64, bytes: u64, @@ -175,71 +146,29 @@ pub(crate) fn with_span_metadata( let length = u64::try_from(length.ok_or_else(|| invalid("canonical span lacks logical length"))?) .map_err(|_| invalid("negative canonical byte length"))?; - let (slot, material) = match (body_reference, slot_offset, body_id, raw_bytes) { - (None, None, None, None) => (None, None), - (Some(reference), Some(position), Some(id), Some(length)) - if reference == id && position >= 0 && id > 0 && length > 0 => + let (slot, id) = match (body_reference, slot_offset, body_id, raw_bytes) { + (Some(reference), Some(position), Some(id), Some(bytes)) + if reference == id && position >= 0 && id > 0 && bytes > 0 => { ( - Some(Slot { + Slot { offset: position as u64, - bytes: length as u64, - }), - Some((id, length as u64)), + bytes: bytes as u64, + }, + id, ) } - _ => return Err(invalid("canonical span has an incomplete body slot")), + _ => return Err(invalid("canonical tool projection lacks its body slot")), }; - let layout = Layout::new(length, slot)?; + Layout::new(length, Some(slot))?; let end = offset .checked_add(bytes) .ok_or_else(|| invalid("canonical span length overflows"))?; - let segments = layout.segments(offset..end)?; - let mut parts = Parts { - parts: std::array::from_fn(|_| None), - current: 0, - }; - let result = (|| { - for (index, segment) in segments.into_iter().enumerate() { - let Some(segment) = segment else { continue }; - let (table, column, rowid, expected, offset, bytes) = match segment { - Segment::Header { offset, bytes } => ( - "event_log", - "event_header", - sequence, - layout.header_bytes(), - offset, - bytes, - ), - Segment::Body { offset, bytes } => { - let (id, raw_bytes) = material - .ok_or_else(|| invalid("canonical range requires missing body material"))?; - ("event_bodies", "payload", id, raw_bytes, offset, bytes) - } - }; - let blob = raw.blob_open("main", table, column, rowid, true)?; - // Install ownership before any validation or seek can fail. - parts.parts[index] = Some(blob.take(bytes)); - let reader = parts.parts[index] - .as_mut() - .expect("installed canonical reader"); - if reader.get_ref().len() as u64 != expected { - return Err(invalid("canonical material length changed")); - } - reader.get_mut().seek(SeekFrom::Start(offset))?; - } - consume(&mut parts) - })(); - let mut closing = Ok(()); - for part in parts.parts.into_iter().flatten() { - if let Err(error) = part.into_inner().close() { - closing = match closing { - Ok(()) => Err(error), - Err(previous) => Err(rusqlite::Error::ToSqlConversionFailure(Box::new( - io::Error::other(format!("{previous}; {error}")), - ))), - }; - } + if offset < slot.offset || end > slot.offset + slot.bytes { + return Err(invalid("canonical tool range exceeds its body slot")); } - settle(result, closing) + with_material(raw, id, slot.bytes, |reader| { + reader.seek(SeekFrom::Start(offset - slot.offset))?; + consume(&mut reader.take(bytes)) + }) } diff --git a/platform/event-log/src/canonical/write.rs b/platform/event-log/src/canonical/write.rs index 61ee884137..03099b5e7f 100644 --- a/platform/event-log/src/canonical/write.rs +++ b/platform/event-log/src/canonical/write.rs @@ -22,6 +22,7 @@ use super::{recipe::Projection, source}; use crate::StoreError; use sha2::{Digest, Sha256}; use sqlx::sqlite::SqliteWorkerConnection; +use std::io::Read; pub(crate) async fn prepare( worker: &mut SqliteWorkerConnection<'_>, projection: &Projection<'_>, diff --git a/platform/event-log/src/context/semantic/captured.rs b/platform/event-log/src/context/semantic/captured.rs index 4f6ef31f22..28da3f8e79 100644 --- a/platform/event-log/src/context/semantic/captured.rs +++ b/platform/event-log/src/context/semantic/captured.rs @@ -848,79 +848,6 @@ fn fragment_backing(raw: &RawConnection, key: RowPosition) -> Result Err(invalid("captured semantic fragment is missing or invalid")), } } -fn with_canonical_range( - raw: &RawConnection, - key: RowPosition, - offset: u64, - bytes: u64, - start: u64, - count: u64, - consume: impl FnOnce(&mut dyn Read) -> Result, -) -> Result { - let query = format!("SELECT f.storage_kind,f.byte_offset,f.byte_len,{} FROM context_semantic_fragments f - LEFT JOIN event_log e ON e.sequence=f.source_sequence AND f.storage_kind='canonical_tool_projection' {} - WHERE f.source_sequence=? AND f.source_index=?", crate::canonical::SPAN_COLUMNS, crate::canonical::JOINS); - raw.query_row( - &query, - params![key.source_sequence as i64, key.item_index as i64], - |row| { - Ok((|| { - let kind: String = row.get("storage_kind")?; - let source = super::source::SemanticPayloadSource::from_metadata( - key.source_sequence as i64, - key.item_index as i64, - &kind, - row.get("byte_offset")?, - row.get("byte_len")?, - )?; - let (actual, length) = match source { - super::source::SemanticPayloadSource::CanonicalToolProjection(span) => { - (span.offset(), span.len()) - } - super::source::SemanticPayloadSource::CanonicalModelPart(span) => { - (span.offset(), span.len()) - } - super::source::SemanticPayloadSource::Pieces => { - return Err(invalid("captured canonical source changed")); - } - }; - if actual as u64 != offset || length as u64 != bytes { - return Err(invalid("captured canonical source changed")); - } - if start.checked_add(count).is_none_or(|end| end > bytes) { - return Err(invalid("canonical range exceeds captured span")); - } - let position = offset - .checked_add(start) - .ok_or_else(|| invalid("canonical span offset overflows"))?; - match source { - super::source::SemanticPayloadSource::CanonicalModelPart(_) => with_blob( - raw, - "event_log", - "event_header", - key.source_sequence as i64, - position, - count, - consume, - ), - super::source::SemanticPayloadSource::CanonicalToolProjection(_) => { - crate::canonical::with_span_on_row( - raw, - row, - key.source_sequence as i64, - position, - count, - consume, - ) - } - super::source::SemanticPayloadSource::Pieces => { - unreachable!("validated canonical source") - } - } - })()) - }, - )? -} fn check_pieces(raw: &RawConnection, key: RowPosition) -> Result<(), StoreError> { let valid:bool=raw.query_row("SELECT EXISTS(SELECT 1 FROM context_semantic_fragments WHERE source_sequence=? AND source_index=? AND storage_kind='pieces')",params![key.source_sequence as i64,key.item_index as i64],|r|r.get(0))?; if valid { diff --git a/platform/event-log/src/context/semantic/captured/artifact_identity.rs b/platform/event-log/src/context/semantic/captured/artifact_identity.rs index 49bb73dd14..0e014a6947 100644 --- a/platform/event-log/src/context/semantic/captured/artifact_identity.rs +++ b/platform/event-log/src/context/semantic/captured/artifact_identity.rs @@ -54,9 +54,10 @@ fn with_range( length, consume, ), - Backing::Canonical { key, offset, bytes } => { - with_canonical_range(raw, key, offset, bytes, range.start, length, consume) - } + Backing::Canonical { key, offset, bytes } => CanonicalRead::load_on_handle( + raw, key, offset, bytes, + )? + .with_range(raw, range.start, length, consume), Backing::Pieces { key, .. } => { check_pieces(raw, key)?; let mut reader = PieceReader { diff --git a/platform/event-log/src/context/semantic/captured/canonical.rs b/platform/event-log/src/context/semantic/captured/canonical.rs index 8197e67fac..cd0b14c862 100644 --- a/platform/event-log/src/context/semantic/captured/canonical.rs +++ b/platform/event-log/src/context/semantic/captured/canonical.rs @@ -18,11 +18,18 @@ */ //! Callback-local source metadata uses the original worker's statement cache. +use super::super::source::SemanticPayloadSource; use super::{Backing, RawConnection, RowPosition, StoreError, invalid, with_blob}; use crate::canonical::SpanMetadata; use sqlx::{Row, sqlite::SqliteWorkerConnection}; use std::io::Read; +fn query() -> String { + format!("SELECT f.storage_kind,f.byte_offset,f.byte_len,{} FROM context_semantic_fragments f + LEFT JOIN event_log e ON e.sequence=f.source_sequence AND f.storage_kind='canonical_tool_projection' {} + WHERE f.source_sequence=? AND f.source_index=?", crate::canonical::SPAN_COLUMNS, crate::canonical::JOINS) +} + pub(super) struct CanonicalRead { key: RowPosition, offset: u64, @@ -37,45 +44,69 @@ impl CanonicalRead { let Backing::Canonical { key, offset, bytes } = *backing else { return Ok(None); }; - let query = format!("SELECT f.storage_kind,f.byte_offset,f.byte_len,{} FROM context_semantic_fragments f - LEFT JOIN event_log e ON e.sequence=f.source_sequence AND f.storage_kind='canonical_tool_projection' {} - WHERE f.source_sequence=? AND f.source_index=?", crate::canonical::SPAN_COLUMNS, crate::canonical::JOINS); - let row = sqlx::query(sqlx::AssertSqlSafe(query)) + let row = sqlx::query(sqlx::AssertSqlSafe(query())) .bind(key.source_sequence as i64) .bind(key.item_index as i64) .fetch_one(&mut *worker) .await?; - let source = super::super::source::SemanticPayloadSource::from_metadata( + let source = SemanticPayloadSource::from_metadata( key.source_sequence as i64, key.item_index as i64, row.try_get("storage_kind")?, row.try_get("byte_offset")?, row.try_get("byte_len")?, )?; - let (actual, length, model) = match source { - super::super::source::SemanticPayloadSource::CanonicalToolProjection(span) => { - (span.offset(), span.len(), false) - } - super::super::source::SemanticPayloadSource::CanonicalModelPart(span) => { - (span.offset(), span.len(), true) + Self::checked(key, offset, bytes, source, SpanMetadata::from_row(&row)?).map(Some) + } + pub(super) fn load_on_handle( + raw: &RawConnection, + key: RowPosition, + offset: u64, + bytes: u64, + ) -> Result { + raw.query_row( + &query(), + [key.source_sequence as i64, key.item_index as i64], + |row| { + Ok((|| { + let kind: String = row.get("storage_kind")?; + let source = SemanticPayloadSource::from_metadata( + key.source_sequence as i64, + key.item_index as i64, + &kind, + row.get("byte_offset")?, + row.get("byte_len")?, + )?; + Self::checked(key, offset, bytes, source, SpanMetadata::from_handle(row)?) + })()) + }, + )? + } + fn checked( + key: RowPosition, + offset: u64, + bytes: u64, + source: SemanticPayloadSource, + material: SpanMetadata, + ) -> Result { + let (actual, length, material) = match source { + SemanticPayloadSource::CanonicalToolProjection(span) => { + (span.offset(), span.len(), Some(material)) } - super::super::source::SemanticPayloadSource::Pieces => { + SemanticPayloadSource::CanonicalModelPart(span) => (span.offset(), span.len(), None), + SemanticPayloadSource::Pieces => { return Err(invalid("captured canonical source changed")); } }; if actual as u64 != offset || length as u64 != bytes { return Err(invalid("captured canonical source changed")); } - Ok(Some(Self { + Ok(Self { key, offset, bytes, - material: if model { - None - } else { - Some(SpanMetadata::from_row(&row)?) - }, - })) + material, + }) } pub(super) fn with_range( &self, @@ -92,14 +123,9 @@ impl CanonicalRead { .checked_add(start) .ok_or_else(|| invalid("canonical span offset overflows"))?; match self.material { - Some(material) => crate::canonical::with_span_metadata( - raw, - self.key.source_sequence as i64, - material, - position, - count, - consume, - ), + Some(material) => { + crate::canonical::with_span_metadata(raw, material, position, count, consume) + } None => with_blob( raw, "event_log", diff --git a/plugins/agent/engine/src/prepared_prompt/media/tests.rs b/plugins/agent/engine/src/prepared_prompt/media/tests.rs index c3150e8c79..4b87aff8f5 100644 --- a/plugins/agent/engine/src/prepared_prompt/media/tests.rs +++ b/plugins/agent/engine/src/prepared_prompt/media/tests.rs @@ -483,6 +483,87 @@ async fn request_size(prompt: &PreparedPrompt, provider: &ProviderConfig) -> u64 .largest() } +#[tokio::test] +async fn canonical_tool_artifact_tokens_require_complete_original_json_strings() { + use maka_agent_facts::model::prompt_source::source::ChargedBuffer; + use maka_event_log::context::semantic::{ + ArtifactIdentityValidity, ArtifactLookup, ArtifactLookupResult, PayloadEncoding, + SourcePosition, + }; + let fixture = Fixture::new(true, 8, 8).await; + { + let source = fixture + .log + .capture_source(fixture.selection.clone()) + .await + .unwrap(); + let mut page = source + .directory_page(None, ChargedBuffer::new(vec![0; 4096], ())) + .await + .unwrap(); + let mut payload = None; + while let Some(record) = page.next_row().unwrap() { + if record.row.id.source_sequence == fixture.selection.through { + payload = source.describe_payload(&record.row).unwrap(); + break; + } + } + let payload = payload.expect("canonical tool artifact projection"); + assert_eq!(payload.encoding(), PayloadEncoding::CanonicalProjection); + let mut chunk = source + .read_chunk( + &payload, + SourcePosition::default(), + ChargedBuffer::new(vec![0; 4096], ()), + ) + .await + .unwrap(); + let json = std::str::from_utf8(&chunk.buffer.as_mut()[..chunk.bytes_read]).unwrap(); + let token = |literal: &str| { + let start = json.find(literal).expect("original identity token") as u64; + start..start + literal.len() as u64 + }; + let session = token("\"session\""); + let artifact = token("\"source-image\""); + let lookup = |artifact| { + ArtifactLookup::new( + payload.clone(), + session.clone(), + artifact, + ChargedBuffer::new(vec![0; 64], ()), + ) + .unwrap() + }; + let found = source + .with_artifact_capture(lookup(artifact.clone()), |_, _, reader| { + let mut bytes = Vec::new(); + std::io::Read::read_to_end(reader, &mut bytes)?; + Ok(bytes) + }) + .await + .unwrap(); + match found { + ArtifactLookupResult::Captured(bytes) => assert_eq!(bytes, fixture.original), + _ => panic!("original canonical identity must resolve its artifact"), + } + let truncated = artifact.start..artifact.end - 1; + assert!( + source + .with_artifact_capture(lookup(truncated), |_, _, _| Ok(())) + .await + .is_err() + ); + assert_eq!( + source + .validate_artifact_identity(lookup(artifact)) + .await + .unwrap(), + ArtifactIdentityValidity::Valid + ); + } + fixture.close().await; +} + #[tokio::test] async fn captured_user_and_tool_images_share_one_whole_view_across_protocols_and_replay() { let mut expected = None; From 9341667e5105d19648df29166e27c4d7b336d1d8 Mon Sep 17 00:00:00 2001 From: M4n5ter Date: Sun, 1 Nov 2026 12:26:12 +0800 Subject: [PATCH 8/8] fix(release): refresh reviewed notice input fingerprints Regenerate notices against the final dependency graph and bundled product. The license document is unchanged; refresh the three Cargo inputs and two inherited SDK fingerprints. Generated-by: Codex --- products/maka/cli/notices.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/products/maka/cli/notices.json b/products/maka/cli/notices.json index 9f288b6566..b9ed9e51c4 100644 --- a/products/maka/cli/notices.json +++ b/products/maka/cli/notices.json @@ -1,14 +1,14 @@ { "inputs": { - "Cargo.lock": "ca5a7b90dfd8670beabdcd13809e1f19fd2835b3d4b4e001701c77e3a80618c2", - "Cargo.toml": "daf18767edc88e0f704553cf594f8fd222fbf8edab5118ade55a208b05d532d1", + "Cargo.lock": "d0efdcc97e0c5b5ccd3d1683d730bd7518fd786bc6b9b5e02f9c0fbc2dab7f28", + "Cargo.toml": "be5bfb825c62cb348a1eff36a6c839a7055e5c5fbd5b79f3db9c34bbb6ace4b2", "package-lock.json": "863e20d94f46507befd14d7abe94982bb110dff3efaa13b9d1c3b60333f8d64a", "package.json": "a513ba49c73a1cd6d8060d9a4f2edb47f5286469553cf3b3f4959b9ceb64fa58", "packages/plugin-sdk/src/composer-view.ts": "67a3873beb21778ddbba561f949ebe840c6af33fa0cdb3f3a9da8f6fb4cb03d3", "packages/plugin-sdk/src/configurations.ts": "5cd4b679d786a4766f97460c8fbe070a92415b5210561344c71de4f61c471f5e", - "packages/plugin-sdk/src/contracts.ts": "63134585c74106b6dbf6d149598cfb2aa93f76fdfe86beed62d859bbb66291ca", + "packages/plugin-sdk/src/contracts.ts": "5d5d464f2c002c9f7ebacce0dc1593a7b445b43ee5a6a5a166aa772e1255388f", "packages/plugin-sdk/src/generated/methods.ts": "83c5100a927372eeb44f53a714f0796bb812a74979f290b170026752536ad370", - "packages/plugin-sdk/src/generated/runtime.ts": "9508dc5cda141731e27eacaeaec3a268e3dd0617b6804b33ce5c3142678b1f0c", + "packages/plugin-sdk/src/generated/runtime.ts": "324025c34d57671b745e7dba9dd301b35257d3ada34c7631cc22495374119520", "packages/plugin-sdk/src/index.ts": "813f3f63b2bc47a4ce3fc9e1deb9fcad2f51dd13a26b37eb1341202e0804dc81", "packages/plugin-sdk/src/native.ts": "6a354ce1068d3eb288ebaf4e3f0e4a2ef00479690e895e53fa4be3f1789fb620", "packages/plugin-sdk/src/workbench-views.ts": "5b0e829bc8df658994187384a5788227c578437e15fc1cb3082915009856e08c", @@ -20,7 +20,7 @@ "platform/code-runtime/Cargo.toml": "30a8e555effeaa063b11d255d60d6f858d53e3bd950e2f162382324ef0a7cc20", "platform/computer-use/Cargo.toml": "cb406d00b3bfa195b44267fb290ab1910e9700bebce3d1829232e312edc8b2ed", "platform/computer-use/THIRD_PARTY_NOTICES": "8b25853c9fe8f668531e01e69c6c803e511b7beac5b642f52b4fdf545bfa5dea", - "platform/event-log/Cargo.toml": "e8d149e2fee903a4fd155931cd3191f986416797c4e6ddf60be4e556583e2e14", + "platform/event-log/Cargo.toml": "8cfe0b94df9350888eabac043377fec86d53d4bd117102631770c38c742c7684", "platform/host-js/Cargo.toml": "881688e22e1325226dc54c1d94ad370fef2b4ce68fbc0c61b3574fbbae29541f", "platform/host-runtime/Cargo.toml": "846b01af213572f65070500b441b9fdd6ae7b132d530a241086594abf44a627f", "platform/js-runtime/Cargo.toml": "268eb3ed9f689cb895ab94de85c7365c3f54bb5eb5a4abaf32b4636af6085156",