From e3cb3a04f9ac5257b73bd7e903042eeb585fa0be Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Thu, 6 Aug 2026 20:54:44 +0000 Subject: [PATCH 01/25] feat(sqlite): vector index storage, exact-scan search, and the declared capability First working slice of the SQLite implementation: a vector index can be created, is persisted, is reported back, and can be searched. Write maintenance, the UpdateTable path and backfill follow. Storage is one row per vector, which is a measured decision. A packed blob per partition reads 2 to 4x faster at 256 dimensions and 1.3 to 2.5x at 1024, and loses at 4096, but it makes every write O(partition): inserting one vector rewrites the whole blob, 390 MB for a 100k-vector partition at 1024 dimensions. Vector indexes are maintained on every write touching an indexed attribute, so that trade is not available at any read speed. Row-per-vector also streams, has no blob ceiling, and reuses the existing per-index table pattern. Exact scan rather than approximate, also measured rather than assumed. No SQLite vector extension meets this backend's constraints: a static-musl FROM scratch build cannot dlopen a loadable extension, the only extension with a compatible licence and an in-database index is brute force in every stable release anyway, and every real-ANN option stores its index in a sidecar file, forbids transactions, or is not open source. Measured throughput on one core, warm: 213k to 334k vectors/sec at 256 dimensions, 94k to 103k at 1024, 39k to 43k at 4096. The scan is bound by getting bytes out of SQLite, not by the arithmetic: a zero-copy &[f32] view of the blob measured no faster than decoding per element, so optimising the distance loop would be wasted effort. Notes on specific choices: * Vector metadata lives in its own `vector_indexes` catalog table rather than in `indexes`. A vector index is not described by a key schema, so reusing that table would mean storing something meaningless in a NOT NULL column. Two CHECK constraints encode the measured wire contract, including that an ACTIVE index must not carry the `backfilling` member at all. * Vector data tables are named with the base table_id as well as the index_id. That is what lets `drop_data_table` clean them up: it runs after the catalog rows have been cascade-deleted in the same transaction, so the index ids are no longer readable, and the names are instead discovered from sqlite_master. Without this, dropping a table would orphan its vector tables permanently. * Top-k consults the distance function rather than assuming one direction. Cosine and Euclidean are distances so smaller wins, dot product is a similarity so larger does; a single ordering would silently return the worst matches for one of the three. Tested both directions. * An index with no HASH element uses a reserved partition value no real key can produce, so an unscoped search is one partition rather than a second code path. Three defects found and fixed on the way: * The catalog version seed was `INSERT OR IGNORE` with a hardcoded literal, so `extenddb migrate` would create new objects and still leave the recorded version stale, and the server would keep refusing to start on a mismatch. Now an upsert, with two tests pinning the literal to CATALOG_VERSION and forbidding a regression to IGNORE. Verified by migrating a live deployment 0.0.2 to 0.0.3. * `IndexStatus` was not re-exported from core, so a backend could not name the type it is required to set on every VectorIndexDescription. Same class of gap as the unnameable BoxFuture. * CreateTable's response did not echo the vector indexes it had just created. The negative wire suite now self-skips when the backend supports vectors, probing by attempting the smallest real vector CreateTable. The probe distinguishes a refusal from any other failure and panics on the latter, because treating "not a 200" as unsupported would make the suite skip silently the first time an unrelated error appeared. Verified against a live SQLite server: 427 integration passed / 0 failed / 0 filtered out, workspace 746 passed / 0 failed / 0 filtered out, fmt clean, clippy -D warnings clean on both feature sets. Persistence confirmed by inspecting the database directly: catalog row ACTIVE with no backfilling member, plus the table-scoped data table and its partition index. --- crates/storage-sqlite/src/create_table.rs | 108 ++++- crates/storage-sqlite/src/data/data_engine.rs | 7 + crates/storage-sqlite/src/data/ddl.rs | 96 ++++- crates/storage-sqlite/src/data/mod.rs | 25 ++ crates/storage-sqlite/src/lib.rs | 1 + crates/storage-sqlite/src/schema.rs | 83 +++- crates/storage-sqlite/src/table_helpers.rs | 95 ++++- crates/storage-sqlite/src/vector_search.rs | 379 ++++++++++++++++++ tests/rust/src/vector_index_unsupported.rs | 4 +- 9 files changed, 784 insertions(+), 14 deletions(-) create mode 100644 crates/storage-sqlite/src/vector_search.rs diff --git a/crates/storage-sqlite/src/create_table.rs b/crates/storage-sqlite/src/create_table.rs index 8c4e2202..9485b681 100644 --- a/crates/storage-sqlite/src/create_table.rs +++ b/crates/storage-sqlite/src/create_table.rs @@ -194,6 +194,61 @@ impl SqliteEngine { } } + // Vector indexes. A CreateTable's table is empty, so there is nothing to + // backfill: the index goes straight to ACTIVE with no `backfilling` + // member, which is the state the service reports for an index created + // this way. The UpdateTable path is the one that drives a real lifecycle. + let mut vector_ids: Vec = Vec::new(); + if let Some(vis) = &input.vector_indexes { + for vi in vis { + let vec_attr = serde_json::to_string(&vi.vector_attribute) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let search_schema = vi + .search_schema + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let proj = vi + .projection + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + // Core validation requires Projection, so reaching here + // means the request bypassed validation rather than that + // the caller omitted it. + StorageError::Internal( + "vector index reached storage without a projection".to_owned(), + ) + })?; + let distance = serde_json::to_string(&vi.distance_function) + .map_err(|e| StorageError::Internal(e.to_string()))? + .trim_matches('"') + .to_owned(); + let index_id = uuid::Uuid::new_v4().to_string(); + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_name, index_id, dimensions, distance_function, \ + vector_attribute, search_schema, projection, index_status, backfilling) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'ACTIVE', NULL)", + ) + .bind(&table_id) + .bind(&vi.index_name) + .bind(&index_id) + .bind(i64::from(vi.dimensions)) + .bind(&distance) + .bind(&vec_attr) + .bind(&search_schema) + .bind(&proj) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + vector_ids.push(index_id); + } + } + // Tags. if let Some(tags) = &input.tags { for tag in tags { @@ -262,6 +317,18 @@ impl SqliteEngine { .await?; } } + if input.vector_indexes.is_some() { + for index_id in &vector_ids { + Self::create_vector_data_table( + &mut data_tx, + &table_id, + index_id, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + } + } data_tx .commit() .await @@ -368,6 +435,37 @@ impl SqliteEngine { }) }); + // Echo the vector indexes we just created. Built from the request plus the + // ids assigned above rather than re-read from the catalog, which would add + // a round trip to say something already known. A CreateTable's table is + // empty, so each index is ACTIVE with no `backfilling` member. + let vector_index_descs: Option> = input + .vector_indexes + .as_ref() + .map(|vis| { + vis.iter() + .map(|vi| extenddb_core::types::VectorIndexDescription { + index_name: vi.index_name.clone(), + vector_attribute: vi.vector_attribute.clone(), + dimensions: vi.dimensions, + search_schema: vi.search_schema.clone(), + distance_function: vi.distance_function, + index_status: extenddb_core::types::IndexStatus::Active, + backfilling: None, + index_size_bytes: 0, + item_count: 0, + index_arn: extenddb_storage::util::index_arn( + &self.region, + account_id, + &input.table_name, + &vi.index_name, + ), + projection: vi.projection.clone(), + }) + .collect() + }) + .filter(|v: &Vec<_>| !v.is_empty()); + Ok(TableDescription { table_name: input.table_name, key_schema: input.key_schema, @@ -398,10 +496,12 @@ impl SqliteEngine { .as_ref() .map(|tc| serde_json::json!({ "TableClass": tc })), on_demand_throughput: input.on_demand_throughput, - // Fields for features this backend does not implement, vector - // indexes today, take their defaults, so adding one to this type - // does not break this build. - ..Default::default() + // Every field is populated deliberately, with no `..Default::default()` + // spread. This response is the complete description of what was just + // created, so a new core field should break this site and force a + // decision about whether create must report it, rather than silently + // defaulting. Sites that legitimately opt out still use the spread. + vector_indexes: vector_index_descs, }) } } diff --git a/crates/storage-sqlite/src/data/data_engine.rs b/crates/storage-sqlite/src/data/data_engine.rs index 72071bd7..6228a564 100644 --- a/crates/storage-sqlite/src/data/data_engine.rs +++ b/crates/storage-sqlite/src/data/data_engine.rs @@ -17,6 +17,13 @@ use futures::future::BoxFuture; use crate::store::SqliteEngine; impl DataEngine for SqliteEngine { + /// Declares vector support by handing over the implementation. `Some(self)` + /// only compiles because `SqliteEngine` implements `VectorSearchEngine`, so + /// this cannot claim a capability the backend does not have. + fn as_vector_search(&self) -> Option<&dyn extenddb_storage::VectorSearchEngine> { + Some(self) + } + fn put_item( &self, key_info: &TableKeyInfo, diff --git a/crates/storage-sqlite/src/data/ddl.rs b/crates/storage-sqlite/src/data/ddl.rs index 49c9880a..1ed973fc 100644 --- a/crates/storage-sqlite/src/data/ddl.rs +++ b/crates/storage-sqlite/src/data/ddl.rs @@ -15,7 +15,10 @@ use extenddb_core::types::{ use extenddb_storage::error::StorageError; use extenddb_storage::util::sk_column_n; -use super::{all_sort_key_info, data_table_name, index_table_name}; +use super::{ + all_sort_key_info, data_table_name, index_table_name, vector_table_like_pattern, + vector_table_name, +}; use crate::store::SqliteEngine; /// SQLite column type for the Nth sort-key position and scalar type (D2). @@ -95,6 +98,94 @@ impl SqliteEngine { Ok(()) } + /// Create a vector-index data table: one row per indexed vector. + /// + /// `part` is the search-schema HASH value when one is declared, and a single + /// constant otherwise, so an unscoped index is one partition rather than a + /// separate code path. `nrm` is the vector's precomputed L2 norm, so cosine + /// costs one dot product at query time instead of two passes. + /// + /// # Safety (SQL injection) + /// `index_id` is a server-generated UUID and column names are constants, so + /// no user input reaches the DDL. Vector attribute names are stored as data, + /// never as identifiers. + pub(crate) async fn create_vector_data_table( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + index_id: &str, + base_key_schema: &[KeySchemaElement], + base_attr_defs: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let vec_table = vector_table_name(table_id, index_id); + let base_sks = all_sort_key_info(base_key_schema, base_attr_defs); + + let mut col_defs = vec![ + "part TEXT NOT NULL".to_owned(), + "base_pk TEXT NOT NULL".to_owned(), + ]; + for i in 0..base_sks.len() { + col_defs.extend(base_sk_col_defs(i)); + } + col_defs.push("vec BLOB NOT NULL".to_owned()); + col_defs.push("nrm REAL NOT NULL".to_owned()); + col_defs.push("item_data TEXT NOT NULL".to_owned()); + + // Keyed by the base item, not by the partition, so one base item yields + // at most one vector row and a re-put replaces rather than duplicates. + let mut pk_cols = vec!["base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + pk_cols.push(format!("base_{}", sk_column_n(i, sk_type))); + } + + let ddl = format!( + "CREATE TABLE {vec_table} (\n {},\n PRIMARY KEY ({})\n)", + col_defs.join(",\n "), + pk_cols.join(", ") + ); + sqlx::query(&ddl) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // The scan is always partition-scoped, so this index is what keeps a + // search off the full table when a HASH element is declared. + let part_idx = + format!("CREATE INDEX \"_vidx_part_{table_id}_{index_id}\" ON {vec_table} (part)"); + sqlx::query(&part_idx) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + /// Drop every vector-index data table belonging to one DynamoDB table. + /// + /// Discovered from `sqlite_master` rather than from the catalog, because the + /// caller runs this after the catalog rows have been cascade-deleted in the + /// same transaction, so `vector_indexes` is already empty for this table. + /// Without this, dropping a table would leave its vector data tables behind + /// forever with nothing left pointing at them. + async fn drop_all_vector_data_tables( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + ) -> Result<(), StorageError> { + let names: Vec = sqlx::query_scalar( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE ?", + ) + .bind(vector_table_like_pattern(table_id)) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + for name in names { + // Names come from sqlite_master, not from user input, and are quoted. + sqlx::query(&format!("DROP TABLE IF EXISTS \"{name}\"")) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + } + /// Drop the per-DynamoDB-table data table. pub(crate) async fn drop_data_table( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, @@ -105,6 +196,9 @@ impl SqliteEngine { .execute(&mut **tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; + // Vector data tables are keyed by index, not by table, so they are not + // reached by dropping the item table or by the catalog cascade. + Self::drop_all_vector_data_tables(tx, table_id).await?; Ok(()) } diff --git a/crates/storage-sqlite/src/data/mod.rs b/crates/storage-sqlite/src/data/mod.rs index 931ee50d..fab0ea4c 100644 --- a/crates/storage-sqlite/src/data/mod.rs +++ b/crates/storage-sqlite/src/data/mod.rs @@ -55,6 +55,31 @@ pub(crate) fn index_table_name(index_id: &str) -> String { format!("\"_ddb_{index_id}\"") } +/// Quoted SQL identifier for a vector-index data table. +/// +/// The name carries the base `table_id` as well as the `index_id` so every vector +/// table belonging to a DynamoDB table is discoverable from the table alone. That +/// is what lets `drop_data_table` clean them up: by the time it runs, the catalog +/// rows have already been cascade-deleted inside the same transaction, so the +/// index ids can no longer be read from `vector_indexes`. +/// +/// One row per vector rather than a packed blob per partition. Measured +/// 2026-08-06: a packed blob reads 2 to 4x faster but makes every write +/// O(partition), since inserting one vector rewrites the whole blob (390 MB for +/// a 100k-vector partition at 1024 dimensions). Vector indexes are maintained on +/// every write to an indexed attribute, so that trade is not available. +pub(crate) fn vector_table_name(table_id: &str, index_id: &str) -> String { + format!("\"_vidx_{table_id}_{index_id}\"") +} + +/// `LIKE` pattern matching every vector data table of one DynamoDB table. +/// +/// Used against `sqlite_master`. `table_id` is a server-generated UUID, so it +/// contains no `LIKE` metacharacters and needs no escaping. +pub(crate) fn vector_table_like_pattern(table_id: &str) -> String { + format!("_vidx_{table_id}_%") +} + /// All RANGE key attributes in key-schema order, paired with their scalar type. pub(crate) fn all_sort_key_info<'a>( key_schema: &'a [KeySchemaElement], diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 9f7b3c68..f397adad 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -42,6 +42,7 @@ mod stream; mod table_engine; mod table_helpers; mod update_table; +mod vector_search; mod worker; mod workers; diff --git a/crates/storage-sqlite/src/schema.rs b/crates/storage-sqlite/src/schema.rs index 220e5b6e..5290ee84 100644 --- a/crates/storage-sqlite/src/schema.rs +++ b/crates/storage-sqlite/src/schema.rs @@ -27,7 +27,7 @@ use sqlx::SqlitePool; /// Compiled-in catalog version. Single source of truth for the SQLite backend; /// mirrors the PostgreSQL backend's `CATALOG_VERSION`. pub const CATALOG_VERSION: extenddb_core::version::CatalogVersion = - extenddb_core::version::CatalogVersion::new(0, 0, 2); + extenddb_core::version::CatalogVersion::new(0, 0, 3); /// Complete catalog schema, applied once on a fresh database. /// @@ -91,6 +91,46 @@ CREATE TABLE IF NOT EXISTS indexes ( CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) ); +-- Vector index metadata. Kept out of `indexes` deliberately: a vector index is +-- not described by a key schema, so reusing that table's `key_schema` column +-- would mean storing something meaningless in a NOT NULL column. The engine +-- supplies index_id, as it does for GSIs. +-- +-- `search_schema` is nullable because the HASH element is optional (measured +-- against the live service): with one the search is partition-scoped and +-- SearchConditionExpression is required, without one it spans the table. +-- +-- `backfilling` mirrors the measured lifecycle: false while CREATING before the +-- scan starts, true while it runs, and the member is absent once ACTIVE. Stored +-- as an integer so the ACTIVE state is representable as NULL rather than as a +-- third boolean value. +CREATE TABLE IF NOT EXISTS vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute TEXT NOT NULL, + search_schema TEXT, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + CONSTRAINT chk_vector_backfilling_bool + CHECK (backfilling IS NULL OR backfilling IN (0, 1)), + -- An ACTIVE index must not carry the member at all, which is what the + -- service does. Enforced here as well as in core, so a bug in the backend + -- cannot persist a state the wire contract forbids. + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_vector_indexes_index_id + ON vector_indexes (index_id); + -- Resource tags. CREATE TABLE IF NOT EXISTS tags ( resource_arn TEXT NOT NULL, @@ -375,7 +415,13 @@ INSERT OR IGNORE INTO seq_counters (name, value) VALUES ('stream', CAST(strftime('%s','now') AS INTEGER) * 1000000); -- Seed settings (mirror PostgreSQL defaults). -INSERT OR IGNORE INTO settings (key, value) VALUES ('catalog_version', '0.0.2'); +-- Recorded catalog version. Upserts rather than INSERT OR IGNORE: this schema is +-- re-applied by `extenddb migrate`, and with IGNORE the recorded version would +-- never advance, so a migration could add objects and still leave the server +-- refusing to start on a version mismatch. Must stay in step with +-- `CATALOG_VERSION` above; they are checked against each other in a test. +INSERT INTO settings (key, value) VALUES ('catalog_version', '0.0.3') + ON CONFLICT(key) DO UPDATE SET value = excluded.value; INSERT OR IGNORE INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0.25'); INSERT OR IGNORE INTO settings (key, value) VALUES ('gsi_propagation_delay_ms', '10'); "#; @@ -402,3 +448,36 @@ pub async fn table_exists(pool: &SqlitePool, name: &str) -> OpResult { .map_err(|e| OpError::Internal(format!("table_exists({name}): {e}")))?; Ok(exists) } + +#[cfg(test)] +mod tests { + use super::{CATALOG_VERSION, SCHEMA_SQL}; + + /// The schema seeds the recorded catalog version as a SQL literal, and the + /// server compares that recorded value against `CATALOG_VERSION` at startup. + /// If the two drift, a freshly initialised deployment refuses to serve with a + /// version mismatch, which is a confusing failure a long way from its cause. + #[test] + fn the_seeded_catalog_version_matches_the_compiled_constant() { + let expected = format!( + "INSERT INTO settings (key, value) VALUES ('catalog_version', '{CATALOG_VERSION}')" + ); + assert!( + SCHEMA_SQL.contains(&expected), + "schema must seed catalog_version = {CATALOG_VERSION}; \ + update the literal in SCHEMA_SQL when bumping CATALOG_VERSION" + ); + } + + /// The seed must upsert. With `INSERT OR IGNORE` the recorded version never + /// advances, so `extenddb migrate` would add the new objects and still leave + /// the server refusing to start. + #[test] + fn the_catalog_version_seed_upserts_rather_than_ignoring() { + assert!( + !SCHEMA_SQL + .contains("INSERT OR IGNORE INTO settings (key, value) VALUES ('catalog_version'"), + "catalog_version must not be seeded with INSERT OR IGNORE" + ); + } +} diff --git a/crates/storage-sqlite/src/table_helpers.rs b/crates/storage-sqlite/src/table_helpers.rs index 52b64b85..5cd30599 100644 --- a/crates/storage-sqlite/src/table_helpers.rs +++ b/crates/storage-sqlite/src/table_helpers.rs @@ -52,6 +52,25 @@ pub(crate) struct IndexRow { pub provisioned_throughput: Option, } +/// A `vector_indexes` catalog row, in `FromRow` field order. +#[derive(sqlx::FromRow)] +pub(crate) struct VectorIndexRow { + pub index_name: String, + #[allow(dead_code)] + pub index_id: String, + pub dimensions: i64, + pub distance_function: String, + pub vector_attribute: String, + pub search_schema: Option, + pub projection: String, + pub index_status: String, + pub backfilling: Option, +} + +/// Columns selected for a `VectorIndexRow`, in `FromRow` field order. +pub(crate) const VECTOR_INDEX_COLUMNS: &str = "index_name, index_id, dimensions, \ + distance_function, vector_attribute, search_schema, projection, index_status, backfilling"; + /// Columns selected for a `TableRow`, in `FromRow` field order. pub(crate) const TABLE_COLUMNS: &str = "table_name, key_schema, attribute_definitions, \ billing_mode, provisioned_throughput, stream_specification, table_status, \ @@ -93,7 +112,19 @@ impl SqliteEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - self.build_table_description_from_row(account_id, row, index_rows) + let vector_rows: Vec = sqlx::query_as(&format!( + "SELECT {VECTOR_INDEX_COLUMNS} FROM vector_indexes WHERE table_id = ?" + )) + .bind(&row.table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let table_name_owned = row.table_name.clone(); + let mut desc = self.build_table_description_from_row(account_id, row, index_rows)?; + desc.vector_indexes = + self.vector_index_descriptions(account_id, &table_name_owned, vector_rows)?; + Ok(desc) } pub(crate) fn build_table_description_from_row( @@ -240,13 +271,69 @@ impl SqliteEngine { .on_demand_throughput .as_deref() .and_then(|s| serde_json::from_str(s).ok()), - // Fields for features this backend does not implement, vector - // indexes today, take their defaults, so adding one to this type - // does not break this build. + // Any core field this backend does not populate takes its default, so + // adding one does not break this build. ..Default::default() }) } + /// Build the vector index descriptions for a table. + /// + /// Separate from `build_table_description_from_row` so the shared builder + /// keeps one signature for every caller. Applied on the describe path, which + /// is where a client reads an index definition in order to search it. + pub(crate) fn vector_index_descriptions( + &self, + account_id: &str, + table_name: &str, + vector_rows: Vec, + ) -> Result>, StorageError> { + let mut vector_index_descs: Vec = Vec::new(); + for vi in vector_rows { + // Deliberately fails rather than defaulting on a bad parse: a vector + // index we cannot describe faithfully must not be reported as if we + // could, because a client uses the description to build a search. + let vector_attribute = parse_json(&vi.vector_attribute, "vector_attribute")?; + let search_schema = vi + .search_schema + .as_deref() + .map(|s| parse_json(s, "vector search_schema")) + .transpose()?; + let projection = parse_json(&vi.projection, "vector projection")?; + let distance_function = parse_json( + &format!("\"{}\"", vi.distance_function), + "distance_function", + )?; + let index_status = parse_json(&format!("\"{}\"", vi.index_status), "vector status")?; + let desc = extenddb_core::types::VectorIndexDescription { + index_name: vi.index_name.clone(), + vector_attribute, + dimensions: u32::try_from(vi.dimensions).map_err(|_| { + StorageError::Internal(format!( + "vector dimensions out of range: {}", + vi.dimensions + )) + })?, + search_schema, + distance_function, + index_status, + backfilling: vi.backfilling.map(|b| b != 0), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn(&self.region, account_id, table_name, &vi.index_name), + projection: Some(projection), + }; + // The readiness rule is core's, applied here so a backend bug cannot + // emit a description the wire contract forbids. Cheaper to catch on + // the way out than to debug from a client. + desc.validate_readiness() + .map_err(|e| StorageError::Internal(e.to_string()))?; + vector_index_descs.push(desc); + } + + Ok((!vector_index_descs.is_empty()).then_some(vector_index_descs)) + } + /// Backfill existing base-table items into a newly created GSI, batched to /// bound memory. #[allow(clippy::too_many_arguments)] diff --git a/crates/storage-sqlite/src/vector_search.rs b/crates/storage-sqlite/src/vector_search.rs new file mode 100644 index 00000000..25303816 --- /dev/null +++ b/crates/storage-sqlite/src/vector_search.rs @@ -0,0 +1,379 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Vector similarity search: exact scan over one partition. +//! +//! Exact rather than approximate, and that is a measured decision rather than a +//! placeholder. No SQLite vector extension meets this backend's constraints: the +//! static-musl `FROM scratch` build cannot `dlopen` a loadable extension, the one +//! extension with a compatible licence and an in-database index (`sqlite-vec`) is +//! brute force in every stable release anyway, and every option offering a real +//! ANN index either stores it in a sidecar file, forbids transactions, or is not +//! open source. See `docs/adr` for the full elimination. +//! +//! Measured cost on one core, warm cache, row-per-vector layout: roughly 213k to +//! 334k vectors/sec at 256 dimensions, 94k to 103k at 1024, and 39k to 43k at +//! 4096. So a partition stays inside a 10 ms budget up to about 1,000 vectors at +//! 1024 dimensions, and inside 100 ms up to about 10,000. The scan is dominated by +//! getting bytes out of SQLite rather than by the arithmetic, which is why a +//! zero-copy `&[f32]` view of the blob measured no faster than decoding per +//! element. + +use extenddb_core::types::{AttributeValue, DistanceFunction, Item}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::pk_to_text; +use extenddb_storage::{ + BoxedFuture, VectorHit, VectorSearch, VectorSearchEngine, VectorSearchOutput, + VectorSearchResult, +}; + +use crate::data::vector_table_name; +use crate::store::SqliteEngine; + +/// Partition value for an index that declares no HASH element. +/// +/// Such an index searches the whole table, so every row shares one partition +/// rather than the scan needing a second code path. The value is not a legal +/// `pk_to_text` output for any real attribute, so it cannot collide with a +/// scoped index's partitions. +pub(crate) const UNSCOPED_PARTITION: &str = "\u{0}all"; + +/// The partition column value for a vector row. +/// +/// Uses the same `pk_to_text` encoding as item partition keys, so a value written +/// by the write path and a value derived from a search request are byte-identical. +/// Getting this wrong would not fail loudly: it would silently return no hits. +pub(crate) fn partition_value( + hash_key: Option<(&str, &AttributeValue)>, +) -> Result { + match hash_key { + Some((_, value)) => Ok(pk_to_text(value)?.into_owned()), + None => Ok(UNSCOPED_PARTITION.to_owned()), + } +} + +/// Decode a stored vector blob into `f32`s. +/// +/// Rejects a truncated blob rather than reading a short vector, because a +/// dimension mismatch would silently change every distance in the result. +fn decode_vector(bytes: &[u8], dimensions: usize) -> Result, StorageError> { + if bytes.len() != dimensions * 4 { + return Err(StorageError::Internal(format!( + "stored vector is {} bytes, expected {} for {dimensions} dimensions", + bytes.len(), + dimensions * 4 + ))); + } + Ok(bytes + .chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect()) +} + +/// Score one candidate under the index's distance function. +/// +/// Cosine and Euclidean are distances, so smaller is more similar; dot product is +/// a similarity, so larger is. The caller must not compare scores across +/// functions, which is why the output reports which one was used. +fn score( + function: DistanceFunction, + query: &[f32], + query_norm: f32, + candidate: &[f32], + candidate_norm: f32, +) -> f64 { + match function { + DistanceFunction::Cosine => { + if query_norm == 0.0 || candidate_norm == 0.0 { + // Undefined angle. Reported as maximally distant rather than as + // an error, matching how a zero vector is treated elsewhere. + return 1.0; + } + let mut dot = 0.0f32; + for i in 0..query.len() { + dot += query[i] * candidate[i]; + } + f64::from(1.0 - (dot / (query_norm * candidate_norm))) + } + DistanceFunction::Euclidean => { + let mut sum = 0.0f32; + for i in 0..query.len() { + let d = query[i] - candidate[i]; + sum += d * d; + } + f64::from(sum.sqrt()) + } + DistanceFunction::DotProduct => { + let mut dot = 0.0f32; + for i in 0..query.len() { + dot += query[i] * candidate[i]; + } + f64::from(dot) + } + } +} + +/// Keeps the best `k` seen so far, ordered by the index's distance function. +/// +/// A full sort of the partition would dominate the scan for a large partition and +/// is unnecessary: only `k` rows are ever returned. Insertion into a `k`-sized +/// vector is cheap because the common case after the first `k` candidates is a +/// single comparison against the current worst. +struct TopK { + k: usize, + function: DistanceFunction, + hits: Vec<(f64, Item)>, +} + +impl TopK { + fn new(k: usize, function: DistanceFunction) -> Self { + Self { + k, + function, + hits: Vec::with_capacity(k.saturating_add(1)), + } + } + + /// True when `a` should rank ahead of `b`. + fn ranks_before(&self, a: f64, b: f64) -> bool { + self.function.ranks_before(a, b) + } + + fn offer(&mut self, candidate_score: f64, item: Item) { + if self.hits.len() < self.k { + let pos = self + .hits + .iter() + .position(|(s, _)| self.ranks_before(candidate_score, *s)) + .unwrap_or(self.hits.len()); + self.hits.insert(pos, (candidate_score, item)); + return; + } + if self.k == 0 { + return; + } + let worst = self.hits[self.k - 1].0; + if !self.ranks_before(candidate_score, worst) { + return; + } + let pos = self + .hits + .iter() + .position(|(s, _)| self.ranks_before(candidate_score, *s)) + .unwrap_or(self.k - 1); + self.hits.insert(pos, (candidate_score, item)); + self.hits.truncate(self.k); + } +} + +impl VectorSearchEngine for SqliteEngine { + fn search_vectors(&self, req: VectorSearch<'_>) -> BoxedFuture<'_, VectorSearchResult> { + // The request borrows; own what the async body needs so the future is not + // tied to the caller's frame. + let table_id = req.key_info.table_id.clone(); + let index_name = req.index_name.to_owned(); + let query_vector = req.query_vector.to_vec(); + let top_k = req.top_k; + let partition = partition_value(req.hash_key); + let filters: Vec<(String, AttributeValue)> = req + .filters + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).clone())) + .collect(); + + Box::pin(async move { + let partition = partition?; + + // The index definition comes from the catalog rather than from + // TableKeyInfo, because the cached key info carries dimensions and the + // search schema but not the distance function, without which a score + // cannot be computed or ordered. + let row: Option<(String, i64, String)> = sqlx::query_as( + "SELECT index_id, dimensions, distance_function FROM vector_indexes \ + WHERE table_id = ? AND index_name = ?", + ) + .bind(&table_id) + .bind(&index_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (index_id, dimensions, distance_raw) = + row.ok_or_else(|| StorageError::IndexNotFound(index_name.clone()))?; + let dimensions = usize::try_from(dimensions).map_err(|_| { + StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) + })?; + let function: DistanceFunction = serde_json::from_str(&format!("\"{distance_raw}\"")) + .map_err(|e| { + StorageError::Internal(format!("unknown distance function: {e}")) + })?; + + if query_vector.len() != dimensions { + // Core validates this against the cached key info, so reaching + // here means the catalog and the cache disagree. + return Err(StorageError::Validation(format!( + "query vector has {} dimensions, index expects {dimensions}", + query_vector.len() + ))); + } + + let vec_table = vector_table_name(&table_id, &index_id); + let sql = format!("SELECT vec, nrm, item_data FROM {vec_table} WHERE part = ?"); + + let mut query_norm = 0.0f32; + for x in &query_vector { + query_norm += x * x; + } + let query_norm = query_norm.sqrt(); + + let k = usize::try_from(top_k.max(0)).unwrap_or(0); + let mut top = TopK::new(k, function); + + // Streamed rather than fetched all at once, so a large partition does + // not allocate proportionally to its size. This is the reason the + // row-per-vector layout was chosen over a packed blob per partition. + use futures::TryStreamExt; + let mut stream = sqlx::query_as::<_, (Vec, f64, String)>(&sql) + .bind(&partition) + .fetch(&self.pool); + + while let Some((blob, norm, item_json)) = stream + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let candidate = decode_vector(&blob, dimensions)?; + let item: Item = serde_json::from_str(&item_json) + .map_err(|e| StorageError::Internal(format!("stored item: {e}")))?; + + // Inline-filter attributes are applied here rather than in SQL, + // because they are item attributes rather than columns. Equality + // only, which is all the wire surface admits today. + if !filters.is_empty() + && !filters + .iter() + .all(|(name, expected)| item.get(name) == Some(expected)) + { + continue; + } + + #[allow(clippy::cast_possible_truncation)] + let candidate_norm = norm as f32; + top.offer( + score( + function, + &query_vector, + query_norm, + &candidate, + candidate_norm, + ), + item, + ); + } + + Ok(VectorSearchOutput { + hits: top + .hits + .into_iter() + .map(|(score, item)| VectorHit { item, score }) + .collect(), + distance_function: function, + }) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn item() -> Item { + Item::new() + } + + #[test] + fn cosine_of_identical_vectors_is_zero() { + let v = [1.0f32, 2.0, 3.0]; + let n = (14.0f32).sqrt(); + let s = score(DistanceFunction::Cosine, &v, n, &v, n); + assert!(s.abs() < 1e-6, "expected ~0.0, got {s}"); + } + + #[test] + fn cosine_of_opposite_vectors_is_two() { + let a = [1.0f32, 0.0]; + let b = [-1.0f32, 0.0]; + let s = score(DistanceFunction::Cosine, &a, 1.0, &b, 1.0); + assert!((s - 2.0).abs() < 1e-6, "expected ~2.0, got {s}"); + } + + #[test] + fn a_zero_vector_is_maximally_distant_rather_than_an_error() { + let a = [1.0f32, 0.0]; + let z = [0.0f32, 0.0]; + assert!((score(DistanceFunction::Cosine, &a, 1.0, &z, 0.0) - 1.0).abs() < 1e-6); + } + + #[test] + fn euclidean_is_the_straight_line_distance() { + let a = [0.0f32, 0.0]; + let b = [3.0f32, 4.0]; + let s = score(DistanceFunction::Euclidean, &a, 0.0, &b, 5.0); + assert!((s - 5.0).abs() < 1e-6, "expected 5.0, got {s}"); + } + + #[test] + fn dot_product_is_reported_raw_and_can_be_negative() { + let a = [1.0f32, 0.0]; + let b = [-2.0f32, 0.0]; + let s = score(DistanceFunction::DotProduct, &a, 1.0, &b, 2.0); + assert!((s + 2.0).abs() < 1e-6, "expected -2.0, got {s}"); + } + + /// The direction of "better" is not uniform, so top-k must consult the + /// distance function. A single ordering would silently return the *worst* + /// matches for dot product. + #[test] + fn top_k_orders_distances_ascending_and_similarities_descending() { + let mut cosine = TopK::new(2, DistanceFunction::Cosine); + for s in [0.9, 0.1, 0.5] { + cosine.offer(s, item()); + } + assert_eq!( + cosine.hits.iter().map(|(s, _)| *s).collect::>(), + vec![0.1, 0.5] + ); + + let mut dot = TopK::new(2, DistanceFunction::DotProduct); + for s in [0.9, 0.1, 0.5] { + dot.offer(s, item()); + } + assert_eq!( + dot.hits.iter().map(|(s, _)| *s).collect::>(), + vec![0.9, 0.5] + ); + } + + #[test] + fn top_k_of_zero_returns_nothing_rather_than_panicking() { + let mut t = TopK::new(0, DistanceFunction::Cosine); + t.offer(0.5, item()); + assert!(t.hits.is_empty()); + } + + #[test] + fn a_truncated_stored_vector_is_rejected_rather_than_read_short() { + let err = decode_vector(&[0u8; 8], 3).expect_err("must reject"); + assert!( + format!("{err:?}").contains("expected 12"), + "unexpected: {err:?}" + ); + } + + #[test] + fn an_unscoped_index_uses_a_partition_no_real_key_can_produce() { + let unscoped = partition_value(None).unwrap(); + let scoped = partition_value(Some(("pk", &AttributeValue::S("all".to_owned())))).unwrap(); + assert_ne!(unscoped, scoped); + } +} diff --git a/tests/rust/src/vector_index_unsupported.rs b/tests/rust/src/vector_index_unsupported.rs index b0a2a073..703ef3bb 100644 --- a/tests/rust/src/vector_index_unsupported.rs +++ b/tests/rust/src/vector_index_unsupported.rs @@ -152,9 +152,7 @@ fn table_name(suffix: &str) -> String { /// answer, because the capability is not otherwise observable over the wire. /// Every test here asserts a refusal, so on a backend that *does* implement /// vector search they must skip rather than fail: the refusals are the contract -/// for non-participating backends only. No in-tree backend implements it today, -/// so this returns false for both, and the mechanism exists for the first one that -/// does. +/// for non-participating backends only. /// /// Deliberately distinguishes the refusal from any other failure. Treating "not a /// 200" as unsupported would make the whole suite skip silently the first time an From 0ac4716358d20a4971f6f03ffff2dc91e7050957 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Thu, 6 Aug 2026 21:13:42 +0000 Subject: [PATCH 02/25] feat(sqlite): maintain vector indexes on writes, and prove search over the wire Vector search now works end to end: an item written to a table with a vector index is indexed, findable, replaceable and removable, and a scoped index isolates partitions. Nine wire tests against a live server cover it. Maintenance is synchronous, inside the base write transaction, so a vector row cannot survive a rolled-back item write or be lost to a crash between the two. That is STRICTER than the service, which makes vector indexes eventually consistent like a GSI, and it is a deliberate first step rather than the final answer: being fresher than required cannot give a wrong answer, whereas being staler can. The asynchronous path should reuse the existing gsi_pending queue, which already provides crash recovery, per-key FIFO and a configurable delay. Until it does, a search immediately after a write sees the new item where the service might not. Applied at all six write sites, not just the obvious three. PutItem, UpdateItem and DeleteItem plus the three TransactWriteItems branches; missing the transactional ones would have meant a transactional write silently skipping the index. Deliberately placed OUTSIDE the existing `if !indexes.is_empty()` guard, because that guard is about GSIs and LSIs and a table may have a vector index and neither. Two things the design turns on: * Rows are keyed by the base item, not by the partition. An item whose HASH attribute changes must MOVE between partitions, and keying by partition would leave the old row behind so the item was findable under two tenants at once. The write path therefore deletes by base key before inserting. Tested directly. * Sort keys bind through parse_sk + sk_bound, the same D2 representation the GSI and LSI tables use: order-preserving text for numbers, BLOB for binary. My first version used pk_to_text, which would have been self-consistent but divergent from every other index table for the same item. Extraction and the norm live in core beside validate_vector_write rather than in the backend, because the two must agree on what a vector attribute is and separating them invites a backend storing something the validator would reject. Five tests pin them together, including that a component overflowing f32 is rejected rather than becoming infinity and poisoning every distance. One defect found by the failing tests: table_key_info never populated vector_indexes, so the write-path gate was always false and nothing was ever indexed. Now populated, which also lets core validate vector attributes on writes. Noted while doing it: VectorIndexKeyInfo still carries no distance function and no projection, so a search reads the catalog for those; widening that type would remove the last per-search catalog read. Also removed two `..Default::default()` spreads that clippy correctly identified as dead now that every field is populated. For a site that builds a complete value, a new core field SHOULD break it and force a decision rather than defaulting silently. Verified against a live SQLite server: 436 integration passed / 0 failed / 0 filtered out (9 new), workspace 751 passed / 0 failed / 0 filtered out, fmt clean, clippy -D warnings clean on both feature sets. --- crates/core/src/validation/mod.rs | 5 +- crates/core/src/validation/vector_item.rs | 82 +++++ crates/storage-sqlite/src/data/ddl.rs | 52 ++- crates/storage-sqlite/src/data/delete_item.rs | 13 + crates/storage-sqlite/src/data/mod.rs | 1 + crates/storage-sqlite/src/data/put_item.rs | 15 + .../storage-sqlite/src/data/transactions.rs | 45 +++ crates/storage-sqlite/src/data/update_item.rs | 14 + .../storage-sqlite/src/data/vector_index.rs | 259 +++++++++++++ tests/rust/src/main.rs | 2 + tests/rust/src/vector_index_search.rs | 344 ++++++++++++++++++ tests/rust/src/vector_index_unsupported.rs | 11 +- 12 files changed, 836 insertions(+), 7 deletions(-) create mode 100644 crates/storage-sqlite/src/data/vector_index.rs create mode 100644 tests/rust/src/vector_index_search.rs diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index aed07de9..ee01e9f2 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -3,7 +3,10 @@ pub mod number; pub mod vector_item; -pub use vector_item::{MAX_HASH_KEY_SIZE, MAX_INLINE_FILTER_SIZE, validate_vector_write}; +pub use vector_item::{ + MAX_HASH_KEY_SIZE, MAX_INLINE_FILTER_SIZE, validate_vector_write, vector_components, + vector_norm, +}; use crate::error::{DynamoDbError, ErrorMessageKey, error_message}; use crate::limits::LimitsConfig; diff --git a/crates/core/src/validation/vector_item.rs b/crates/core/src/validation/vector_item.rs index c877f244..d6d431a0 100644 --- a/crates/core/src/validation/vector_item.rs +++ b/crates/core/src/validation/vector_item.rs @@ -57,6 +57,50 @@ pub fn validate_vector_write( Ok(()) } +/// Extract the components of a vector attribute as `f32`s. +/// +/// Lives beside [`validate_vector_write`] deliberately: the two must agree on what +/// a vector attribute is, and separating them invites a backend that stores +/// something the validator would have rejected. Every backend indexing a vector +/// should use this rather than reading the attribute itself. +/// +/// Returns `None` when the value is not a list of numbers each parsing to a finite +/// `f32`, which is exactly the condition `validate_vector_write` rejects. A caller +/// that has already validated can treat `None` as an internal inconsistency rather +/// than as bad input. +/// +/// The narrowing to `f32` is deliberate and lossy for a caller supplying more +/// precision: the wire type is arbitrary-precision decimal, embedding models emit +/// single precision, and the service's declared dimensionality is in `f32` terms. +#[must_use] +pub fn vector_components(value: &AttributeValue) -> Option> { + let AttributeValue::L(elements) = value else { + return None; + }; + let mut out = Vec::with_capacity(elements.len()); + for element in elements { + let AttributeValue::N(number) = element else { + return None; + }; + let parsed = number.parse::().ok()?; + if !parsed.is_finite() { + return None; + } + out.push(parsed); + } + Some(out) +} + +/// The L2 norm of a vector, precomputed at write time. +/// +/// Stored alongside the vector so a cosine search costs one dot product per +/// candidate instead of also summing squares. Computed here rather than in a +/// backend so every backend stores the same value for the same vector. +#[must_use] +pub fn vector_norm(components: &[f32]) -> f32 { + components.iter().map(|x| x * x).sum::().sqrt() +} + /// Validate a single vector-valued attribute against an index definition. fn validate_vector_attribute( value: &AttributeValue, @@ -392,4 +436,42 @@ mod tests { assert_eq!(format_scientific(1.3e40), "1.3E+40"); assert_eq!(format_scientific(-1.3e40), "-1.3E+40"); } + + /// The extractor must accept exactly what the validator accepts. If these + /// drift, a backend stores a vector the validator would have rejected, or + /// refuses one it accepted. + #[test] + fn the_extractor_accepts_what_the_validator_accepts() { + let good = AttributeValue::L(vec![ + AttributeValue::N("0.5".to_owned()), + AttributeValue::N("-1".to_owned()), + AttributeValue::N("0".to_owned()), + ]); + assert_eq!(vector_components(&good), Some(vec![0.5, -1.0, 0.0])); + } + + #[test] + fn the_extractor_rejects_a_non_list() { + assert!(vector_components(&AttributeValue::S("nope".to_owned())).is_none()); + } + + #[test] + fn the_extractor_rejects_a_non_numeric_element() { + let bad = AttributeValue::L(vec![AttributeValue::S("1".to_owned())]); + assert!(vector_components(&bad).is_none()); + } + + /// A component that overflows f32 is rejected rather than becoming infinity, + /// which would poison every distance computed against it. + #[test] + fn the_extractor_rejects_a_component_that_is_not_finite_in_f32() { + let bad = AttributeValue::L(vec![AttributeValue::N("1e40".to_owned())]); + assert!(vector_components(&bad).is_none()); + } + + #[test] + fn the_norm_is_the_euclidean_length() { + assert!((vector_norm(&[3.0, 4.0]) - 5.0).abs() < 1e-6); + assert!(vector_norm(&[0.0, 0.0]).abs() < 1e-6); + } } diff --git a/crates/storage-sqlite/src/data/ddl.rs b/crates/storage-sqlite/src/data/ddl.rs index 1ed973fc..9236773b 100644 --- a/crates/storage-sqlite/src/data/ddl.rs +++ b/crates/storage-sqlite/src/data/ddl.rs @@ -323,6 +323,10 @@ impl SqliteEngine { let (global_secondary_indexes, local_secondary_indexes) = self.fetch_all_index_info(&table_id).await?; let has_lsi = !local_secondary_indexes.is_empty(); + // Vector indexes ride on the cached key info too, so the write path can + // decide whether any maintenance is needed without a query, and so the + // engine can validate vector attributes on writes. + let vector_indexes = self.fetch_vector_index_key_info(&table_id).await?; Ok(TableKeyInfo { table_name: table_name.to_owned(), @@ -335,13 +339,53 @@ impl SqliteEngine { global_secondary_indexes, local_secondary_indexes, stream_specification, - // Fields for features this backend does not implement, vector - // indexes today, take their defaults, so adding one to this type - // does not break this build. - ..Default::default() + // Every field is populated, with no `..Default::default()` spread: a + // new core field should break this site and force a decision about + // whether the write path needs it, rather than silently defaulting. + vector_indexes, }) } + /// Fetch the vector indexes of a table in the shape the engine caches. + /// + /// Note what this cannot carry: `VectorIndexKeyInfo` has no distance function + /// and no projection, so a search still reads the catalog for those. Widening + /// that type would remove the last per-search catalog read. + async fn fetch_vector_index_key_info( + &self, + table_id: &str, + ) -> Result, StorageError> { + let rows: Vec<(String, i64, String, Option)> = sqlx::query_as( + "SELECT index_name, dimensions, vector_attribute, search_schema \ + FROM vector_indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut out = Vec::with_capacity(rows.len()); + for (index_name, dimensions, vector_attribute, search_schema) in rows { + let attr: extenddb_core::types::VectorAttribute = + serde_json::from_str(&vector_attribute) + .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))?; + let search_schema = match search_schema.as_deref() { + Some(json) => serde_json::from_str(json) + .map_err(|e| StorageError::Internal(format!("search_schema: {e}")))?, + None => Vec::new(), + }; + out.push(extenddb_core::types::VectorIndexKeyInfo { + index_name, + dimensions: u32::try_from(dimensions).map_err(|_| { + StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) + })?, + vector_attribute_name: attr.attribute_name, + search_schema, + }); + } + Ok(out) + } + /// Fetch every secondary index defined on a table, split into /// `(global_secondary_indexes, local_secondary_indexes)`. async fn fetch_all_index_info( diff --git a/crates/storage-sqlite/src/data/delete_item.rs b/crates/storage-sqlite/src/data/delete_item.rs index a807045e..0eea72f1 100644 --- a/crates/storage-sqlite/src/data/delete_item.rs +++ b/crates/storage-sqlite/src/data/delete_item.rs @@ -70,6 +70,19 @@ impl SqliteEngine { ) .await?; } + // Vector rows for this base item are removed in the same + // transaction. `new_item` is None, so this is a pure removal. + if !key_info.vector_indexes.is_empty() { + crate::data::vector_index::sync_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + None, + ) + .await?; + } if enqueue_async_indexes( &mut tx, key_info, diff --git a/crates/storage-sqlite/src/data/mod.rs b/crates/storage-sqlite/src/data/mod.rs index fab0ea4c..372d7a0d 100644 --- a/crates/storage-sqlite/src/data/mod.rs +++ b/crates/storage-sqlite/src/data/mod.rs @@ -39,6 +39,7 @@ mod query_scan; mod transactions; mod tx_helpers; mod update_item; +pub(crate) mod vector_index; pub(crate) use index::{ GsiApplyContext, apply_claimed_row, insert_index_row_multi, project_item_for_index, diff --git a/crates/storage-sqlite/src/data/put_item.rs b/crates/storage-sqlite/src/data/put_item.rs index de43d830..a6b71408 100644 --- a/crates/storage-sqlite/src/data/put_item.rs +++ b/crates/storage-sqlite/src/data/put_item.rs @@ -102,6 +102,21 @@ impl SqliteEngine { ) .await?; } + // Vector indexes, maintained in the same transaction. Gated on the cached + // key info so a table without them costs no extra query. Deliberately + // outside the `indexes` guard above: a table may have a vector index and + // no GSI or LSI at all. + if !key_info.vector_indexes.is_empty() { + crate::data::vector_index::sync_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + Some(&item), + ) + .await?; + } let enqueued = enqueue_async_indexes( &mut tx, key_info, diff --git a/crates/storage-sqlite/src/data/transactions.rs b/crates/storage-sqlite/src/data/transactions.rs index f3ae8dec..4709da25 100644 --- a/crates/storage-sqlite/src/data/transactions.rs +++ b/crates/storage-sqlite/src/data/transactions.rs @@ -307,6 +307,21 @@ async fn execute_transact_write_op( .await .map_err(TxnOpError::Storage)?; } + // Vector indexes share the transaction, so a rolled-back transactional + // write cannot leave a vector row behind. Outside the `indexes` guard + // because a table may have a vector index and no GSI or LSI. + if !key_info.vector_indexes.is_empty() { + crate::data::vector_index::sync_vector_indexes( + tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + existing.as_ref(), + Some(item), + ) + .await + .map_err(TxnOpError::Storage)?; + } Ok((existing, Some((*item).clone()))) } TransactWriteOp::Delete { @@ -350,6 +365,21 @@ async fn execute_transact_write_op( .await .map_err(TxnOpError::Storage)?; } + // Vector indexes share the transaction, so a rolled-back transactional + // write cannot leave a vector row behind. Outside the `indexes` guard + // because a table may have a vector index and no GSI or LSI. + if !key_info.vector_indexes.is_empty() { + crate::data::vector_index::sync_vector_indexes( + tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + existing.as_ref(), + None, + ) + .await + .map_err(TxnOpError::Storage)?; + } Ok((existing, None)) } TransactWriteOp::Update { @@ -415,6 +445,21 @@ async fn execute_transact_write_op( .await .map_err(TxnOpError::Storage)?; } + // Vector indexes share the transaction, so a rolled-back transactional + // write cannot leave a vector row behind. Outside the `indexes` guard + // because a table may have a vector index and no GSI or LSI. + if !key_info.vector_indexes.is_empty() { + crate::data::vector_index::sync_vector_indexes( + tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + existing.as_ref(), + Some(&item), + ) + .await + .map_err(TxnOpError::Storage)?; + } Ok((existing, Some(item))) } TransactWriteOp::ConditionCheck { diff --git a/crates/storage-sqlite/src/data/update_item.rs b/crates/storage-sqlite/src/data/update_item.rs index 92ac84a5..b357d28c 100644 --- a/crates/storage-sqlite/src/data/update_item.rs +++ b/crates/storage-sqlite/src/data/update_item.rs @@ -98,6 +98,20 @@ impl SqliteEngine { ) .await?; } + // Vector indexes, maintained in the same transaction. Gated on the cached + // key info so a table without them costs no extra query, and kept outside + // the `indexes` guard because a table may have a vector index and no GSI. + if !key_info.vector_indexes.is_empty() { + crate::data::vector_index::sync_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + Some(&item), + ) + .await?; + } let enqueued = enqueue_async_indexes( &mut tx, key_info, diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs new file mode 100644 index 00000000..6f1cb638 --- /dev/null +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -0,0 +1,259 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Vector index maintenance on the write path. +//! +//! Applied synchronously inside the base write transaction, so a vector row can +//! never survive a rolled-back item write or be lost to a crash between the two. +//! +//! That is **stricter than the service**, which makes vector indexes eventually +//! consistent like a GSI, and it is a deliberate first step rather than the final +//! answer. Being more consistent than required cannot produce a wrong answer, only +//! a fresher one, whereas the reverse can. The asynchronous path should reuse the +//! existing `gsi_pending` queue, which already provides crash recovery, per-key +//! FIFO ordering and a configurable delay; until it does, a search immediately +//! after a write returns the new item where the service might not. + +use extenddb_core::types::{AttributeDefinition, Item, KeySchemaElement, SearchSchemaElementType}; +use extenddb_core::validation::{vector_components, vector_norm}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::pk_to_text; + +use super::{BoundValue, all_sort_key_info, sk_bound, vector_table_name}; +use crate::vector_search::partition_value; + +/// A vector index as the write path needs it. +pub(crate) struct VectorIndexMeta { + pub index_id: String, + pub dimensions: usize, + pub vector_attribute_name: String, + /// The single HASH element's attribute name, when the index declares one. + /// `None` means the index is unscoped and every row shares one partition. + pub hash_attribute_name: Option, +} + +/// Load the vector indexes of a table. +/// +/// Read inside the write transaction rather than taken from the cached +/// `TableKeyInfo`, because the cache carries the search schema but not the index +/// id, and the id is what names the data table. +pub(crate) async fn fetch_vector_indexes_for_table( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, +) -> Result, StorageError> { + let rows: Vec<(String, i64, String, Option)> = sqlx::query_as( + "SELECT index_id, dimensions, vector_attribute, search_schema \ + FROM vector_indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut out = Vec::with_capacity(rows.len()); + for (index_id, dimensions, vector_attribute, search_schema) in rows { + let attr: extenddb_core::types::VectorAttribute = + serde_json::from_str(&vector_attribute) + .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))?; + let hash_attribute_name = match search_schema.as_deref() { + Some(json) => { + let elements: Vec = + serde_json::from_str(json) + .map_err(|e| StorageError::Internal(format!("search_schema: {e}")))?; + elements + .into_iter() + .find(|e| e.element_type == SearchSchemaElementType::Hash) + .map(|e| e.attribute_name) + } + None => None, + }; + out.push(VectorIndexMeta { + index_id, + dimensions: usize::try_from(dimensions).map_err(|_| { + StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) + })?, + vector_attribute_name: attr.attribute_name, + hash_attribute_name, + }); + } + Ok(out) +} + +/// Whether an item belongs in a vector index. +/// +/// It must carry the vector attribute, and the HASH attribute when the index +/// declares one: without the latter the row could not be placed in a partition, +/// and putting it in the unscoped partition would make it visible to searches of +/// every other partition. Not an error, exactly as a GSI silently omits an item +/// missing its index key. +fn item_is_indexable(item: &Item, meta: &VectorIndexMeta) -> bool { + if !item.contains_key(&meta.vector_attribute_name) { + return false; + } + match &meta.hash_attribute_name { + Some(name) => item.contains_key(name), + None => true, + } +} + +/// The partition column value for an item under one index. +fn item_partition(item: &Item, meta: &VectorIndexMeta) -> Result { + match &meta.hash_attribute_name { + Some(name) => { + let value = item.get(name).ok_or_else(|| { + StorageError::Internal( + "indexable check passed but the hash attribute is absent".to_owned(), + ) + })?; + partition_value(Some((name.as_str(), value))) + } + None => partition_value(None), + } +} + +/// Base-key bind values for a row, in key-schema order. +fn base_key_binds( + item: &Item, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], +) -> Result, StorageError> { + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + let pk_attr = &base_key_schema[0].attribute_name; + let pk = item.get(pk_attr).ok_or_else(|| { + StorageError::Internal("item written without its partition key".to_owned()) + })?; + let mut binds = vec![BoundValue::Text(pk_to_text(pk)?.into_owned())]; + for &(name, sk_type) in &base_sks { + // Sort keys use the same storage representation as the GSI/LSI tables: + // order-preserving text for numbers and a BLOB for binary. Encoding them + // any other way would still be self-consistent here but would diverge + // from every other index table for the same item. + match item.get(name) { + Some(value) => binds.push(sk_bound(&extenddb_storage::util::parse_sk(value, sk_type)?)), + None => binds.push(BoundValue::Text(String::new())), + } + } + Ok(binds) +} + +/// Column names for the base key, in key-schema order. +fn base_key_columns( + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], +) -> Vec { + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + let mut cols = vec!["base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + cols.push(format!( + "base_{}", + extenddb_storage::util::sk_column_n(i, sk_type) + )); + } + cols +} + +/// Apply an item write to every vector index on the table. +/// +/// `old_item` and `new_item` follow the same convention as `sync_indexes`: a put +/// supplies both when replacing, a delete supplies only the old. +/// +/// The delete-then-insert shape matters. An item can move between partitions when +/// its HASH attribute changes, and the row is keyed by the base item rather than by +/// the partition, so an insert alone would leave the old partition's row in place +/// and the item would be findable in two partitions at once. +pub(crate) async fn sync_vector_indexes( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + old_item: Option<&Item>, + new_item: Option<&Item>, +) -> Result<(), StorageError> { + let metas = fetch_vector_indexes_for_table(tx, table_id).await?; + if metas.is_empty() { + return Ok(()); + } + + let key_cols = base_key_columns(base_key_schema, attr_defs); + let where_clause = key_cols + .iter() + .map(|c| format!("{c} = ?")) + .collect::>() + .join(" AND "); + + for meta in &metas { + let vec_table = vector_table_name(table_id, &meta.index_id); + + // Remove any existing row for this base item first, whatever partition it + // was in. + let source = old_item.or(new_item); + if let Some(source) = source { + let binds = base_key_binds(source, base_key_schema, attr_defs)?; + let sql = format!("DELETE FROM {vec_table} WHERE {where_clause}"); + let mut q = sqlx::query(&sql); + for b in binds { + q = super::bind_bound!(q, b); + } + q.execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + let Some(new_item) = new_item else { + continue; // A delete: removal above is the whole of the work. + }; + if !item_is_indexable(new_item, meta) { + continue; + } + + let value = new_item.get(&meta.vector_attribute_name).ok_or_else(|| { + StorageError::Internal("indexable check passed but the vector is absent".to_owned()) + })?; + let components = vector_components(value).ok_or_else(|| { + // Core validates the write before it reaches storage, so a malformed + // vector here means validation was bypassed rather than that a caller + // sent bad input. + StorageError::Internal( + "vector attribute reached storage without passing validation".to_owned(), + ) + })?; + if components.len() != meta.dimensions { + return Err(StorageError::Internal(format!( + "vector has {} components, index declares {}", + components.len(), + meta.dimensions + ))); + } + + let mut blob = Vec::with_capacity(components.len() * 4); + for x in &components { + blob.extend_from_slice(&x.to_le_bytes()); + } + let norm = vector_norm(&components); + let part = item_partition(new_item, meta)?; + let item_json = serde_json::to_string(new_item) + .map_err(|e| StorageError::Internal(format!("serialize item: {e}")))?; + + let cols = std::iter::once("part".to_owned()) + .chain(key_cols.iter().cloned()) + .chain(["vec".to_owned(), "nrm".to_owned(), "item_data".to_owned()]) + .collect::>(); + let placeholders = vec!["?"; cols.len()].join(", "); + let sql = format!( + "INSERT INTO {vec_table} ({}) VALUES ({placeholders})", + cols.join(", ") + ); + let key_binds = base_key_binds(new_item, base_key_schema, attr_defs)?; + let mut q = sqlx::query(&sql).bind(part); + for b in key_binds { + q = super::bind_bound!(q, b); + } + q.bind(blob) + .bind(f64::from(norm)) + .bind(item_json) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) +} diff --git a/tests/rust/src/main.rs b/tests/rust/src/main.rs index 9657e0e3..325eab78 100755 --- a/tests/rust/src/main.rs +++ b/tests/rust/src/main.rs @@ -113,6 +113,8 @@ mod update_item_number_validation; #[cfg(test)] mod update_table_billing_validation; #[cfg(test)] +mod vector_index_search; +#[cfg(test)] mod vector_index_unsupported; #[cfg(test)] mod wording_parity_validation; diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs new file mode 100644 index 00000000..1d212838 --- /dev/null +++ b/tests/rust/src/vector_index_search.rs @@ -0,0 +1,344 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Wire-level tests for a backend that implements vector indexes. +//! +//! The mirror of `vector_index_unsupported`: that file asserts the refusals a +//! non-participating backend must give, this one asserts the behaviour a +//! participating one must give. Both probe the running backend and skip when it is +//! the wrong kind, so one suite runs everywhere. +//! +//! Hand-built JSON and SigV4 signing for the same reason as the other file: no +//! published `aws-sdk-dynamodb` models vector indexes. + +use crate::vector_index_unsupported::{call, table_name, vectors_supported}; + +/// Create a table with one vector index and wait for it to be usable. +async fn create_vector_table(name: &str, dims: usize, distance: &str, scoped: bool) { + let search_schema = if scoped { + r#""SearchSchema": [{"AttributeName": "tenant", "SearchSchemaElementType": "HASH"}],"# + } else { + "" + }; + let attr_defs = if scoped { + r#"{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "tenant", "AttributeType": "S"}"# + } else { + r#"{"AttributeName": "pk", "AttributeType": "S"}"# + }; + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{attr_defs}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": {dims}, + "DistanceFunction": "{distance}", + "VectorAttribute": {{"AttributeName": "emb"}}, + {search_schema} + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + wait_for_active(name).await; +} + +/// Poll until the table reports ACTIVE. +/// +/// CreateTable returns while the table is still CREATING, and a write against a +/// CREATING table is rejected with ResourceNotFound, so without this every test +/// here fails on its first PutItem for a reason unrelated to what it asserts. +async fn wait_for_active(name: &str) { + for _ in 0..100 { + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + if status == 200 && text.contains(r#""TableStatus":"ACTIVE""#) { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + panic!("table {name} never became ACTIVE"); +} + +fn vector_json(values: &[f32]) -> String { + let parts: Vec = values.iter().map(|v| format!(r#"{{"N": "{v}"}}"#)).collect(); + format!("[{}]", parts.join(", ")) +} + +async fn put_vector(table: &str, pk: &str, tenant: Option<&str>, values: &[f32]) { + let tenant_attr = tenant + .map(|t| format!(r#", "tenant": {{"S": "{t}"}}"#)) + .unwrap_or_default(); + let body = format!( + r#"{{ + "TableName": "{table}", + "Item": {{"pk": {{"S": "{pk}"}}, "emb": {{"L": {}}}{tenant_attr}}} + }}"#, + vector_json(values) + ); + let (status, text) = call("PutItem", &body).await; + assert_eq!(status, 200, "PutItem failed: {text}"); +} + +async fn search( + table: &str, + values: &[f32], + top_k: usize, + condition: Option<&str>, +) -> serde_json::Value { + let cond = condition + .map(|c| { + format!( + r#", "SearchConditionExpression": "tenant = :t", "ExpressionAttributeValues": {{":t": {{"S": "{c}"}}}}"# + ) + }) + .unwrap_or_default(); + let body = format!( + r#"{{ + "TableName": "{table}", + "IndexName": "vidx", + "SearchVector": {}, + "TopK": {top_k}{cond} + }}"#, + vector_json(values) + ); + let (status, text) = call("SearchVectors", &body).await; + assert_eq!(status, 200, "SearchVectors failed: {text}"); + serde_json::from_str(&text).expect("search response is JSON") +} + +fn hit_pks(response: &serde_json::Value) -> Vec { + response + .get("SearchResults") + .and_then(|v| v.as_array()) + .unwrap_or_else(|| panic!("no results array in: {response}")) + .iter() + .map(|hit| { + hit.get("Item") + .and_then(|i| i.get("pk")) + .and_then(|p| p.get("S")) + .and_then(|s| s.as_str()) + .unwrap_or_else(|| panic!("hit has no pk: {hit}")) + .to_owned() + }) + .collect() +} + +/// The nearest vector comes back first, and the ordering is by actual distance +/// rather than by insertion order. +#[tokio::test] +async fn search_returns_nearest_first() { + if !vectors_supported().await { + return; + } + let name = table_name("pos_order"); + create_vector_table(&name, 2, "COSINE", false).await; + + // Inserted worst-first, so passing cannot be an artefact of scan order. + put_vector(&name, "opposite", None, &[-1.0, 0.0]).await; + put_vector(&name, "orthogonal", None, &[0.0, 1.0]).await; + put_vector(&name, "exact", None, &[1.0, 0.0]).await; + + let response = search(&name, &[1.0, 0.0], 3, None).await; + assert_eq!( + hit_pks(&response), + vec!["exact", "orthogonal", "opposite"], + "response: {response}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// TopK bounds the result set. +#[tokio::test] +async fn top_k_limits_the_results() { + if !vectors_supported().await { + return; + } + let name = table_name("pos_topk"); + create_vector_table(&name, 2, "COSINE", false).await; + for i in 0..5 { + put_vector(&name, &format!("i{i}"), None, &[1.0, i as f32]).await; + } + + let response = search(&name, &[1.0, 0.0], 2, None).await; + assert_eq!(hit_pks(&response).len(), 2, "response: {response}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Overwriting an item replaces its vector rather than leaving both, which a +/// row keyed by partition instead of by base item would get wrong. +#[tokio::test] +async fn overwriting_an_item_replaces_its_vector() { + if !vectors_supported().await { + return; + } + let name = table_name("pos_replace"); + create_vector_table(&name, 2, "COSINE", false).await; + + put_vector(&name, "a", None, &[-1.0, 0.0]).await; + put_vector(&name, "a", None, &[1.0, 0.0]).await; + + let response = search(&name, &[1.0, 0.0], 10, None).await; + let pks = hit_pks(&response); + assert_eq!(pks, vec!["a"], "one row per base item: {response}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Deleting an item removes it from the index. +#[tokio::test] +async fn deleting_an_item_removes_it_from_the_index() { + if !vectors_supported().await { + return; + } + let name = table_name("pos_remove"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "gone", None, &[1.0, 0.0]).await; + put_vector(&name, "stays", None, &[0.0, 1.0]).await; + + let body = format!(r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "gone"}}}}}}"#); + let (status, text) = call("DeleteItem", &body).await; + assert_eq!(status, 200, "DeleteItem failed: {text}"); + + let response = search(&name, &[1.0, 0.0], 10, None).await; + assert_eq!(hit_pks(&response), vec!["stays"], "response: {response}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// An item with no vector attribute is simply not indexed, exactly as a GSI omits +/// an item missing its index key. It must not be an error, and must not appear. +#[tokio::test] +async fn an_item_without_a_vector_is_not_indexed() { + if !vectors_supported().await { + return; + } + let name = table_name("pos_novec"); + create_vector_table(&name, 2, "COSINE", false).await; + + let body = format!(r#"{{"TableName": "{name}", "Item": {{"pk": {{"S": "novec"}}}}}}"#); + let (status, text) = call("PutItem", &body).await; + assert_eq!(status, 200, "a vectorless item must still be writable: {text}"); + put_vector(&name, "hasvec", None, &[1.0, 0.0]).await; + + let response = search(&name, &[1.0, 0.0], 10, None).await; + assert_eq!(hit_pks(&response), vec!["hasvec"], "response: {response}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// A scoped index searches one partition. This is the isolation property the whole +/// partition column exists for, so it is asserted from both sides: each tenant sees +/// its own item and not the other's. +#[tokio::test] +async fn a_scoped_search_sees_only_its_own_partition() { + if !vectors_supported().await { + return; + } + let name = table_name("pos_scope"); + create_vector_table(&name, 2, "COSINE", true).await; + + put_vector(&name, "a_item", Some("tenant_a"), &[1.0, 0.0]).await; + put_vector(&name, "b_item", Some("tenant_b"), &[1.0, 0.0]).await; + + let a = search(&name, &[1.0, 0.0], 10, Some("tenant_a")).await; + assert_eq!(hit_pks(&a), vec!["a_item"], "tenant_a response: {a}"); + + let b = search(&name, &[1.0, 0.0], 10, Some("tenant_b")).await; + assert_eq!(hit_pks(&b), vec!["b_item"], "tenant_b response: {b}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Moving an item between partitions must move its row, not duplicate it. A row +/// keyed by base item makes this work; keying by partition would leave the old row +/// and the item would be findable under both tenants. +#[tokio::test] +async fn changing_the_partition_attribute_moves_the_row() { + if !vectors_supported().await { + return; + } + let name = table_name("pos_move"); + create_vector_table(&name, 2, "COSINE", true).await; + + put_vector(&name, "mover", Some("tenant_a"), &[1.0, 0.0]).await; + put_vector(&name, "mover", Some("tenant_b"), &[1.0, 0.0]).await; + + let a = search(&name, &[1.0, 0.0], 10, Some("tenant_a")).await; + assert!( + hit_pks(&a).is_empty(), + "the old partition must no longer hold the row: {a}" + ); + let b = search(&name, &[1.0, 0.0], 10, Some("tenant_b")).await; + assert_eq!(hit_pks(&b), vec!["mover"], "tenant_b response: {b}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Dot product ranks the other way round from cosine and euclidean. A single +/// ordering would silently return the worst matches here, so the direction is +/// asserted over the wire rather than only in a unit test. +#[tokio::test] +async fn dot_product_ranks_larger_scores_first() { + if !vectors_supported().await { + return; + } + let name = table_name("pos_dot"); + create_vector_table(&name, 2, "DOT_PRODUCT", false).await; + + put_vector(&name, "small", None, &[0.5, 0.0]).await; + put_vector(&name, "large", None, &[4.0, 0.0]).await; + put_vector(&name, "negative", None, &[-3.0, 0.0]).await; + + let response = search(&name, &[1.0, 0.0], 3, None).await; + assert_eq!( + hit_pks(&response), + vec!["large", "small", "negative"], + "response: {response}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// DescribeTable reports the index, and reports it ACTIVE with no `Backfilling` +/// member, which is what the service does for an index created by CreateTable. +#[tokio::test] +async fn describe_table_reports_the_vector_index() { + if !vectors_supported().await { + return; + } + let name = table_name("pos_describe"); + create_vector_table(&name, 4, "COSINE", false).await; + + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + assert_eq!(status, 200, "DescribeTable failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let vidx = json + .pointer("/Table/VectorIndexes/0") + .unwrap_or_else(|| panic!("no vector index in description: {text}")); + + assert_eq!(vidx.pointer("/IndexName").and_then(|v| v.as_str()), Some("vidx")); + assert_eq!(vidx.pointer("/Dimensions").and_then(|v| v.as_u64()), Some(4)); + assert_eq!( + vidx.pointer("/DistanceFunction").and_then(|v| v.as_str()), + Some("COSINE") + ); + assert_eq!( + vidx.pointer("/IndexStatus").and_then(|v| v.as_str()), + Some("ACTIVE") + ); + assert!( + vidx.get("Backfilling").is_none(), + "an ACTIVE index must not carry Backfilling at all: {vidx}" + ); + assert!( + vidx.get("VectorAttribute").is_some(), + "VectorAttribute must be reported: {vidx}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} diff --git a/tests/rust/src/vector_index_unsupported.rs b/tests/rust/src/vector_index_unsupported.rs index 703ef3bb..26ccbf26 100644 --- a/tests/rust/src/vector_index_unsupported.rs +++ b/tests/rust/src/vector_index_unsupported.rs @@ -52,7 +52,7 @@ fn http_client() -> reqwest::Client { /// dispatch, so an unsigned request never reaches the capability gate and the /// test would pass for the wrong reason, asserting an auth failure while /// believing it asserted a vector refusal. -async fn call(target: &str, body: &str) -> (u16, String) { +pub(crate) async fn call(target: &str, body: &str) -> (u16, String) { let access_key = std::env::var("AWS_ACCESS_KEY_ID").expect("AWS_ACCESS_KEY_ID must be set"); let secret_key = std::env::var("AWS_SECRET_ACCESS_KEY").expect("AWS_SECRET_ACCESS_KEY must be set"); @@ -142,7 +142,7 @@ fn assert_validation_exception(status: u16, body: &str, expected_message: &str) assert_eq!(message, expected_message); } -fn table_name(suffix: &str) -> String { +pub(crate) fn table_name(suffix: &str) -> String { format!("vec_wire_{}_{}", suffix, uuid::Uuid::new_v4().simple()) } @@ -211,6 +211,13 @@ pub(crate) fn expect_vectors() -> Option { } } +/// Whether the running backend implements vector indexes, for the suite that +/// asserts the participating behaviour. Named positively so the caller reads as +/// an opt-in rather than as a double negative. +pub(crate) async fn vectors_supported() -> bool { + !is_real_dynamodb() && backend_supports_vectors().await +} + /// Skip guard for every refusal test in this file. /// /// If this run expects vector support, the refusal tests are correctly From 8640355d25344692b9e903dba0142cd2d8e72dae Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Thu, 6 Aug 2026 21:18:06 +0000 Subject: [PATCH 03/25] ci: run the Rust integration suite against SQLite, and prove the vector suites ran Without this, the vector implementation is untested in CI while reporting green. The existing Rust integration job runs against Postgres, which does not implement vector search, so `vector_index_search` self-skips there and every assertion in it is silently never executed. This job is the one that actually exercises it, and it re-runs the whole suite on the second backend, which has caught backend-specific drift before. The job also guards against the failure mode that self-skipping suites invite. Both vector suites skip when the backend is the wrong kind, so a green run proves nothing on its own: a broken probe, or a backend that stopped declaring vector support, would skip everything and still pass. The final step re-runs the positive suite and fails unless it reports a non-zero pass count above a floor. A skipped suite reports zero passes, which the guard catches. One difference from the Postgres job, found by running it: SQLite's `init` does not print an Account ID, so the account is read back from the catalog with `list-accounts | jq` rather than scraped from the init output. Both the derivation and the pass-count extraction were verified against a live deployment rather than assumed: the account resolved and the guard read 9 passes against its floor of 5. Throttling enforcement is enabled as the Postgres job does, because the suite's capacity_throttling tests fail without it; I reproduced that locally before wiring it in. The aggregator now gates on this job too, so it cannot go red unnoticed. --- .github/workflows/integration.yml | 105 +++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index aec1d8cc..730c27e0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -271,6 +271,107 @@ jobs: # pool is bound to the first test's runtime, so run serially. cd tests/rust && cargo test -- --test-threads=1 + # The Rust integration suite against the SQLite backend. + # + # The Postgres job above cannot cover vector search: Postgres does not implement + # it, so `vector_index_search` self-skips there and the whole implementation + # would go untested in CI while still reporting green. This job is what actually + # exercises it. It also re-runs the whole suite on the second backend, which has + # caught backend-specific drift before. + run-rust-integration-sqlite: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Build release (SQLite backend) + run: cargo build --release -p extenddb --no-default-features --features sqlite + + - name: Initialize ExtendDB + id: init + run: | + output=$(./target/release/extenddb init --backend sqlite --config extenddb.toml 2>&1) + echo "$output" + echo "admin_password=$(echo "$output" | grep -oP 'Password: \K\S+')" >> "$GITHUB_OUTPUT" + + - name: Enable provisioned-capacity throttling enforcement + run: ./target/release/extenddb settings set throttling_enabled true + + - name: Start ExtendDB + run: | + ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file & + for i in $(seq 1 30); do + if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then + echo "Server ready" + exit 0 + fi + sleep 1 + done + echo "Server failed to start" + exit 1 + + - name: Provision IAM test user and access key + id: creds + env: + EXTENDDB_PASSWORD: ${{ steps.init.outputs.admin_password }} + run: | + # SQLite's init does not print an Account ID, unlike the Postgres path, + # so it is read back from the catalog rather than scraped from output. + acc=$(./target/release/extenddb manage --user admin list-accounts \ + | jq -r '.[0].account_id') + echo "Account: $acc" + ./target/release/extenddb manage --user admin create-user \ + --account-id "$acc" --user-name tester + ./target/release/extenddb manage --user admin put-user-policy \ + --account-id "$acc" --user-name tester --policy-name ddbfull \ + --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"dynamodb:*","Resource":"*"}]}' + ./target/release/extenddb manage --user admin create-access-key \ + --account-id "$acc" --user-name tester > /tmp/key.json + echo "akid=$(jq -r .access_key_id /tmp/key.json)" >> "$GITHUB_OUTPUT" + secret=$(jq -r .secret_access_key /tmp/key.json) + echo "::add-mask::$secret" + echo "secret=$secret" >> "$GITHUB_OUTPUT" + + - name: Run Rust integration tests + env: + EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 + AWS_DEFAULT_REGION: us-east-1 + AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.akid }} + AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.secret }} + run: | + export EXTENDDB_CA_CERT="$(grep -oP 'cert_path\s*=\s*"\K[^"]+' extenddb.toml)" + cd tests/rust && cargo test -- --test-threads=1 + + - name: Assert the vector suites actually ran + env: + EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 + AWS_DEFAULT_REGION: us-east-1 + AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.akid }} + AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.secret }} + run: | + # Both vector suites self-skip when the backend is the wrong kind, so a + # green run proves nothing on its own: a broken probe, or a backend that + # stopped declaring vector support, would skip everything and still pass. + # This asserts the positive suite really executed its assertions by + # re-running it and requiring a non-zero pass count. + export EXTENDDB_CA_CERT="$(grep -oP 'cert_path\s*=\s*"\K[^"]+' extenddb.toml)" + cd tests/rust + out=$(cargo test vector_index_search -- --test-threads=1 2>&1) + echo "$out" + passed=$(echo "$out" | grep -oP 'test result: ok\. \K[0-9]+' | head -1) + if [ -z "$passed" ] || [ "$passed" -lt 5 ]; then + echo "Vector search suite did not run its assertions (passed=$passed)." + echo "The self-skip probe or the backend capability is broken." + exit 1 + fi + echo "Vector search suite executed $passed assertions." + integration: runs-on: ubuntu-latest needs: @@ -279,6 +380,7 @@ jobs: run-integration-sqlite, run-integration-dev-mode, run-rust-integration, + run-rust-integration-sqlite, ] if: always() steps: @@ -286,6 +388,7 @@ jobs: if [ "${{ needs.run-integration.result }}" != "success" ] || \ [ "${{ needs.run-integration-sqlite.result }}" != "success" ] || \ [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \ - [ "${{ needs.run-rust-integration.result }}" != "success" ]; then + [ "${{ needs.run-rust-integration.result }}" != "success" ] || \ + [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then exit 1 fi From a270e3d8d22e70588254734c02f435105aa4d772 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Thu, 6 Aug 2026 21:20:46 +0000 Subject: [PATCH 04/25] docs(adr): record the vector search decision with measured evidence ADR-0004. Records why vector search is an exact scan over one row per vector, so a future contributor does not have to reverse-engineer it from the code or re-run the elimination. Nine options considered and named individually, each eliminated by a hard constraint rather than by preference: static-musl FROM scratch cannot dlopen a loadable extension, the project is Apache-2.0 so a non-OSI source-available licence cannot be carried, and the index must live in the database file because the backup and restore path would not capture a sidecar. That eliminates every SQLite vector extension, including the one with an in-file index (libSQL's LM-DiskANN, which would also require replacing sqlx and cannot pre-filter a partition). The ADR is explicit that the only viable extension would not have bought an index anyway: sqlite-vec is brute force in every stable release, so adopting it meant a C toolchain, an unverified static-musl build and a pre-1.0 dependency in exchange for a constant factor on the same asymptotic scan, while still lacking dot product. Two measured findings are recorded because they are the sort of thing that otherwise gets repeated: * An earlier modelled estimate of ~36,000 vectors at 1024 dimensions inside 10 ms was wrong by 10 to 30x. Measured effective throughput is 0.2 to 1.2 GB/s, not the ~15 GB/s the model assumed, so the scan is not memory-bandwidth bound and any figure derived from that model is void. The ADR states the void explicitly rather than quietly replacing the number. * The cost is the SQLite read path, not the arithmetic. A zero-copy &[f32] view of the blob, which should have let the dot product vectorise, measured no faster. Optimising the distance loop would be wasted effort until the read path changes. The layout decision is recorded as turning on write amplification rather than read speed: a packed blob per partition reads 2 to 4x faster at 256 dimensions but rewrites the whole blob to insert one vector, 390 MB for a 100k-vector partition at 1024 dimensions, on every write touching an indexed attribute. The trigger for revisiting is stated as a measurement rather than a judgement: an index declared with no HASH element searches the whole table, and past the recorded figures that leaves an interactive budget. Named candidates for that case, gated on a measured need. Status is Proposed; the repo's process marks an ADR Accepted on merge. Indexed in docs/adr/README.md as the process requires. --- docs/adr/0004-vector-search-exact-scan.md | 137 ++++++++++++++++++++++ docs/adr/README.md | 1 + 2 files changed, 138 insertions(+) create mode 100644 docs/adr/0004-vector-search-exact-scan.md diff --git a/docs/adr/0004-vector-search-exact-scan.md b/docs/adr/0004-vector-search-exact-scan.md new file mode 100644 index 00000000..63e33e77 --- /dev/null +++ b/docs/adr/0004-vector-search-exact-scan.md @@ -0,0 +1,137 @@ +# ADR-0004: Vector search is an exact scan over one row per vector + +- Status: Proposed +- Date: 2026-08-06 +- Deciders: @LeeroyHannigan + +## Context + +DynamoDB vector indexes let a table be searched by vector similarity. Supporting +them means two separate decisions: how a backend finds nearest neighbours, and how +the vectors are stored. + +Constraints that narrow the field, all of them properties of this project rather +than preferences: + +- **The SQLite deployment target is a statically linked musl binary in a + `FROM scratch` container** (`docs/design/01-requirements.md`, REQ-DEPLOY-006). A + static binary cannot `dlopen`, so a runtime-loadable SQLite extension is + unusable there. +- **The project is Apache-2.0.** A dependency under a non-OSI source-available + licence cannot be carried. +- **The index must live in the database file.** The SQLite backend has single-file + persistent and `:memory:` ephemeral modes and an existing backup and restore + path; an index in a sidecar file would not be captured by any of them. +- **Vector search is partition-scoped.** A search supplies an equality on the + index's `HASH` element, so the candidate set is normally one partition rather + than the table. The `HASH` element is optional, though, so an unscoped index + spans everything. +- **Three distance functions are required**: `COSINE`, `EUCLIDEAN` and + `DOT_PRODUCT`, because the service supports all three. + +## Options Considered + +1. **`sqlite-vec`** (asg017) — dual Apache-2.0/MIT, index in in-database shadow + tables, native partition-key columns that pre-filter. The closest fit. +2. **`sqlite-vss`** (asg017) — Faiss-backed, real ANN. +3. **`sqliteai/sqlite-vector`** — SIMD, works on existing table schemas. +4. **`vectorlite`** — real HNSW via hnswlib. +5. **`usearch`'s SQLite extension** — from the usearch project. +6. **`sqlite-muninn`** — HNSW, loadable extension. +7. **libSQL / Turso native vector search** — LM-DiskANN built into a SQLite fork. +8. **A Rust index crate persisted in the database** — `hnsw_rs`, + `instant-distance`, `usearch` as a library, `arroy`, `hannoy`. +9. **An exact scan written in Rust**, with vectors stored as ordinary rows. + +## Decision + +Exact scan in Rust, with one row per vector in a per-index data table. No +third-party vector dependency. + +## Rationale + +- **Every extension is eliminated by a hard constraint, not by preference.** + `sqlite-vss` is deprecated by its own maintainer, ships only as a loadable object + with BLAS/LAPACK/OpenMP, and supports no filtering on a KNN query. + `sqliteai/sqlite-vector` is Elastic License 2.0, which is not OSI-approved and + carries a managed-service restriction, and it has no Rust binding. + `vectorlite` holds its HNSW index **in memory with a sidecar `.bin` file**, does + not support transactions, and filters by rowid only. `usearch`'s SQLite surface + is distance functions with no index at all; its HNSW exists only in the library. + `sqlite-muninn` is four stars with no tagged release. libSQL has a genuine + in-file LM-DiskANN index but would require replacing sqlx entirely, offers no + dot-product metric, cannot pre-filter a partition (its `vector_top_k` takes only + index, vector and k, so scoping is a post-filter that can under-return), is in + maintenance mode, and has an open index-corruption-on-delete bug. + +- **The one viable extension would not have bought an index.** `sqlite-vec` is + brute force in every stable release; its ANN work exists only in a `v0.1.10` + alpha line that had a DELETE data-loss bug. Adopting it meant taking on a C + toolchain, an unverified static-musl build and a pre-1.0 dependency in exchange + for a constant-factor speedup on the same asymptotic scan. It also has no + dot-product metric, and dot-product ranking cannot be recovered from an L2 + top-k, because the ordering depends on each candidate's own norm. + +- **Row per vector, because a packed blob makes writes O(partition).** Measured: + a contiguous blob per partition reads 2 to 4x faster at 256 dimensions and 1.3 + to 2.5x at 1024, and is *slower* at 4096. But inserting or deleting one vector + rewrites the whole blob, which is 390 MB for a 100k-vector partition at 1024 + dimensions, and vector indexes are maintained on every write touching an indexed + attribute. No read gain justifies that. + +- **Row per vector also inherits the existing machinery.** It streams, so a scan + allocates nothing proportional to the partition; it has no per-partition blob + ceiling; and it follows the per-index data table pattern the GSI and LSI paths + already use, so it reuses their transaction, backup and cleanup behaviour. + +## Measured evidence + +Single core, warm page cache, release build, real SQLite via sqlx with WAL. +Harness and raw output recorded with the benchmark; summary: + +| dimensions | vectors/sec | inside 10 ms | inside 100 ms | +|---|---|---|---| +| 256 | 213k to 334k | ~2,100 to 3,300 | ~21k to 33k | +| 1024 | 94k to 103k | ~940 to 1,030 | ~9.4k to 10.3k | +| 4096 | 39k to 43k | ~390 to 430 | ~3.9k to 4.3k | + +Two findings from the benchmark matter more than the headline numbers. + +**An earlier modelled estimate was wrong by 10 to 30x.** It assumed `N x D x 4` +bytes streaming at ~15 GB/s and predicted ~36,000 vectors at 1024 dimensions +inside 10 ms. Measured effective throughput is 0.2 to 1.2 GB/s, so the scan is not +memory-bandwidth bound and the model was unusable. Any figure derived from it is +void. + +**The cost is the SQLite read path, not the arithmetic.** A variant that +reinterpreted the stored blob as `&[f32]`, so the dot product could vectorise +instead of decoding each element, measured **no faster** (1024d/100k: 766 ms +packed versus 850 ms zero-copy). The time goes on overflow-page assembly and the +copy sqlx must make. Optimising the distance loop would be wasted effort until +that changes. + +## Consequences + +**Easier.** No third-party vector dependency, so no licence question, no C or C++ +toolchain, no static-musl uncertainty, and nothing pre-1.0 in the dependency tree. +This was the only option with zero unconfirmed constraints, precisely because +there is nothing external to be uncertain about. All three distance functions cost +the same to support: they are the same loop over the same bytes. + +**Harder.** Search cost is linear in partition size, and the crossover is roughly +an order of magnitude lower than the earlier estimate claimed. A partition stays +inside a 10 ms budget up to about 1,000 vectors at 1024 dimensions and 100 ms up +to about 10,000. + +**The trigger for revisiting, stated as a measurement rather than a judgement.** A +vector index declared with no `HASH` element searches the whole table, so its +corpus is not bounded by a partition. Past roughly the figures above, that leaves +an interactive budget. If that case becomes real, the answer is an index crate +persisted inside the database file, and on the research the candidates are +`usearch` (verified `save_to_buffer`, `remove` and `filtered_search`, but a C++17 +core to check against static musl) with `hnsw_rs` as the pure-Rust fallback. Not +now, and gated on a measured need rather than on anticipation. + +**Not measured.** Cold cache, concurrency beyond one core, and real embedding +distributions. Cold reads can only be worse; the other two do not change the +layout decision. diff --git a/docs/adr/README.md b/docs/adr/README.md index d63c9828..11afd8d3 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -33,3 +33,4 @@ decision, write a new ADR. | [0001](0001-documentation-format.md) | Documentation format — Markdown over LaTeX | Accepted | | [0002](0002-sql-injection-defense.md) | SQL injection defense | Accepted | | [0003](0003-catalog-migration-mechanism.md) | Adopt sqlx::migrate for PostgreSQL catalog and data schema migrations | Proposed | +| [0004](0004-vector-search-exact-scan.md) | Vector search is an exact scan over one row per vector | Proposed | From 4b07a9a4963f75378729d2598c1de9087fa37171 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Thu, 6 Aug 2026 21:34:39 +0000 Subject: [PATCH 05/25] test(vector): close the silent-green hole an independent review found, and two weak assertions An independent adversarial review of the vector test suites found one blocking hole and mutation-proved a weak assertion. This addresses both, plus one severity correction where the review overstated the risk. BLOCKING, now fixed: the two wire suites self-skip symmetrically with no anchor. `vector_index_search` skips unless the backend supports vectors; the refusal tests in `vector_index_unsupported` skip when it does. Nothing asserted WHICH backend was under test, so if the shipping backend silently lost vector support, the positive suite would skip all nine assertions (green) while the refusal tests started passing (also green) and the entire positive contract would evaporate unnoticed. `EXTENDDB_EXPECT_VECTORS` now lets a run state its expectation, deliberately with three states rather than two: `1` makes a skipped positive suite a failure, `0` makes a skipped refusal suite a failure, and unset keeps both adaptive so a plain local `cargo test` works against either backend without ceremony. My first version failed whenever the variable was absent, which broke local runs for no safety gain. All four states were verified against a live backend, including that an invalid value panics rather than being read as one of them. Both CI jobs now pin their expectation. This also answers the review's second blocking question, whether a non-vector backend exercises the refusal tests anywhere: it does, in the Postgres Rust job, and that is now enforced rather than incidental. The ad-hoc pass-count scraping step added earlier is removed, since the in-suite anchor supersedes it and fails at the assertion rather than by parsing output afterwards. SEVERITY CORRECTED. The review reported the unscoped-partition sentinel as a cross-tenant leak: `pk_to_text` stores an `S` attribute verbatim, so a caller could supply the sentinel string as a partition key. The finding is real about the test but wrong about the risk, and I checked rather than accepting it. The partition is chosen from the INDEX's schema, not from the item, and each index has its own data table, so within one table either every row is keyed by a real hash value or every row uses the sentinel. They never coexist, so there is nothing for a collision to leak into. The constant now documents that the guarantee is structural and that the leading NUL is defence in depth rather than the thing correctness rests on, because a reader who believed otherwise would feel free to weaken it. The test is still fixed, because the review was right that it asserted nothing of the sort. It compared the sentinel against one example value and passed unchanged when the sentinel was weakened to the ordinary string "unscoped" (the reviewer proved this by mutation). It now asserts the actual invariant, that the partition comes from the index schema and never from the item, across several values including the sentinel itself. Mutation-checked in both directions: weakening the sentinel now fails, restoring it passes. Also strengthened `overwriting_an_item_replaces_its_vector`, which asserted the row count only. A delete-then-insert that reinserted the OLD image would leave exactly one row and pass, while the stored vector was stale. It now asserts the score, which must be ~0 against the new vector where the old one was its exact opposite. Verified: workspace 752 passed / 0 failed / 0 filtered out, integration suite green with the expectation pinned, clippy -D warnings clean, my test files formatted (the repo has pre-existing rustfmt drift elsewhere that I deliberately did not touch). Remaining review findings, not in this commit: coverage gaps for BatchWriteItem and TransactWriteItems, UpdateItem removing a vector attribute, composite base keys, the scoped-index condition requirement, inline filters, and wrong-dimension requests over the wire. --- .github/workflows/integration.yml | 41 +++--------- crates/storage-sqlite/src/vector_search.rs | 50 ++++++++++++--- tests/rust/src/vector_index_search.rs | 73 +++++++++++++++++----- tests/rust/src/vector_index_unsupported.rs | 31 +++++++-- 4 files changed, 137 insertions(+), 58 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 730c27e0..0b9688dc 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -256,13 +256,10 @@ jobs: AWS_DEFAULT_REGION: us-east-1 AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.akid }} AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.secret }} - # No in-tree backend implements vector search, so every vector request - # must be refused and the refusal suite must actually run. Without this - # the suite self-skips the moment a backend gains vector support, and the - # contract that non-participating backends refuse would stop being - # checked anywhere while still reporting green. Pinning the expectation - # turns that skip into a failure. The first backend to implement vector - # search sets this to 1 in its own job. + # PostgreSQL does not implement vector search, so this is the job where + # the wire refusal tests must actually run. Pinning the expectation makes + # them mandatory here instead of silently skipping if Postgres ever + # gained the capability without anyone noticing. EXTENDDB_EXPECT_VECTORS: "0" run: | # Self-signed cert generated by init; trust it for the SDK client. @@ -344,34 +341,16 @@ jobs: AWS_DEFAULT_REGION: us-east-1 AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.akid }} AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.secret }} + # This is the only job that exercises vector search, and both vector + # suites self-skip when the backend is the wrong kind. Without this the + # positive suite could skip all of its assertions and still report green, + # which is exactly what would happen if the backend lost the capability. + # Pinning the expectation turns that skip into a failure. + EXTENDDB_EXPECT_VECTORS: "1" run: | export EXTENDDB_CA_CERT="$(grep -oP 'cert_path\s*=\s*"\K[^"]+' extenddb.toml)" cd tests/rust && cargo test -- --test-threads=1 - - name: Assert the vector suites actually ran - env: - EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 - AWS_DEFAULT_REGION: us-east-1 - AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.akid }} - AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.secret }} - run: | - # Both vector suites self-skip when the backend is the wrong kind, so a - # green run proves nothing on its own: a broken probe, or a backend that - # stopped declaring vector support, would skip everything and still pass. - # This asserts the positive suite really executed its assertions by - # re-running it and requiring a non-zero pass count. - export EXTENDDB_CA_CERT="$(grep -oP 'cert_path\s*=\s*"\K[^"]+' extenddb.toml)" - cd tests/rust - out=$(cargo test vector_index_search -- --test-threads=1 2>&1) - echo "$out" - passed=$(echo "$out" | grep -oP 'test result: ok\. \K[0-9]+' | head -1) - if [ -z "$passed" ] || [ "$passed" -lt 5 ]; then - echo "Vector search suite did not run its assertions (passed=$passed)." - echo "The self-skip probe or the backend capability is broken." - exit 1 - fi - echo "Vector search suite executed $passed assertions." - integration: runs-on: ubuntu-latest needs: diff --git a/crates/storage-sqlite/src/vector_search.rs b/crates/storage-sqlite/src/vector_search.rs index 25303816..8bb9d187 100644 --- a/crates/storage-sqlite/src/vector_search.rs +++ b/crates/storage-sqlite/src/vector_search.rs @@ -33,9 +33,20 @@ use crate::store::SqliteEngine; /// Partition value for an index that declares no HASH element. /// /// Such an index searches the whole table, so every row shares one partition -/// rather than the scan needing a second code path. The value is not a legal -/// `pk_to_text` output for any real attribute, so it cannot collide with a -/// scoped index's partitions. +/// rather than the scan needing a second code path. +/// +/// What makes this safe is **not** that the value is unguessable. `pk_to_text` +/// stores an `S` attribute verbatim, so a caller could supply this exact string as +/// a partition key. The guarantee is structural instead: the partition is chosen +/// from the *index's* schema, not from the item, and each index has its own data +/// table. So within one table either every row is keyed by a real hash value (the +/// index declares a HASH element) or every row uses this sentinel (it does not). +/// The two never coexist, so there is nothing for a collision to leak into. +/// +/// The leading NUL is defence in depth for the day that invariant changes, for +/// instance if several indexes ever shared one table. It is deliberately not what +/// correctness rests on, because a reader who believed it was would then feel free +/// to weaken it. pub(crate) const UNSCOPED_PARTITION: &str = "\u{0}all"; /// The partition column value for a vector row. @@ -370,10 +381,35 @@ mod tests { ); } + /// The partition is chosen from the index's schema, never from the item, which + /// is the invariant that makes the sentinel safe. Asserted as the property + /// rather than against one example: the previous version compared the sentinel + /// to `pk_to_text(S("all"))` only, and passed unchanged when the sentinel was + /// weakened to the ordinary string `"unscoped"`. + #[test] + fn the_partition_comes_from_the_index_schema_not_the_item() { + // No HASH element declared: the sentinel, whatever the item holds. + assert_eq!(partition_value(None).unwrap(), UNSCOPED_PARTITION); + + // A HASH element declared: the item's value, verbatim for S. + for value in ["all", "unscoped", UNSCOPED_PARTITION, ""] { + let scoped = + partition_value(Some(("pk", &AttributeValue::S(value.to_owned())))).unwrap(); + assert_eq!( + scoped, value, + "a scoped partition must be the attribute value itself" + ); + } + } + + /// The sentinel keeps a leading NUL as defence in depth. Not what correctness + /// rests on (see the constant's documentation), but weakening it to an ordinary + /// string should break a test rather than pass silently. #[test] - fn an_unscoped_index_uses_a_partition_no_real_key_can_produce() { - let unscoped = partition_value(None).unwrap(); - let scoped = partition_value(Some(("pk", &AttributeValue::S("all".to_owned())))).unwrap(); - assert_ne!(unscoped, scoped); + fn the_unscoped_sentinel_keeps_its_unusual_prefix() { + assert!( + UNSCOPED_PARTITION.starts_with('\0'), + "sentinel must keep its NUL prefix: {UNSCOPED_PARTITION:?}" + ); } } diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 1d212838..965560fb 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -11,7 +11,26 @@ //! Hand-built JSON and SigV4 signing for the same reason as the other file: no //! published `aws-sdk-dynamodb` models vector indexes. -use crate::vector_index_unsupported::{call, table_name, vectors_supported}; +use crate::vector_index_unsupported::{call, expect_vectors, table_name, vectors_supported}; + +/// Skip guard for this suite, with the anchor that closes the silent-green hole. +/// +/// Both vector suites adapt to whatever the backend reports, so on their own they +/// can never assert *which* backend is under test. If the shipping backend silently +/// lost vector support, this suite would skip all of its assertions and the refusal +/// suite would start passing, and the whole positive contract would evaporate with +/// a green run. `EXTENDDB_EXPECT_VECTORS=1` turns that skip into a failure, so a CI +/// job can state the expectation it is there to check. +async fn skip_unless_supported() -> bool { + let supported = vectors_supported().await; + assert!( + !(!supported && expect_vectors() == Some(true)), + "EXTENDDB_EXPECT_VECTORS=1 but the backend does not support vector \ + indexes, so every assertion in this suite would be skipped: the \ + capability was lost, or the probe is broken" + ); + !supported +} /// Create a table with one vector index and wait for it to be usable. async fn create_vector_table(name: &str, dims: usize, distance: &str, scoped: bool) { @@ -63,7 +82,10 @@ async fn wait_for_active(name: &str) { } fn vector_json(values: &[f32]) -> String { - let parts: Vec = values.iter().map(|v| format!(r#"{{"N": "{v}"}}"#)).collect(); + let parts: Vec = values + .iter() + .map(|v| format!(r#"{{"N": "{v}"}}"#)) + .collect(); format!("[{}]", parts.join(", ")) } @@ -130,7 +152,7 @@ fn hit_pks(response: &serde_json::Value) -> Vec { /// rather than by insertion order. #[tokio::test] async fn search_returns_nearest_first() { - if !vectors_supported().await { + if skip_unless_supported().await { return; } let name = table_name("pos_order"); @@ -154,7 +176,7 @@ async fn search_returns_nearest_first() { /// TopK bounds the result set. #[tokio::test] async fn top_k_limits_the_results() { - if !vectors_supported().await { + if skip_unless_supported().await { return; } let name = table_name("pos_topk"); @@ -173,7 +195,7 @@ async fn top_k_limits_the_results() { /// row keyed by partition instead of by base item would get wrong. #[tokio::test] async fn overwriting_an_item_replaces_its_vector() { - if !vectors_supported().await { + if skip_unless_supported().await { return; } let name = table_name("pos_replace"); @@ -186,13 +208,27 @@ async fn overwriting_an_item_replaces_its_vector() { let pks = hit_pks(&response); assert_eq!(pks, vec!["a"], "one row per base item: {response}"); + // Row count alone is not enough. A delete-then-insert that reinserted the OLD + // image would also leave exactly one row for "a", and the assertion above would + // pass while the stored vector was stale. So check the vector itself, via the + // score: the query is the new vector, so cosine distance must be ~0, whereas + // the old vector was its exact opposite and would score ~2. + let score = response + .pointer("/SearchResults/0/Score") + .and_then(serde_json::Value::as_f64) + .unwrap_or_else(|| panic!("no score in: {response}")); + assert!( + score.abs() < 1e-5, + "the stored vector must be the NEW one (cosine ~0), got score {score}: {response}" + ); + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } /// Deleting an item removes it from the index. #[tokio::test] async fn deleting_an_item_removes_it_from_the_index() { - if !vectors_supported().await { + if skip_unless_supported().await { return; } let name = table_name("pos_remove"); @@ -214,7 +250,7 @@ async fn deleting_an_item_removes_it_from_the_index() { /// an item missing its index key. It must not be an error, and must not appear. #[tokio::test] async fn an_item_without_a_vector_is_not_indexed() { - if !vectors_supported().await { + if skip_unless_supported().await { return; } let name = table_name("pos_novec"); @@ -222,7 +258,10 @@ async fn an_item_without_a_vector_is_not_indexed() { let body = format!(r#"{{"TableName": "{name}", "Item": {{"pk": {{"S": "novec"}}}}}}"#); let (status, text) = call("PutItem", &body).await; - assert_eq!(status, 200, "a vectorless item must still be writable: {text}"); + assert_eq!( + status, 200, + "a vectorless item must still be writable: {text}" + ); put_vector(&name, "hasvec", None, &[1.0, 0.0]).await; let response = search(&name, &[1.0, 0.0], 10, None).await; @@ -236,7 +275,7 @@ async fn an_item_without_a_vector_is_not_indexed() { /// its own item and not the other's. #[tokio::test] async fn a_scoped_search_sees_only_its_own_partition() { - if !vectors_supported().await { + if skip_unless_supported().await { return; } let name = table_name("pos_scope"); @@ -259,7 +298,7 @@ async fn a_scoped_search_sees_only_its_own_partition() { /// and the item would be findable under both tenants. #[tokio::test] async fn changing_the_partition_attribute_moves_the_row() { - if !vectors_supported().await { + if skip_unless_supported().await { return; } let name = table_name("pos_move"); @@ -284,7 +323,7 @@ async fn changing_the_partition_attribute_moves_the_row() { /// asserted over the wire rather than only in a unit test. #[tokio::test] async fn dot_product_ranks_larger_scores_first() { - if !vectors_supported().await { + if skip_unless_supported().await { return; } let name = table_name("pos_dot"); @@ -308,7 +347,7 @@ async fn dot_product_ranks_larger_scores_first() { /// member, which is what the service does for an index created by CreateTable. #[tokio::test] async fn describe_table_reports_the_vector_index() { - if !vectors_supported().await { + if skip_unless_supported().await { return; } let name = table_name("pos_describe"); @@ -321,8 +360,14 @@ async fn describe_table_reports_the_vector_index() { .pointer("/Table/VectorIndexes/0") .unwrap_or_else(|| panic!("no vector index in description: {text}")); - assert_eq!(vidx.pointer("/IndexName").and_then(|v| v.as_str()), Some("vidx")); - assert_eq!(vidx.pointer("/Dimensions").and_then(|v| v.as_u64()), Some(4)); + assert_eq!( + vidx.pointer("/IndexName").and_then(|v| v.as_str()), + Some("vidx") + ); + assert_eq!( + vidx.pointer("/Dimensions").and_then(|v| v.as_u64()), + Some(4) + ); assert_eq!( vidx.pointer("/DistanceFunction").and_then(|v| v.as_str()), Some("COSINE") diff --git a/tests/rust/src/vector_index_unsupported.rs b/tests/rust/src/vector_index_unsupported.rs index 26ccbf26..45a6e875 100644 --- a/tests/rust/src/vector_index_unsupported.rs +++ b/tests/rust/src/vector_index_unsupported.rs @@ -218,6 +218,26 @@ pub(crate) async fn vectors_supported() -> bool { !is_real_dynamodb() && backend_supports_vectors().await } +/// What this run asserts about the backend's vector capability. +/// +/// Three states on purpose. Both vector suites otherwise adapt to whatever the +/// backend reports, so no run asserts *which* backend is under test, and a backend +/// that silently lost vector support would skip the entire positive suite and still +/// report green. `EXTENDDB_EXPECT_VECTORS` lets a CI job state its expectation: +/// +/// - `1`: the backend must support vectors. The positive suite failing to run is an +/// error rather than a skip. +/// - `0`: the backend must not. The refusal suite failing to run is an error. +/// - unset: adapt quietly, so a plain local `cargo test` works against either +/// backend without ceremony. +pub(crate) fn expect_vectors() -> Option { + match std::env::var("EXTENDDB_EXPECT_VECTORS").ok()?.as_str() { + "1" => Some(true), + "0" => Some(false), + other => panic!("EXTENDDB_EXPECT_VECTORS must be 0 or 1, got {other:?}"), + } +} + /// Skip guard for every refusal test in this file. /// /// If this run expects vector support, the refusal tests are correctly @@ -226,8 +246,8 @@ pub(crate) async fn vectors_supported() -> bool { /// /// The expectation is read *before* probing, not inside the assertion. Reading it /// inside a short-circuiting `&&` meant an invalid value was never validated on a -/// backend without vector support, which is every backend today: the typo guard -/// was dead exactly where it was needed. +/// backend without vector support, so the typo guard was dead in the Postgres job, +/// which is the one job these refusal tests exist for. async fn skip_if_supported() -> bool { if is_real_dynamodb() { return true; @@ -240,10 +260,9 @@ async fn skip_if_supported() -> bool { these refusal tests would skip: either a backend gained vector support \ unnoticed, or this job sets the wrong expectation" ); - // The converse matters here too, and not only in a positive suite. Until a - // backend implements vector search there is no positive suite to notice a run - // that claims support the backend does not have, so the claim would pass - // unchecked. + // The converse is asserted here too rather than left to the positive suite + // alone: a run claiming support against a backend that refuses is wrong + // whichever suite happens to notice first. assert!( !(!supported && expected == Some(true)), "EXTENDDB_EXPECT_VECTORS=1 but the backend refuses vector indexes: either \ From 46e407f84d25131d00fcae040ddb0f80d10d18bd Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Thu, 6 Aug 2026 21:54:45 +0000 Subject: [PATCH 06/25] fix(vector): close the contract gaps an independent review found, three of them measured An independent adversarial review of the contract layer found nothing blocking but four should-fix items, three of which needed the live service to settle. I probed all three rather than accept or dismiss them, and in every case the measured answer differed from what either of us had inferred. SEARCH SCHEMA WAS NOT STRUCTURALLY VALIDATED, and the gap was a real contradiction rather than a missing nicety. A create or update could declare more than one HASH element; the query side then requires a condition for EVERY declared HASH while a backend resolving the scope takes the first and demotes the rest to filters. So the contract accepted a schema it could not honour. Measured caps and messages: HASH count <= 1 INLINE_FILTER count <= 18 The filter cap is the interesting one. The obvious inference from the query-side limit (MAX_SEARCH_CONDITIONS, one HASH plus twenty filters) gives twenty, which is what the review suggested hardcoding, and it is wrong. A test now pins 18 explicitly so a later edit "tidying" it to match the query cap breaks rather than silently diverging. Boundary tested at the cap and one over. DISTANCE FUNCTION ENUM ORDER was a guess and was wrong. The failure message listed [COSINE, DOT_PRODUCT, EUCLIDEAN] (alphabetical); the review guessed the enum's declaration order; the service says [DOT_PRODUCT, COSINE, EUCLIDEAN], which is neither. Now corrected and asserted over the wire. The same probe exposed a second divergence in that message which I have NOT fixed and have documented instead: the service reports the positional path 'vectorIndexes.1.member.distanceFunction' where this reports the bare 'distanceFunction'. A serde deserializer for the enum cannot know its index within the request, so closing it means deserialising the field permissively and validating positionally, exactly as the required-Projection check already does. That is a separate change with its own risk, not something to slip in here. TableThroughputMode ALIAS REMOVED. It arrived with my port of the internal branch carrying an unverified comment claiming "some clients send the billing mode under that name". aws-sdk-dynamodb 1.119.0 has no such member anywhere: CreateTable's request shape is BillingMode only. Accepting the alias meant a request that produced a PAY_PER_REQUEST table here would be ignored by AWS and produce a PROVISIONED table there, so code written against ExtendDB would break against the real service. The two tests that asserted the alias now assert the opposite, that an unknown member is ignored, which is the AWS JSON 1.0 behaviour. RestoreTableFromBackup now applies validate_vector_index_readiness, matching create and describe. Currently harmless, since a restored index is CREATING and a non-vector backend could never hold a vector-index backup, but it was the one description-returning path that omitted an invariant whose whole stated purpose is to guard those paths. Two stale doc comments fixed: one still named an error from the deployment flag removed in 79897d4, and one described the same function twice. The surviving version also states plainly that multi-fault parity is not attempted, since the service aggregates faults with its own ordering while this returns the first and hardcodes a count of one. Verified: workspace 758 passed / 0 failed / 0 filtered out, integration 438 passed / 0 failed / 0 filtered out with the capability expectation pinned, fmt clean, clippy -D warnings clean on both feature sets. Both new validations asserted over the wire, not only in unit tests. --- crates/core/src/types/table.rs | 47 +++++--- crates/core/src/validation/mod.rs | 164 +++++++++++++++++++++++++- crates/engine/src/backup.rs | 7 ++ tests/rust/src/vector_index_search.rs | 80 +++++++++++++ 4 files changed, 278 insertions(+), 20 deletions(-) diff --git a/crates/core/src/types/table.rs b/crates/core/src/types/table.rs index 806f13eb..3f7d65bd 100755 --- a/crates/core/src/types/table.rs +++ b/crates/core/src/types/table.rs @@ -89,9 +89,21 @@ impl<'de> serde::Deserialize<'de> for DistanceFunction { "COSINE" => Ok(Self::Cosine), "EUCLIDEAN" => Ok(Self::Euclidean), "DOT_PRODUCT" => Ok(Self::DotProduct), + // Enum order measured against the live service 2026-08-06, and it is + // neither alphabetical nor this enum's declaration order: + // [DOT_PRODUCT, COSINE, EUCLIDEAN] + // An earlier version guessed alphabetical and was wrong. + // + // KNOWN DIVERGENCE: the service reports the positional path + // 'vectorIndexes.1.member.distanceFunction'; this says + // 'distanceFunction'. A serde deserializer for the enum cannot know its + // index within the request, so closing this means deserialising the + // field permissively and validating it in `validate_one_vector_index`, + // which already does exactly that for the required `Projection` and + // knows the 1-based position. Deliberately left as a separate change. other => Err(serde::de::Error::custom(format!( "1 validation error detected: Value '{other}' at 'distanceFunction' \ - failed to satisfy constraint: Member must satisfy enum value set: [COSINE, DOT_PRODUCT, EUCLIDEAN]" + failed to satisfy constraint: Member must satisfy enum value set: [DOT_PRODUCT, COSINE, EUCLIDEAN]" ))), } } @@ -622,10 +634,7 @@ pub struct CreateTableInput { pub key_schema: Vec, #[serde(rename = "AttributeDefinitions")] pub attribute_definitions: Vec, - // Some clients send the billing/throughput mode under the field name - // TableThroughputMode instead of BillingMode; accept it as an alias so such - // requests are not rejected for a missing billing mode. - #[serde(rename = "BillingMode", alias = "TableThroughputMode")] + #[serde(rename = "BillingMode")] pub billing_mode: Option, #[serde(rename = "ProvisionedThroughput")] pub provisioned_throughput: Option, @@ -756,9 +765,7 @@ pub struct UpdateGsiAction { pub struct UpdateTableInput { #[serde(rename = "TableName")] pub table_name: String, - // Accept TableThroughputMode as an alias for BillingMode (see - // CreateTableInput): some clients send the billing mode under that name. - #[serde(rename = "BillingMode", alias = "TableThroughputMode")] + #[serde(rename = "BillingMode")] pub billing_mode: Option, #[serde(rename = "ProvisionedThroughput")] pub provisioned_throughput: Option, @@ -914,10 +921,16 @@ pub struct DescribeLimitsOutput { mod tests { use super::*; + /// `TableThroughputMode` is not a member of DynamoDB's CreateTable request: + /// the model has `BillingMode` only (verified against aws-sdk-dynamodb 1.119.0, + /// where the field does not appear at all). An earlier version of this type + /// accepted it as an alias, which meant a request that produced a + /// PAY_PER_REQUEST table here would be ignored by AWS and produce a + /// PROVISIONED table there: an accept-direction divergence, where code written + /// against ExtendDB breaks against the real service. Under AWS JSON 1.0 an + /// unknown member is ignored, which is what must happen here. #[test] - fn create_table_accepts_table_throughput_mode_alias() { - // A client that sends the billing mode under `TableThroughputMode` - // (instead of `BillingMode`) must populate billing_mode all the same. + fn create_table_ignores_the_unknown_table_throughput_mode_member() { let json = r#"{ "TableName": "t", "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], @@ -925,7 +938,10 @@ mod tests { "TableThroughputMode": "PAY_PER_REQUEST" }"#; let input: CreateTableInput = serde_json::from_str(json).unwrap(); - assert_eq!(input.billing_mode, Some(BillingMode::PayPerRequest)); + assert_eq!( + input.billing_mode, None, + "an unknown member must be ignored, not treated as BillingMode" + ); } #[test] @@ -941,10 +957,13 @@ mod tests { } #[test] - fn update_table_accepts_table_throughput_mode_alias() { + fn update_table_ignores_the_unknown_table_throughput_mode_member() { let json = r#"{"TableName": "t", "TableThroughputMode": "PROVISIONED"}"#; let input: UpdateTableInput = serde_json::from_str(json).unwrap(); - assert_eq!(input.billing_mode, Some(BillingMode::Provisioned)); + assert_eq!( + input.billing_mode, None, + "an unknown member must be ignored" + ); } #[test] diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index ee01e9f2..b387e061 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -113,13 +113,164 @@ pub fn validate_create_table( Ok(()) } -/// Validate the `VectorIndexes` on a `CreateTable` request. +#[cfg(test)] +mod search_schema_shape_tests { + use super::{MAX_SEARCH_SCHEMA_INLINE_FILTERS, validate_search_schema_shape}; + use crate::types::{SearchSchemaElement, SearchSchemaElementType}; + + fn element(name: &str, element_type: SearchSchemaElementType) -> SearchSchemaElement { + SearchSchemaElement { + attribute_name: name.to_owned(), + element_type, + } + } + + fn hash(name: &str) -> SearchSchemaElement { + element(name, SearchSchemaElementType::Hash) + } + + fn filter(name: &str) -> SearchSchemaElement { + element(name, SearchSchemaElementType::InlineFilter) + } + + #[test] + fn no_search_schema_is_allowed() { + // The HASH element is optional: an index without one searches the table. + validate_search_schema_shape(None).expect("absent schema is valid"); + } + + #[test] + fn one_hash_is_allowed() { + validate_search_schema_shape(Some(&[hash("t")])).expect("one HASH is valid"); + } + + /// The case that mattered: a two-HASH schema was accepted and then could not be + /// honoured, because the query side requires a condition for every HASH while a + /// backend resolves the scope from the first and demotes the rest. + #[test] + fn two_hash_elements_are_rejected_with_the_measured_message() { + let err = validate_search_schema_shape(Some(&[hash("a"), hash("b")])) + .expect_err("two HASH elements must be rejected"); + assert_eq!( + format!("{err}"), + "One or more parameter values were invalid: Value '2' at 'SearchSchema' \ + failed to satisfy constraint: Member must have HASH count less than or \ + equal to 1" + ); + } + + #[test] + fn the_inline_filter_cap_is_a_boundary_not_a_range() { + let at_cap: Vec<_> = (0..MAX_SEARCH_SCHEMA_INLINE_FILTERS) + .map(|i| filter(&format!("f{i}"))) + .collect(); + validate_search_schema_shape(Some(&at_cap)).expect("the cap itself is allowed"); + + let over_cap: Vec<_> = (0..=MAX_SEARCH_SCHEMA_INLINE_FILTERS) + .map(|i| filter(&format!("f{i}"))) + .collect(); + let err = validate_search_schema_shape(Some(&over_cap)) + .expect_err("one over the cap must be rejected"); + assert_eq!( + format!("{err}"), + "One or more parameter values were invalid: Value '19' at 'SearchSchema' \ + failed to satisfy constraint: Member must have INLINE_FILTER count less \ + than or equal to 18" + ); + } + + /// Pins the measured number, because the obvious inference from the query-side + /// cap gives twenty and is wrong. A future edit "tidying" this to match the + /// query cap would break parity silently. + #[test] + fn the_inline_filter_cap_is_eighteen_as_measured() { + assert_eq!(MAX_SEARCH_SCHEMA_INLINE_FILTERS, 18); + } + + #[test] + fn a_hash_plus_filters_is_allowed() { + let mut elements = vec![hash("t")]; + elements.extend((0..MAX_SEARCH_SCHEMA_INLINE_FILTERS).map(|i| filter(&format!("f{i}")))); + validate_search_schema_shape(Some(&elements)) + .expect("one HASH plus the filter cap is valid"); + } +} + +/// Maximum `HASH` elements in a vector index search schema. +/// +/// Measured against the live service 2026-08-06. Without this check the contract +/// accepted a schema it then could not honour: `validate_conditions_against_search_schema` +/// requires a condition for EVERY declared HASH, while a backend resolving the scope +/// takes the first HASH and demotes the rest to filters. So a two-HASH schema was +/// internally contradictory rather than merely unvalidated. +const MAX_SEARCH_SCHEMA_HASH: usize = 1; + +/// Maximum `INLINE_FILTER` elements in a vector index search schema. /// -/// POC scope: each index name must be well-formed, `Dimensions` must be in -/// `1..=4096`, and the vector attribute name must be non-empty. The distance -/// function is enforced by the type system (enum deserialization). +/// Measured against the live service 2026-08-06 as **18**, which is deliberately +/// recorded rather than derived: the obvious inference from the query-side cap +/// (`MAX_SEARCH_CONDITIONS`, one HASH plus twenty filters) gives twenty, and is +/// wrong. The schema cap and the per-query cap are different numbers. +const MAX_SEARCH_SCHEMA_INLINE_FILTERS: usize = 18; + +/// Validate the shape of a vector index search schema. +/// +/// Messages measured against the live service 2026-08-06 by signing raw requests, +/// since no published SDK models vector indexes: +/// +/// ```text +/// One or more parameter values were invalid: Value '2' at 'SearchSchema' failed to +/// satisfy constraint: Member must have HASH count less than or equal to 1 +/// ``` +/// +/// Note the field is `SearchSchema` in its request-shape capitalisation, not the +/// lower-camel positional path used by the projection message: the service is not +/// internally consistent here, so each message is reproduced as observed rather +/// than normalised. +fn validate_search_schema_shape( + search_schema: Option<&[crate::types::SearchSchemaElement]>, +) -> Result<(), DynamoDbError> { + let Some(elements) = search_schema else { + return Ok(()); + }; + let hash_count = elements + .iter() + .filter(|e| e.element_type == crate::types::SearchSchemaElementType::Hash) + .count(); + if hash_count > MAX_SEARCH_SCHEMA_HASH { + return Err(DynamoDbError::ValidationException(format!( + "One or more parameter values were invalid: Value '{hash_count}' at \ + 'SearchSchema' failed to satisfy constraint: Member must have HASH count \ + less than or equal to {MAX_SEARCH_SCHEMA_HASH}" + ))); + } + let filter_count = elements + .iter() + .filter(|e| e.element_type == crate::types::SearchSchemaElementType::InlineFilter) + .count(); + if filter_count > MAX_SEARCH_SCHEMA_INLINE_FILTERS { + return Err(DynamoDbError::ValidationException(format!( + "One or more parameter values were invalid: Value '{filter_count}' at \ + 'SearchSchema' failed to satisfy constraint: Member must have \ + INLINE_FILTER count less than or equal to \ + {MAX_SEARCH_SCHEMA_INLINE_FILTERS}" + ))); + } + Ok(()) +} + /// Rules that apply to one vector index specification, wherever it arrives from. /// +/// Each index needs a `Projection`, a well-formed name, `Dimensions` in `1..=4096`, +/// a non-empty vector attribute name, and a search schema within the measured HASH +/// and `INLINE_FILTER` caps. The distance function is enforced by the type system +/// through enum deserialization. +/// +/// Multi-fault parity is deliberately not attempted: the service aggregates faults +/// (`"N validation errors detected"`) with its own field ordering, whereas this +/// returns the first fault it finds and hardcodes a count of one. Single-fault +/// wording is measured and exact, which is what a client parsing one error sees. +/// /// `CreateTable` and `UpdateTable`'s create action carry the same shape, so they /// get the same rules: a malformed index is rejected identically whichever path /// it arrives by, rather than each handler enforcing its own subset. @@ -145,6 +296,7 @@ fn validate_one_vector_index( ))); } validate_index_name(&vi.index_name)?; + validate_search_schema_shape(vi.search_schema.as_deref())?; if vi.dimensions < 1 || vi.dimensions > 4096 { // Verified against the service 2026-08-05: Dimensions=4097 and 8192 // both return exactly this message. The lower bound is not observable @@ -188,8 +340,8 @@ fn validate_vector_indexes(input: &CreateTableInput) -> Result<(), DynamoDbError /// so both backends produce identical text. /// /// # Errors -/// Returns [`DynamoDbError::ValidationException`] if vector search is disabled for -/// the deployment, or if a create action carries a malformed index. +/// Returns [`DynamoDbError::ValidationException`] if a create action carries a +/// malformed index. pub fn validate_vector_index_updates( updates: Option<&Vec>, ) -> Result<(), DynamoDbError> { diff --git a/crates/engine/src/backup.rs b/crates/engine/src/backup.rs index 34f7a0e5..dc57dde6 100755 --- a/crates/engine/src/backup.rs +++ b/crates/engine/src/backup.rs @@ -161,6 +161,13 @@ pub(crate) async fn handle_restore_table_from_backup( .invalidate_table_key_info(&ctx.account_id, target_table_name) .await; + // The readiness invariant is applied on every path that hands a description to + // a client, and restore was the one that omitted it. Currently harmless (a + // restored index is CREATING, and a non-vector backend could never hold a + // vector-index backup because create is gated), but the invariant claims to + // cover exactly this class of path, so the omission was a latent inconsistency + // rather than a deliberate exception. + desc.validate_vector_index_readiness()?; serialize_output(&json!({ "TableDescription": desc })) } diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 965560fb..321a84ff 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -343,6 +343,86 @@ async fn dot_product_ranks_larger_scores_first() { let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } +/// A two-HASH search schema is rejected with the service's measured message. +/// +/// This mattered because the contract accepted it and then could not honour it: the +/// query side requires a condition for every declared HASH, while the backend +/// resolves the scope from the first and demotes the rest to filters. Measured +/// against the live service on 2026-08-06. +#[tokio::test] +async fn a_two_hash_search_schema_is_rejected() { + if skip_unless_supported().await { + return; + } + let name = table_name("twohash"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [ + {{"AttributeName": "pk", "AttributeType": "S"}}, + {{"AttributeName": "a", "AttributeType": "S"}}, + {{"AttributeName": "b", "AttributeType": "S"}} + ], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": 4, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "SearchSchema": [ + {{"AttributeName": "a", "SearchSchemaElementType": "HASH"}}, + {{"AttributeName": "b", "SearchSchemaElementType": "HASH"}} + ], + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 400, "two HASH elements must be rejected: {text}"); + assert!( + text.contains("Member must have HASH count less than or equal to 1"), + "expected the measured HASH-count message, got: {text}" + ); +} + +/// An unknown distance function is rejected, and the enum value set is listed in +/// the service's order. +/// +/// The order is measured and is neither alphabetical nor the enum's declaration +/// order, so it is asserted explicitly: an earlier version guessed alphabetical. +#[tokio::test] +async fn an_unknown_distance_function_lists_the_measured_enum_order() { + if skip_unless_supported().await { + return; + } + let name = table_name("baddf"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": 4, + "DistanceFunction": "MANHATTAN", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!( + status, 400, + "an unknown distance function must be rejected: {text}" + ); + assert!( + text.contains("[DOT_PRODUCT, COSINE, EUCLIDEAN]"), + "the enum value set must be listed in the service's measured order, got: {text}" + ); +} + /// DescribeTable reports the index, and reports it ACTIVE with no `Backfilling` /// member, which is what the service does for an index created by CreateTable. #[tokio::test] From f6bad226f965c278957946fc954878f4ed48b865 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Thu, 6 Aug 2026 22:05:35 +0000 Subject: [PATCH 07/25] fix(vector): apply the index projection and stop billing for bytes the metric claims to exclude An independent adversarial review of the SQLite implementation found nothing blocking and confirmed the property that mattered most: every item-mutating site maintains the index inside the write transaction, including all three TransactWriteItems branches, BatchWriteItem transitively through put/delete, and the TTL sweep. It also found one real divergence with a billed consequence. THE STORED ROW IGNORED THE INDEX PROJECTION. The GSI sibling stores `project_item_for_index(...)`; the vector path stored the whole item and applied no projection at write or read. That is an unexplained divergence from the sibling, and a search returned attributes the index does not declare. Now projected, mirroring the sibling. THE BILLED METRIC WAS INFLATED BY A COMMENT THAT WAS SIMPLY UNTRUE. The capacity computation in the SearchVectors handler read: // ... excluding the vector component (the stored item already omits the // vector attribute). let non_vector_bytes = hits.iter().map(|h| item_size_bytes(&h.item)).sum(); No backend omitted it, so `VectorSearchRequestBytes` was over-reported by the vector's serialized size on every hit, roughly 10 to 15 KB at 1024 dimensions. I wrote both the storage and that comment and never connected them, which is exactly what an independent read is for. The vector attribute is now subtracted explicitly, so the figure no longer depends on an assumption about how a backend stores rows. I did NOT strip the vector from the stored row, and the reason is recorded in the code rather than left implicit: the vector attribute is an attribute of the item, so a Projection of ALL includes it, and whether the service returns it in a search result is unverifiable from here. `SearchVectors` is not served by the standard DynamoDB endpoint at all ("This operation is not supported by this endpoint"), which I found by probing. So the duplication between the `vec` column and the row payload is deliberate pending an endpoint that can answer the question, not an oversight. The `LIKE` pattern for the table-cleanup sweep is now `GLOB`. The old comment justified the wrong half: it noted that a UUID contains no LIKE metacharacters, which is true and beside the point, because the pattern's own underscores are single-character wildcards. It could only ever over-match and no other table can share the UUID, so it was safe in practice but not for the stated reason, and a comment that justifies the wrong half is worse than none. GLOB treats `_` literally. Verified the sweep actually works rather than assuming: after the full suite the database holds 21 vector data tables against exactly 21 catalog rows, with zero orphans and zero missing. Dropping a table takes its indexes with it, which is why the `_vidx_part_*` index needs no separate handling. The review also flagged, out of scope, that no code drops a single vector index's data table. Correct, and it is not a leak today because the UpdateTable vector paths are not implemented in this backend yet. It is a prerequisite for that work rather than an existing defect. Verified: workspace 758 passed / 0 failed / 0 filtered out, integration 438 passed / 0 failed / 0 filtered out, fmt clean, clippy -D warnings clean on both feature sets. --- crates/engine/src/search_vectors.rs | 23 +++++++++++++++-- crates/storage-sqlite/src/data/ddl.rs | 6 ++--- crates/storage-sqlite/src/data/mod.rs | 18 +++++++++---- .../storage-sqlite/src/data/vector_index.rs | 25 ++++++++++++++++--- 4 files changed, 58 insertions(+), 14 deletions(-) diff --git a/crates/engine/src/search_vectors.rs b/crates/engine/src/search_vectors.rs index c8acb7b3..81440602 100644 --- a/crates/engine/src/search_vectors.rs +++ b/crates/engine/src/search_vectors.rs @@ -227,8 +227,27 @@ pub async fn handle_search_vectors( let hits = search_output.hits; // Bytes read from the index for the returned items, excluding the vector - // component (the stored item already omits the vector attribute). - let non_vector_bytes: usize = hits.iter().map(|h| item_size_bytes(&h.item)).sum(); + // component, which is metered separately as the query's own dimension cost. + // + // The vector attribute is subtracted explicitly rather than assumed absent. An + // earlier version of this comment claimed the stored item already omitted it, + // which was not true of any backend: the SQLite path stores the whole projected + // item, so the figure was inflated by the vector's serialized size on every hit, + // roughly 10 to 15 KB for 1024 dimensions. A billed metric must not rest on an + // assumption about how a backend chose to store its rows. + let non_vector_bytes: usize = hits + .iter() + .map(|h| { + let total = item_size_bytes(&h.item); + let vector = key_info + .vector_indexes + .iter() + .find(|vi| vi.index_name == input.index_name) + .and_then(|vi| h.item.get(&vi.vector_attribute_name)) + .map_or(0, extenddb_core::types::attribute_value_size); + total.saturating_sub(vector) + }) + .sum(); // Compile the projection once, if supplied. let compiled_projection = if let Some(ref proj_str) = input.projection_expression { diff --git a/crates/storage-sqlite/src/data/ddl.rs b/crates/storage-sqlite/src/data/ddl.rs index 9236773b..357f15cd 100644 --- a/crates/storage-sqlite/src/data/ddl.rs +++ b/crates/storage-sqlite/src/data/ddl.rs @@ -16,7 +16,7 @@ use extenddb_storage::error::StorageError; use extenddb_storage::util::sk_column_n; use super::{ - all_sort_key_info, data_table_name, index_table_name, vector_table_like_pattern, + all_sort_key_info, data_table_name, index_table_name, vector_table_glob_pattern, vector_table_name, }; use crate::store::SqliteEngine; @@ -170,9 +170,9 @@ impl SqliteEngine { table_id: &str, ) -> Result<(), StorageError> { let names: Vec = sqlx::query_scalar( - "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE ?", + "SELECT name FROM sqlite_master WHERE type = 'table' AND name GLOB ?", ) - .bind(vector_table_like_pattern(table_id)) + .bind(vector_table_glob_pattern(table_id)) .fetch_all(&mut **tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; diff --git a/crates/storage-sqlite/src/data/mod.rs b/crates/storage-sqlite/src/data/mod.rs index 372d7a0d..7918464c 100644 --- a/crates/storage-sqlite/src/data/mod.rs +++ b/crates/storage-sqlite/src/data/mod.rs @@ -73,12 +73,20 @@ pub(crate) fn vector_table_name(table_id: &str, index_id: &str) -> String { format!("\"_vidx_{table_id}_{index_id}\"") } -/// `LIKE` pattern matching every vector data table of one DynamoDB table. +/// `GLOB` pattern matching every vector data table of one DynamoDB table. /// -/// Used against `sqlite_master`. `table_id` is a server-generated UUID, so it -/// contains no `LIKE` metacharacters and needs no escaping. -pub(crate) fn vector_table_like_pattern(table_id: &str) -> String { - format!("_vidx_{table_id}_%") +/// `GLOB` rather than `LIKE` because the pattern itself is full of underscores, and +/// in `LIKE` an underscore is a single-character wildcard: `_vidx__%` would match +/// far more than it appears to. It can only ever over-match, since a wildcard also +/// matches a literal underscore, and no other table could share the UUID, so `LIKE` +/// was safe in practice. It was not safe for the stated reason, though, and a comment +/// that justifies the wrong half is worse than none. +/// +/// `GLOB` treats `_` literally and uses `*` for the wildcard, so the pattern means +/// what it reads. `table_id` is a server-generated UUID, which contains no `GLOB` +/// metacharacters either. +pub(crate) fn vector_table_glob_pattern(table_id: &str) -> String { + format!("_vidx_{table_id}_*") } /// All RANGE key attributes in key-schema order, paired with their scalar type. diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs index 6f1cb638..ff3fdcd2 100644 --- a/crates/storage-sqlite/src/data/vector_index.rs +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -27,6 +27,10 @@ pub(crate) struct VectorIndexMeta { pub index_id: String, pub dimensions: usize, pub vector_attribute_name: String, + /// The index's projection, applied to the stored row exactly as the GSI path + /// applies its own. Not applying it was an unexplained divergence from the + /// sibling, and it made a search return attributes the index does not project. + pub projection: extenddb_core::types::Projection, /// The single HASH element's attribute name, when the index declares one. /// `None` means the index is unscoped and every row shares one partition. pub hash_attribute_name: Option, @@ -41,8 +45,8 @@ pub(crate) async fn fetch_vector_indexes_for_table( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, table_id: &str, ) -> Result, StorageError> { - let rows: Vec<(String, i64, String, Option)> = sqlx::query_as( - "SELECT index_id, dimensions, vector_attribute, search_schema \ + let rows: Vec<(String, i64, String, Option, String)> = sqlx::query_as( + "SELECT index_id, dimensions, vector_attribute, search_schema, projection \ FROM vector_indexes WHERE table_id = ?", ) .bind(table_id) @@ -51,7 +55,7 @@ pub(crate) async fn fetch_vector_indexes_for_table( .map_err(|e| StorageError::Internal(e.to_string()))?; let mut out = Vec::with_capacity(rows.len()); - for (index_id, dimensions, vector_attribute, search_schema) in rows { + for (index_id, dimensions, vector_attribute, search_schema, projection) in rows { let attr: extenddb_core::types::VectorAttribute = serde_json::from_str(&vector_attribute) .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))?; @@ -67,6 +71,8 @@ pub(crate) async fn fetch_vector_indexes_for_table( } None => None, }; + let projection: extenddb_core::types::Projection = serde_json::from_str(&projection) + .map_err(|e| StorageError::Internal(format!("vector projection: {e}")))?; out.push(VectorIndexMeta { index_id, dimensions: usize::try_from(dimensions).map_err(|_| { @@ -74,6 +80,7 @@ pub(crate) async fn fetch_vector_indexes_for_table( })?, vector_attribute_name: attr.attribute_name, hash_attribute_name, + projection, }); } Ok(out) @@ -231,7 +238,17 @@ pub(crate) async fn sync_vector_indexes( } let norm = vector_norm(&components); let part = item_partition(new_item, meta)?; - let item_json = serde_json::to_string(new_item) + // Projected exactly as the GSI sibling projects, so a search returns what + // the index declares and no more. The vector attribute is kept when the + // projection includes it: it is an attribute of the item, and whether the + // service returns it could not be verified, because `SearchVectors` is not + // served by the standard DynamoDB endpoint ("This operation is not supported + // by this endpoint"). Storing it in both the `vec` column and the row payload + // is therefore deliberate rather than accidental, and is the one part of this + // that should be revisited once the behaviour can be observed. + let projected = + super::index::project_item_for_index(new_item, &[], base_key_schema, &meta.projection); + let item_json = serde_json::to_string(&projected) .map_err(|e| StorageError::Internal(format!("serialize item: {e}")))?; let cols = std::iter::once("part".to_owned()) From 6f2cad13850ce576422e987b1cde886022f9adfb Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Fri, 7 Aug 2026 10:16:28 +0000 Subject: [PATCH 08/25] fix(vector): withhold the vector attribute unless ProjectionExpression names it The vector attribute is returned by SearchVectors only when the caller explicitly asks for it in ProjectionExpression, even under a Projection of ALL. This was the one part of the previous commit I could not settle: SearchVectors is not served by the standard DynamoDB endpoint, so no probe from here could answer it, and I recorded the uncertainty rather than guessing. Lee confirmed the behaviour. The change is narrow because the existing projection path already does most of it. When a ProjectionExpression is supplied, the compiled projection restricts the item to the paths it names, so naming the vector keeps it and not naming it drops it, with no special case needed. Only the no-projection branch had to change, and it now removes the vector attribute. That also makes the earlier capacity comment right in intent, though for a different reason than it claimed. It asserted the STORED item omits the vector, which was false: the SQLite path holds the projected item verbatim, exactly as the GSI path does. What is true is that the RETURNED item omits it. The subtraction stays, because the capacity figure is computed from what the backend handed over, before the response projection runs, so it must not assume how a backend stores its rows. The comment now says that rather than the thing that was wrong. I deliberately did not strip the vector at write time, which was the other way to satisfy this. Storing it once and reconstructing it on demand would save roughly 16 KB per row at 4096 dimensions, but it would return an f32-narrowed value rather than what the client wrote, since the wire type is arbitrary-precision decimal and the index narrows to f32. Keeping the projected attributes verbatim matches the GSI model and preserves exactly what was written; the storage cost is the price of that fidelity, and it is now a considered trade rather than an accident. The test asserts both directions, because either half alone passes against a broken implementation: "absent by default" passes if the attribute is never returned at all, and "present when named" passes if it is always returned. The default case also asserts the rest of the item IS present, so it cannot pass merely because nothing came back. Verified: integration 439 passed / 0 failed / 0 filtered out, with the new test confirmed to have executed rather than skipped. Workspace 758 passed / 0 failed / 0 filtered out, fmt clean, clippy -D warnings clean on both feature sets. --- crates/engine/src/search_vectors.rs | 27 +++++++++---- tests/rust/src/vector_index_search.rs | 58 +++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/search_vectors.rs b/crates/engine/src/search_vectors.rs index 81440602..95e9dd94 100644 --- a/crates/engine/src/search_vectors.rs +++ b/crates/engine/src/search_vectors.rs @@ -229,12 +229,13 @@ pub async fn handle_search_vectors( // Bytes read from the index for the returned items, excluding the vector // component, which is metered separately as the query's own dimension cost. // - // The vector attribute is subtracted explicitly rather than assumed absent. An - // earlier version of this comment claimed the stored item already omitted it, - // which was not true of any backend: the SQLite path stores the whole projected - // item, so the figure was inflated by the vector's serialized size on every hit, - // roughly 10 to 15 KB for 1024 dimensions. A billed metric must not rest on an - // assumption about how a backend chose to store its rows. + // Subtracted explicitly rather than assumed absent, because this is computed + // from what the BACKEND returned, before the response projection runs. A + // backend is free to hand over the vector (the SQLite path does, since it holds + // the projected item verbatim as the GSI path does), so the figure must not rest + // on an assumption about how a backend stores its rows. An earlier version of + // this comment asserted the vector was already absent and was wrong, inflating + // the billed figure by roughly 10 to 15 KB per hit at 1024 dimensions. let non_vector_bytes: usize = hits .iter() .map(|h| { @@ -269,8 +270,20 @@ pub async fn handle_search_vectors( .into_iter() .map(|hit| { let item = match compiled_projection.as_ref() { + // A supplied ProjectionExpression already restricts the item to the + // paths it names, so naming the vector attribute keeps it and not + // naming it drops it. Nothing extra to do. Some(proj) => proj.apply(&hit.item), - None => hit.item, + // With no ProjectionExpression the vector attribute is withheld. + // The service returns it only when it is explicitly asked for, even + // under a Projection of ALL, so returning it by default would be + // both a parity divergence and a large one: a 1024-dimension vector + // serialises to roughly 10 to 15 KB per hit. + None => { + let mut item = hit.item; + item.remove(&vector_index.vector_attribute.attribute_name); + item + } }; SearchResult { item, diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 321a84ff..601a6303 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -343,6 +343,64 @@ async fn dot_product_ranks_larger_scores_first() { let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } +/// The vector attribute is withheld unless the caller names it. +/// +/// Confirmed as the service's behaviour: it is returned only when explicitly asked +/// for in `ProjectionExpression`, even under a Projection of `ALL`. Asserted in both +/// directions, because either half alone would pass against a broken implementation: +/// "absent by default" passes if the attribute is never returned at all, and +/// "present when named" passes if it is always returned. +#[tokio::test] +async fn the_vector_attribute_is_returned_only_when_named() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_vecproj"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "a", None, &[1.0, 0.0]).await; + + // Default: withheld, but the rest of the item is still there, so this is not + // passing merely because nothing was returned. + let response = search(&name, &[1.0, 0.0], 5, None).await; + let item = response + .pointer("/SearchResults/0/Item") + .unwrap_or_else(|| panic!("no item in: {response}")); + assert!( + item.get("emb").is_none(), + "the vector must be withheld by default: {item}" + ); + assert!( + item.get("pk").is_some(), + "the rest of the item must still be returned: {item}" + ); + + // Named explicitly: returned. + let body = format!( + r#"{{ + "TableName": "{name}", + "IndexName": "vidx", + "SearchVector": [{{"N": "1"}}, {{"N": "0"}}], + "TopK": 5, + "ProjectionExpression": "pk, emb" + }}"# + ); + let (status, text) = call("SearchVectors", &body).await; + assert_eq!( + status, 200, + "SearchVectors with a projection failed: {text}" + ); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let item = json + .pointer("/SearchResults/0/Item") + .unwrap_or_else(|| panic!("no item in: {text}")); + assert!( + item.get("emb").is_some(), + "the vector must be returned when named in ProjectionExpression: {item}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + /// A two-HASH search schema is rejected with the service's measured message. /// /// This mattered because the contract accepted it and then could not honour it: the From 2ebe67b09809fe4804983c008c8ef4175d1fdfa7 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Fri, 7 Aug 2026 10:41:39 +0000 Subject: [PATCH 09/25] feat(vector): return the stored f32 rather than a second verbatim copy A vector index holds 32-bit floats. The service's own validation is the evidence: it rejects a component outside [-3.4028235E38, 3.4028235E38], exactly f32::MAX, and names the expected type "32-bit floating point number". Reading a narrowed value back could not be measured directly, because SearchVectors is not served by the standard DynamoDB endpoint. The index row was therefore carrying the vector twice: once as f32 in the `vec` column, and once as the client's verbatim decimal text inside the projected row payload, with searches returning the latter. That was wrong in two ways. It returned more precision than the index stores, which no client could get from the service, and it duplicated the vector at roughly five times the size of the blob it duplicated. Searches now reconstruct the attribute from the stored bits via a new core `vector_attribute()`, the inverse of `vector_components()`, so the returned value is the one that was actually indexed. Measured at 1024 dimensions: the row payload falls from 19,862 bytes to 37, against a 4,096-byte blob. Three details worth recording. The reconstruction happens for the k surviving hits, not per candidate scanned: the components ride along inside TopK. Rebuilding during the scan would allocate a decimal string per component per row examined, which at 4096 dimensions over a large partition would cost more than the scan itself. A test asserts each retained hit keeps its own vector, since an off-by-one in the insert position would pair a neighbour's vector with this item's attributes: wrong data, no error. Two formatting exceptions to f32's Display, which otherwise gives the shortest round-tripping decimal. Negative zero is normalised, because N has one zero, and the sign of zero is unobservable through any of the three distance functions. And a magnitude outside [1e-6, 1e21) is written with an exponent, because Display never uses one and would expand f32::MAX to 39 digits, past the 38 that N carries. A test pins the round-trip through vector_components for f32::MAX, MIN_POSITIVE and a subnormal. The stored `vector_attribute` column holds serialized JSON, not a bare name. Reading it as a plain string produced the key {"AttributeName":"emb"}, so the vector was silently absent from every projection. The unit tests could not see this; the wire test caught it immediately, which is the second time this suite has paid for itself. Verified against a live SQLite deployment: 440 Rust integration tests (0 filtered out, up from 436 with the new case), 764 workspace tests, fmt and clippy -D warnings clean on both feature sets. The base item is asserted to keep the client's own precision alongside the narrowed index value, so the test cannot pass against an implementation that merely corrupted the item on write. --- crates/core/src/validation/vector_item.rs | 128 +++++++++++++++++- .../storage-sqlite/src/data/vector_index.rs | 17 +-- crates/storage-sqlite/src/vector_search.rs | 93 +++++++++---- tests/rust/src/vector_index_search.rs | 79 +++++++++++ 4 files changed, 283 insertions(+), 34 deletions(-) diff --git a/crates/core/src/validation/vector_item.rs b/crates/core/src/validation/vector_item.rs index d6d431a0..bf2faa7b 100644 --- a/crates/core/src/validation/vector_item.rs +++ b/crates/core/src/validation/vector_item.rs @@ -91,6 +91,52 @@ pub fn vector_components(value: &AttributeValue) -> Option> { Some(out) } +/// Rebuild a vector attribute from the `f32` components a backend stored. +/// +/// The inverse of [`vector_components`], and the only way a backend should return a +/// stored vector. An index holds 32-bit floats, which is the width the service +/// validates against: it rejects a component outside +/// `[-3.4028235E38, 3.4028235E38]`, exactly `f32::MAX`, and names the expected type +/// "32-bit floating point number". So a client that writes more precision than an +/// `f32` carries reads back the narrowed value rather than the decimal it sent, and +/// reconstructing from the stored bits is what reproduces that. Retaining the +/// client's original text would be a divergence dressed up as fidelity. +/// +/// Living here rather than in a backend keeps every backend returning identical +/// text for identical stored bits. +#[must_use] +pub fn vector_attribute(components: &[f32]) -> AttributeValue { + AttributeValue::L( + components + .iter() + .map(|component| AttributeValue::N(format_component(*component))) + .collect(), + ) +} + +/// Canonical `N` text for one stored component. +/// +/// `f32`'s `Display` gives the shortest decimal that round-trips to the same bits, +/// which is what is wanted, with two exceptions. Negative zero is normalised, +/// because `N` has a single zero. And a magnitude far from 1 is written in +/// scientific notation, because `Display` never uses an exponent and would expand +/// `f32::MAX` to a 39-digit integer, past the 38 digits `N` carries. The thresholds +/// sit well outside the range embeddings occupy, so a real vector always takes the +/// plain path. +fn format_component(value: f32) -> String { + if value == 0.0 { + // Covers -0.0, which compares equal to 0.0. + return "0".to_owned(); + } + // Outside this window `Display` either pads with leading zeros or expands to + // more digits than `N` carries. + const PLAIN: std::ops::Range = 1e-6..1e21; + if !PLAIN.contains(&value.abs()) { + return with_exponent_sign(format!("{value:E}")); + } + format!("{value}") +} + /// The L2 norm of a vector, precomputed at write time. /// /// Stored alongside the vector so a cosine search costs one dot product per @@ -227,7 +273,11 @@ fn attribute_type_token(value: &AttributeValue) -> &'static str { /// Format a float in upper-case scientific notation with an explicit exponent /// sign, e.g. `1.3E+40` or `-1.3E+40`. fn format_scientific(value: f64) -> String { - let formatted = format!("{value:E}"); + with_exponent_sign(format!("{value:E}")) +} + +/// Rust omits the `+` on a positive exponent; the service includes it. +fn with_exponent_sign(formatted: String) -> String { if let Some(exponent_pos) = formatted.find('E') { let (mantissa, exponent) = formatted.split_at(exponent_pos); let digits = &exponent[1..]; @@ -437,6 +487,82 @@ mod tests { assert_eq!(format_scientific(-1.3e40), "-1.3E+40"); } + /// The invariant that makes reconstruction safe: whatever text is produced for a + /// stored component must parse back to the identical bits. If this fails, a + /// search returns a vector that is not the one indexed. + /// + /// Negative zero is the one deliberate exception and is asserted separately. + #[test] + fn rebuilding_a_vector_round_trips_the_stored_bits() { + let components = [ + 0.0f32, + 1.0, + -1.0, + 0.1, + 0.123_456_79, + f32::MAX, + f32::MIN, + f32::MIN_POSITIVE, + 1e-40, // subnormal + 3.4e38, + -1.5e-30, + ]; + let rebuilt = vector_attribute(&components); + let parsed = vector_components(&rebuilt).expect("rebuilt vector must revalidate"); + assert_eq!(parsed.len(), components.len()); + for (original, back) in components.iter().zip(parsed.iter()) { + assert_eq!( + original.to_bits(), + back.to_bits(), + "component {original} did not round-trip" + ); + } + } + + /// `N` carries one zero, so a stored `-0.0` returns as `0`. The sign of zero is + /// unobservable through any of the three distance functions, so normalising it + /// cannot change a score or an ordering. + #[test] + fn negative_zero_is_normalised() { + assert_eq!(format_component(-0.0), "0"); + assert_eq!(format_component(0.0), "0"); + let parsed = vector_components(&vector_attribute(&[-0.0f32])).expect("valid"); + assert_eq!(parsed[0].to_bits(), 0.0f32.to_bits()); + } + + /// Ordinary embedding magnitudes must not be dressed up in exponents: the plain + /// form is what a client wrote and what it expects to read. + #[test] + fn ordinary_magnitudes_stay_in_plain_decimal() { + assert_eq!(format_component(0.5), "0.5"); + assert_eq!(format_component(-1.25), "-1.25"); + assert_eq!(format_component(1.0), "1"); + assert_eq!(format_component(0.000_123), "0.000123"); + } + + /// `Display` never uses an exponent, so `f32::MAX` would expand to 39 digits and + /// exceed what `N` carries. + #[test] + fn extreme_magnitudes_use_an_exponent() { + assert_eq!(format_component(f32::MAX), "3.4028235E+38"); + assert!(format_component(1e-40).contains("E-")); + assert!(!format_component(f32::MAX).contains("00000")); + } + + /// A client writing more precision than an `f32` carries reads back the narrowed + /// value, which is what the service does, rather than its own decimal string. + #[test] + fn excess_client_precision_is_narrowed_not_preserved() { + let written = + AttributeValue::L(vec![AttributeValue::N("0.12345678901234567890".to_owned())]); + let stored = vector_components(&written).expect("valid vector"); + let returned = vector_attribute(&stored); + assert_eq!( + returned, + AttributeValue::L(vec![AttributeValue::N("0.12345679".to_owned())]) + ); + } + /// The extractor must accept exactly what the validator accepts. If these /// drift, a backend stores a vector the validator would have rejected, or /// refuses one it accepted. diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs index ff3fdcd2..0b0b867c 100644 --- a/crates/storage-sqlite/src/data/vector_index.rs +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -239,15 +239,16 @@ pub(crate) async fn sync_vector_indexes( let norm = vector_norm(&components); let part = item_partition(new_item, meta)?; // Projected exactly as the GSI sibling projects, so a search returns what - // the index declares and no more. The vector attribute is kept when the - // projection includes it: it is an attribute of the item, and whether the - // service returns it could not be verified, because `SearchVectors` is not - // served by the standard DynamoDB endpoint ("This operation is not supported - // by this endpoint"). Storing it in both the `vec` column and the row payload - // is therefore deliberate rather than accidental, and is the one part of this - // that should be revisited once the behaviour can be observed. - let projected = + // the index declares and no more. + let mut projected = super::index::project_item_for_index(new_item, &[], base_key_schema, &meta.projection); + // The vector itself is not kept in the payload: it is already in the `vec` + // column as `f32`, which is the width the service validates against, and the + // search path rebuilds the attribute from those bits. Keeping a verbatim + // decimal copy here duplicated 10 to 15 KB per row at 1024 dimensions and + // would have returned the client's original precision where the service + // returns the narrowed value. + projected.remove(&meta.vector_attribute_name); let item_json = serde_json::to_string(&projected) .map_err(|e| StorageError::Internal(format!("serialize item: {e}")))?; diff --git a/crates/storage-sqlite/src/vector_search.rs b/crates/storage-sqlite/src/vector_search.rs index 8bb9d187..b6ee2093 100644 --- a/crates/storage-sqlite/src/vector_search.rs +++ b/crates/storage-sqlite/src/vector_search.rs @@ -133,7 +133,12 @@ fn score( struct TopK { k: usize, function: DistanceFunction, - hits: Vec<(f64, Item)>, + /// The decoded components ride along with each retained hit so the returned + /// attribute is rebuilt only for the `k` survivors. Rebuilding during the scan + /// would allocate a decimal string per component per row examined, which at + /// 4096 dimensions over a large partition would cost far more than the scan. + /// Moving the already-decoded vector in is free. + hits: Vec<(f64, Item, Vec)>, } impl TopK { @@ -150,14 +155,14 @@ impl TopK { self.function.ranks_before(a, b) } - fn offer(&mut self, candidate_score: f64, item: Item) { + fn offer(&mut self, candidate_score: f64, item: Item, components: Vec) { if self.hits.len() < self.k { let pos = self .hits .iter() - .position(|(s, _)| self.ranks_before(candidate_score, *s)) + .position(|(s, _, _)| self.ranks_before(candidate_score, *s)) .unwrap_or(self.hits.len()); - self.hits.insert(pos, (candidate_score, item)); + self.hits.insert(pos, (candidate_score, item, components)); return; } if self.k == 0 { @@ -170,9 +175,9 @@ impl TopK { let pos = self .hits .iter() - .position(|(s, _)| self.ranks_before(candidate_score, *s)) + .position(|(s, _, _)| self.ranks_before(candidate_score, *s)) .unwrap_or(self.k - 1); - self.hits.insert(pos, (candidate_score, item)); + self.hits.insert(pos, (candidate_score, item, components)); self.hits.truncate(self.k); } } @@ -199,9 +204,9 @@ impl VectorSearchEngine for SqliteEngine { // TableKeyInfo, because the cached key info carries dimensions and the // search schema but not the distance function, without which a score // cannot be computed or ordered. - let row: Option<(String, i64, String)> = sqlx::query_as( - "SELECT index_id, dimensions, distance_function FROM vector_indexes \ - WHERE table_id = ? AND index_name = ?", + let row: Option<(String, i64, String, String)> = sqlx::query_as( + "SELECT index_id, dimensions, distance_function, vector_attribute \ + FROM vector_indexes WHERE table_id = ? AND index_name = ?", ) .bind(&table_id) .bind(&index_name) @@ -209,8 +214,17 @@ impl VectorSearchEngine for SqliteEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let (index_id, dimensions, distance_raw) = + let (index_id, dimensions, distance_raw, vector_attribute_json) = row.ok_or_else(|| StorageError::IndexNotFound(index_name.clone()))?; + // Stored as the serialized `VectorAttribute`, not a bare name, so it is + // deserialized exactly as the write path does. Treating the column as a + // plain string yields the key `{"AttributeName":"emb"}`. + let vector_attribute_name = + serde_json::from_str::( + &vector_attribute_json, + ) + .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))? + .attribute_name; let dimensions = usize::try_from(dimensions).map_err(|_| { StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) })?; @@ -270,23 +284,33 @@ impl VectorSearchEngine for SqliteEngine { #[allow(clippy::cast_possible_truncation)] let candidate_norm = norm as f32; - top.offer( - score( - function, - &query_vector, - query_norm, - &candidate, - candidate_norm, - ), - item, + let candidate_score = score( + function, + &query_vector, + query_norm, + &candidate, + candidate_norm, ); + top.offer(candidate_score, item, candidate); } Ok(VectorSearchOutput { hits: top .hits .into_iter() - .map(|(score, item)| VectorHit { item, score }) + .map(|(score, mut item, components)| { + // Reinstated from the stored `f32`s rather than from a + // second copy in the payload, so what comes back is the + // narrowed value that was actually indexed. The engine drops + // it again unless a `ProjectionExpression` names it, and the + // billed byte count subtracts it, so putting it here does not + // change either the default response or the metric. + item.insert( + vector_attribute_name.clone(), + extenddb_core::validation::vector_item::vector_attribute(&components), + ); + VectorHit { item, score } + }) .collect(), distance_function: function, }) @@ -348,27 +372,46 @@ mod tests { fn top_k_orders_distances_ascending_and_similarities_descending() { let mut cosine = TopK::new(2, DistanceFunction::Cosine); for s in [0.9, 0.1, 0.5] { - cosine.offer(s, item()); + cosine.offer(s, item(), vec![]); } assert_eq!( - cosine.hits.iter().map(|(s, _)| *s).collect::>(), + cosine.hits.iter().map(|(s, _, _)| *s).collect::>(), vec![0.1, 0.5] ); let mut dot = TopK::new(2, DistanceFunction::DotProduct); for s in [0.9, 0.1, 0.5] { - dot.offer(s, item()); + dot.offer(s, item(), vec![]); } assert_eq!( - dot.hits.iter().map(|(s, _)| *s).collect::>(), + dot.hits.iter().map(|(s, _, _)| *s).collect::>(), vec![0.9, 0.5] ); } + /// Each retained hit must keep its *own* vector. The components are inserted at + /// a computed position alongside the score, so an off-by-one there would return + /// a neighbour's vector against this item's attributes: wrong data, no error. + #[test] + fn a_retained_hit_keeps_its_own_vector() { + let mut t = TopK::new(3, DistanceFunction::Cosine); + // Offered worst-first so every insert lands at the front and the pairing is + // exercised rather than incidentally correct. + for (score, tag) in [(0.9f64, 9.0f32), (0.5, 5.0), (0.1, 1.0)] { + t.offer(score, item(), vec![tag]); + } + let paired: Vec<(f64, f32)> = t + .hits + .iter() + .map(|(s, _, components)| (*s, components[0])) + .collect(); + assert_eq!(paired, vec![(0.1, 1.0), (0.5, 5.0), (0.9, 9.0)]); + } + #[test] fn top_k_of_zero_returns_nothing_rather_than_panicking() { let mut t = TopK::new(0, DistanceFunction::Cosine); - t.offer(0.5, item()); + t.offer(0.5, item(), vec![]); assert!(t.hits.is_empty()); } diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 601a6303..c16db413 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -401,6 +401,85 @@ async fn the_vector_attribute_is_returned_only_when_named() { let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } +/// A vector index holds 32-bit floats, so a client writing more precision than an +/// `f32` carries reads the narrowed value back from a search, while the base item +/// keeps exactly what was written. +/// +/// The service's own validation is the evidence for the width: it rejects a +/// component outside `[-3.4028235E38, 3.4028235E38]`, which is exactly `f32::MAX`, +/// and names the expected type "32-bit floating point number". Reading the narrowed +/// value back could not be measured directly, because `SearchVectors` is not served +/// by the standard DynamoDB endpoint. +/// +/// Both halves are asserted deliberately. Without the base-table half this would +/// also pass against an implementation that had simply corrupted the item on write. +#[tokio::test] +async fn the_index_narrows_a_vector_to_f32_but_the_item_keeps_its_own_precision() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_vecf32"); + create_vector_table(&name, 2, "COSINE", false).await; + + // More decimal places than an f32 carries. Nearest f32 is 0.12345679. No + // trailing zero, because `N` normalisation trims one and that would confound + // the base-item assertion below with a second, unrelated effect. + let written = "0.1234567890123456789"; + let body = format!( + r#"{{ + "TableName": "{name}", + "Item": {{"pk": {{"S": "a"}}, "emb": {{"L": [{{"N": "{written}"}}, {{"N": "0"}}]}}}} + }}"# + ); + let (status, text) = call("PutItem", &body).await; + assert_eq!(status, 200, "PutItem failed: {text}"); + + // The base item is untouched: narrowing belongs to the index, not the write. + let (status, text) = call( + "GetItem", + &format!( + r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "a"}}}}, "ConsistentRead": true}}"# + ), + ) + .await; + assert_eq!(status, 200, "GetItem failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + assert_eq!( + json.pointer("/Item/emb/L/0/N").and_then(|v| v.as_str()), + Some(written), + "the base item must keep the client's own precision: {text}" + ); + + // The search returns what the index stored, which is the f32. + let body = format!( + r#"{{ + "TableName": "{name}", + "IndexName": "vidx", + "SearchVector": [{{"N": "1"}}, {{"N": "0"}}], + "TopK": 5, + "ProjectionExpression": "pk, emb" + }}"# + ); + let (status, text) = call("SearchVectors", &body).await; + assert_eq!(status, 200, "SearchVectors failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + assert_eq!( + json.pointer("/SearchResults/0/Item/emb/L/0/N") + .and_then(|v| v.as_str()), + Some("0.12345679"), + "the index must return the narrowed f32: {text}" + ); + // The exactly-representable component must not acquire a decimal point. + assert_eq!( + json.pointer("/SearchResults/0/Item/emb/L/1/N") + .and_then(|v| v.as_str()), + Some("0"), + "an exact component must round-trip unchanged: {text}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + /// A two-HASH search schema is rejected with the service's measured message. /// /// This mattered because the contract accepted it and then could not honour it: the From fcbc27dac23da45db8c0574212df3a5291d43031 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Fri, 7 Aug 2026 10:57:39 +0000 Subject: [PATCH 10/25] fix(vector): correct four write-path facts measured against real DynamoDB Everything below was measured on 2026-08-07 against the live service in us-east-1, using raw signed requests because the installed CLI predates vector indexes. Three of the four were wrong by inference, and none of the existing tests could see it. Number formatting. The service NEVER returns an exponent in an N value, whatever form it was sent: 3.4028235E+38 reads back as 340282350000000000000000000000000000000, and 1E-40 as 0.0000000000000000000000000000000000000001. Both are exactly what f32's Display produces, so returning a reconstructed vector is now plain decimal always. The previous commit wrote extreme magnitudes in scientific notation on the reasoning that a 39-digit expansion would exceed the 38 digits N carries. That reading was wrong: the limit is on SIGNIFICANT digits, confirmed by a separate error for a 39-significant-digit value ("Attempting to store more than 38 significant digits in a Number") and by a 38-digit value being accepted and returned verbatim. An f32's shortest round-tripping form carries at most 9 significant digits, so no stored component can ever approach the limit however long the expansion runs. Three error messages diverged. The non-list message was invented rather than measured, and was wrong in three ways at once: a colon after "invalid" where the service uses a full stop, the phrase "a list of numbers" where the service says "32-bit floating point number list", and a full stop before IndexName that the service omits. N, S and NS in the vector position all produce the same text. The out-of-range message omitted the "One or more parameter values were invalid. " prefix entirely. The wrong-type message used a colon after "invalid" where the service uses a full stop. The tests could not catch any of this because they asserted fragments that began after the prefix, which is exactly where the drift was. All three now assert the whole string by equality. The one message that was already pinned whole, the dimension mismatch, was correct. Confirmed while measuring, and unchanged: the value in the out-of-range message IS normalised to scientific notation whatever form it arrived in (a plain 39-digit integer drew "Value: 4E+38"), a component at exactly f32::MAX is accepted, excess decimal precision is accepted rather than rejected, an absent vector attribute is accepted, an empty list draws the size message with Actual: 0, and negative zero reads back as 0, matching the normalisation in format_component. N also trims one trailing zero, which the new wire test had to account for. 440 Rust integration tests (0 filtered out), 764 workspace tests, fmt and clippy -D warnings clean on both feature sets. Probe tables deleted. --- crates/core/src/validation/vector_item.rs | 96 +++++++++++++++------- tests/rust/src/vector_index_unsupported.rs | 28 +------ 2 files changed, 68 insertions(+), 56 deletions(-) diff --git a/crates/core/src/validation/vector_item.rs b/crates/core/src/validation/vector_item.rs index bf2faa7b..a460efa7 100644 --- a/crates/core/src/validation/vector_item.rs +++ b/crates/core/src/validation/vector_item.rs @@ -116,24 +116,25 @@ pub fn vector_attribute(components: &[f32]) -> AttributeValue { /// Canonical `N` text for one stored component. /// -/// `f32`'s `Display` gives the shortest decimal that round-trips to the same bits, -/// which is what is wanted, with two exceptions. Negative zero is normalised, -/// because `N` has a single zero. And a magnitude far from 1 is written in -/// scientific notation, because `Display` never uses an exponent and would expand -/// `f32::MAX` to a 39-digit integer, past the 38 digits `N` carries. The thresholds -/// sit well outside the range embeddings occupy, so a real vector always takes the -/// plain path. +/// Plain decimal, always, which is `f32`'s `Display`. Measured against the live +/// service on 2026-08-07: DynamoDB never returns an exponent, whatever it was sent. +/// `3.4028235E+38` reads back as `340282350000000000000000000000000000000` and +/// `1E-40` as `0.0000000000000000000000000000000000000001`, both of which are +/// exactly what `Display` produces for the same `f32`. +/// +/// This corrects an earlier reading of the 38-digit limit as a limit on characters. +/// It bounds *significant* digits, and an `f32`'s shortest round-tripping form +/// carries at most 9 of them, so no stored component can approach it however long +/// the expansion runs. A 38-significant-digit value was accepted and returned +/// verbatim in the same probe. +/// +/// The one departure from `Display` is negative zero, which the service also +/// normalises: `-0` reads back as `0`. fn format_component(value: f32) -> String { if value == 0.0 { // Covers -0.0, which compares equal to 0.0. return "0".to_owned(); } - // Outside this window `Display` either pads with leading zeros or expands to - // more digits than `N` carries. - const PLAIN: std::ops::Range = 1e-6..1e21; - if !PLAIN.contains(&value.abs()) { - return with_exponent_sign(format!("{value:E}")); - } format!("{value}") } @@ -157,9 +158,13 @@ fn validate_vector_attribute( let dimensions = index.dimensions as usize; let AttributeValue::L(elements) = value else { + // Measured 2026-08-07 against N, S and NS in the vector position, all three + // of which produce this exact text. Note "32-bit floating point number + // list" rather than any phrasing of "list of numbers", and no full stop + // before IndexName, matching the size message below. return Err(invalid(format!( - "One or more parameter values were invalid: Invalid type for parameter {attr}, \ - Expected: a list of numbers. IndexName: {index_name}" + "One or more parameter values were invalid. Invalid type for parameter {attr}, \ + Expected: 32-bit floating point number list IndexName: {index_name}" ))); }; @@ -191,14 +196,15 @@ fn validate_vector_attribute( .map(format_scientific) .unwrap_or_else(|_| number.clone()); return Err(invalid(format!( - "Invalid value for parameter {attr}[{position}], Value: {display} is \ - outside valid range [-3.4028235E38, 3.4028235E38]. IndexName: {index_name}" + "One or more parameter values were invalid. Invalid value for parameter \ + {attr}[{position}], Value: {display} is outside valid range \ + [-3.4028235E38, 3.4028235E38]. IndexName: {index_name}" ))); } } other => { return Err(invalid(format!( - "One or more parameter values were invalid: Invalid type for parameter \ + "One or more parameter values were invalid. Invalid type for parameter \ {attr}[{position}], Expected: 32-bit floating point number, Actual: {}. \ IndexName: {index_name}", attribute_type_token(other) @@ -398,10 +404,19 @@ mod tests { assert!(message.contains("Expected: 5, Actual: 0")); } + /// Asserted whole rather than by fragment. Every message below was measured + /// against the live service on 2026-08-07, and the previous fragment assertions + /// excluded the "One or more parameter values were invalid" prefix, which is + /// precisely where three of the four had drifted: two used a colon where the + /// service uses a full stop, and one omitted the prefix entirely. #[test] fn rejects_non_list_vector() { let message = err(&item_with_vector(AttributeValue::N("0.1".to_owned()))); - assert!(message.contains("Invalid type for parameter ProductEmbedding")); + assert_eq!( + message, + "One or more parameter values were invalid. Invalid type for parameter \ + ProductEmbedding, Expected: 32-bit floating point number list IndexName: ProductIndex" + ); } #[test] @@ -414,10 +429,16 @@ mod tests { AttributeValue::N("0.5".to_owned()), ]); let message = err(&item_with_vector(value)); - assert!(message.contains("Invalid type for parameter ProductEmbedding[2]")); - assert!(message.contains("Expected: 32-bit floating point number, Actual: S")); + assert_eq!( + message, + "One or more parameter values were invalid. Invalid type for parameter \ + ProductEmbedding[2], Expected: 32-bit floating point number, Actual: S. \ + IndexName: ProductIndex" + ); } + /// The offending value is echoed in scientific notation whatever form it was + /// sent in: the service answered `Value: 4E+38` to a plain 39-digit integer. #[test] fn rejects_value_out_of_range() { let value = AttributeValue::L(vec![ @@ -428,10 +449,12 @@ mod tests { AttributeValue::N("0.5".to_owned()), ]); let message = err(&item_with_vector(value)); - assert!(message.contains( - "Invalid value for parameter ProductEmbedding[1], Value: 1.3E+40 is outside \ - valid range [-3.4028235E38, 3.4028235E38]. IndexName: ProductIndex" - )); + assert_eq!( + message, + "One or more parameter values were invalid. Invalid value for parameter \ + ProductEmbedding[1], Value: 1.3E+40 is outside valid range \ + [-3.4028235E38, 3.4028235E38]. IndexName: ProductIndex" + ); } #[test] @@ -540,13 +563,24 @@ mod tests { assert_eq!(format_component(0.000_123), "0.000123"); } - /// `Display` never uses an exponent, so `f32::MAX` would expand to 39 digits and - /// exceed what `N` carries. + /// `Display` never uses an exponent, and neither does the service. These exact + /// strings were read back from real DynamoDB on 2026-08-07 after sending + /// `3.4028235E+38` and `1E-40`, so matching them is measured parity rather than + /// a formatting preference. #[test] - fn extreme_magnitudes_use_an_exponent() { - assert_eq!(format_component(f32::MAX), "3.4028235E+38"); - assert!(format_component(1e-40).contains("E-")); - assert!(!format_component(f32::MAX).contains("00000")); + fn extreme_magnitudes_match_the_plain_form_the_service_returns() { + assert_eq!( + format_component(f32::MAX), + "340282350000000000000000000000000000000" + ); + assert_eq!( + format_component(1e-40), + "0.0000000000000000000000000000000000000001" + ); + assert!( + !format_component(f32::MAX).contains('E'), + "an exponent is never returned by the service" + ); } /// A client writing more precision than an `f32` carries reads back the narrowed diff --git a/tests/rust/src/vector_index_unsupported.rs b/tests/rust/src/vector_index_unsupported.rs index 45a6e875..af7ddd36 100644 --- a/tests/rust/src/vector_index_unsupported.rs +++ b/tests/rust/src/vector_index_unsupported.rs @@ -186,31 +186,6 @@ async fn backend_supports_vectors() -> bool { panic!("vector support probe failed for an unrelated reason: {status} {text}"); } -/// What this run asserts about the backend's vector capability. -/// -/// Three states on purpose. A suite that adapts to whatever the backend reports -/// never asserts *which* backend is under test, so a backend that silently gained -/// or lost vector support would change what is exercised and still report green. -/// `EXTENDDB_EXPECT_VECTORS` lets a CI job state its expectation: -/// -/// - `1`: the backend must support vectors, so a positive suite failing to run is -/// an error rather than a skip. No such suite exists yet; it arrives with the -/// first participating backend, and the variable is defined here so that suite -/// inherits the guard rather than inventing one. -/// - `0`: the backend must not, so these refusal tests failing to run is an error. -/// - unset: adapt quietly, so a plain local `cargo test` works against either -/// backend without ceremony. -/// -/// An unrecognised value panics rather than being read as one of the two, because -/// a typo that silently meant "unset" would disable the guard it was added for. -pub(crate) fn expect_vectors() -> Option { - match std::env::var("EXTENDDB_EXPECT_VECTORS").ok()?.as_str() { - "1" => Some(true), - "0" => Some(false), - other => panic!("EXTENDDB_EXPECT_VECTORS must be 0 or 1, got {other:?}"), - } -} - /// Whether the running backend implements vector indexes, for the suite that /// asserts the participating behaviour. Named positively so the caller reads as /// an opt-in rather than as a double negative. @@ -230,6 +205,9 @@ pub(crate) async fn vectors_supported() -> bool { /// - `0`: the backend must not. The refusal suite failing to run is an error. /// - unset: adapt quietly, so a plain local `cargo test` works against either /// backend without ceremony. +/// +/// An unrecognised value panics rather than being read as one of the two, because +/// a typo that silently meant "unset" would disable the guard it was added for. pub(crate) fn expect_vectors() -> Option { match std::env::var("EXTENDDB_EXPECT_VECTORS").ok()?.as_str() { "1" => Some(true), From 21abd09bafaaefd7ad39178913e1dc8c7a4c9fa1 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Fri, 7 Aug 2026 13:15:21 +0000 Subject: [PATCH 11/25] feat(sqlite): UpdateTable vector index create and delete, with backfill Closes the hole the engine post-condition guard exposed. The backend declared vector support and never read vector_index_updates, so a Create returned 200 and created nothing, and a Delete returned 200 while the index stayed ACTIVE and kept returning hits. Structured as the GSI sibling is, two-phase: the catalog row commits first at CREATING, then the data table and backfill run, then the row flips to ACTIVE. A crash in between leaves a CREATING row for the startup reconciler rather than an ACTIVE index over a missing or partial table. The status sequence is the one measured against the service on 2026-08-06 by seeding 3000 items of 1024 dimensions so the backfill took 8.5 minutes and could be observed: CREATING with Backfilling false, then CREATING with true, then ACTIVE with the member absent. Writing false first rather than jumping to true matters because presence does not imply backfilling, so a client must read the value; and the flag is set outside the backfill transaction, since a flag that only exists inside the transaction doing the work cannot be observed by anyone while the work happens. The row writer is now shared, not duplicated. insert_vector_row is called by both the write path and the backfill, because they are the only two producers of a vector row and a second copy would be free to drift: a backfilled row shaped differently from a live-written one searches correctly right up until the difference matters, with nothing to catch it. A test writes one item before the index exists and one after, and requires both to be found, which is what actually pins the two paths together. Also adds drop_vector_data_table_by_id, which an earlier review flagged as missing. The table-drop path sweeps sqlite_master because the catalog rows have already cascade-deleted by then and the index ids are unreadable; here the id is known, so the name is derived directly. Crash recovery reconcile_incomplete_vector_indexes runs at startup beside the GSI reconciler, as a separate pass rather than a shared one: the two live in different catalog tables and are built by different code, so a failure to reconcile one must not skip the other. It drops and rebuilds rather than resuming, and there is a test for exactly that, because the crash that actually happens leaves the data table holding SOME of the rows. Resuming would index those items a second time and a search would return the same item twice. The simpler test, whose simulated crash leaves no data table at all, cannot catch this: the drop is a no-op there. Both were mutation-checked, removing the drop fails the partial-table test and restoring it passes. Verification The two backfill wire tests were mutation-checked as well: with the backfill neutered to write nothing, both fail, and the second reports only the later-written item, so it genuinely distinguishes backfilled rows from live-written ones. Five new wire tests: backfill of pre-existing items including one without a vector that must be skipped rather than break the scan, later writes still indexed, delete stops both serving and reporting, delete leaves the base items untouched, and duplicate-create plus missing-delete rejected. Three reconciler unit tests, one pre-existing. 445 Rust integration tests and 773 workspace tests, 0 filtered out, fmt and clippy -D warnings clean on both feature sets. --- crates/storage-sqlite/src/data/ddl.rs | 19 + .../storage-sqlite/src/data/vector_index.rs | 207 +++++-- crates/storage-sqlite/src/lib.rs | 11 + crates/storage-sqlite/src/update_table.rs | 546 ++++++++++++++++++ tests/rust/src/vector_index_search.rs | 276 ++++++++- 5 files changed, 998 insertions(+), 61 deletions(-) diff --git a/crates/storage-sqlite/src/data/ddl.rs b/crates/storage-sqlite/src/data/ddl.rs index 357f15cd..f1c12f96 100644 --- a/crates/storage-sqlite/src/data/ddl.rs +++ b/crates/storage-sqlite/src/data/ddl.rs @@ -165,6 +165,25 @@ impl SqliteEngine { /// same transaction, so `vector_indexes` is already empty for this table. /// Without this, dropping a table would leave its vector data tables behind /// forever with nothing left pointing at them. + /// Drop one vector index's data table. + /// + /// The table-drop path sweeps `sqlite_master` instead, because by the time it + /// runs the catalog rows have already cascade-deleted and the index ids are + /// unreadable. Here the id is known, so the name is derived directly rather + /// than matched by pattern. + pub(crate) async fn drop_vector_data_table_by_id( + pool: &sqlx::SqlitePool, + table_id: &str, + index_id: &str, + ) -> Result<(), StorageError> { + let vec_table = vector_table_name(table_id, index_id); + sqlx::query(&format!("DROP TABLE IF EXISTS {vec_table}")) + .execute(pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + async fn drop_all_vector_data_tables( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, table_id: &str, diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs index 0b0b867c..3e818c9a 100644 --- a/crates/storage-sqlite/src/data/vector_index.rs +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -209,69 +209,158 @@ pub(crate) async fn sync_vector_indexes( let Some(new_item) = new_item else { continue; // A delete: removal above is the whole of the work. }; - if !item_is_indexable(new_item, meta) { - continue; - } + insert_vector_row( + tx, + table_id, + meta, + new_item, + base_key_schema, + attr_defs, + &key_cols, + ) + .await?; + } + Ok(()) +} - let value = new_item.get(&meta.vector_attribute_name).ok_or_else(|| { - StorageError::Internal("indexable check passed but the vector is absent".to_owned()) - })?; - let components = vector_components(value).ok_or_else(|| { - // Core validates the write before it reaches storage, so a malformed - // vector here means validation was bypassed rather than that a caller - // sent bad input. - StorageError::Internal( - "vector attribute reached storage without passing validation".to_owned(), - ) - })?; - if components.len() != meta.dimensions { - return Err(StorageError::Internal(format!( - "vector has {} components, index declares {}", - components.len(), - meta.dimensions - ))); - } +/// Write one item's row into one vector index. +/// +/// Shared by the write path and by backfill deliberately. These are the only two +/// producers of a vector row, and a second copy of this logic would be free to +/// drift: a backfilled row shaped differently from a live-written one would search +/// correctly right up until the difference mattered, with nothing to catch it. +/// +/// A non-indexable item is a no-op rather than an error, which is what makes a +/// backfill over a table where only some items carry the vector work. +pub(crate) async fn insert_vector_row( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + meta: &VectorIndexMeta, + item: &Item, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + key_cols: &[String], +) -> Result<(), StorageError> { + if !item_is_indexable(item, meta) { + return Ok(()); + } + let vec_table = vector_table_name(table_id, &meta.index_id); - let mut blob = Vec::with_capacity(components.len() * 4); - for x in &components { - blob.extend_from_slice(&x.to_le_bytes()); - } - let norm = vector_norm(&components); - let part = item_partition(new_item, meta)?; - // Projected exactly as the GSI sibling projects, so a search returns what - // the index declares and no more. - let mut projected = - super::index::project_item_for_index(new_item, &[], base_key_schema, &meta.projection); - // The vector itself is not kept in the payload: it is already in the `vec` - // column as `f32`, which is the width the service validates against, and the - // search path rebuilds the attribute from those bits. Keeping a verbatim - // decimal copy here duplicated 10 to 15 KB per row at 1024 dimensions and - // would have returned the client's original precision where the service - // returns the narrowed value. - projected.remove(&meta.vector_attribute_name); - let item_json = serde_json::to_string(&projected) - .map_err(|e| StorageError::Internal(format!("serialize item: {e}")))?; + let value = item.get(&meta.vector_attribute_name).ok_or_else(|| { + StorageError::Internal("indexable check passed but the vector is absent".to_owned()) + })?; + let components = vector_components(value).ok_or_else(|| { + // Core validates the write before it reaches storage, so a malformed + // vector here means validation was bypassed rather than that a caller + // sent bad input. + StorageError::Internal( + "vector attribute reached storage without passing validation".to_owned(), + ) + })?; + if components.len() != meta.dimensions { + return Err(StorageError::Internal(format!( + "vector has {} components, index declares {}", + components.len(), + meta.dimensions + ))); + } - let cols = std::iter::once("part".to_owned()) - .chain(key_cols.iter().cloned()) - .chain(["vec".to_owned(), "nrm".to_owned(), "item_data".to_owned()]) - .collect::>(); - let placeholders = vec!["?"; cols.len()].join(", "); - let sql = format!( - "INSERT INTO {vec_table} ({}) VALUES ({placeholders})", - cols.join(", ") - ); - let key_binds = base_key_binds(new_item, base_key_schema, attr_defs)?; - let mut q = sqlx::query(&sql).bind(part); - for b in key_binds { - q = super::bind_bound!(q, b); - } - q.bind(blob) - .bind(f64::from(norm)) - .bind(item_json) - .execute(&mut **tx) + let mut blob = Vec::with_capacity(components.len() * 4); + for x in &components { + blob.extend_from_slice(&x.to_le_bytes()); + } + let norm = vector_norm(&components); + let part = item_partition(item, meta)?; + // Projected exactly as the GSI sibling projects, so a search returns what + // the index declares and no more. + let mut projected = + super::index::project_item_for_index(item, &[], base_key_schema, &meta.projection); + // The vector itself is not kept in the payload: it is already in the `vec` + // column as `f32`, which is the width the service validates against, and the + // search path rebuilds the attribute from those bits. Keeping a verbatim + // decimal copy here duplicated 10 to 15 KB per row at 1024 dimensions and + // would have returned the client's original precision where the service + // returns the narrowed value. + projected.remove(&meta.vector_attribute_name); + let item_json = serde_json::to_string(&projected) + .map_err(|e| StorageError::Internal(format!("serialize item: {e}")))?; + + let cols = std::iter::once("part".to_owned()) + .chain(key_cols.iter().cloned()) + .chain(["vec".to_owned(), "nrm".to_owned(), "item_data".to_owned()]) + .collect::>(); + let placeholders = vec!["?"; cols.len()].join(", "); + let sql = format!( + "INSERT INTO {vec_table} ({}) VALUES ({placeholders})", + cols.join(", ") + ); + let key_binds = base_key_binds(item, base_key_schema, attr_defs)?; + let mut q = sqlx::query(&sql).bind(part); + for b in key_binds { + q = super::bind_bound!(q, b); + } + q.bind(blob) + .bind(f64::from(norm)) + .bind(item_json) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) +} + +/// Populate a newly created vector index from the base table. +/// +/// Batched by offset exactly as `backfill_gsi` is, and for the same reason: a table +/// large enough to be worth indexing is too large to hold in memory. Returns the +/// number of rows written, which is what distinguishes "backfilled nothing because +/// no item carries the vector" from "backfilled nothing because the scan is broken". +pub(crate) async fn backfill_vector_index( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + meta: &VectorIndexMeta, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], +) -> Result { + const BATCH: i64 = 500; + let base_table = super::data_table_name(table_id); + let key_cols = base_key_columns(base_key_schema, attr_defs); + let sql = format!("SELECT item_data FROM {base_table} ORDER BY pk LIMIT ? OFFSET ?"); + + let mut offset: i64 = 0; + let mut written = 0usize; + loop { + let rows: Vec<(String,)> = sqlx::query_as(&sql) + .bind(BATCH) + .bind(offset) + .fetch_all(&mut **tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; + if rows.is_empty() { + break; + } + let fetched = i64::try_from(rows.len()).unwrap_or(BATCH); + for (item_json,) in rows { + let item: Item = serde_json::from_str(&item_json) + .map_err(|e| StorageError::Internal(format!("stored item: {e}")))?; + if !item_is_indexable(&item, meta) { + continue; + } + insert_vector_row( + tx, + table_id, + meta, + &item, + base_key_schema, + attr_defs, + &key_cols, + ) + .await?; + written += 1; + } + if fetched < BATCH { + break; + } + offset += fetched; } - Ok(()) + Ok(written) } diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index f397adad..5d8f7195 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -234,6 +234,17 @@ fn sqlite_server_components_factory( Err(e) => tracing::error!("Failed to reconcile incomplete GSIs: {e}"), } + // Same for a vector index. A separate pass rather than a shared one, because + // the two live in different catalog tables and are built by different code; + // a failure to reconcile one must not skip the other. + match engine.reconcile_incomplete_vector_indexes().await { + Ok(n) if n > 0 => { + tracing::info!("Reconciled {n} incomplete vector index(es) at startup"); + } + Ok(_) => {} + Err(e) => tracing::error!("Failed to reconcile incomplete vector indexes: {e}"), + } + let control_plane_notify = engine.control_plane_notify(); let engine = Arc::new(engine); diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index 149a8344..09c0271c 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -9,9 +9,13 @@ //! replaces `FOR UPDATE`. GSI data tables are created (and backfilled) or //! dropped after the catalog transaction commits, with catalog cleanup on //! a data-DDL failure. +//! +//! Vector index create/delete follows the same two-phase shape, with the +//! backfill lifecycle the service was measured to report. use extenddb_core::types::{ AttributeDefinition, BillingMode, KeySchemaElement, TableDescription, UpdateTableInput, + VectorIndexSpecification, }; use extenddb_storage::error::StorageError; @@ -293,6 +297,90 @@ impl SqliteEngine { } } + // Vector index create/delete. Structured exactly like the GSI block above + // and for the same reason: the catalog row is committed first at CREATING, + // and the data table plus backfill happen after, so a crash in between + // leaves a CREATING row the startup reconciler rebuilds rather than an + // ACTIVE index with a partial table. + let mut vec_created: Vec<(String, VectorIndexSpecification)> = Vec::new(); + let mut vec_deleted: Vec = Vec::new(); + if let Some(updates) = &input.vector_index_updates { + for update in updates { + if let Some(create) = &update.create { + let dup: Option<(String,)> = sqlx::query_as( + "SELECT index_name FROM vector_indexes \ + WHERE table_id = ? AND index_name = ?", + ) + .bind(&table_id) + .bind(&create.index_name) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if dup.is_some() { + return Err(StorageError::IndexAlreadyExists(create.index_name.clone())); + } + let index_id = uuid::Uuid::new_v4().to_string(); + let vec_attr = serde_json::to_string(&create.vector_attribute) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let search_schema = create + .search_schema + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let projection = serde_json::to_string(&create.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let dimensions = i64::from(create.dimensions); + let distance = serde_json::to_string(&create.distance_function) + .map_err(|e| StorageError::Internal(e.to_string()))? + .trim_matches('"') + .to_owned(); + // `backfilling` starts at false rather than absent or true. + // Measured against the service on 2026-08-06: the member appears + // as false while the index exists but its backfill has not + // started, flips to true during, and is removed once ACTIVE. + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, \ + vector_attribute, search_schema, projection, index_status, backfilling) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'CREATING', 0)", + ) + .bind(&table_id) + .bind(&index_id) + .bind(&create.index_name) + .bind(dimensions) + .bind(&distance) + .bind(&vec_attr) + .bind(&search_schema) + .bind(&projection) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + vec_created.push((index_id, create.clone())); + } + if let Some(delete) = &update.delete { + let existing: Option<(String,)> = sqlx::query_as( + "SELECT index_id FROM vector_indexes \ + WHERE table_id = ? AND index_name = ?", + ) + .bind(&table_id) + .bind(&delete.index_name) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let (del_id,) = existing + .ok_or_else(|| StorageError::IndexNotFound(delete.index_name.clone()))?; + sqlx::query("DELETE FROM vector_indexes WHERE table_id = ? AND index_name = ?") + .bind(&table_id) + .bind(&delete.index_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + vec_deleted.push(del_id); + } + } + } + tx.commit() .await .map_err(|e| StorageError::Internal(e.to_string()))?; @@ -391,10 +479,142 @@ impl SqliteEngine { } } + // Vector index data DDL and backfill, after the catalog commit. + if !vec_created.is_empty() || !vec_deleted.is_empty() { + let base_ks: Vec = serde_json::from_str(&ks_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let base_ad: Vec = serde_json::from_str(&ad_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let effective_ad = input.attribute_definitions.as_deref().unwrap_or(&base_ad); + + for (index_id, create) in &vec_created { + let result = self + .build_vector_index(&table_id, index_id, create, &base_ks, effective_ad) + .await; + if let Err(e) = result { + tracing::error!( + "Failed to build vector index '{}' on '{}', cleaning up catalog: {e}", + create.index_name, + input.table_name + ); + // Same cleanup as the GSI path: leaving the CREATING row behind + // would have the reconciler retry a build that just failed + // deterministically, on every startup. + let _ = sqlx::query( + "DELETE FROM vector_indexes WHERE table_id = ? AND index_name = ?", + ) + .bind(&table_id) + .bind(&create.index_name) + .execute(&self.pool) + .await; + let _ = + Self::drop_vector_data_table_by_id(&self.pool, &table_id, index_id).await; + return Err(e); + } + } + + for index_id in &vec_deleted { + Self::drop_vector_data_table_by_id(&self.pool, &table_id, index_id).await?; + } + } + self.build_table_description(account_id, &input.table_name) .await } + /// Create a vector index's data table and populate it, then mark it ready. + /// + /// The status sequence is the one measured against the service on 2026-08-06: + /// `CREATING` with `Backfilling: false`, then `CREATING` with `true` while the + /// scan runs, then `ACTIVE` with the member absent. Writing `false` first rather + /// than jumping straight to `true` matters because a client is documented to + /// read the value rather than test for presence, so an index that exists but + /// has not started backfilling must say so. + /// + /// The data table and the backfill share one transaction, so a search can never + /// see a half-populated table: it sees no table (index still `CREATING`, so the + /// engine will not route to it) or a complete one. + async fn build_vector_index( + &self, + table_id: &str, + index_id: &str, + create: &VectorIndexSpecification, + base_ks: &[KeySchemaElement], + effective_ad: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let mut data_tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Self::create_vector_data_table(&mut data_tx, table_id, index_id, base_ks, effective_ad) + .await?; + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // The scan is about to start, so the member becomes true. Set outside the + // backfill transaction, otherwise no observer could see it: the whole point + // of the flag is to be readable while the scan is in progress. + sqlx::query( + "UPDATE vector_indexes SET backfilling = 1 WHERE table_id = ? AND index_id = ?", + ) + .bind(table_id) + .bind(index_id) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut fill_tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let metas = + crate::data::vector_index::fetch_vector_indexes_for_table(&mut fill_tx, table_id) + .await? + .into_iter() + .find(|m| m.index_id == index_id) + .ok_or_else(|| { + StorageError::Internal( + "the vector index catalog row vanished between commit and backfill" + .to_owned(), + ) + })?; + let written = crate::data::vector_index::backfill_vector_index( + &mut fill_tx, + table_id, + &metas, + base_ks, + effective_ad, + ) + .await?; + fill_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + tracing::info!( + index_name = %create.index_name, + vectors_indexed = written, + "vector index backfill complete" + ); + + // Populated, so the index can serve. `backfilling` is cleared to NULL rather + // than set to 0, because the service removes the member once ACTIVE, and the + // catalog CHECK constraint enforces that pairing. + sqlx::query( + "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL \ + WHERE table_id = ? AND index_id = ?", + ) + .bind(table_id) + .bind(index_id) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + /// Rebuild any GSI left in `CREATING` by a crash between the catalog commit /// and the completion of its data-table backfill. Runs once at startup: for /// each such index it drops any partial data table, recreates and backfills @@ -468,6 +688,93 @@ impl SqliteEngine { } Ok(rebuilt) } + + /// Rebuild any vector index left in `CREATING` by a crash between the catalog + /// commit and the end of its backfill. + /// + /// Same contract as [`Self::reconcile_incomplete_gsis`]: an `ACTIVE` vector index + /// with a missing or partial data table can never be observed, and nothing is + /// left permanently stuck in `CREATING`. Idempotent, because the data table is + /// dropped and rebuilt rather than appended to; without the drop, a retry would + /// duplicate every row it had already written before the crash, and a search + /// would return the same item several times. + pub(crate) async fn reconcile_incomplete_vector_indexes(&self) -> Result { + let rows: Vec<(String, String, String, String)> = sqlx::query_as( + "SELECT v.index_id, v.table_id, t.key_schema, t.attribute_definitions \ + FROM vector_indexes v JOIN tables t ON v.table_id = t.table_id \ + WHERE v.index_status = 'CREATING'", + ) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut rebuilt = 0usize; + for (index_id, table_id, base_ks_json, base_ad_json) in rows { + let base_key_schema: Vec = serde_json::from_str(&base_ks_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let attr_defs: Vec = serde_json::from_str(&base_ad_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let _writer = self.write_lock.lock().await; + Self::drop_vector_data_table_by_id(&self.pool, &table_id, &index_id).await?; + + let mut data_tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Self::create_vector_data_table( + &mut data_tx, + &table_id, + &index_id, + &base_key_schema, + &attr_defs, + ) + .await?; + // The definition is read from the catalog rather than reconstructed, + // because the request that created it is long gone. + let meta = crate::data::vector_index::fetch_vector_indexes_for_table( + &mut data_tx, + &table_id, + ) + .await? + .into_iter() + .find(|m| m.index_id == index_id) + .ok_or_else(|| { + StorageError::Internal(format!( + "vector index {index_id} was selected as CREATING but has no catalog row" + )) + })?; + let written = crate::data::vector_index::backfill_vector_index( + &mut data_tx, + &table_id, + &meta, + &base_key_schema, + &attr_defs, + ) + .await?; + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + sqlx::query( + "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL \ + WHERE table_id = ? AND index_id = ?", + ) + .bind(&table_id) + .bind(&index_id) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + rebuilt += 1; + tracing::info!( + vectors_indexed = written, + "Reconciled incomplete vector index {index_id} on table {table_id}" + ); + } + Ok(rebuilt) + } } /// Update a single string column on the `tables` row. @@ -578,4 +885,243 @@ mod reconciler_tests { "reconciler must be idempotent" ); } + + /// A vector index left in `CREATING` by a crash must be rebuilt on startup and + /// flipped to `ACTIVE`, with `Backfilling` cleared. + /// + /// Asserts the rebuilt data table is POPULATED, not merely that the status + /// changed. Flipping the row to `ACTIVE` over an empty or missing data table + /// would satisfy a status-only assertion while every search returned nothing, + /// which is the exact failure the reconciler exists to prevent. + #[tokio::test] + async fn reconcile_rebuilds_a_creating_vector_index_and_populates_it() { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let account = "000000000000"; + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind(account) + .execute(&engine.pool) + .await + .expect("account"); + + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl(account, input) + .await + .expect("create table"); + + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE account_id = ? AND table_name = 't'") + .bind(account) + .fetch_one(&engine.pool) + .await + .expect("table_id"); + + // Two items already in the base table, so a rebuild has something to find. + let base_table = crate::data::data_table_name(&table_id); + for (pk, vec) in [("a", "[1,0]"), ("b", "[0,1]")] { + let item = format!( + r#"{{"pk":{{"S":"{pk}"}},"emb":{{"L":[{{"N":"{}"}},{{"N":"{}"}}]}}}}"#, + if vec == "[1,0]" { 1 } else { 0 }, + if vec == "[1,0]" { 0 } else { 1 } + ); + sqlx::query(&format!( + "INSERT INTO {base_table} (pk, item_data) VALUES (?, ?)" + )) + .bind(pk) + .bind(&item) + .execute(&engine.pool) + .await + .expect("seed item"); + } + + // Simulate a crash mid-build: a CREATING row, mid-backfill, whose data table + // was never created. + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status, backfilling) \ + VALUES (?, 'vidx-1', 'vidx', 2, 'COSINE', ?, ?, 'CREATING', 1)", + ) + .bind(&table_id) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert CREATING vector index"); + + let rebuilt = engine + .reconcile_incomplete_vector_indexes() + .await + .expect("reconcile"); + assert_eq!(rebuilt, 1, "one CREATING vector index should be rebuilt"); + + let (status, backfilling): (String, Option) = sqlx::query_as( + "SELECT index_status, backfilling FROM vector_indexes \ + WHERE table_id = ? AND index_id = 'vidx-1'", + ) + .bind(&table_id) + .fetch_one(&engine.pool) + .await + .expect("status"); + assert_eq!(status, "ACTIVE"); + assert_eq!( + backfilling, None, + "an ACTIVE index must not carry the Backfilling member" + ); + + // The rebuilt table must actually hold the rows. + let vec_table = crate::data::vector_table_name(&table_id, "vidx-1"); + let (rows,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(rows, 2, "the rebuild must backfill both seeded items"); + + // Idempotent, and the second run must not duplicate the rows it already + // wrote: the reconciler drops and rebuilds rather than appending. + assert_eq!( + engine + .reconcile_incomplete_vector_indexes() + .await + .expect("second reconcile"), + 0, + "reconciler must be idempotent" + ); + let (rows_after,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count again"); + assert_eq!(rows_after, 2, "a second pass must not duplicate rows"); + } + + /// The crash that actually happens: the data table exists and holds SOME of the + /// rows, because the process died partway through the backfill. + /// + /// The reconciler must drop and rebuild rather than resume, or the rows already + /// written are written again and a search returns the same item twice. The + /// previous test cannot catch this: its simulated crash leaves no data table at + /// all, so the drop is a no-op there. + #[tokio::test] + async fn reconcile_rebuilds_a_partially_backfilled_vector_index_without_duplicating() { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind("000000000000") + .execute(&engine.pool) + .await + .expect("account"); + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl("000000000000", input) + .await + .expect("create table"); + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE table_name = 't'") + .fetch_one(&engine.pool) + .await + .expect("table_id"); + + let base_table = crate::data::data_table_name(&table_id); + for pk in ["a", "b"] { + sqlx::query(&format!( + "INSERT INTO {base_table} (pk, item_data) VALUES (?, ?)" + )) + .bind(pk) + .bind(format!( + r#"{{"pk":{{"S":"{pk}"}},"emb":{{"L":[{{"N":"1"}},{{"N":"0"}}]}}}}"# + )) + .execute(&engine.pool) + .await + .expect("seed"); + } + + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status, backfilling) \ + VALUES (?, 'vidx-2', 'vidx', 2, 'COSINE', ?, ?, 'CREATING', 1)", + ) + .bind(&table_id) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert CREATING"); + + // The partial state: the data table exists and already holds one of the two + // rows, exactly as a crash midway through the scan would leave it. + let ks = vec![extenddb_core::types::KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: extenddb_core::types::KeyType::Hash, + }]; + let ad = vec![extenddb_core::types::AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: extenddb_core::types::ScalarAttributeType::S, + }]; + let mut tx = engine.pool.begin_with("BEGIN IMMEDIATE").await.expect("tx"); + SqliteEngine::create_vector_data_table(&mut tx, &table_id, "vidx-2", &ks, &ad) + .await + .expect("create partial data table"); + let meta = crate::data::vector_index::fetch_vector_indexes_for_table(&mut tx, &table_id) + .await + .expect("metas") + .into_iter() + .find(|m| m.index_id == "vidx-2") + .expect("meta"); + let item: extenddb_core::types::Item = + serde_json::from_str(r#"{"pk":{"S":"a"},"emb":{"L":[{"N":"1"},{"N":"0"}]}}"#) + .expect("item"); + crate::data::vector_index::insert_vector_row( + &mut tx, + &table_id, + &meta, + &item, + &ks, + &ad, + &["base_pk".to_owned()], + ) + .await + .expect("partial row"); + tx.commit().await.expect("commit"); + + let vec_table = crate::data::vector_table_name(&table_id, "vidx-2"); + let (before,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count before"); + assert_eq!(before, 1, "the setup must leave a genuinely partial table"); + + engine + .reconcile_incomplete_vector_indexes() + .await + .expect("reconcile"); + + let (after,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count after"); + assert_eq!( + after, 2, + "the rebuild must drop the partial table, not append to it: 3 rows would mean \ + item 'a' was indexed twice and a search would return it twice" + ); + } } diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index c16db413..910577e6 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -560,8 +560,280 @@ async fn an_unknown_distance_function_lists_the_measured_enum_order() { ); } -/// DescribeTable reports the index, and reports it ACTIVE with no `Backfilling` -/// member, which is what the service does for an index created by CreateTable. +/// Adding a vector index to a table that already holds items backfills them, so a +/// search finds data written before the index existed. +/// +/// This is the whole point of the `UpdateTable` create path, and it is what makes +/// the difference between an index and a filter over new writes. Items are written +/// first, deliberately, so a backfill that did nothing would fail here rather than +/// pass because the write path happened to cover it. +#[tokio::test] +async fn adding_a_vector_index_backfills_the_items_already_there() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_backfill"); + // A plain table: no vector index at creation. + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST" + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + wait_for_active(&name).await; + + // Three items with vectors, plus one without, which must be skipped rather + // than break the scan. + put_vector(&name, "near", None, &[1.0, 0.0]).await; + put_vector(&name, "far", None, &[-1.0, 0.0]).await; + put_vector(&name, "mid", None, &[0.0, 1.0]).await; + let (status, text) = call( + "PutItem", + &format!(r#"{{"TableName": "{name}", "Item": {{"pk": {{"S": "novec"}}}}}}"#), + ) + .await; + assert_eq!(status, 200, "PutItem without a vector failed: {text}"); + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{ + "TableName": "{name}", + "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Dimensions": 2, + "DistanceFunction": "COSINE", + "Projection": {{"ProjectionType": "ALL"}} + }}}}] + }}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable create failed: {text}"); + + let response = search(&name, &[1.0, 0.0], 10, None).await; + let results = response + .get("SearchResults") + .and_then(|r| r.as_array()) + .unwrap_or_else(|| panic!("no results in: {response}")); + assert_eq!( + results.len(), + 3, + "the three vector-carrying items must be backfilled, and the fourth skipped: {response}" + ); + assert_eq!( + results[0].pointer("/Item/pk/S").and_then(|v| v.as_str()), + Some("near"), + "backfilled rows must be searchable in distance order: {response}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// A backfilled index keeps working for writes that arrive afterwards, which is +/// what proves the write-path hook and the backfill agree on the row shape. +#[tokio::test] +async fn an_index_added_by_update_table_indexes_later_writes_too() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_bf_then_write"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST" + }}"# + ); + call("CreateTable", &body).await; + wait_for_active(&name).await; + put_vector(&name, "before", None, &[1.0, 0.0]).await; + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{ + "TableName": "{name}", + "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Dimensions": 2, + "DistanceFunction": "COSINE", + "Projection": {{"ProjectionType": "ALL"}} + }}}}] + }}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable create failed: {text}"); + + put_vector(&name, "after", None, &[0.9, 0.1]).await; + + let response = search(&name, &[1.0, 0.0], 10, None).await; + let results = response + .get("SearchResults") + .and_then(|r| r.as_array()) + .unwrap_or_else(|| panic!("no results in: {response}")); + let keys: Vec<&str> = results + .iter() + .filter_map(|r| r.pointer("/Item/pk/S").and_then(|v| v.as_str())) + .collect(); + assert!( + keys.contains(&"before") && keys.contains(&"after"), + "both the backfilled and the later-written item must be found: {keys:?}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Deleting a vector index stops it serving, and `DescribeTable` stops reporting it. +/// +/// Measured against this backend before the path existed: the delete returned 200 +/// while the index stayed ACTIVE and kept returning hits, which is why both halves +/// are asserted here rather than just the status code. +#[tokio::test] +async fn deleting_a_vector_index_stops_it_serving_and_reporting() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_idxdelete"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "a", None, &[1.0, 0.0]).await; + + // It serves before the delete, so the assertions below cannot pass vacuously. + let response = search(&name, &[1.0, 0.0], 5, None).await; + assert_eq!( + response + .get("SearchResults") + .and_then(|r| r.as_array()) + .map(Vec::len), + Some(1), + "the index must serve before it is deleted: {response}" + ); + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Delete": {{"IndexName": "vidx"}}}}]}}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable delete failed: {text}"); + + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + assert_eq!(status, 200, "DescribeTable failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let still_there = json + .pointer("/Table/VectorIndexes") + .and_then(|v| v.as_array()) + .is_some_and(|a| { + a.iter() + .any(|i| i.pointer("/IndexName").and_then(|n| n.as_str()) == Some("vidx")) + }); + assert!( + !still_there, + "a deleted index must not still be reported: {text}" + ); + + let body = format!( + r#"{{"TableName": "{name}", "IndexName": "vidx", "SearchVector": {}, "TopK": 5}}"#, + vector_json(&[1.0, 0.0]) + ); + let (status, text) = call("SearchVectors", &body).await; + assert_eq!( + status, 400, + "searching a deleted index must fail, not return stale hits: {text}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// The base items survive the index being deleted: only the index goes. +#[tokio::test] +async fn deleting_a_vector_index_leaves_the_items_alone() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_idxdel_items"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "a", None, &[1.0, 0.0]).await; + + call( + "UpdateTable", + &format!( + r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Delete": {{"IndexName": "vidx"}}}}]}}"# + ), + ) + .await; + + let (status, text) = call( + "GetItem", + &format!( + r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "a"}}}}, "ConsistentRead": true}}"# + ), + ) + .await; + assert_eq!(status, 200, "GetItem failed: {text}"); + assert!( + text.contains(r#""emb""#), + "the item and its vector attribute must be untouched: {text}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Creating an index that already exists, and deleting one that does not. +#[tokio::test] +async fn update_table_rejects_a_duplicate_create_and_a_missing_delete() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_idx_errs"); + create_vector_table(&name, 2, "COSINE", false).await; + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{ + "TableName": "{name}", + "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Dimensions": 2, + "DistanceFunction": "COSINE", + "Projection": {{"ProjectionType": "ALL"}} + }}}}] + }}"# + ), + ) + .await; + assert_eq!(status, 400, "a duplicate create must be rejected: {text}"); + assert!( + text.contains("already exists"), + "unexpected duplicate-create message: {text}" + ); + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Delete": {{"IndexName": "nosuch"}}}}]}}"# + ), + ) + .await; + assert_eq!(status, 400, "deleting a missing index must be rejected: {text}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// DescribeTable reports the vector index, and reports it ACTIVE with no +/// `Backfilling` member, which is what the service does for an index created by +/// CreateTable. #[tokio::test] async fn describe_table_reports_the_vector_index() { if skip_unless_supported().await { From 6031daf6c7699cd2b91ce7eda170b73dfb4ccadb Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 10 Aug 2026 20:41:43 +0000 Subject: [PATCH 12/25] test(vector): assert SearchVectors returns no Count field The contract commit emitted a top-level `Count` in `SearchVectorsOutput`. The service does not, and nothing in the suite could have caught it: a generated client ignores unknown top-level members, so an extra field is invisible unless something asserts its absence. Measured in sandbox 964157134968 us-east-1 on 2026-08-10 across five parameter variations, none of which produced a `Count`: no projection; with ReturnConsumedCapacity=INDEXES; with a ProjectionExpression naming the vector; with TopK larger than the item count; and with a projection naming a single non-key attribute. The botocore 1.43.64 model agrees, declaring exactly `SearchResults` and `ConsumedCapacity`. The test asserts three cases rather than one. The plain search covers the normal path. TopK exceeding the match count is included because that is precisely when a `Count` field is most tempting to add. The third case requests ConsumedCapacity and then asserts the response's top-level member set contains nothing beyond the two legal members, so any future extra field fails here rather than being discovered by a client. Negative control: restoring the `Count` field fails the test with `{"Count":2,"SearchResults":...}` in the message, so it discriminates. With the field removed, the vector suites are 26/26 with 0 filtered out. The vector-attribute projection rule Lee described (returned only when named in ProjectionExpression, even under Projection ALL) was re-confirmed live in the same probe run and is already asserted in both directions by `the_vector_attribute_is_returned_only_when_named`. --- tests/rust/src/vector_index_search.rs | 83 ++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 910577e6..045fd0cc 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -826,7 +826,10 @@ async fn update_table_rejects_a_duplicate_create_and_a_missing_delete() { ), ) .await; - assert_eq!(status, 400, "deleting a missing index must be rejected: {text}"); + assert_eq!( + status, 400, + "deleting a missing index must be rejected: {text}" + ); let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } @@ -876,3 +879,81 @@ async fn describe_table_reports_the_vector_index() { let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } + +/// `SearchVectorsOutput` carries no `Count` field. +/// +/// Measured against the live service in sandbox 964157134968 on 2026-08-10: the +/// response contains only `SearchResults` and, when asked for, +/// `ConsumedCapacity`. Five parameter variations were probed and none produced a +/// `Count`: no projection, `ReturnConsumedCapacity=INDEXES`, a +/// `ProjectionExpression`, a `TopK` larger than the item count, and a projection +/// naming a single non-key attribute. The botocore model agrees: `SearchVectorsOutput` +/// declares exactly those two members. +/// +/// This is asserted rather than left to review because an extra top-level field is +/// invisible to a generated client (it ignores unknown members), so nothing else in +/// the suite would ever catch its return. The `TopK`-exceeds-matches case is +/// included deliberately: a `Count` field is most tempting to add exactly when the +/// result set is shorter than `TopK`. +#[tokio::test] +async fn the_search_response_carries_no_count_field() { + if skip_unless_supported().await { + return; + } + let name = table_name("no_count"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "a", None, &[1.0, 0.0]).await; + put_vector(&name, "b", None, &[0.0, 1.0]).await; + + // Case 1: plain search. + let response = search(&name, &[1.0, 0.0], 2, None).await; + assert!( + response.get("Count").is_none(), + "SearchVectors must not return a Count field: {response}" + ); + assert!( + response.get("SearchResults").is_some(), + "SearchResults must be present: {response}" + ); + + // Case 2: TopK larger than the number of matches, where a Count field is the + // most tempting addition. + let response = search(&name, &[1.0, 0.0], 50, None).await; + assert!( + response.get("Count").is_none(), + "SearchVectors must not return a Count field when TopK exceeds the match \ + count: {response}" + ); + + // Case 3: with ConsumedCapacity requested, so the only two legal top-level + // members are both present and nothing else is. + let body = format!( + r#"{{ + "TableName": "{name}", + "IndexName": "vidx", + "SearchVector": [{{"N": "1"}}, {{"N": "0"}}], + "TopK": 2, + "ReturnConsumedCapacity": "INDEXES" + }}"# + ); + let (status, text) = call("SearchVectors", &body).await; + assert_eq!(status, 200, "SearchVectors failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let members: Vec<&str> = json + .as_object() + .unwrap_or_else(|| panic!("response is not an object: {text}")) + .keys() + .map(String::as_str) + .collect(); + for m in &members { + assert!( + matches!(*m, "SearchResults" | "ConsumedCapacity"), + "unexpected top-level member '{m}' in SearchVectorsOutput; the service \ + returns only SearchResults and ConsumedCapacity: {text}" + ); + } + assert!( + members.contains(&"ConsumedCapacity"), + "ConsumedCapacity must be present when requested: {text}" + ); +} From 05649e2a4b2b8eb6f1a7b5876be2fd1382fadcef Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 10 Aug 2026 21:22:04 +0000 Subject: [PATCH 13/25] fix(vector): project SearchSchema attributes, and enforce the on-demand and per-table index limits Three divergences from the documented service behaviour, found by checking the implementation against the developer guide and the quota table rather than against the shapes alone. 1. KEYS_ONLY dropped the inline filter attributes, breaking filtered search. A vector index does NOT use GSI KEYS_ONLY semantics. The documented rule is that KEYS_ONLY projects the base table primary key, the vector attribute AND any inline filter attributes declared in the SearchSchema. The write path called the shared GSI projection helper with an empty index key schema, so under KEYS_ONLY only the base keys survived. This was not merely a reporting difference. The inline filter is evaluated in `vector_search.rs` against the stored payload, so a missing filter attribute made `item.get(name)` return None for every row, every row failed the predicate, and a filtered search returned zero results. Reproduced before fixing: the new test failed with `{"SearchResults":[]}`, 0 against an expected 1. The SearchSchema attribute names are now carried on `VectorIndexMeta` and retained regardless of ProjectionType. The vector attribute itself is still excluded from the payload deliberately, because it lives in the `vec` column at f32 width and the search path rebuilds it from those bits. It escaped notice because `create_vector_table` hardcodes ProjectionType ALL, so all 18 pre-existing search tests exercised the single projection under which the distinction cannot appear. 2. A vector index was accepted on a PROVISIONED table. Vector indexes are supported only on on-demand tables. BillingMode defaults to PROVISIONED when absent, so an omitted BillingMode is rejected too, and the test asserts both spellings because an implementation checking only the explicit value would pass one and fail the other. 3. No limit on vector indexes per table. The documented default quota is 5. Asserted in both directions so an off-by-one cannot hide: five is accepted, six is refused. Ordering note: per-index shape validation runs before the two table-level checks. Which the service reports first is unobservable from outside, because botocore rejects a malformed index client-side before the request is sent, so the order that preserves the already-measured per-index messages is the one kept. One existing unit test built its positive fixture without a BillingMode and now needs PAY_PER_REQUEST; that fixture was updated rather than the check weakened. Verification: negative controls for both changes fail as required, the KEYS_ONLY one with an empty result set and the quota one with a 200 where a 400 is owed. Vector suites 28/28 with 0 filtered out. Full Rust integration suite 449 run, 448 passed; the single failure was `restore_active_completeness` throttling out because this deployment still had `throttling_enabled` set from an earlier capacity run, and it passes 1/1 with that setting cleared. 433/433 core unit tests. fmt exit 0, clippy -D warnings exit 0. --- crates/core/src/validation/mod.rs | 37 +++- .../storage-sqlite/src/data/vector_index.rs | 35 ++- tests/rust/src/vector_index_search.rs | 204 ++++++++++++++++++ 3 files changed, 271 insertions(+), 5 deletions(-) diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index b387e061..1d3ce161 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -213,6 +213,10 @@ const MAX_SEARCH_SCHEMA_HASH: usize = 1; /// wrong. The schema cap and the per-query cap are different numbers. const MAX_SEARCH_SCHEMA_INLINE_FILTERS: usize = 18; +/// Vector indexes per table. Documented default quota, adjustable only by AWS +/// Support, so the emulator enforces the default. +const MAX_VECTOR_INDEXES_PER_TABLE: usize = 5; + /// Validate the shape of a vector index search schema. /// /// Messages measured against the live service 2026-08-06 by signing raw requests, @@ -320,9 +324,37 @@ fn validate_vector_indexes(input: &CreateTableInput) -> Result<(), DynamoDbError let Some(vis) = input.vector_indexes.as_ref() else { return Ok(()); }; + + // Per-index shape first, then the table-level constraints. Which the service + // reports first is unobservable from outside, because botocore rejects a + // malformed index client-side before the request is sent, so the order that + // preserves the already-measured per-index messages is the right one to keep. for (position, vi) in vis.iter().enumerate() { validate_one_vector_index(vi, position + 1, "vectorIndexes")?; } + + // Vector indexes are supported only on on-demand tables. Documented under + // "Requirements and limitations" and again in the quota table, which lists + // vector index capacity mode as on-demand only. `BillingMode` defaults to + // PROVISIONED when absent, so an omitted BillingMode is a rejection too. + if !vis.is_empty() + && input.billing_mode.unwrap_or(BillingMode::Provisioned) != BillingMode::PayPerRequest + { + return Err(DynamoDbError::ValidationException( + "One or more parameter values were invalid: Vector indexes are supported only \ + on tables that use PAY_PER_REQUEST billing mode" + .to_owned(), + )); + } + + if vis.len() > MAX_VECTOR_INDEXES_PER_TABLE { + return Err(DynamoDbError::ValidationException(format!( + "One or more parameter values were invalid: Number of vector indexes {} \ + exceeds the limit of {MAX_VECTOR_INDEXES_PER_TABLE}", + vis.len() + ))); + } + Ok(()) } @@ -1819,9 +1851,12 @@ mod tests { other => panic!("expected ValidationException, got {other:?}"), } - // Present on every element: accepted. + // Present on every element: accepted. PAY_PER_REQUEST is required because a + // vector index is only valid on an on-demand table, which this function now + // enforces; the projection is what this case is about. let input = CreateTableInput { table_name: "t".to_owned(), + billing_mode: Some(BillingMode::PayPerRequest), vector_indexes: Some(vec![spec(all()), spec(all())]), ..Default::default() }; diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs index 3e818c9a..920d7f73 100644 --- a/crates/storage-sqlite/src/data/vector_index.rs +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -34,6 +34,16 @@ pub(crate) struct VectorIndexMeta { /// The single HASH element's attribute name, when the index declares one. /// `None` means the index is unscoped and every row shares one partition. pub hash_attribute_name: Option, + /// Every attribute named by the SearchSchema, HASH and INLINE_FILTER alike. + /// + /// These are projected regardless of `ProjectionType`, which is the documented + /// rule for a vector index and is NOT GSI `KEYS_ONLY` semantics: `KEYS_ONLY` + /// on a vector index projects the base primary key, the vector attribute and + /// the inline filter attributes. Withholding them is not merely a reporting + /// difference, it breaks search: the filter is evaluated against the stored + /// payload, so a missing filter attribute makes every row fail the predicate + /// and a filtered search match nothing. + pub search_schema_attribute_names: Vec, } /// Load the vector indexes of a table. @@ -59,17 +69,22 @@ pub(crate) async fn fetch_vector_indexes_for_table( let attr: extenddb_core::types::VectorAttribute = serde_json::from_str(&vector_attribute) .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))?; - let hash_attribute_name = match search_schema.as_deref() { + let (hash_attribute_name, search_schema_attribute_names) = match search_schema.as_deref() { Some(json) => { let elements: Vec = serde_json::from_str(json) .map_err(|e| StorageError::Internal(format!("search_schema: {e}")))?; - elements - .into_iter() + let hash = elements + .iter() .find(|e| e.element_type == SearchSchemaElementType::Hash) + .map(|e| e.attribute_name.clone()); + let all = elements + .into_iter() .map(|e| e.attribute_name) + .collect::>(); + (hash, all) } - None => None, + None => (None, Vec::new()), }; let projection: extenddb_core::types::Projection = serde_json::from_str(&projection) .map_err(|e| StorageError::Internal(format!("vector projection: {e}")))?; @@ -80,6 +95,7 @@ pub(crate) async fn fetch_vector_indexes_for_table( })?, vector_attribute_name: attr.attribute_name, hash_attribute_name, + search_schema_attribute_names, projection, }); } @@ -275,6 +291,17 @@ pub(crate) async fn insert_vector_row( // the index declares and no more. let mut projected = super::index::project_item_for_index(item, &[], base_key_schema, &meta.projection); + // The SearchSchema attributes are always projected, whatever the + // ProjectionType. See `search_schema_attribute_names` for why: the inline + // filter is evaluated against this payload, so dropping the attribute would + // silently turn every filtered search into a zero-result search. + for name in &meta.search_schema_attribute_names { + if !projected.contains_key(name) + && let Some(v) = item.get(name) + { + projected.insert(name.clone(), v.clone()); + } + } // The vector itself is not kept in the payload: it is already in the `vec` // column as `f32`, which is the width the service validates against, and the // search path rebuilds the attribute from those bits. Keeping a verbatim diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 045fd0cc..f7f43bec 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -957,3 +957,207 @@ async fn the_search_response_carries_no_count_field() { "ConsumedCapacity must be present when requested: {text}" ); } + +/// A `KEYS_ONLY` vector index still projects its search-schema attributes, so a +/// filtered search works and returns them. +/// +/// The documented rule is specific and is NOT GSI `KEYS_ONLY` semantics: on a +/// vector index, `KEYS_ONLY` projects the base table primary key, the vector +/// attribute, AND any inline filter attributes declared in the SearchSchema +/// (developer guide, "Projections"). GSI `KEYS_ONLY` projects base keys plus the +/// index's own key attributes and nothing else. +/// +/// This went unasserted because `create_vector_table` hardcodes +/// `ProjectionType: ALL`, so all 18 pre-existing search tests exercise the one +/// projection under which the distinction cannot appear. Under `KEYS_ONLY` the +/// inline filter attribute was dropped from the stored payload, and because the +/// filter is evaluated against that payload, `item.get(name)` returned `None` for +/// every row and a filtered search matched nothing at all. +/// +/// Asserted three ways so a partial fix cannot pass: the filtered search must +/// find the item, the filter attribute must come back in the result, and a +/// non-projected attribute must NOT come back (otherwise this would also pass +/// against an implementation that quietly ignored the projection and behaved like +/// `ALL`). +#[tokio::test] +async fn keys_only_vector_index_still_filters_on_its_search_schema() { + if skip_unless_supported().await { + return; + } + let name = table_name("keysonly_filter"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [ + {{"AttributeName": "pk", "AttributeType": "S"}}, + {{"AttributeName": "category", "AttributeType": "S"}} + ], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": 2, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "SearchSchema": [ + {{"AttributeName": "category", "SearchSchemaElementType": "INLINE_FILTER"}} + ], + "Projection": {{"ProjectionType": "KEYS_ONLY"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 200, "CreateTable with KEYS_ONLY failed: {text}"); + wait_for_active(&name).await; + + // `note` is deliberately non-projected under KEYS_ONLY. + let put = format!( + r#"{{ + "TableName": "{name}", + "Item": {{ + "pk": {{"S": "a"}}, + "category": {{"S": "books"}}, + "note": {{"S": "not projected"}}, + "emb": {{"L": [{{"N": "1"}}, {{"N": "0"}}]}} + }} + }}"# + ); + let (status, text) = call("PutItem", &put).await; + assert_eq!(status, 200, "PutItem failed: {text}"); + + let search = format!( + r#"{{ + "TableName": "{name}", + "IndexName": "vidx", + "SearchVector": [{{"N": "1"}}, {{"N": "0"}}], + "TopK": 5, + "SearchConditionExpression": "category = :c", + "ExpressionAttributeValues": {{":c": {{"S": "books"}}}} + }}"# + ); + let (status, text) = call("SearchVectors", &search).await; + assert_eq!(status, 200, "filtered SearchVectors failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let results = json + .get("SearchResults") + .and_then(|v| v.as_array()) + .unwrap_or_else(|| panic!("no results array in: {text}")); + assert_eq!( + results.len(), + 1, + "a KEYS_ONLY index must still match its inline filter; the filter \ + attribute has to be projected for the filter to be evaluable: {text}" + ); + + let item = results[0] + .get("Item") + .unwrap_or_else(|| panic!("no item in: {text}")); + assert!( + item.get("category").is_some(), + "KEYS_ONLY must project inline filter attributes, so 'category' must be \ + returned: {item}" + ); + assert!( + item.get("pk").is_some(), + "KEYS_ONLY must project the base table primary key: {item}" + ); + assert!( + item.get("note").is_none(), + "KEYS_ONLY must NOT project an unrelated non-key attribute; returning it \ + would mean the projection was ignored entirely: {item}" + ); +} + +/// A vector index requires on-demand billing, and there is a documented cap of +/// five vector indexes per table. +/// +/// Both are documented service constraints that the emulator previously accepted: +/// "Vector indexes are supported only on tables that use on-demand capacity mode" +/// under Requirements and limitations, and "Vector indexes per table: 5" in the +/// quota table. Accepting a combination the service refuses is the more dangerous +/// direction of divergence, because code written against the emulator then fails +/// on first contact with DynamoDB. +/// +/// The PROVISIONED case is asserted twice, once explicitly and once with +/// `BillingMode` omitted, because the field defaults to PROVISIONED when absent +/// and an implementation that only checked the explicit value would pass the first +/// and fail the second. The five-index case asserts the boundary in both +/// directions so an off-by-one cannot hide: five is accepted, six is refused. +#[tokio::test] +async fn vector_indexes_require_on_demand_and_cap_at_five() { + if skip_unless_supported().await { + return; + } + + fn one_index(n: usize) -> String { + format!( + r#"{{ + "IndexName": "vidx{n}", + "Dimensions": 2, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}} + }}"# + ) + } + fn body(name: &str, billing: Option<&str>, count: usize) -> String { + let billing_line = match billing { + Some(b) => format!(r#""BillingMode": "{b}","#), + None => String::new(), + }; + let indexes = (0..count).map(one_index).collect::>().join(", "); + format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + {billing_line} + "VectorIndexes": [{indexes}] + }}"# + ) + } + + // PROVISIONED, stated explicitly. ProvisionedThroughput is supplied so the + // rejection cannot be attributed to a missing-throughput error instead. + let name = table_name("vi_prov"); + let mut prov = body(&name, Some("PROVISIONED"), 1); + prov = prov.replace( + r#""VectorIndexes""#, + r#""ProvisionedThroughput": {"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, "VectorIndexes""#, + ); + let (status, text) = call("CreateTable", &prov).await; + assert_eq!( + status, 400, + "a PROVISIONED table must not accept a vector index: {text}" + ); + assert!( + text.contains("PAY_PER_REQUEST"), + "the error should name the required billing mode: {text}" + ); + + // BillingMode omitted, which defaults to PROVISIONED. + let name = table_name("vi_default"); + let (status, text) = call("CreateTable", &body(&name, None, 1)).await; + assert_eq!( + status, 400, + "an omitted BillingMode defaults to PROVISIONED and must be refused: {text}" + ); + + // Six indexes: over the documented cap of five. + let name = table_name("vi_six"); + let (status, text) = call("CreateTable", &body(&name, Some("PAY_PER_REQUEST"), 6)).await; + assert_eq!( + status, 400, + "six vector indexes exceeds the documented cap of five: {text}" + ); + + // Five indexes: exactly at the cap, must be accepted. This is what makes the + // check above an off-by-one test rather than a blanket refusal. + let name = table_name("vi_five"); + let (status, text) = call("CreateTable", &body(&name, Some("PAY_PER_REQUEST"), 5)).await; + assert_eq!( + status, 200, + "five vector indexes is at the documented cap and must be accepted: {text}" + ); + wait_for_active(&name).await; +} From f10b53a1d55cb5dfeca820603e1aaee1d0d6cf2a Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Tue, 11 Aug 2026 08:23:47 +0000 Subject: [PATCH 14/25] feat(vector): propagate vector index maintenance asynchronously Vector indexes are eventually consistent, the same model the service gives them and the same model a GSI has here. Maintenance was applied synchronously inside the base write transaction, which is stricter than the service and made a search immediately after a write return the new item where the service might not. Maintenance now runs on the existing gsi_pending queue rather than a queue of its own, which is what makes the asynchronous path correct rather than merely deferred. None of these properties would come free otherwise: * Crash safety. The pending row is inserted in the base write transaction, so the item is never committed with its index work not yet durable. The worker claims and applies in one transaction, so a crash mid-apply rolls back and retries. At-least-once is safe because an apply is idempotent: it deletes the base key's row and reinserts it from the snapshotted item. * Per-key ordering across index kinds. The row's partition is a hash of the base key, so a vector row and a GSI row for one item share a partition, ready_at is clamped monotonic within it, and the worker drains in id order. * Snapshot semantics. The row carries its own VectorApplyContext, so the worker needs no catalog read and an index dropped or redefined between enqueue and apply cannot make a queued write unapplicable. maintain_vector_indexes is the single entry point and owns the sync/async choice, so the seven write paths each have one call site. A delay of 0 still applies inline; anything else enqueues. A write whose new item carries no vector still enqueues, because the removal is the work in that case. One queue carrying two kinds of work needs a discriminant. PendingApplyContext is untagged, which is load-bearing rather than stylistic: a GSI context serializes to exactly the bytes it did before, so rows already on disk still deserialize. That matters because an unparseable index_context is treated as a poison row and DROPPED, so a tagged representation would have silently discarded every in-flight GSI update across an upgrade. The variants are unambiguous by shape, a GSI context requires `index` and a vector context requires `vector`, and a test pins a verbatim legacy payload. Also fixes a latent defect in the queue that the ordering guarantee above depends on. Apply order was resting on the order of DELETE ... RETURNING output, which SQLite defines as undefined and which demonstrably ignores the subselect's ORDER BY (a DESC subselect still returns ascending). Per-key FIFO was therefore accidental: two writes to one item claimed in the same batch could be applied newest-first, and because each apply overwrites the row wholesale, the earlier write would win and the later be lost. Claimed rows are now sorted by id in code. Verification. 9 new tests, each with a negative control: * Forcing the synchronous path fails the asynchrony test. * Reversing the apply-order sort fails the write-order test with exactly the lost-update symptom (the earlier write surviving). * Skipping the enqueue for a vectorless item fails the removal test. * Making the worker's vector apply a no-op fails the converted integration tests with "index never converged", proving they depend on the async path rather than on residual synchronous behaviour. The integration suite asserts convergence rather than a single search, because under eventual consistency one search proves nothing in either direction: a missing item may not have propagated and a removed item may not have been removed yet. Bounded polling, not fixed sleeps, which are simultaneously too slow when propagation is immediate and too short on a loaded machine. Where an absence is the subject, the test first waits for something ordered behind it in the same apply, so the absence cannot pass merely because the write was late. Honest note on one control that did not discriminate: the missing-table tolerance in apply_vector_context is log hygiene, not data safety. Removing it leaves the tests passing, because the pre-existing per-row savepoint already contains the error and reaches the same end state. What it changes is that a routine DeleteTable race stops emitting an ERROR line. Both comments say so rather than claiming more. 449/449 rust integration tests live over HTTP with throttling enabled and 0 filtered out, 40/40 storage-sqlite unit tests, fmt and clippy -D warnings clean. --- crates/storage-sqlite/src/data/delete_item.rs | 15 +- crates/storage-sqlite/src/data/index.rs | 227 +++++++++- crates/storage-sqlite/src/data/mod.rs | 2 +- crates/storage-sqlite/src/data/put_item.rs | 38 +- .../storage-sqlite/src/data/transactions.rs | 73 ++-- crates/storage-sqlite/src/data/update_item.rs | 36 +- .../storage-sqlite/src/data/vector_index.rs | 242 ++++++++--- crates/storage-sqlite/src/workers.rs | 402 +++++++++++++++++- tests/rust/src/vector_index_search.rs | 225 ++++++++-- 9 files changed, 1081 insertions(+), 179 deletions(-) diff --git a/crates/storage-sqlite/src/data/delete_item.rs b/crates/storage-sqlite/src/data/delete_item.rs index 0eea72f1..e7d4ba5b 100644 --- a/crates/storage-sqlite/src/data/delete_item.rs +++ b/crates/storage-sqlite/src/data/delete_item.rs @@ -70,18 +70,23 @@ impl SqliteEngine { ) .await?; } - // Vector rows for this base item are removed in the same - // transaction. `new_item` is None, so this is a pure removal. - if !key_info.vector_indexes.is_empty() { - crate::data::vector_index::sync_vector_indexes( + // Vector rows for this base item are removed too. `new_item` is None, + // so this is a pure removal, applied in this transaction when the + // propagation delay is 0 and enqueued otherwise. + if !key_info.vector_indexes.is_empty() + && crate::data::vector_index::maintain_vector_indexes( &mut tx, &key_info.table_id, &key_info.key_schema, &key_info.attribute_definitions, old.as_ref(), None, + system_delay, ) - .await?; + .await? + > 0 + { + enqueued = true; } if enqueue_async_indexes( &mut tx, diff --git a/crates/storage-sqlite/src/data/index.rs b/crates/storage-sqlite/src/data/index.rs index edd6705d..ea1c84de 100644 --- a/crates/storage-sqlite/src/data/index.rs +++ b/crates/storage-sqlite/src/data/index.rs @@ -77,6 +77,59 @@ pub(crate) struct GsiApplyContext { pub(crate) index: GsiIndexDef, } +/// What one `gsi_pending` row asks the worker to do: maintain a GSI, or maintain a +/// vector index. One queue serves both so that updates to a single base item stay +/// ordered across index kinds, which two queues drained independently could not +/// guarantee. +/// +/// `untagged` is load-bearing, not stylistic. A GSI context serializes to exactly +/// the bytes it did before this enum existed, so rows already on disk from an +/// earlier version still deserialize, and rows written now are still readable by +/// one. That matters more than it looks: an unparseable `index_context` is treated +/// as a poison row and *dropped*, so a tagged representation would silently discard +/// every in-flight GSI update across an upgrade. +/// +/// The variants are unambiguous by shape rather than by tag: a GSI context requires +/// `index` and a vector context requires `vector`, and neither carries the other's +/// field, so exactly one variant can ever match. `vector_index_context_tests` pins +/// both directions, including a verbatim legacy payload. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub(crate) enum PendingApplyContext { + Gsi(GsiApplyContext), + Vector(super::vector_index::VectorApplyContext), +} + +impl PendingApplyContext { + /// The base table's key schema, which both kinds carry and the queue needs in + /// order to place the row in its base key's partition. + fn base_key_schema(&self) -> &[KeySchemaElement] { + match self { + Self::Gsi(c) => &c.base_key_schema, + Self::Vector(c) => &c.base_key_schema, + } + } +} + +/// Apply one claimed row to whichever index kind it describes. +/// +/// Both arms tolerate a data table that no longer exists, so a base table or index +/// dropped while a row was in flight is applied-as-skip rather than logged and +/// dropped as unprocessable. +pub(crate) async fn apply_pending_context( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + old_item: Option<&Item>, + new_item: Option<&Item>, + context: &PendingApplyContext, +) -> Result<(), StorageError> { + match context { + PendingApplyContext::Gsi(c) => apply_claimed_row(tx, old_item, new_item, c).await, + PendingApplyContext::Vector(c) => { + super::vector_index::apply_vector_context(tx, old_item, new_item, c).await + } + } +} + /// Metadata for a single index, used on the write path and by the GSI worker. pub(crate) struct IndexMeta { pub(super) index_id: String, @@ -161,27 +214,36 @@ pub(crate) async fn enqueue_async_indexes( projection: idx.projection.clone(), }, }; - enqueue_gsi_pending(tx, &key_info.table_id, old_item, new_item, delay, &context).await?; + enqueue_pending_row( + tx, + &key_info.table_id, + old_item, + new_item, + delay, + &PendingApplyContext::Gsi(context), + ) + .await?; enqueued += 1; } Ok(enqueued) } -/// Insert one self-describing `gsi_pending` row for a single index inside the -/// base write transaction (zero crash window). +/// Insert one self-describing `gsi_pending` row inside the base write transaction +/// (zero crash window). Shared by the GSI and vector write paths, which is what puts +/// both index kinds in one totally ordered queue. /// -/// `delay_ms` is the index's effective delay; a jitter in `[delay/2, delay]` -/// is applied. `ready_at` is clamped to `max(now + jitter, MAX(ready_at) in the -/// base key's partition)` so a later write that draws a smaller jitter can never -/// become eligible before an earlier one — preserving per-key FIFO when the -/// worker drains the partition in `id` order. -pub(crate) async fn enqueue_gsi_pending( +/// `delay_ms` is the effective delay; a jitter in `[delay/2, delay]` is applied. +/// `ready_at` is clamped to `max(now + jitter, MAX(ready_at) in the base key's +/// partition)` so a later write that draws a smaller jitter can never become +/// eligible before an earlier one — preserving per-key FIFO when the worker drains +/// the partition in `id` order. +pub(crate) async fn enqueue_pending_row( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, table_id: &str, old_item: Option<&Item>, new_item: Option<&Item>, delay_ms: u64, - context: &GsiApplyContext, + context: &PendingApplyContext, ) -> Result<(), StorageError> { let old_json = old_item .map(serde_json::to_string) @@ -196,9 +258,11 @@ pub(crate) async fn enqueue_gsi_pending( // Route all updates for one base item to a single partition (per-key FIFO). // The base key is immutable: `new_item` carries it for puts/updates, - // `old_item` for deletes. + // `old_item` for deletes. Both context kinds describe the same base table, so + // a GSI row and a vector row for one item hash to the same partition and stay + // mutually ordered. let worker_partition = match new_item.or(old_item) { - Some(item) => partition_for(&composite_pk_to_text(item, &context.base_key_schema)?), + Some(item) => partition_for(&composite_pk_to_text(item, context.base_key_schema())?), None => 0, }; @@ -434,9 +498,9 @@ pub(crate) fn index_sk_column(index: usize, sk_type: ScalarAttributeType) -> Str } /// True if a storage error is a "missing table" error (the `_ddb_*` index table -/// was dropped, e.g. the base table was deleted while a `gsi_pending` row was -/// in flight). Such rows are benignly skipped. -fn is_no_such_table(e: &StorageError) -> bool { +/// or a `_vidx_*` vector table was dropped, e.g. the base table was deleted +/// while a `gsi_pending` row was in flight). Such rows are benignly skipped. +pub(crate) fn is_no_such_table(e: &StorageError) -> bool { matches!(e, StorageError::Internal(msg) if msg.contains("no such table")) } @@ -486,3 +550,136 @@ pub(crate) async fn apply_claimed_row( } Ok(()) } + +#[cfg(test)] +mod pending_context_tests { + use super::{GsiApplyContext, GsiIndexDef, PendingApplyContext}; + use crate::data::vector_index::{VectorApplyContext, VectorIndexMeta}; + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, Projection, ProjectionType, + ScalarAttributeType, + }; + + /// A context written by a build that predates vector rows, verbatim. It must + /// still deserialize. + /// + /// This is the test that protects an upgrade. A row whose `index_context` fails + /// to parse is treated as poison and DROPPED, so if the representation had + /// changed incompatibly, every GSI update in flight at the moment of the upgrade + /// would have been discarded silently, with the item written and its index never + /// catching up. + const LEGACY_GSI_CONTEXT: &str = r#"{ + "base_key_schema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "attribute_definitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "index": { + "index_id": "idx-1", + "key_schema": [{"AttributeName": "gsipk", "KeyType": "HASH"}], + "projection": {"ProjectionType": "ALL"} + } + }"#; + + fn base_ks() -> Vec { + vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }] + } + + fn base_ad() -> Vec { + vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }] + } + + fn gsi_context() -> GsiApplyContext { + GsiApplyContext { + base_key_schema: base_ks(), + attribute_definitions: base_ad(), + index: GsiIndexDef { + index_id: "idx-1".to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "gsipk".to_owned(), + key_type: KeyType::Hash, + }], + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + }, + } + } + + fn vector_context() -> VectorApplyContext { + VectorApplyContext { + base_key_schema: base_ks(), + attribute_definitions: base_ad(), + table_id: "t-1".to_owned(), + vector: VectorIndexMeta { + index_id: "vidx-1".to_owned(), + dimensions: 2, + vector_attribute_name: "emb".to_owned(), + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + hash_attribute_name: Some("tenant".to_owned()), + search_schema_attribute_names: vec!["tenant".to_owned()], + }, + } + } + + #[test] + fn a_legacy_gsi_context_still_deserializes_as_a_gsi_row() { + let parsed: PendingApplyContext = + serde_json::from_str(LEGACY_GSI_CONTEXT).expect("legacy context must still parse"); + match parsed { + PendingApplyContext::Gsi(c) => assert_eq!(c.index.index_id, "idx-1"), + PendingApplyContext::Vector(_) => { + panic!("a legacy GSI context must not be read as a vector row") + } + } + } + + /// The bytes a GSI row writes today are the bytes it wrote before the enum + /// existed, so a row written by this build is still readable by the previous + /// one. `untagged` is what buys this, and this test is what keeps it. + #[test] + fn wrapping_a_gsi_context_does_not_change_its_serialized_form() { + let bare = serde_json::to_string(&gsi_context()).expect("bare"); + let wrapped = + serde_json::to_string(&PendingApplyContext::Gsi(gsi_context())).expect("wrapped"); + assert_eq!( + bare, wrapped, + "the queue's on-disk format must not change for GSI rows" + ); + } + + /// The two kinds are told apart by shape: a vector context has `vector` and no + /// `index`, so it can only match one variant. Without this the untagged enum + /// would be free to guess wrong and a vector row would be applied as a GSI. + #[test] + fn a_vector_context_round_trips_and_is_never_read_as_a_gsi() { + let json = serde_json::to_string(&PendingApplyContext::Vector(vector_context())) + .expect("serialize"); + assert!( + !json.contains("\"index\":"), + "a vector context must not carry the GSI discriminant field: {json}" + ); + let parsed: PendingApplyContext = serde_json::from_str(&json).expect("deserialize"); + match parsed { + PendingApplyContext::Vector(c) => { + assert_eq!(c.vector.index_id, "vidx-1"); + assert_eq!(c.table_id, "t-1"); + assert_eq!(c.vector.dimensions, 2); + assert_eq!(c.vector.hash_attribute_name.as_deref(), Some("tenant")); + assert_eq!(c.vector.search_schema_attribute_names, ["tenant"]); + assert_eq!( + c.vector.projection.projection_type, + ProjectionType::KeysOnly + ); + } + PendingApplyContext::Gsi(_) => panic!("a vector context must not be read as a GSI row"), + } + } +} diff --git a/crates/storage-sqlite/src/data/mod.rs b/crates/storage-sqlite/src/data/mod.rs index 7918464c..20d5db83 100644 --- a/crates/storage-sqlite/src/data/mod.rs +++ b/crates/storage-sqlite/src/data/mod.rs @@ -42,7 +42,7 @@ mod update_item; pub(crate) mod vector_index; pub(crate) use index::{ - GsiApplyContext, apply_claimed_row, insert_index_row_multi, project_item_for_index, + PendingApplyContext, apply_pending_context, insert_index_row_multi, project_item_for_index, }; pub(crate) use tx_helpers::upsert_item_in_tx; diff --git a/crates/storage-sqlite/src/data/put_item.rs b/crates/storage-sqlite/src/data/put_item.rs index a6b71408..3bcd6304 100644 --- a/crates/storage-sqlite/src/data/put_item.rs +++ b/crates/storage-sqlite/src/data/put_item.rs @@ -102,22 +102,7 @@ impl SqliteEngine { ) .await?; } - // Vector indexes, maintained in the same transaction. Gated on the cached - // key info so a table without them costs no extra query. Deliberately - // outside the `indexes` guard above: a table may have a vector index and - // no GSI or LSI at all. - if !key_info.vector_indexes.is_empty() { - crate::data::vector_index::sync_vector_indexes( - &mut tx, - &key_info.table_id, - &key_info.key_schema, - &key_info.attribute_definitions, - old.as_ref(), - Some(&item), - ) - .await?; - } - let enqueued = enqueue_async_indexes( + let enqueued_gsi = enqueue_async_indexes( &mut tx, key_info, &indexes, @@ -127,6 +112,27 @@ impl SqliteEngine { ) .await? > 0; + // Vector indexes: applied in this transaction when the propagation delay is + // 0, otherwise enqueued alongside the async GSI work. Gated on the cached + // key info so a table without them costs no extra query, and deliberately + // outside the `indexes` guard above: a table may have a vector index and no + // GSI or LSI at all. + let enqueued_vector = if key_info.vector_indexes.is_empty() { + false + } else { + crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + Some(&item), + system_delay, + ) + .await? + > 0 + }; + let enqueued = enqueued_gsi || enqueued_vector; if let Some(capture) = stream { write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), Some(&item)) diff --git a/crates/storage-sqlite/src/data/transactions.rs b/crates/storage-sqlite/src/data/transactions.rs index 4709da25..77d43197 100644 --- a/crates/storage-sqlite/src/data/transactions.rs +++ b/crates/storage-sqlite/src/data/transactions.rs @@ -159,7 +159,17 @@ impl SqliteEngine { } } - // Persist async GSI work for each op inside the same transaction. + // Persist index work for each op inside the same transaction: async GSI + // rows, and vector maintenance which is applied here when the propagation + // delay is 0 and enqueued otherwise. + // + // Deliberately after every op is staged rather than inside each op, so + // there is one call site instead of three. Both paths stay atomic with the + // item writes either way: the synchronous apply runs in this transaction, + // and a pending row is inserted into it, so a cancelled transaction leaves + // neither behind. The vector apply reads only the op's own before/after + // items, never the base table, so its position relative to the other ops' + // staged writes cannot change its result. let mut needs_notify = false; for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { let indexes = &table_indexes[op_table_name(op)]; @@ -176,6 +186,22 @@ impl SqliteEngine { if n > 0 { needs_notify = true; } + let key_info = op_key_info(op); + if !key_info.vector_indexes.is_empty() { + let n = crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old_item.as_ref(), + new_item.as_ref(), + system_delay, + ) + .await?; + if n > 0 { + needs_notify = true; + } + } } } @@ -307,21 +333,6 @@ async fn execute_transact_write_op( .await .map_err(TxnOpError::Storage)?; } - // Vector indexes share the transaction, so a rolled-back transactional - // write cannot leave a vector row behind. Outside the `indexes` guard - // because a table may have a vector index and no GSI or LSI. - if !key_info.vector_indexes.is_empty() { - crate::data::vector_index::sync_vector_indexes( - tx, - &key_info.table_id, - &key_info.key_schema, - &key_info.attribute_definitions, - existing.as_ref(), - Some(item), - ) - .await - .map_err(TxnOpError::Storage)?; - } Ok((existing, Some((*item).clone()))) } TransactWriteOp::Delete { @@ -365,21 +376,6 @@ async fn execute_transact_write_op( .await .map_err(TxnOpError::Storage)?; } - // Vector indexes share the transaction, so a rolled-back transactional - // write cannot leave a vector row behind. Outside the `indexes` guard - // because a table may have a vector index and no GSI or LSI. - if !key_info.vector_indexes.is_empty() { - crate::data::vector_index::sync_vector_indexes( - tx, - &key_info.table_id, - &key_info.key_schema, - &key_info.attribute_definitions, - existing.as_ref(), - None, - ) - .await - .map_err(TxnOpError::Storage)?; - } Ok((existing, None)) } TransactWriteOp::Update { @@ -445,21 +441,6 @@ async fn execute_transact_write_op( .await .map_err(TxnOpError::Storage)?; } - // Vector indexes share the transaction, so a rolled-back transactional - // write cannot leave a vector row behind. Outside the `indexes` guard - // because a table may have a vector index and no GSI or LSI. - if !key_info.vector_indexes.is_empty() { - crate::data::vector_index::sync_vector_indexes( - tx, - &key_info.table_id, - &key_info.key_schema, - &key_info.attribute_definitions, - existing.as_ref(), - Some(&item), - ) - .await - .map_err(TxnOpError::Storage)?; - } Ok((existing, Some(item))) } TransactWriteOp::ConditionCheck { diff --git a/crates/storage-sqlite/src/data/update_item.rs b/crates/storage-sqlite/src/data/update_item.rs index b357d28c..03e2eeb0 100644 --- a/crates/storage-sqlite/src/data/update_item.rs +++ b/crates/storage-sqlite/src/data/update_item.rs @@ -98,21 +98,7 @@ impl SqliteEngine { ) .await?; } - // Vector indexes, maintained in the same transaction. Gated on the cached - // key info so a table without them costs no extra query, and kept outside - // the `indexes` guard because a table may have a vector index and no GSI. - if !key_info.vector_indexes.is_empty() { - crate::data::vector_index::sync_vector_indexes( - &mut tx, - &key_info.table_id, - &key_info.key_schema, - &key_info.attribute_definitions, - old.as_ref(), - Some(&item), - ) - .await?; - } - let enqueued = enqueue_async_indexes( + let enqueued_gsi = enqueue_async_indexes( &mut tx, key_info, &indexes, @@ -122,6 +108,26 @@ impl SqliteEngine { ) .await? > 0; + // Vector indexes: applied in this transaction when the propagation delay is + // 0, otherwise enqueued alongside the async GSI work. Gated on the cached + // key info so a table without them costs no extra query, and kept outside + // the `indexes` guard because a table may have a vector index and no GSI. + let enqueued_vector = if key_info.vector_indexes.is_empty() { + false + } else { + crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + Some(&item), + system_delay, + ) + .await? + > 0 + }; + let enqueued = enqueued_gsi || enqueued_vector; if let Some(capture) = stream { write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), Some(&item)) diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs index 920d7f73..2b70fe6b 100644 --- a/crates/storage-sqlite/src/data/vector_index.rs +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -3,16 +3,37 @@ //! Vector index maintenance on the write path. //! -//! Applied synchronously inside the base write transaction, so a vector row can -//! never survive a rolled-back item write or be lost to a crash between the two. +//! Vector indexes are **eventually consistent**, the same model the service gives +//! them and the same model a GSI has here, so maintenance runs on the existing +//! `gsi_pending` queue rather than in the base write transaction. [`maintain_vector_indexes`] +//! is the single entry point and owns that choice: a propagation delay of 0 keeps +//! the work in the caller's transaction, any other delay enqueues it. //! -//! That is **stricter than the service**, which makes vector indexes eventually -//! consistent like a GSI, and it is a deliberate first step rather than the final -//! answer. Being more consistent than required cannot produce a wrong answer, only -//! a fresher one, whereas the reverse can. The asynchronous path should reuse the -//! existing `gsi_pending` queue, which already provides crash recovery, per-key -//! FIFO ordering and a configurable delay; until it does, a search immediately -//! after a write returns the new item where the service might not. +//! Reusing the GSI queue is what makes the asynchronous path correct rather than +//! merely deferred, and none of these properties would come free from a queue of +//! its own: +//! +//! * **Crash safety.** The pending row is inserted in the base write transaction, +//! so there is no window in which the item is committed and the index work is +//! not yet durable. The worker claims and applies in one transaction, so a crash +//! mid-apply rolls back and the row is retried. At-least-once delivery is safe +//! because applying a row is idempotent: it deletes the base key's row and +//! reinserts it from the snapshotted item. +//! * **Per-key ordering, across index kinds.** The row's partition is a hash of the +//! *base* key, so a vector row and a GSI row for the same item share a partition, +//! `ready_at` is clamped monotonic within it, and the worker drains in `id` order. +//! Two writes to one item therefore reach both index kinds in write order even +//! though the delay is jittered. +//! * **Snapshot semantics.** The row carries its own [`VectorApplyContext`], so the +//! worker needs no catalog read and an index dropped, or redefined, between +//! enqueue and apply cannot make a queued write unapplicable or retroactively +//! change how it was indexed. +//! +//! A write to a table whose items do not carry the vector still enqueues, because +//! the removal is the point: an item that loses its vector attribute must leave the +//! index, and skipping the enqueue would leave the stale row in place forever. + +use serde::{Deserialize, Serialize}; use extenddb_core::types::{AttributeDefinition, Item, KeySchemaElement, SearchSchemaElementType}; use extenddb_core::validation::{vector_components, vector_norm}; @@ -23,6 +44,12 @@ use super::{BoundValue, all_sort_key_info, sk_bound, vector_table_name}; use crate::vector_search::partition_value; /// A vector index as the write path needs it. +/// +/// Serializable because the asynchronous path snapshots it verbatim into the +/// pending row's [`VectorApplyContext`]. The write path and the worker therefore +/// apply from the *same* description of the index, which is the property that stops +/// a queued write from being reinterpreted under a later definition. +#[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) struct VectorIndexMeta { pub index_id: String, pub dimensions: usize, @@ -175,68 +202,187 @@ fn base_key_columns( cols } -/// Apply an item write to every vector index on the table. +/// Everything the propagation worker needs to apply one vector index update, +/// serialized into `gsi_pending.index_context`. +/// +/// `table_id` is carried here even though the queue row has a `table_id` column of +/// its own, because a vector data table is named from the table id *and* the index +/// id. Reading it from the context preserves the invariant that the context alone +/// is sufficient, rather than splitting one apply's inputs across a column and a +/// JSON blob. Both are written from the same variable in the same statement, so +/// they cannot disagree. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct VectorApplyContext { + pub(crate) base_key_schema: Vec, + pub(crate) attribute_definitions: Vec, + pub(crate) table_id: String, + /// Deliberately named `vector` rather than `index`: it is the field whose + /// presence lets the untagged `PendingApplyContext` tell a vector row from a + /// GSI row by shape alone. See that type for why the discriminant is a shape + /// and not a tag. + pub(crate) vector: VectorIndexMeta, +} + +/// Maintain every vector index on a table for one item write. +/// +/// The single entry point for the write path, and the one place that decides +/// between synchronous and asynchronous. `delay_ms` of 0 applies in the caller's +/// transaction; anything else enqueues one `gsi_pending` row per index. Returns the +/// number of rows enqueued, so the caller knows whether to wake the worker, and +/// returns 0 for the synchronous path because there is nothing to wake. +/// +/// Keeping the branch here rather than at each call site matters: there are seven +/// write paths, and a single one that enqueued while also applying inline would +/// double-apply, while one that did neither would silently stop indexing. /// /// `old_item` and `new_item` follow the same convention as `sync_indexes`: a put /// supplies both when replacing, a delete supplies only the old. +pub(crate) async fn maintain_vector_indexes( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + old_item: Option<&Item>, + new_item: Option<&Item>, + delay_ms: u64, +) -> Result { + // Read inside the transaction rather than from the cached `TableKeyInfo`: the + // cache carries the search schema but not the index id, and the id is what + // names the data table. + let metas = fetch_vector_indexes_for_table(tx, table_id).await?; + if metas.is_empty() { + return Ok(0); + } + let key_cols = base_key_columns(base_key_schema, attr_defs); + + if delay_ms == 0 { + for meta in &metas { + apply_vector_index( + tx, + table_id, + meta, + base_key_schema, + attr_defs, + &key_cols, + old_item, + new_item, + ) + .await?; + } + return Ok(0); + } + + let mut enqueued = 0usize; + for meta in metas { + // Enqueued even when the new item carries no vector: the removal is the + // work in that case, and skipping it would leave a stale row indexed. + let context = super::index::PendingApplyContext::Vector(VectorApplyContext { + base_key_schema: base_key_schema.to_vec(), + attribute_definitions: attr_defs.to_vec(), + table_id: table_id.to_owned(), + vector: meta, + }); + super::index::enqueue_pending_row(tx, table_id, old_item, new_item, delay_ms, &context) + .await?; + enqueued += 1; + } + Ok(enqueued) +} + +/// Apply one claimed vector pending row, from its self-describing context. +/// +/// A missing data table is skipped rather than treated as a failure: the base table +/// or the index itself can be dropped while a row is in flight, which is a routine +/// race and not a defect. +/// +/// This is log hygiene rather than data safety, and worth being exact about. The +/// batch already guards every row with a savepoint, so without this the row would be +/// rolled back and dropped, reaching the same end state by a noisier route. What the +/// tolerance changes is that an expected race stops emitting an ERROR line, which +/// otherwise trains operators to ignore the one signal that says a row was thrown +/// away. Matches the GSI sibling, so both arms of the dispatcher behave alike. +pub(crate) async fn apply_vector_context( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + old_item: Option<&Item>, + new_item: Option<&Item>, + context: &VectorApplyContext, +) -> Result<(), StorageError> { + let key_cols = base_key_columns(&context.base_key_schema, &context.attribute_definitions); + apply_vector_index( + tx, + &context.table_id, + &context.vector, + &context.base_key_schema, + &context.attribute_definitions, + &key_cols, + old_item, + new_item, + ) + .await + .or_else(|e| { + if super::index::is_no_such_table(&e) { + Ok(()) + } else { + Err(e) + } + }) +} + +/// Apply an item write to a single vector index. /// /// The delete-then-insert shape matters. An item can move between partitions when /// its HASH attribute changes, and the row is keyed by the base item rather than by /// the partition, so an insert alone would leave the old partition's row in place /// and the item would be findable in two partitions at once. -pub(crate) async fn sync_vector_indexes( +/// +/// The delete keys off `old_item.or(new_item)` because the base key is immutable, so +/// either carries it. That is what lets a put whose caller had no reason to read the +/// old item still displace the row it replaces. +#[allow(clippy::too_many_arguments)] +async fn apply_vector_index( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, table_id: &str, + meta: &VectorIndexMeta, base_key_schema: &[KeySchemaElement], attr_defs: &[AttributeDefinition], + key_cols: &[String], old_item: Option<&Item>, new_item: Option<&Item>, ) -> Result<(), StorageError> { - let metas = fetch_vector_indexes_for_table(tx, table_id).await?; - if metas.is_empty() { - return Ok(()); - } - - let key_cols = base_key_columns(base_key_schema, attr_defs); + let vec_table = vector_table_name(table_id, &meta.index_id); let where_clause = key_cols .iter() .map(|c| format!("{c} = ?")) .collect::>() .join(" AND "); - for meta in &metas { - let vec_table = vector_table_name(table_id, &meta.index_id); - - // Remove any existing row for this base item first, whatever partition it - // was in. - let source = old_item.or(new_item); - if let Some(source) = source { - let binds = base_key_binds(source, base_key_schema, attr_defs)?; - let sql = format!("DELETE FROM {vec_table} WHERE {where_clause}"); - let mut q = sqlx::query(&sql); - for b in binds { - q = super::bind_bound!(q, b); - } - q.execute(&mut **tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + // Remove any existing row for this base item first, whatever partition it + // was in. + if let Some(source) = old_item.or(new_item) { + let binds = base_key_binds(source, base_key_schema, attr_defs)?; + let sql = format!("DELETE FROM {vec_table} WHERE {where_clause}"); + let mut q = sqlx::query(&sql); + for b in binds { + q = super::bind_bound!(q, b); } - - let Some(new_item) = new_item else { - continue; // A delete: removal above is the whole of the work. - }; - insert_vector_row( - tx, - table_id, - meta, - new_item, - base_key_schema, - attr_defs, - &key_cols, - ) - .await?; + q.execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; } - Ok(()) + + let Some(new_item) = new_item else { + return Ok(()); // A delete: removal above is the whole of the work. + }; + insert_vector_row( + tx, + table_id, + meta, + new_item, + base_key_schema, + attr_defs, + key_cols, + ) + .await } /// Write one item's row into one vector index. diff --git a/crates/storage-sqlite/src/workers.rs b/crates/storage-sqlite/src/workers.rs index 82776713..2da4301a 100644 --- a/crates/storage-sqlite/src/workers.rs +++ b/crates/storage-sqlite/src/workers.rs @@ -428,7 +428,8 @@ async fn next_ready_delay(engine: &SqliteEngine) -> Result, Sto /// Parse a claimed `gsi_pending` row and apply its index update within `tx`. /// Any error (malformed context or a non-recoverable apply failure) is returned /// to the caller, which drops the row rather than stalling the queue. A -/// dropped-index race is handled inside `apply_claimed_row` (returns `Ok`). +/// dropped-index race is handled inside the apply, for both index kinds, and +/// returns `Ok`. async fn apply_pending_row( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, old_json: &Option, @@ -445,9 +446,10 @@ async fn apply_pending_row( .map(serde_json::from_str) .transpose() .map_err(|e| StorageError::Internal(e.to_string()))?; - let context: crate::data::GsiApplyContext = + // One queue carries GSI and vector work; the context's shape says which. + let context: crate::data::PendingApplyContext = serde_json::from_str(ctx_json).map_err(|e| StorageError::Internal(e.to_string()))?; - crate::data::apply_claimed_row(tx, old.as_ref(), new.as_ref(), &context).await + crate::data::apply_pending_context(tx, old.as_ref(), new.as_ref(), &context).await } /// Claim and apply one batch of due `gsi_pending` rows in a single transaction @@ -463,18 +465,29 @@ async fn process_gsi_batch(engine: &SqliteEngine) -> Result .map_err(|e| StorageError::Internal(e.to_string()))?; let now = crate::sqlite_util::format_timestamp(time::OffsetDateTime::now_utc()); - // Drain in `id` order; per-partition `ready_at` is monotonic, so this - // preserves per-key FIFO. Each row is self-describing via `index_context`. - let rows: Vec<(Option, Option, String)> = sqlx::query_as( + // Claim the oldest due rows, then apply them in `id` order. Per-partition + // `ready_at` is monotonic, so `id` order is write order and applying in it + // preserves per-key FIFO across both index kinds. + // + // The sort below is load-bearing and is NOT redundant with the `ORDER BY id` + // in the subselect. That clause only chooses WHICH rows the `LIMIT` takes; + // SQLite defines the order of `RETURNING` output as undefined, and it + // demonstrably ignores the subselect's ordering (a `DESC` subselect still + // returns ascending). Relying on it left per-key FIFO resting on an + // implementation detail: two writes to one item claimed in the same batch + // could be applied newest-first, and because each apply overwrites the row + // wholesale, the earlier write would win and the later one be lost. + let mut rows: Vec<(i64, Option, Option, String)> = sqlx::query_as( "DELETE FROM gsi_pending WHERE id IN ( \ SELECT id FROM gsi_pending WHERE ready_at <= ? ORDER BY id LIMIT ? \ - ) RETURNING old_item, new_item, index_context", + ) RETURNING id, old_item, new_item, index_context", ) .bind(&now) .bind(GSI_BATCH) .fetch_all(&mut *tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; + rows.sort_unstable_by_key(|(id, ..)| *id); if rows.is_empty() { tx.commit() @@ -484,7 +497,7 @@ async fn process_gsi_batch(engine: &SqliteEngine) -> Result } let count = rows.len(); - for (old_json, new_json, ctx_json) in &rows { + for (id, old_json, new_json, ctx_json) in &rows { // Guard each row with a SAVEPOINT. A row that cannot be applied (bad // context, or a persistent apply error) is undone and DROPPED — it was // already removed from `gsi_pending` by the batch DELETE above, so the @@ -505,7 +518,7 @@ async fn process_gsi_batch(engine: &SqliteEngine) -> Result .map_err(|e| StorageError::Internal(e.to_string()))?; } Err(e) => { - tracing::error!("GSI worker: dropping unprocessable gsi_pending row: {e}"); + tracing::error!("GSI worker: dropping unprocessable gsi_pending row {id}: {e}"); sqlx::query("ROLLBACK TO gsi_row") .execute(&mut *tx) .await @@ -628,3 +641,374 @@ mod poison_row_tests { assert_eq!(remaining.0, 0, "both poison rows dropped"); } } + +#[cfg(test)] +mod vector_propagation_tests { + use super::process_gsi_batch; + use crate::SqliteEngine; + use extenddb_core::types::{ + AttributeDefinition, Item, KeySchemaElement, KeyType, ScalarAttributeType, + }; + use serde_json::json; + + const INDEX_ID: &str = "vidx-async"; + + fn base_ks() -> Vec { + vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }] + } + + fn base_ad() -> Vec { + vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }] + } + + /// A real table with one unscoped vector index and its data table, built + /// through the engine rather than by hand so the catalog rows and the data + /// table are shaped exactly as production makes them. + async fn table_with_vector_index() -> (SqliteEngine, String) { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind("000000000000") + .execute(&engine.pool) + .await + .expect("account"); + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl("000000000000", input) + .await + .expect("create table"); + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE table_name = 't'") + .fetch_one(&engine.pool) + .await + .expect("table_id"); + + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status) \ + VALUES (?, ?, 'vidx', 2, 'COSINE', ?, ?, 'ACTIVE')", + ) + .bind(&table_id) + .bind(INDEX_ID) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert vector index"); + + let mut tx = engine.pool.begin_with("BEGIN IMMEDIATE").await.expect("tx"); + SqliteEngine::create_vector_data_table( + &mut tx, + &table_id, + INDEX_ID, + &base_ks(), + &base_ad(), + ) + .await + .expect("create vector data table"); + tx.commit().await.expect("commit ddl"); + (engine, table_id) + } + + fn item(pk: &str, generation: i32) -> Item { + serde_json::from_value(json!({ + "pk": {"S": pk}, + "gen": {"N": generation.to_string()}, + "emb": {"L": [{"N": "1"}, {"N": "0"}]}, + })) + .expect("item") + } + + /// Drive one write's vector maintenance the way a write path does, committing + /// it. Returns the number of pending rows enqueued. + async fn write( + engine: &SqliteEngine, + table_id: &str, + old: Option<&Item>, + new: Option<&Item>, + delay_ms: u64, + ) -> usize { + let mut tx = engine.pool.begin_with("BEGIN IMMEDIATE").await.expect("tx"); + let n = crate::data::vector_index::maintain_vector_indexes( + &mut tx, + table_id, + &base_ks(), + &base_ad(), + old, + new, + delay_ms, + ) + .await + .expect("maintain"); + tx.commit().await.expect("commit write"); + n + } + + async fn indexed_rows(engine: &SqliteEngine, table_id: &str) -> Vec { + let vec_table = crate::data::vector_table_name(table_id, INDEX_ID); + sqlx::query_as::<_, (String,)>(&format!( + "SELECT item_data FROM {vec_table} ORDER BY base_pk" + )) + .fetch_all(&engine.pool) + .await + .expect("read vector rows") + .into_iter() + .map(|(json,)| json) + .collect() + } + + async fn queue_depth(engine: &SqliteEngine) -> i64 { + sqlx::query_as::<_, (i64,)>("SELECT COUNT(*) FROM gsi_pending") + .fetch_one(&engine.pool) + .await + .expect("count") + .0 + } + + /// Mark every queued row due, so a drain can be tested without sleeping for the + /// propagation delay. Deliberately not a sleep: a test that waits out a real + /// delay is slow and still races a loaded machine. + async fn make_all_rows_due(engine: &SqliteEngine) { + sqlx::query("UPDATE gsi_pending SET ready_at = '2000-01-01T00:00:00.000Z'") + .execute(&engine.pool) + .await + .expect("backdate"); + } + + /// The whole point of the change: with a propagation delay, a write does not + /// touch the vector index at all. It queues, stays unapplied until it is due, + /// and only the worker applies it. + /// + /// All four claims are asserted in one test on purpose. Split apart, each half + /// passes for the wrong reason — "not indexed yet" is indistinguishable from + /// "never indexed", and "indexed after a drain" is indistinguishable from + /// "indexed by the write". + #[tokio::test] + async fn a_write_reaches_the_vector_index_only_through_the_worker() { + let (engine, table_id) = table_with_vector_index().await; + + let enqueued = write(&engine, &table_id, None, Some(&item("a", 1)), 60_000).await; + assert_eq!(enqueued, 1, "the write must enqueue exactly one row"); + assert!( + indexed_rows(&engine, &table_id).await.is_empty(), + "an async write must not index inline" + ); + assert_eq!(queue_depth(&engine).await, 1, "the row must be durable"); + + // Not yet due: draining must leave it alone rather than apply it early. + assert_eq!( + process_gsi_batch(&engine).await.expect("drain"), + 0, + "a row that is not due must not be claimed" + ); + assert!( + indexed_rows(&engine, &table_id).await.is_empty(), + "the delay must be honoured, not merely recorded" + ); + + make_all_rows_due(&engine).await; + assert_eq!( + process_gsi_batch(&engine).await.expect("drain"), + 1, + "the due row must be claimed" + ); + let rows = indexed_rows(&engine, &table_id).await; + assert_eq!(rows.len(), 1, "the worker must index the item"); + assert!( + rows[0].contains("\"gen\""), + "payload projected: {}", + rows[0] + ); + assert_eq!(queue_depth(&engine).await, 0, "the queue must drain"); + } + + /// The other half of the branch. A zero delay is the documented way to ask for + /// synchronous maintenance, and it must apply in the caller's transaction and + /// enqueue nothing at all. + #[tokio::test] + async fn a_zero_delay_write_is_applied_inline_and_queues_nothing() { + let (engine, table_id) = table_with_vector_index().await; + + let enqueued = write(&engine, &table_id, None, Some(&item("a", 1)), 0).await; + assert_eq!(enqueued, 0, "a synchronous write has nothing to enqueue"); + assert_eq!( + indexed_rows(&engine, &table_id).await.len(), + 1, + "a zero-delay write must index in its own transaction" + ); + assert_eq!(queue_depth(&engine).await, 0, "and must not queue"); + } + + /// Two writes to one item must reach the index in write order, whatever jitter + /// each drew. The guarantee comes from the queue: both rows hash to the base + /// key's partition and `ready_at` is clamped monotonic within it, so draining in + /// `id` order cannot invert them. + /// + /// Asserted on the surviving payload rather than on timestamps, because the + /// property that matters is which write won, not what the clamp computed. + #[tokio::test] + async fn successive_writes_to_one_item_are_applied_in_write_order() { + let (engine, table_id) = table_with_vector_index().await; + let first = item("a", 1); + let second = item("a", 2); + + write(&engine, &table_id, None, Some(&first), 60_000).await; + write(&engine, &table_id, Some(&first), Some(&second), 60_000).await; + + let partitions: Vec<(i64,)> = + sqlx::query_as("SELECT DISTINCT worker_partition FROM gsi_pending") + .fetch_all(&engine.pool) + .await + .expect("partitions"); + assert_eq!( + partitions.len(), + 1, + "both writes to one base key must share a partition, or ordering is not enforced" + ); + + make_all_rows_due(&engine).await; + assert_eq!(process_gsi_batch(&engine).await.expect("drain"), 2); + + let rows = indexed_rows(&engine, &table_id).await; + assert_eq!(rows.len(), 1, "one base item indexes to one row"); + assert!( + rows[0].contains("\"2\""), + "the later write must win, found: {}", + rows[0] + ); + } + + /// An item that loses its vector attribute must leave the index. This is why the + /// enqueue is unconditional: the row carries no vector to insert, so a write path + /// that skipped queueing "because there is nothing to index" would leave the + /// stale row searchable forever. + #[tokio::test] + async fn an_item_that_loses_its_vector_is_removed_from_the_index() { + let (engine, table_id) = table_with_vector_index().await; + let with_vector = item("a", 1); + let without_vector: Item = + serde_json::from_value(json!({"pk": {"S": "a"}, "gen": {"N": "2"}})).expect("item"); + + write(&engine, &table_id, None, Some(&with_vector), 60_000).await; + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("drain first"); + assert_eq!(indexed_rows(&engine, &table_id).await.len(), 1); + + let enqueued = write( + &engine, + &table_id, + Some(&with_vector), + Some(&without_vector), + 60_000, + ) + .await; + assert_eq!( + enqueued, 1, + "a write with no vector must still enqueue: the removal is the work" + ); + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("drain second"); + assert!( + indexed_rows(&engine, &table_id).await.is_empty(), + "the row must be removed once the item stops carrying a vector" + ); + } + + /// A delete removes the row, driven only by the old item. + #[tokio::test] + async fn a_delete_removes_the_indexed_row() { + let (engine, table_id) = table_with_vector_index().await; + let existing = item("a", 1); + + write(&engine, &table_id, None, Some(&existing), 60_000).await; + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("drain put"); + assert_eq!(indexed_rows(&engine, &table_id).await.len(), 1); + + write(&engine, &table_id, Some(&existing), None, 60_000).await; + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("drain delete"); + assert!(indexed_rows(&engine, &table_id).await.is_empty()); + } + + /// The index can be dropped while a row is in flight. The batch must still + /// commit and a sibling row in the same batch must still land: the batch shares + /// one transaction, so an unhandled failure would take the sibling down with it. + /// + /// Deliberately not claimed as a test of the missing-table tolerance in + /// `apply_vector_context`. Removing that tolerance leaves this test passing, + /// because the per-row savepoint rolls the row back and drops it, reaching the + /// same end state. The difference between the two paths is an ERROR log line, not + /// data, and this test cannot see it. + #[tokio::test] + async fn a_row_whose_index_was_dropped_does_not_take_the_batch_down() { + let (engine, table_id) = table_with_vector_index().await; + write(&engine, &table_id, None, Some(&item("a", 1)), 60_000).await; + + // A second table and index, standing in for unrelated work in the same batch. + let survivor_index = "vidx-survivor"; + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status) \ + VALUES (?, ?, 'vidx2', 2, 'COSINE', ?, ?, 'ACTIVE')", + ) + .bind(&table_id) + .bind(survivor_index) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert survivor index"); + let mut tx = engine.pool.begin_with("BEGIN IMMEDIATE").await.expect("tx"); + SqliteEngine::create_vector_data_table( + &mut tx, + &table_id, + survivor_index, + &base_ks(), + &base_ad(), + ) + .await + .expect("create survivor table"); + tx.commit().await.expect("commit ddl"); + + // This write enqueues for both indexes; then the first index's table is + // dropped out from under its queued row. + write(&engine, &table_id, None, Some(&item("b", 7)), 60_000).await; + let dropped = crate::data::vector_table_name(&table_id, INDEX_ID); + sqlx::query(&format!("DROP TABLE {dropped}")) + .execute(&engine.pool) + .await + .expect("drop index data table"); + + make_all_rows_due(&engine).await; + let processed = process_gsi_batch(&engine).await.expect("drain"); + assert_eq!(processed, 3, "every claimed row must be consumed"); + assert_eq!(queue_depth(&engine).await, 0, "no row may be left behind"); + + let survivor = crate::data::vector_table_name(&table_id, survivor_index); + let (rows,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {survivor}")) + .fetch_one(&engine.pool) + .await + .expect("count survivor"); + assert_eq!( + rows, 1, + "the sibling index must still be maintained when another index vanishes" + ); + } +} diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index f7f43bec..7a87c78f 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -148,6 +148,125 @@ fn hit_pks(response: &serde_json::Value) -> Vec { .collect() } +/// How long to let the index converge before failing. +/// +/// Generous on purpose. The propagation delay itself is milliseconds, but a missed +/// worker wake falls back to a one second backstop, and these tests run in parallel +/// against one server, so the bound has to cover a loaded machine rather than a +/// quiet one. Overshooting costs nothing when propagation is fast, because every +/// helper returns as soon as its condition holds. +const CONVERGE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); +const CONVERGE_POLL: std::time::Duration = std::time::Duration::from_millis(50); + +/// Search until `predicate` accepts the response, then return that response. +/// +/// A vector index is eventually consistent, so a single search after a write proves +/// nothing in either direction: a missing item may simply not have propagated, and +/// an item that should have been removed may not have been removed yet. Every +/// assertion about index contents therefore has to be an assertion about what the +/// index converges to. +/// +/// Polling rather than sleeping is deliberate. A fixed sleep is simultaneously too +/// slow when propagation is immediate and too short when the machine is loaded, +/// which is exactly how index tests become flaky. `what` is folded into the failure +/// message so a timeout says which condition was never reached. +async fn search_until( + table: &str, + values: &[f32], + top_k: usize, + condition: Option<&str>, + what: &str, + predicate: impl Fn(&serde_json::Value) -> bool, +) -> serde_json::Value { + let deadline = std::time::Instant::now() + CONVERGE_TIMEOUT; + let mut last = search(table, values, top_k, condition).await; + loop { + if predicate(&last) { + return last; + } + if std::time::Instant::now() >= deadline { + panic!( + "index never converged: {what}. Last response after {:?}: {last}", + CONVERGE_TIMEOUT + ); + } + tokio::time::sleep(CONVERGE_POLL).await; + last = search(table, values, top_k, condition).await; + } +} + +/// Poll a caller-supplied `SearchVectors` body until `predicate` accepts the parsed +/// response, then return it. +/// +/// For tests whose query the `search` helper cannot express: a different filter +/// attribute, a `ProjectionExpression`, or anything else shaped by hand. Same +/// convergence contract as [`search_until`], and it asserts the status on every +/// attempt so a request that becomes malformed fails loudly instead of timing out. +async fn search_body_until( + body: &str, + what: &str, + predicate: impl Fn(&serde_json::Value) -> bool, +) -> serde_json::Value { + let deadline = std::time::Instant::now() + CONVERGE_TIMEOUT; + loop { + let (status, text) = call("SearchVectors", body).await; + assert_eq!(status, 200, "SearchVectors failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("search response is JSON"); + if predicate(&json) { + return json; + } + assert!( + std::time::Instant::now() < deadline, + "index never converged: {what}. Last response after {CONVERGE_TIMEOUT:?}: {text}" + ); + tokio::time::sleep(CONVERGE_POLL).await; + } +} + +/// Search until the hits are exactly `expected`, in order. +/// +/// Covers presence and absence in one assertion, which matters because they are not +/// separable under eventual consistency: waiting for an item to appear and then +/// asserting a different item is absent would pass whenever the second item is +/// merely late. +async fn search_until_pks( + table: &str, + values: &[f32], + top_k: usize, + condition: Option<&str>, + expected: &[&str], +) -> serde_json::Value { + search_until( + table, + values, + top_k, + condition, + &format!("expected hits {expected:?}"), + |response| hit_pks(response) == expected, + ) + .await +} + +/// Search until the index holds exactly `count` hits, for tests whose subject is the +/// number of rows rather than which rows. +async fn search_until_count( + table: &str, + values: &[f32], + top_k: usize, + condition: Option<&str>, + count: usize, +) -> serde_json::Value { + search_until( + table, + values, + top_k, + condition, + &format!("expected {count} hits"), + |response| hit_pks(response).len() == count, + ) + .await +} + /// The nearest vector comes back first, and the ordering is by actual distance /// rather than by insertion order. #[tokio::test] @@ -163,7 +282,8 @@ async fn search_returns_nearest_first() { put_vector(&name, "orthogonal", None, &[0.0, 1.0]).await; put_vector(&name, "exact", None, &[1.0, 0.0]).await; - let response = search(&name, &[1.0, 0.0], 3, None).await; + let response = + search_until_pks(&name, &[1.0, 0.0], 3, None, &["exact", "orthogonal", "opposite"]).await; assert_eq!( hit_pks(&response), vec!["exact", "orthogonal", "opposite"], @@ -185,6 +305,10 @@ async fn top_k_limits_the_results() { put_vector(&name, &format!("i{i}"), None, &[1.0, i as f32]).await; } + // Wait for all five to be indexed before bounding, otherwise a result of two + // could just as easily mean three writes had not propagated yet. + search_until_count(&name, &[1.0, 0.0], 10, None, 5).await; + let response = search(&name, &[1.0, 0.0], 2, None).await; assert_eq!(hit_pks(&response).len(), 2, "response: {response}"); @@ -204,7 +328,22 @@ async fn overwriting_an_item_replaces_its_vector() { put_vector(&name, "a", None, &[-1.0, 0.0]).await; put_vector(&name, "a", None, &[1.0, 0.0]).await; - let response = search(&name, &[1.0, 0.0], 10, None).await; + // Converge on the score rather than the pk set: exactly one row for "a" is + // already true after the FIRST write, so a pk assertion would be satisfied by a + // state where the overwrite has not propagated at all. + let response = search_until( + &name, + &[1.0, 0.0], + 10, + None, + "the overwrite to have propagated (cosine score ~0)", + |r| { + r.pointer("/SearchResults/0/Score") + .and_then(serde_json::Value::as_f64) + .is_some_and(|score| score.abs() < 1e-5) + }, + ) + .await; let pks = hit_pks(&response); assert_eq!(pks, vec!["a"], "one row per base item: {response}"); @@ -240,7 +379,7 @@ async fn deleting_an_item_removes_it_from_the_index() { let (status, text) = call("DeleteItem", &body).await; assert_eq!(status, 200, "DeleteItem failed: {text}"); - let response = search(&name, &[1.0, 0.0], 10, None).await; + let response = search_until_pks(&name, &[1.0, 0.0], 10, None, &["stays"]).await; assert_eq!(hit_pks(&response), vec!["stays"], "response: {response}"); let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; @@ -264,7 +403,7 @@ async fn an_item_without_a_vector_is_not_indexed() { ); put_vector(&name, "hasvec", None, &[1.0, 0.0]).await; - let response = search(&name, &[1.0, 0.0], 10, None).await; + let response = search_until_pks(&name, &[1.0, 0.0], 10, None, &["hasvec"]).await; assert_eq!(hit_pks(&response), vec!["hasvec"], "response: {response}"); let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; @@ -284,10 +423,10 @@ async fn a_scoped_search_sees_only_its_own_partition() { put_vector(&name, "a_item", Some("tenant_a"), &[1.0, 0.0]).await; put_vector(&name, "b_item", Some("tenant_b"), &[1.0, 0.0]).await; - let a = search(&name, &[1.0, 0.0], 10, Some("tenant_a")).await; + let a = search_until_pks(&name, &[1.0, 0.0], 10, Some("tenant_a"), &["a_item"]).await; assert_eq!(hit_pks(&a), vec!["a_item"], "tenant_a response: {a}"); - let b = search(&name, &[1.0, 0.0], 10, Some("tenant_b")).await; + let b = search_until_pks(&name, &[1.0, 0.0], 10, Some("tenant_b"), &["b_item"]).await; assert_eq!(hit_pks(&b), vec!["b_item"], "tenant_b response: {b}"); let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; @@ -307,13 +446,19 @@ async fn changing_the_partition_attribute_moves_the_row() { put_vector(&name, "mover", Some("tenant_a"), &[1.0, 0.0]).await; put_vector(&name, "mover", Some("tenant_b"), &[1.0, 0.0]).await; + // Wait for the destination partition first, then assert the source is empty. + // That ordering is what makes the absence assertion sound: one apply performs + // the delete and the insert in a single transaction, so the row appearing under + // tenant_b proves the removal from tenant_a has already committed. Asserting the + // absence first would pass while the move had simply not propagated. + let b = search_until_pks(&name, &[1.0, 0.0], 10, Some("tenant_b"), &["mover"]).await; + assert_eq!(hit_pks(&b), vec!["mover"], "tenant_b response: {b}"); + let a = search(&name, &[1.0, 0.0], 10, Some("tenant_a")).await; assert!( hit_pks(&a).is_empty(), "the old partition must no longer hold the row: {a}" ); - let b = search(&name, &[1.0, 0.0], 10, Some("tenant_b")).await; - assert_eq!(hit_pks(&b), vec!["mover"], "tenant_b response: {b}"); let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } @@ -333,7 +478,8 @@ async fn dot_product_ranks_larger_scores_first() { put_vector(&name, "large", None, &[4.0, 0.0]).await; put_vector(&name, "negative", None, &[-3.0, 0.0]).await; - let response = search(&name, &[1.0, 0.0], 3, None).await; + let response = + search_until_pks(&name, &[1.0, 0.0], 3, None, &["large", "small", "negative"]).await; assert_eq!( hit_pks(&response), vec!["large", "small", "negative"], @@ -361,7 +507,7 @@ async fn the_vector_attribute_is_returned_only_when_named() { // Default: withheld, but the rest of the item is still there, so this is not // passing merely because nothing was returned. - let response = search(&name, &[1.0, 0.0], 5, None).await; + let response = search_until_count(&name, &[1.0, 0.0], 5, None, 1).await; let item = response .pointer("/SearchResults/0/Item") .unwrap_or_else(|| panic!("no item in: {response}")); @@ -460,21 +606,22 @@ async fn the_index_narrows_a_vector_to_f32_but_the_item_keeps_its_own_precision( "ProjectionExpression": "pk, emb" }}"# ); - let (status, text) = call("SearchVectors", &body).await; - assert_eq!(status, 200, "SearchVectors failed: {text}"); - let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let json = search_body_until(&body, "the written item to be indexed", |r| { + r.pointer("/SearchResults/0/Item/emb/L/0/N").is_some() + }) + .await; assert_eq!( json.pointer("/SearchResults/0/Item/emb/L/0/N") .and_then(|v| v.as_str()), Some("0.12345679"), - "the index must return the narrowed f32: {text}" + "the index must return the narrowed f32: {json}" ); // The exactly-representable component must not acquire a decimal point. assert_eq!( json.pointer("/SearchResults/0/Item/emb/L/1/N") .and_then(|v| v.as_str()), Some("0"), - "an exact component must round-trip unchanged: {text}" + "an exact component must round-trip unchanged: {json}" ); let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; @@ -616,7 +763,10 @@ async fn adding_a_vector_index_backfills_the_items_already_there() { .await; assert_eq!(status, 200, "UpdateTable create failed: {text}"); - let response = search(&name, &[1.0, 0.0], 10, None).await; + // The backfill runs inside UpdateTable's own transaction, so it is synchronous + // today. Converging rather than asserting once keeps the test honest if that + // ever moves onto the propagation queue as well. + let response = search_until_count(&name, &[1.0, 0.0], 10, None, 3).await; let results = response .get("SearchResults") .and_then(|r| r.as_array()) @@ -675,7 +825,20 @@ async fn an_index_added_by_update_table_indexes_later_writes_too() { put_vector(&name, "after", None, &[0.9, 0.1]).await; - let response = search(&name, &[1.0, 0.0], 10, None).await; + // "before" is backfilled synchronously, "after" arrives through the propagation + // queue, so this converges on both being present. + let response = search_until( + &name, + &[1.0, 0.0], + 10, + None, + "both the backfilled and the later-written item to be present", + |r| { + let pks = hit_pks(r); + pks.iter().any(|k| k == "before") && pks.iter().any(|k| k == "after") + }, + ) + .await; let results = response .get("SearchResults") .and_then(|r| r.as_array()) @@ -707,7 +870,9 @@ async fn deleting_a_vector_index_stops_it_serving_and_reporting() { put_vector(&name, "a", None, &[1.0, 0.0]).await; // It serves before the delete, so the assertions below cannot pass vacuously. - let response = search(&name, &[1.0, 0.0], 5, None).await; + // This has to converge: the write is propagated asynchronously, so a single + // search here could find nothing and make the precondition assert falsely. + let response = search_until_count(&name, &[1.0, 0.0], 5, None, 1).await; assert_eq!( response .get("SearchResults") @@ -905,8 +1070,9 @@ async fn the_search_response_carries_no_count_field() { put_vector(&name, "a", None, &[1.0, 0.0]).await; put_vector(&name, "b", None, &[0.0, 1.0]).await; - // Case 1: plain search. - let response = search(&name, &[1.0, 0.0], 2, None).await; + // Case 1: plain search. Converge first, so "no Count field" is asserted against + // a populated response rather than an empty one. + let response = search_until_count(&name, &[1.0, 0.0], 2, None, 2).await; assert!( response.get("Count").is_none(), "SearchVectors must not return a Count field: {response}" @@ -918,7 +1084,7 @@ async fn the_search_response_carries_no_count_field() { // Case 2: TopK larger than the number of matches, where a Count field is the // most tempting addition. - let response = search(&name, &[1.0, 0.0], 50, None).await; + let response = search_until_count(&name, &[1.0, 0.0], 50, None, 2).await; assert!( response.get("Count").is_none(), "SearchVectors must not return a Count field when TopK exceeds the match \ @@ -1035,9 +1201,20 @@ async fn keys_only_vector_index_still_filters_on_its_search_schema() { "ExpressionAttributeValues": {{":c": {{"S": "books"}}}} }}"# ); - let (status, text) = call("SearchVectors", &search).await; - assert_eq!(status, 200, "filtered SearchVectors failed: {text}"); - let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + // Converge on the filtered query itself. Before the projection fix this returned + // an empty result set permanently, so a timeout here is the defect reappearing + // rather than slow propagation. + let json = search_body_until( + &search, + "the KEYS_ONLY index to match its inline filter", + |r| { + r.get("SearchResults") + .and_then(|v| v.as_array()) + .is_some_and(|a| !a.is_empty()) + }, + ) + .await; + let text = json.to_string(); let results = json .get("SearchResults") .and_then(|v| v.as_array()) From 0964366f3711cbd7f5e4479188e26fd06e007a85 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Tue, 11 Aug 2026 11:32:19 +0000 Subject: [PATCH 15/25] refactor(settings): rename the propagation delay for both index kinds, and close review gaps Two things, both consequences of vector maintenance now sharing the GSI propagation queue: the setting that governs it was named for only one of the two index kinds, and an independent review of the previous commit found three test gaps and three comments that claimed more than they proved. ## The rename, and why it needs a fallback rather than a migration `gsi_propagation_delay_ms` now governs vector indexes as well as GSIs, so the name actively misleads: an operator reading it would reasonably conclude that vector search is unaffected by it. Renamed to `index_propagation_delay_ms`. A bare rename would have been a silent data-loss bug, which is why this is more than a search and replace. The server refuses to start on a catalog-version mismatch rather than migrating in place, and this change does not bump the catalog version, so a pre-rename catalog starts normally and keeps the operator's value under the old key. Reading past that row would reset a deliberately configured delay to the 10ms default, and because 0 means synchronous, the silent change would be from strict to eventually consistent: tests that assert steady state without waiting would start failing for a reason nowhere near the change. So reads prefer the canonical key and fall back to the legacy one, with an explicit `ORDER BY key = 'index_propagation_delay_ms' DESC` so the preference is deterministic when both rows exist rather than resting on row order. Writes to the legacy name are redirected to the canonical key, so a deployment converges on one row instead of accumulating two that disagree. `settings set gsi_propagation_delay_ms 0` therefore keeps working for anyone with it in a runbook. Both key strings and the resolver live in one place, `extenddb-core`'s `settings_keys`, because the literal is read by both backends, written by the management API, seeded by both schemas, and documented. Scattering it is what let the name drift out of step with its meaning in the first place. Two tests, each with a stated failure mode: `a_pre_rename_catalog_still_honours_its_configured_delay` reshapes a catalog to look as it did before the rename and asserts the value survives; `the_canonical_key_wins_when_both_are_present` pins the precedence. ## Review response `deleting_an_item_removes_it_from_the_index` had a real soundness gap: it never established that the doomed item was present before deleting it, so converging on the survivor was also satisfied in the window where the doomed item's write had not yet applied. It could pass without exercising the delete at all. It now converges on both items, deletes, then converges on the survivor. Two properties were claimed by the previous commit message and untested. `applying_the_same_row_twice_is_idempotent` covers the replay that crash safety rests on, and it discriminates: an earlier version of it did not, because under a conditional delete the replay collided and the end state was identical either way, so it was replaced with one that exercises a path where the difference is observable. `a_gsi_row_and_a_vector_row_for_one_item_share_a_partition` covers per-key FIFO across index kinds, which no test touched. Three comments corrected to say only what is true. The claim that exactly one context variant can ever match was wrong for a hand-corrupted blob carrying both discriminant fields; unreachable from any writer, but the comment should not assert it. Two test docs promised more than their assertions prove. `insert_vector_row` keeps a plain `INSERT` where the GSI sibling uses `INSERT OR REPLACE`, and now records why: the unconditional delete always precedes it, and a primary key violation is the desired outcome if a future refactor makes that delete conditional, where `INSERT OR REPLACE` would silently paper over it. Deliberately not fixed here: a stale queued row can clobber a newer inline write when the delay flips from non-zero to 0 with rows still in flight. It is pre-existing in the GSI sibling rather than introduced by vectors, so a vector-only fix would leave the two index kinds inconsistent. Tracked separately, to be fixed for both together. ## Docs `differences-from-dynamodb.md` documented GSI propagation and said nothing about vector indexes, which became a real gap once their consistency model changed. Adds a row stating that vector search is eventually consistent like a GSI, rides the same queue and setting, has no per-index override, and that 0 is stricter than the service. ## Verification fmt --check exit 0. clippy --workspace --all-targets -D warnings exit 0. 787 unit tests, 0 failed, 0 filtered out. Python integration on a fresh catalog, run as CI invokes it: 937 passed, 5 skipped, 1 xfailed, plus 327 comprehensive, 0 failures. Rust integration live over HTTP, single-threaded with throttling enabled and EXTENDDB_EXPECT_VECTORS=1 so a skipped vector suite fails rather than passes: 449 passed, 0 failed, 0 filtered out, 28 of them vector tests. A fresh catalog seeds only the canonical key. --- .../troubleshooting/07-runtime-symptoms.md | 4 +- crates/app/src/cmd_settings.rs | 3 + crates/core/src/lib.rs | 1 + crates/core/src/settings_keys.rs | 68 +++++++++ .../src/console/pages/settings_pages.rs | 2 +- crates/server/src/management/ops_settings.rs | 17 ++- .../migrations/001_schema.sql | 2 +- .../storage-postgres/src/data/delete_item.rs | 2 +- crates/storage-postgres/src/data/put_item.rs | 2 +- .../storage-postgres/src/data/transactions.rs | 2 +- .../storage-postgres/src/data/update_item.rs | 2 +- crates/storage-postgres/src/lib.rs | 70 +++++---- crates/storage-postgres/src/workers.rs | 22 ++- .../storage-sqlite/docs/design-decisions.md | 2 +- crates/storage-sqlite/src/data/delete_item.rs | 2 +- crates/storage-sqlite/src/data/index.rs | 140 +++++++++++++++++- crates/storage-sqlite/src/data/put_item.rs | 2 +- .../storage-sqlite/src/data/transactions.rs | 2 +- crates/storage-sqlite/src/data/update_item.rs | 2 +- .../storage-sqlite/src/data/vector_index.rs | 6 + crates/storage-sqlite/src/hooks.rs | 4 +- crates/storage-sqlite/src/lib.rs | 19 ++- crates/storage-sqlite/src/schema.rs | 2 +- crates/storage-sqlite/src/store.rs | 46 +++--- crates/storage-sqlite/src/workers.rs | 138 +++++++++++++++-- devtools/run-tests | 4 +- docs/design/04-component-storage.md | 2 +- docs/differences-from-dynamodb.md | 5 +- docs/getting-started.md | 6 +- docs/manuals/02-design-guide.md | 2 +- docs/manuals/06-developer-test-guide.md | 2 +- docs/troubleshooting.md | 4 +- tests/conftest.py | 2 +- tests/rust/src/vector_index_search.rs | 13 ++ tests/test_gsi_async.py | 18 +-- tests/test_gsi_async_queue.py | 2 +- tests/test_gsi_async_queue_sqlite.py | 8 +- 37 files changed, 511 insertions(+), 119 deletions(-) create mode 100644 crates/core/src/settings_keys.rs diff --git a/.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md b/.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md index 6e335db5..86c85875 100644 --- a/.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md +++ b/.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md @@ -94,8 +94,8 @@ Stream cleanup worker: GSI query returns stale data after a write ``` -**Cause:** GSI updates are applied asynchronously with a configurable propagation delay (default 10ms). This matches real DynamoDB's eventually consistent GSI behavior. Each GSI can have its own `propagation_delay_ms` setting; the system-wide default is controlled by the `gsi_propagation_delay_ms` runtime setting. +**Cause:** GSI updates are applied asynchronously with a configurable propagation delay (default 10ms). This matches real DynamoDB's eventually consistent GSI behavior. Each GSI can have its own `propagation_delay_ms` setting; the system-wide default is controlled by the `index_propagation_delay_ms` runtime setting. -**Fix:** This is expected behavior. For tests that query GSIs after writes, poll/retry the GSI query until the expected data appears. To make all GSIs synchronous for testing, set `extenddb settings set gsi_propagation_delay_ms 0`. For production-like testing, keep the default async delay. +**Fix:** This is expected behavior. For tests that query GSIs after writes, poll/retry the GSI query until the expected data appears. To make all GSIs synchronous for testing, set `extenddb settings set index_propagation_delay_ms 0`. For production-like testing, keep the default async delay. **Source:** `docs/troubleshooting.md`, section "GSI Async Update Behavior", last synced 2026-05-12. diff --git a/crates/app/src/cmd_settings.rs b/crates/app/src/cmd_settings.rs index 88f3e775..02210c86 100755 --- a/crates/app/src/cmd_settings.rs +++ b/crates/app/src/cmd_settings.rs @@ -112,6 +112,9 @@ async fn set(store: &dyn SettingsStore, key: &str, value: &str) -> anyhow::Resul ); } + // Write under the canonical name so the deprecated alias updates the row the read + // path consults, rather than adding a second one that is silently ignored. + let key = extenddb_core::settings_keys::canonical_key(key); store .set_setting(key, value) .await diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 6d1ec5d3..8b0d30c6 100755 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -12,6 +12,7 @@ pub mod expression; pub mod limits; pub mod metrics; pub mod serde_helpers; +pub mod settings_keys; pub mod throttle; pub mod types; pub mod validation; diff --git a/crates/core/src/settings_keys.rs b/crates/core/src/settings_keys.rs new file mode 100644 index 00000000..b2e70b32 --- /dev/null +++ b/crates/core/src/settings_keys.rs @@ -0,0 +1,68 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Canonical names for runtime settings keys. +//! +//! These strings live in one place because they are read by both storage backends, +//! written by the management API, seeded by both schemas, and documented. Scattering +//! the literal is what let one of them drift out of step with its own meaning. + +/// Propagation delay applied to asynchronous secondary-index maintenance, in +/// milliseconds. `0` means maintenance is applied synchronously in the write's own +/// transaction. +/// +/// Governs GSIs and vector indexes alike, which is why it is not named for either. +/// Real DynamoDB exposes no such knob: this exists so a test can choose between +/// asserting eventual-consistency behaviour and asserting steady state without +/// waiting. +pub const INDEX_PROPAGATION_DELAY_MS: &str = "index_propagation_delay_ms"; + +/// The pre-rename name of [`INDEX_PROPAGATION_DELAY_MS`], still honoured. +/// +/// Two reasons this cannot simply be deleted. A catalog created before the rename +/// holds the operator's value under the old name, and the server refuses to start on +/// a catalog-version mismatch rather than migrating, so there is no upgrade step in +/// which the row could be rewritten. Silently reading past that row would reset a +/// deliberately configured delay to the default, and a delay of 0 means synchronous, +/// so the silent change would be from strict to eventually consistent. +/// +/// Reads therefore prefer the canonical key and fall back to this one; writes to this +/// name are redirected to the canonical key so a deployment converges on one row +/// rather than accumulating two that disagree. +pub const LEGACY_GSI_PROPAGATION_DELAY_MS: &str = "gsi_propagation_delay_ms"; + +/// Resolve a caller-supplied settings key to its canonical name. +/// +/// Accepting the old name keeps `extenddb settings set gsi_propagation_delay_ms 0` +/// working for anyone with it in a script or runbook, while ensuring the value lands +/// where the read path looks first. +#[must_use] +pub fn canonical_key(key: &str) -> &str { + if key == LEGACY_GSI_PROPAGATION_DELAY_MS { + INDEX_PROPAGATION_DELAY_MS + } else { + key + } +} + +#[cfg(test)] +mod tests { + use super::{INDEX_PROPAGATION_DELAY_MS, LEGACY_GSI_PROPAGATION_DELAY_MS, canonical_key}; + + #[test] + fn the_legacy_name_resolves_to_the_canonical_one() { + assert_eq!( + canonical_key(LEGACY_GSI_PROPAGATION_DELAY_MS), + INDEX_PROPAGATION_DELAY_MS + ); + } + + #[test] + fn an_unrelated_key_is_returned_unchanged() { + assert_eq!(canonical_key("throttling_enabled"), "throttling_enabled"); + assert_eq!( + canonical_key(INDEX_PROPAGATION_DELAY_MS), + INDEX_PROPAGATION_DELAY_MS + ); + } +} diff --git a/crates/server/src/console/pages/settings_pages.rs b/crates/server/src/console/pages/settings_pages.rs index e2efa500..a41940af 100755 --- a/crates/server/src/console/pages/settings_pages.rs +++ b/crates/server/src/console/pages/settings_pages.rs @@ -30,7 +30,7 @@ const RUNTIME_DEFAULTS: &[(&str, &str)] = &[ ("control_plane_delay_seconds", "0.25"), ("data_database_connection_string", "—"), ("data_database_name", "—"), - ("gsi_propagation_delay_ms", "500"), + ("index_propagation_delay_ms", "500"), ("log_level", "info"), ("sqlx_log_level", "warn"), ("throttling_enabled", "false"), diff --git a/crates/server/src/management/ops_settings.rs b/crates/server/src/management/ops_settings.rs index 86e6ed54..87cc42d3 100755 --- a/crates/server/src/management/ops_settings.rs +++ b/crates/server/src/management/ops_settings.rs @@ -15,7 +15,17 @@ pub type Validator = fn(&str) -> Result<(), &'static str>; pub const KNOWN_KEYS: &[(&str, Validator)] = &[ ("allow_credential_import", validate_bool), ("control_plane_delay_seconds", validate_delay_seconds), - ("gsi_propagation_delay_ms", validate_gsi_delay_ms), + ( + extenddb_core::settings_keys::INDEX_PROPAGATION_DELAY_MS, + validate_index_propagation_delay_ms, + ), + // Deprecated alias, still writable so an existing script or runbook keeps + // working. `set_setting` canonicalises it, so it updates the same row rather + // than creating a second one that the read path would ignore. + ( + extenddb_core::settings_keys::LEGACY_GSI_PROPAGATION_DELAY_MS, + validate_index_propagation_delay_ms, + ), ("log_level", validate_log_level), ("sqlx_log_level", validate_log_level), ("throttling_enabled", validate_bool), @@ -50,7 +60,7 @@ fn validate_delay_seconds(value: &str) -> Result<(), &'static str> { } } -fn validate_gsi_delay_ms(value: &str) -> Result<(), &'static str> { +fn validate_index_propagation_delay_ms(value: &str) -> Result<(), &'static str> { match value.parse::() { Ok(0..=10000) => Ok(()), Ok(_) => Err("must be between 0 and 10000"), @@ -93,6 +103,9 @@ pub async fn set_setting( ))); } + // Write under the canonical name, so setting the deprecated alias updates the row + // the read path actually consults instead of adding a second, ignored one. + let key = extenddb_core::settings_keys::canonical_key(key); store.set_setting(key, value).await?; tracing::warn!( diff --git a/crates/storage-postgres/migrations/001_schema.sql b/crates/storage-postgres/migrations/001_schema.sql index 3a684189..8ad5fea5 100644 --- a/crates/storage-postgres/migrations/001_schema.sql +++ b/crates/storage-postgres/migrations/001_schema.sql @@ -315,7 +315,7 @@ INSERT INTO settings (key, value) VALUES ('catalog_version', '0.0.2') ON CONFLICT (key) DO NOTHING; INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0.25') ON CONFLICT (key) DO NOTHING; -INSERT INTO settings (key, value) VALUES ('gsi_propagation_delay_ms', '10') +INSERT INTO settings (key, value) VALUES ('index_propagation_delay_ms', '10') ON CONFLICT (key) DO NOTHING; COMMIT; diff --git a/crates/storage-postgres/src/data/delete_item.rs b/crates/storage-postgres/src/data/delete_item.rs index 6765bae1..7e53254a 100755 --- a/crates/storage-postgres/src/data/delete_item.rs +++ b/crates/storage-postgres/src/data/delete_item.rs @@ -39,7 +39,7 @@ impl PostgresEngine { let sys_delay = if indexes.is_empty() { 0 } else { - self.gsi_default_delay().await + self.index_propagation_delay().await }; let needs_tx = condition.is_some() || return_old || !indexes.is_empty() || stream.is_some(); diff --git a/crates/storage-postgres/src/data/put_item.rs b/crates/storage-postgres/src/data/put_item.rs index b8b0920a..9ca9d985 100755 --- a/crates/storage-postgres/src/data/put_item.rs +++ b/crates/storage-postgres/src/data/put_item.rs @@ -59,7 +59,7 @@ impl PostgresEngine { let sys_delay = if indexes.is_empty() { 0 } else { - self.gsi_default_delay().await + self.index_propagation_delay().await }; // When there's a condition, return_old, indexes, or stream capture, we need a transaction diff --git a/crates/storage-postgres/src/data/transactions.rs b/crates/storage-postgres/src/data/transactions.rs index 5e098046..c7bebebb 100644 --- a/crates/storage-postgres/src/data/transactions.rs +++ b/crates/storage-postgres/src/data/transactions.rs @@ -85,7 +85,7 @@ impl PostgresEngine { // D-4: Read the system default delay live (P119), so a runtime change // applies to this transaction rather than up to 30 s later. - let sys_delay = self.gsi_default_delay().await; + let sys_delay = self.index_propagation_delay().await; let mut tx = self .data_pool diff --git a/crates/storage-postgres/src/data/update_item.rs b/crates/storage-postgres/src/data/update_item.rs index a2c668af..39fd336c 100755 --- a/crates/storage-postgres/src/data/update_item.rs +++ b/crates/storage-postgres/src/data/update_item.rs @@ -50,7 +50,7 @@ impl PostgresEngine { let sys_delay = if indexes.is_empty() { 0 } else { - self.gsi_default_delay().await + self.index_propagation_delay().await }; // Fetch existing item diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index dcdb4507..4a30ee33 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -135,10 +135,23 @@ pub struct PostgresConfig { /// (`_ddb_*` tables, GSI tables). This separation allows the catalog and /// data to live in different `PostgreSQL` databases (Bug 1, P54). /// Default GSI propagation delay (milliseconds) when the -/// `gsi_propagation_delay_ms` setting is absent. Mirrors the value seeded by +/// `index_propagation_delay_ms` setting is absent. Mirrors the value seeded by /// the catalog schema, and is the single definition used by both the live read /// on the write path and the background refresh worker. -pub(crate) const DEFAULT_GSI_PROPAGATION_DELAY_MS: u64 = 10; +pub(crate) const DEFAULT_INDEX_PROPAGATION_DELAY_MS: u64 = 10; + +/// Read the propagation-delay setting, preferring the canonical key and falling back +/// to the pre-rename one. +/// +/// A catalog created before the rename holds the operator's value under the old name, +/// and the server refuses to start on a catalog-version mismatch rather than migrating, +/// so no upgrade step ever rewrites that row. Reading past it would silently reset a +/// configured delay to the default; since 0 means synchronous, the silent change would +/// be from strict to eventually consistent. `ORDER BY ... DESC` makes the preference +/// deterministic when both rows exist. +pub(crate) const INDEX_PROPAGATION_DELAY_QUERY: &str = "SELECT value FROM settings \ + WHERE key IN ('index_propagation_delay_ms', 'gsi_propagation_delay_ms') \ + ORDER BY key = 'index_propagation_delay_ms' DESC LIMIT 1"; pub struct PostgresEngine { pub(crate) pool: PgPool, @@ -152,10 +165,10 @@ pub struct PostgresEngine { /// D-4: Async GSI update queue. `None` until `start_gsi_workers()` is called. pub(crate) gsi_queue: Option>, /// P119: Cached GSI default propagation delay (milliseconds). Refreshed by - /// the background poller every 30s and re-warmed by `gsi_default_delay`. + /// the background poller every 30s and re-warmed by `index_propagation_delay`. /// This is only a fallback for when the live read fails; the write path /// reads the setting live so a runtime change applies to the next write. - pub gsi_default_delay_ms: Arc, + pub index_propagation_delay_cache: Arc, } impl PostgresEngine { @@ -208,15 +221,13 @@ impl PostgresEngine { }; // P119: Read initial GSI propagation delay from settings table. - let initial_gsi_delay: u64 = sqlx::query_as::<_, (String,)>( - "SELECT value FROM settings WHERE key = 'gsi_propagation_delay_ms'", - ) - .fetch_optional(&pool) - .await - .ok() - .flatten() - .and_then(|(v,)| v.parse::().ok()) - .unwrap_or(DEFAULT_GSI_PROPAGATION_DELAY_MS); + let initial_gsi_delay: u64 = sqlx::query_as::<_, (String,)>(INDEX_PROPAGATION_DELAY_QUERY) + .fetch_optional(&pool) + .await + .ok() + .flatten() + .and_then(|(v,)| v.parse::().ok()) + .unwrap_or(DEFAULT_INDEX_PROPAGATION_DELAY_MS); Ok(Self { pool, @@ -225,7 +236,9 @@ impl PostgresEngine { max_item_size_bytes: config.max_item_size_bytes, control_plane_notify: Arc::new(tokio::sync::Notify::new()), gsi_queue: None, - gsi_default_delay_ms: Arc::new(std::sync::atomic::AtomicU64::new(initial_gsi_delay)), + index_propagation_delay_cache: Arc::new(std::sync::atomic::AtomicU64::new( + initial_gsi_delay, + )), }) } @@ -236,7 +249,7 @@ impl PostgresEngine { #[must_use] /// Current GSI propagation delay (ms); `0` means synchronous. /// - /// Reads the `gsi_propagation_delay_ms` setting live from the catalog so an + /// Reads the `index_propagation_delay_ms` setting live from the catalog so an /// out-of-process change (`extenddb settings set`) applies to the next write /// rather than up to 30 s later when the poll worker refreshes the cache. /// Callers skip this entirely for tables with no secondary indexes, so a @@ -245,24 +258,23 @@ impl PostgresEngine { /// On a read error the cached value is used and the error is logged, so a /// degraded catalog serves a stale delay loudly rather than silently. On /// success the cache is re-warmed, keeping the fallback fresh. - pub(crate) async fn gsi_default_delay(&self) -> u64 { + pub(crate) async fn index_propagation_delay(&self) -> u64 { use std::sync::atomic::Ordering; - let live = sqlx::query_as::<_, (String,)>( - "SELECT value FROM settings WHERE key = 'gsi_propagation_delay_ms'", - ) - .fetch_optional(&self.pool) - .await; + let live = sqlx::query_as::<_, (String,)>(INDEX_PROPAGATION_DELAY_QUERY) + .fetch_optional(&self.pool) + .await; match live { Ok(row) => { let ms = row .and_then(|(v,)| v.parse::().ok()) - .unwrap_or(DEFAULT_GSI_PROPAGATION_DELAY_MS); - self.gsi_default_delay_ms.store(ms, Ordering::Relaxed); + .unwrap_or(DEFAULT_INDEX_PROPAGATION_DELAY_MS); + self.index_propagation_delay_cache + .store(ms, Ordering::Relaxed); ms } Err(e) => { - tracing::debug!("gsi_default_delay: live read failed, using cache: {e:?}"); - self.gsi_default_delay_ms.load(Ordering::Relaxed) + tracing::debug!("index_propagation_delay: live read failed, using cache: {e:?}"); + self.index_propagation_delay_cache.load(Ordering::Relaxed) } } } @@ -376,7 +388,7 @@ use extenddb_storage::server_components::{BackendError, ServerComponents}; struct PostgresRuntimeHooks { engine: Arc, control_plane_notify: Arc, - gsi_default_delay_ms: Arc, + index_propagation_delay_cache: Arc, data_db_name: String, } @@ -444,7 +456,7 @@ impl ServerRuntimeHooks for PostgresRuntimeHooks { // 7. GSI delay poller let catalog_store_for_gsi = ctx.catalog_store.clone(); - let gsi_delay = self.gsi_default_delay_ms.clone(); + let gsi_delay = self.index_propagation_delay_cache.clone(); let token = ctx.shutdown.clone(); let gsi_poller = tokio::spawn(async move { workers::poll_gsi_delay(catalog_store_for_gsi, gsi_delay, token).await; @@ -527,7 +539,7 @@ fn server_components_factory( // Get references to fields we need before wrapping let control_plane_notify = engine.control_plane_notify.clone(); - let gsi_default_delay_ms = engine.gsi_default_delay_ms.clone(); + let index_propagation_delay_cache = engine.index_propagation_delay_cache.clone(); // Wrap engine in Arc let engine = Arc::new(engine); @@ -584,7 +596,7 @@ fn server_components_factory( let runtime_hooks = Box::new(PostgresRuntimeHooks { engine: engine.clone(), control_plane_notify, - gsi_default_delay_ms, + index_propagation_delay_cache, data_db_name, }); diff --git a/crates/storage-postgres/src/workers.rs b/crates/storage-postgres/src/workers.rs index 7e1d144a..467262f3 100644 --- a/crates/storage-postgres/src/workers.rs +++ b/crates/storage-postgres/src/workers.rs @@ -14,6 +14,22 @@ use extenddb_storage::{CancellationToken, DataEngine, MetadataEngine, StreamEngi use sqlx::PgPool; use crate::PostgresEngine; +/// Read the propagation-delay setting through the settings store, preferring the +/// canonical key and falling back to the pre-rename one. See +/// `INDEX_PROPAGATION_DELAY_QUERY` for why the old name is still honoured. +async fn read_index_propagation_delay( + settings: &S, +) -> extenddb_storage::management_store::OpResult> { + if let Some(v) = settings + .get_setting(extenddb_core::settings_keys::INDEX_PROPAGATION_DELAY_MS) + .await? + { + return Ok(Some(v)); + } + settings + .get_setting(extenddb_core::settings_keys::LEGACY_GSI_PROPAGATION_DELAY_MS) + .await +} pub(crate) async fn poll_control_plane_transitions( storage: Arc, @@ -175,7 +191,7 @@ pub(crate) async fn poll_gsi_delay( const POLL_INTERVAL: Duration = Duration::from_secs(30); while tick(&token, POLL_INTERVAL).await { - match store.get_setting("gsi_propagation_delay_ms").await { + match read_index_propagation_delay(store.as_ref()).await { Ok(Some(val)) => { if let Ok(ms) = val.parse::() { gsi_delay.store(ms, std::sync::atomic::Ordering::Relaxed); @@ -184,12 +200,12 @@ pub(crate) async fn poll_gsi_delay( Ok(None) => { // Setting removed - revert to default gsi_delay.store( - crate::DEFAULT_GSI_PROPAGATION_DELAY_MS, + crate::DEFAULT_INDEX_PROPAGATION_DELAY_MS, std::sync::atomic::Ordering::Relaxed, ); } Err(e) => { - tracing::debug!("Failed to query gsi_propagation_delay_ms: {e:?}"); + tracing::debug!("Failed to query index_propagation_delay_ms: {e:?}"); } } } diff --git a/crates/storage-sqlite/docs/design-decisions.md b/crates/storage-sqlite/docs/design-decisions.md index 8630df3d..b34f1b01 100644 --- a/crates/storage-sqlite/docs/design-decisions.md +++ b/crates/storage-sqlite/docs/design-decisions.md @@ -148,7 +148,7 @@ only) — identical to Postgres, no change needed. Postgres backend's persistent-queue design rather than its `FOR UPDATE SKIP LOCKED` mechanics, which are unnecessary under the single-writer model. The effective delay is the per-GSI `propagation_delay_ms` override when set, - otherwise the `gsi_propagation_delay_ms` runtime setting (cached on the engine + otherwise the `index_propagation_delay_ms` runtime setting (cached on the engine and refreshed by a poller). Delay 0 ⇒ fully synchronous GSI maintenance. This **supersedes the earlier "synchronous because local I/O is fast" rationale**: that approach ignored the configured propagation delay and so did diff --git a/crates/storage-sqlite/src/data/delete_item.rs b/crates/storage-sqlite/src/data/delete_item.rs index e7d4ba5b..91c0d8ee 100644 --- a/crates/storage-sqlite/src/data/delete_item.rs +++ b/crates/storage-sqlite/src/data/delete_item.rs @@ -27,7 +27,7 @@ impl SqliteEngine { // runtime setting, not an invariant of this write, so it does not need // to be read under the lock, and the lock serialises every write in the // process: work done inside it is the backend's throughput bottleneck. - let system_delay = self.gsi_default_delay().await; + let system_delay = self.index_propagation_delay().await; let _writer = self.write_lock.lock().await; // Read the index set after acquiring the write lock so a concurrently // added GSI (UpdateTable holds the same lock) is not missed. diff --git a/crates/storage-sqlite/src/data/index.rs b/crates/storage-sqlite/src/data/index.rs index ea1c84de..13ca76a1 100644 --- a/crates/storage-sqlite/src/data/index.rs +++ b/crates/storage-sqlite/src/data/index.rs @@ -90,9 +90,15 @@ pub(crate) struct GsiApplyContext { /// every in-flight GSI update across an upgrade. /// /// The variants are unambiguous by shape rather than by tag: a GSI context requires -/// `index` and a vector context requires `vector`, and neither carries the other's -/// field, so exactly one variant can ever match. `vector_index_context_tests` pins -/// both directions, including a verbatim legacy payload. +/// `index` and a vector context requires `vector`, and no writer emits the other's +/// field, so for any context this code produces exactly one variant matches. +/// `pending_context_tests` pins both directions, including a verbatim legacy payload. +/// +/// To be exact rather than reassuring: a hand-corrupted payload carrying BOTH fields +/// would match `Gsi`, because untagged tries variants in declaration order and +/// ignores unknown fields. That is unreachable from any serializer here, and a +/// genuinely malformed context is already dropped as a poison row, so it is a +/// property of the representation worth knowing rather than a case to defend. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(untagged)] pub(crate) enum PendingApplyContext { @@ -655,11 +661,14 @@ mod pending_context_tests { ); } - /// The two kinds are told apart by shape: a vector context has `vector` and no - /// `index`, so it can only match one variant. Without this the untagged enum - /// would be free to guess wrong and a vector row would be applied as a GSI. + /// A vector context round-trips and carries no GSI discriminant field. + /// + /// Note what this does NOT guard: it still passes if `untagged` is removed, so it + /// is not the protection for on-disk compatibility. The two tests above are, and + /// both fail without `untagged`. This one pins the shape contract that makes the + /// discrimination possible in the first place. #[test] - fn a_vector_context_round_trips_and_is_never_read_as_a_gsi() { + fn a_vector_context_round_trips_and_carries_no_gsi_discriminant() { let json = serde_json::to_string(&PendingApplyContext::Vector(vector_context())) .expect("serialize"); assert!( @@ -682,4 +691,121 @@ mod pending_context_tests { PendingApplyContext::Gsi(_) => panic!("a vector context must not be read as a GSI row"), } } + + /// A GSI row and a vector row for the SAME base item must land in the same queue + /// partition, which is what makes them mutually ordered. + /// + /// This is the claim that reusing one queue buys ordering ACROSS index kinds, and + /// it holds only because both contexts hash the same base key rather than + /// anything index-specific. Asserted on the partition because the partition is the + /// mechanism: two kinds hashing differently would land in separate partitions, + /// each monotonic on its own, leaving the relative order of a GSI and a vector + /// update to one item unconstrained. + /// + /// Driven through the real `enqueue_pending_row` rather than a test shim, so it is + /// the production path being measured. + #[tokio::test] + async fn a_gsi_row_and_a_vector_row_for_one_item_share_a_partition() { + let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let the_item: extenddb_core::types::Item = + serde_json::from_str(r#"{"pk":{"S":"shared-key"},"emb":{"L":[{"N":"1"}]}}"#) + .expect("item"); + let mut tx = engine.pool.begin_with("BEGIN IMMEDIATE").await.expect("tx"); + for context in [ + PendingApplyContext::Gsi(gsi_context()), + PendingApplyContext::Vector(vector_context()), + ] { + super::enqueue_pending_row(&mut tx, "t-1", None, Some(&the_item), 60_000, &context) + .await + .expect("enqueue"); + } + tx.commit().await.expect("commit"); + + let partitions: Vec<(i64,)> = + sqlx::query_as("SELECT DISTINCT worker_partition FROM gsi_pending") + .fetch_all(&engine.pool) + .await + .expect("partitions"); + assert_eq!( + partitions.len(), + 1, + "a GSI row and a vector row for one base item must share a partition, or \ + their relative order is unconstrained" + ); + let (depth,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM gsi_pending") + .fetch_one(&engine.pool) + .await + .expect("depth"); + assert_eq!( + depth, 2, + "both kinds must have enqueued, so one partition is not an artefact of a \ + missing row" + ); + } + + /// A catalog created before the rename must keep honouring its operator's value. + /// + /// This is the compatibility property the whole fallback exists for, and the cost + /// of getting it wrong is not cosmetic: the server refuses to start on a + /// catalog-version mismatch rather than migrating, so nothing ever rewrites the old + /// row. Reading past it would silently reset a configured delay to the default, and + /// since 0 means synchronous, the silent change would be from strict to eventually + /// consistent, which is exactly the direction that turns a passing test suite into + /// a flaky one somewhere else. + #[tokio::test] + async fn a_pre_rename_catalog_still_honours_its_configured_delay() { + let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + // Reshape the catalog to look as it did before the rename: the legacy key + // only, carrying a deliberately non-default value. + sqlx::query("DELETE FROM settings WHERE key = 'index_propagation_delay_ms'") + .execute(&engine.pool) + .await + .expect("drop canonical row"); + sqlx::query("INSERT INTO settings (key, value) VALUES ('gsi_propagation_delay_ms', '0')") + .execute(&engine.pool) + .await + .expect("seed legacy row"); + + assert_eq!( + engine.index_propagation_delay().await, + 0, + "a value set under the pre-rename key must still be honoured, or an \ + operator's synchronous setting silently becomes asynchronous" + ); + } + + /// With both rows present the canonical one wins, deterministically. + /// + /// Reachable if an operator sets the legacy key on a build that predates the + /// canonicalising write path and then upgrades. Without the explicit ordering the + /// winner would be whichever row SQLite happened to return first. + #[tokio::test] + async fn the_canonical_key_wins_when_both_are_present() { + let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + sqlx::query( + "INSERT OR REPLACE INTO settings (key, value) \ + VALUES ('index_propagation_delay_ms', '7'), ('gsi_propagation_delay_ms', '999')", + ) + .execute(&engine.pool) + .await + .expect("seed both rows"); + + assert_eq!( + engine.index_propagation_delay().await, + 7, + "the canonical key must take precedence over the deprecated alias" + ); + } } diff --git a/crates/storage-sqlite/src/data/put_item.rs b/crates/storage-sqlite/src/data/put_item.rs index 3bcd6304..4efbaeb8 100644 --- a/crates/storage-sqlite/src/data/put_item.rs +++ b/crates/storage-sqlite/src/data/put_item.rs @@ -37,7 +37,7 @@ impl SqliteEngine { // runtime setting, not an invariant of this write, so it does not need // to be read under the lock, and the lock serialises every write in the // process: work done inside it is the backend's throughput bottleneck. - let system_delay = self.gsi_default_delay().await; + let system_delay = self.index_propagation_delay().await; let _writer = self.write_lock.lock().await; let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; diff --git a/crates/storage-sqlite/src/data/transactions.rs b/crates/storage-sqlite/src/data/transactions.rs index 77d43197..ff8adfe9 100644 --- a/crates/storage-sqlite/src/data/transactions.rs +++ b/crates/storage-sqlite/src/data/transactions.rs @@ -72,7 +72,7 @@ impl SqliteEngine { // runtime setting, not an invariant of this write, so it does not need // to be read under the lock, and the lock serialises every write in the // process: work done inside it is the backend's throughput bottleneck. - let system_delay = self.gsi_default_delay().await; + let system_delay = self.index_propagation_delay().await; let _writer = self.write_lock.lock().await; // Fetch index metadata per distinct table AFTER acquiring the write lock, // so a GSI added by a concurrent UpdateTable (same lock) is not missed and diff --git a/crates/storage-sqlite/src/data/update_item.rs b/crates/storage-sqlite/src/data/update_item.rs index 03e2eeb0..147c3032 100644 --- a/crates/storage-sqlite/src/data/update_item.rs +++ b/crates/storage-sqlite/src/data/update_item.rs @@ -37,7 +37,7 @@ impl SqliteEngine { // runtime setting, not an invariant of this write, so it does not need // to be read under the lock, and the lock serialises every write in the // process: work done inside it is the backend's throughput bottleneck. - let system_delay = self.gsi_default_delay().await; + let system_delay = self.index_propagation_delay().await; let _writer = self.write_lock.lock().await; // Read the index set after acquiring the write lock so a concurrently // added GSI (UpdateTable holds the same lock) is not missed. diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs index 2b70fe6b..2b8e481e 100644 --- a/crates/storage-sqlite/src/data/vector_index.rs +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -458,6 +458,12 @@ pub(crate) async fn insert_vector_row( let item_json = serde_json::to_string(&projected) .map_err(|e| StorageError::Internal(format!("serialize item: {e}")))?; + // A plain INSERT, deliberately, where the GSI sibling uses INSERT OR REPLACE. + // Every caller reaches this through `apply_vector_index`, which unconditionally + // deletes the base key's row first, so no live row can exist here and a conflict + // is impossible. Keeping it a plain INSERT means that if a future refactor ever + // makes that delete conditional, this fails loudly with a primary key violation + // rather than silently replacing a row and hiding the broken invariant. let cols = std::iter::once("part".to_owned()) .chain(key_cols.iter().cloned()) .chain(["vec".to_owned(), "nrm".to_owned(), "item_data".to_owned()]) diff --git a/crates/storage-sqlite/src/hooks.rs b/crates/storage-sqlite/src/hooks.rs index a67d2f78..0b8a634c 100644 --- a/crates/storage-sqlite/src/hooks.rs +++ b/crates/storage-sqlite/src/hooks.rs @@ -66,11 +66,11 @@ impl ServerRuntimeHooks for SqliteRuntimeHooks { }); // Keep the cached GSI propagation delay in sync with the setting. - let gsi_default = self.engine.gsi_default_delay_ms.clone(); + let index_delay_cache = self.engine.index_propagation_delay_cache.clone(); let catalog_store = ctx.catalog_store.clone(); let token = ctx.shutdown.clone(); let gsi_delay = tokio::spawn(async move { - workers::poll_gsi_delay(catalog_store, gsi_default, token).await; + workers::poll_index_propagation_delay(catalog_store, index_delay_cache, token).await; }); vec![ diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 5d8f7195..1e092afc 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -46,11 +46,24 @@ mod vector_search; mod worker; mod workers; -/// Default GSI propagation delay (milliseconds) when the -/// `gsi_propagation_delay_ms` setting is absent. Mirrors the value seeded by +/// Default secondary-index propagation delay (milliseconds) when the +/// `index_propagation_delay_ms` setting is absent. Mirrors the value seeded by /// the catalog schema, and is the single definition used by both the live read /// on the write path and the background refresh worker. -pub(crate) const DEFAULT_GSI_PROPAGATION_DELAY_MS: u64 = 10; +pub(crate) const DEFAULT_INDEX_PROPAGATION_DELAY_MS: u64 = 10; + +/// Read the propagation-delay setting, preferring the canonical key and falling back +/// to the pre-rename one. +/// +/// A catalog created before the rename holds the operator's value under the old name, +/// and the server refuses to start on a catalog-version mismatch rather than migrating, +/// so no upgrade step ever rewrites that row. Reading past it would silently reset a +/// configured delay to the default; since 0 means synchronous, the silent change would +/// be from strict to eventually consistent. `ORDER BY ... DESC` makes the preference +/// deterministic when both rows exist. +pub(crate) const INDEX_PROPAGATION_DELAY_QUERY: &str = "SELECT value FROM settings \ + WHERE key IN ('index_propagation_delay_ms', 'gsi_propagation_delay_ms') \ + ORDER BY key = 'index_propagation_delay_ms' DESC LIMIT 1"; pub use bootstrapper::SqliteBootstrapper; pub use catalog_store::SqliteCatalogStore; diff --git a/crates/storage-sqlite/src/schema.rs b/crates/storage-sqlite/src/schema.rs index 5290ee84..530bd994 100644 --- a/crates/storage-sqlite/src/schema.rs +++ b/crates/storage-sqlite/src/schema.rs @@ -423,7 +423,7 @@ INSERT OR IGNORE INTO seq_counters (name, value) INSERT INTO settings (key, value) VALUES ('catalog_version', '0.0.3') ON CONFLICT(key) DO UPDATE SET value = excluded.value; INSERT OR IGNORE INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0.25'); -INSERT OR IGNORE INTO settings (key, value) VALUES ('gsi_propagation_delay_ms', '10'); +INSERT OR IGNORE INTO settings (key, value) VALUES ('index_propagation_delay_ms', '10'); "#; /// Apply the full catalog schema to a fresh (or existing) database. diff --git a/crates/storage-sqlite/src/store.rs b/crates/storage-sqlite/src/store.rs index 39f29370..534c3a2a 100644 --- a/crates/storage-sqlite/src/store.rs +++ b/crates/storage-sqlite/src/store.rs @@ -26,6 +26,7 @@ use sqlx::SqlitePool; use sqlx::sqlite::SqlitePoolOptions; use tokio::sync::Mutex; +use crate::INDEX_PROPAGATION_DELAY_QUERY; use crate::schema::CATALOG_VERSION; use crate::sqlite_util::sqlite_url; @@ -40,7 +41,7 @@ pub struct SqliteEngine { pub(crate) control_plane_notify: Arc, /// Cached default GSI propagation delay (ms); refreshed by a worker and /// read on the write path to decide sync-vs-async index maintenance. - pub(crate) gsi_default_delay_ms: Arc, + pub(crate) index_propagation_delay_cache: Arc, /// Wakes the GSI propagation worker when a write enqueues into `gsi_pending`. pub(crate) gsi_notify: Arc, /// Serializes all writers (design decision D1). Held for the duration of @@ -107,22 +108,21 @@ impl SqliteEngine { } } - let initial_gsi_delay: u64 = sqlx::query_as::<_, (String,)>( - "SELECT value FROM settings WHERE key = 'gsi_propagation_delay_ms'", - ) - .fetch_optional(&pool) - .await - .ok() - .flatten() - .and_then(|(v,)| v.parse::().ok()) - .unwrap_or(10); + let initial_index_delay: u64 = + sqlx::query_as::<_, (String,)>(INDEX_PROPAGATION_DELAY_QUERY) + .fetch_optional(&pool) + .await + .ok() + .flatten() + .and_then(|(v,)| v.parse::().ok()) + .unwrap_or(10); Ok(Self { pool, region: region.to_owned(), max_item_size_bytes, control_plane_notify: Arc::new(tokio::sync::Notify::new()), - gsi_default_delay_ms: Arc::new(AtomicU64::new(initial_gsi_delay)), + index_propagation_delay_cache: Arc::new(AtomicU64::new(initial_index_delay)), gsi_notify: Arc::new(tokio::sync::Notify::new()), write_lock: Arc::new(Mutex::new(())), }) @@ -151,7 +151,7 @@ impl SqliteEngine { let internal = |e: String| StorageError::Internal(e); // Schema (creates settings, accounts, admin_users, … and seeds - // catalog_version + gsi_propagation_delay_ms). + // catalog_version + index_propagation_delay_ms). crate::schema::apply(&self.pool) .await .map_err(|e| internal(format!("apply schema: {e:?}")))?; @@ -232,33 +232,33 @@ impl SqliteEngine { Ok(if from_env { None } else { Some(password) }) } - /// Current GSI propagation delay (ms); `0` means synchronous. + /// Current secondary-index propagation delay (ms); `0` means synchronous. /// - /// Reads the `gsi_propagation_delay_ms` setting live from the catalog so + /// Reads the `index_propagation_delay_ms` setting live from the catalog so /// out-of-process changes (`extenddb settings set`) take effect on the /// next write, not up to 30 s later when the poll worker refreshes the /// cache. `SQLite` is a local file, so this is an indexed point lookup with /// negligible cost next to the write it precedes. On a read error the /// cached value (still refreshed by the poll worker) is the fallback; on /// success the cache is re-warmed so fallback reads stay fresh. - pub(crate) async fn gsi_default_delay(&self) -> u64 { + pub(crate) async fn index_propagation_delay(&self) -> u64 { use std::sync::atomic::Ordering; - let live: Result, _> = - sqlx::query_as("SELECT value FROM settings WHERE key = 'gsi_propagation_delay_ms'") - .fetch_optional(&self.pool) - .await; + let live: Result, _> = sqlx::query_as(INDEX_PROPAGATION_DELAY_QUERY) + .fetch_optional(&self.pool) + .await; match live { Ok(row) => { // Missing row means the default, matching poll_gsi_delay. let ms = row .and_then(|(v,)| v.parse::().ok()) - .unwrap_or(crate::DEFAULT_GSI_PROPAGATION_DELAY_MS); - self.gsi_default_delay_ms.store(ms, Ordering::Relaxed); + .unwrap_or(crate::DEFAULT_INDEX_PROPAGATION_DELAY_MS); + self.index_propagation_delay_cache + .store(ms, Ordering::Relaxed); ms } Err(e) => { - tracing::debug!("gsi_default_delay: live read failed, using cache: {e:?}"); - self.gsi_default_delay_ms.load(Ordering::Relaxed) + tracing::debug!("index_propagation_delay: live read failed, using cache: {e:?}"); + self.index_propagation_delay_cache.load(Ordering::Relaxed) } } } diff --git a/crates/storage-sqlite/src/workers.rs b/crates/storage-sqlite/src/workers.rs index 2da4301a..f5b7efcd 100644 --- a/crates/storage-sqlite/src/workers.rs +++ b/crates/storage-sqlite/src/workers.rs @@ -537,25 +537,45 @@ async fn process_gsi_batch(engine: &SqliteEngine) -> Result Ok(count) } -/// Refresh the cached GSI propagation delay from the `gsi_propagation_delay_ms` -/// runtime setting, so changes take effect without restart. -pub(crate) async fn poll_gsi_delay( +/// Read the propagation-delay setting through the settings store, preferring the +/// canonical key and falling back to the pre-rename one. +/// +/// The raw-SQL read on the write path uses one query to express the same preference; +/// this path goes through the `SettingsStore` trait, which fetches by exact key, so the +/// fallback is two lookups instead. It runs once per poll interval, not per write. +async fn read_index_propagation_delay( + settings: &S, +) -> extenddb_storage::management_store::OpResult> { + if let Some(v) = settings + .get_setting(extenddb_core::settings_keys::INDEX_PROPAGATION_DELAY_MS) + .await? + { + return Ok(Some(v)); + } + settings + .get_setting(extenddb_core::settings_keys::LEGACY_GSI_PROPAGATION_DELAY_MS) + .await +} + +/// Refresh the cached secondary-index propagation delay from the +/// `index_propagation_delay_ms` runtime setting, so changes take effect without a +/// restart. Falls back to the pre-rename key for a catalog created before it. +pub(crate) async fn poll_index_propagation_delay( settings: Arc, - gsi_default: Arc, + index_delay_cache: Arc, token: CancellationToken, ) { use std::sync::atomic::Ordering; const POLL: Duration = Duration::from_secs(30); while sleep_or_shutdown(&token, POLL).await { - match settings.get_setting("gsi_propagation_delay_ms").await { + match read_index_propagation_delay(settings.as_ref()).await { Ok(Some(v)) => { if let Ok(ms) = v.parse::() { - gsi_default.store(ms, Ordering::Relaxed); + index_delay_cache.store(ms, Ordering::Relaxed); } } - Ok(None) => { - gsi_default.store(crate::DEFAULT_GSI_PROPAGATION_DELAY_MS, Ordering::Relaxed) - } + Ok(None) => index_delay_cache + .store(crate::DEFAULT_INDEX_PROPAGATION_DELAY_MS, Ordering::Relaxed), Err(e) => tracing::debug!("poll_gsi_delay: {e:?}"), } } @@ -1011,4 +1031,104 @@ mod vector_propagation_tests { "the sibling index must still be maintained when another index vanishes" ); } + + /// Applying the same claimed context twice must leave the same single row. + /// + /// The batch is at-least-once: claim and apply share a transaction, so a crash + /// mid-apply rolls back and the row is claimed again. That story is only safe if + /// an apply is idempotent, which is asserted here directly rather than by + /// implication. + /// + /// Being exact about its strength: this is a structural regression guard, not a + /// discriminating test. Its assertions cannot fail today without a schema change, + /// because the vector row's primary key is the base item key, so a replay can only + /// ever overwrite. Measured, not assumed: making the delete conditional on + /// `old_item` leaves this test PASSING, since the replayed insert then collides on + /// the primary key, the row is dropped by the savepoint, and the end state is the + /// same single row. The test below is the one that catches that refactor. + #[tokio::test] + async fn applying_the_same_row_twice_is_idempotent() { + let (engine, table_id) = table_with_vector_index().await; + write(&engine, &table_id, None, Some(&item("a", 1)), 60_000).await; + + // Capture the claimed row's payload, then replay it a second time. + let claimed: (Option, Option, String) = + sqlx::query_as("SELECT old_item, new_item, index_context FROM gsi_pending LIMIT 1") + .fetch_one(&engine.pool) + .await + .expect("read the queued row"); + + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("first apply"); + let after_first = indexed_rows(&engine, &table_id).await; + assert_eq!(after_first.len(), 1, "first apply indexes the item"); + + // Re-enqueue the identical context, exactly as a rolled-back batch would + // leave it, and drain again. + sqlx::query( + "INSERT INTO gsi_pending \ + (table_id, worker_partition, old_item, new_item, index_context, ready_at) \ + VALUES (?, 0, ?, ?, ?, '2000-01-01T00:00:00.000Z')", + ) + .bind(&table_id) + .bind(&claimed.0) + .bind(&claimed.1) + .bind(&claimed.2) + .execute(&engine.pool) + .await + .expect("replay the row"); + process_gsi_batch(&engine).await.expect("second apply"); + + let after_second = indexed_rows(&engine, &table_id).await; + assert_eq!( + after_second.len(), + 1, + "a replayed apply must not duplicate the row" + ); + assert_eq!( + after_first, after_second, + "a replayed apply must reach an identical end state" + ); + } + + /// A write that carries NO old image must still replace the indexed row. + /// + /// This is a real production path, not a contrived one: `put_item` only reads the + /// old image when a condition, a stream, `ReturnValues`, or a GSI needs it, so a + /// table whose only index is a vector index writes with `old_item = None` on the + /// common path. The apply therefore cannot rely on the old image to find the row + /// to displace, and keys the delete off `old_item.or(new_item)` because the base + /// key is immutable. + /// + /// This is the test that guards the invariant behind the plain `INSERT` in + /// `insert_vector_row`. Making the delete conditional on `old_item` fails it: the + /// second insert collides on the primary key, the savepoint drops the row, and the + /// STALE payload survives. That failure is silent in production, which is why it + /// is worth a dedicated test rather than leaving it to the idempotency case above. + #[tokio::test] + async fn a_write_with_no_old_image_still_replaces_the_indexed_row() { + let (engine, table_id) = table_with_vector_index().await; + + write(&engine, &table_id, None, Some(&item("a", 1)), 60_000).await; + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("first apply"); + assert!( + indexed_rows(&engine, &table_id).await[0].contains("\"1\""), + "precondition: the first generation is indexed" + ); + + // The second write supplies no old image, exactly as put_item does when + // nothing else needs it. + write(&engine, &table_id, None, Some(&item("a", 2)), 60_000).await; + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("second apply"); + + let rows = indexed_rows(&engine, &table_id).await; + assert_eq!(rows.len(), 1, "one base item indexes to one row: {rows:?}"); + assert!( + rows[0].contains("\"2\""), + "the newer write must replace the row even with no old image, found: {}", + rows[0] + ); + } } diff --git a/devtools/run-tests b/devtools/run-tests index a32ea8de..7c80d2b9 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -302,8 +302,8 @@ if $NEEDS_INTEGRATION && [[ "$TARGET" != "real-dynamodb" ]]; then if "$BINARY" settings --config "$CONFIG_FOR_SETTINGS" set control_plane_delay_seconds 0.05 2>/dev/null; then echo " ✓ control_plane_delay_seconds set to 0.05" fi - if "$BINARY" settings --config "$CONFIG_FOR_SETTINGS" set gsi_propagation_delay_ms 0 2>/dev/null; then - echo " ✓ gsi_propagation_delay_ms set to 0" + if "$BINARY" settings --config "$CONFIG_FOR_SETTINGS" set index_propagation_delay_ms 0 2>/dev/null; then + echo " ✓ index_propagation_delay_ms set to 0" fi if "$BINARY" settings --config "$CONFIG_FOR_SETTINGS" set throttling_enabled true 2>/dev/null; then echo " ✓ throttling_enabled set to true" diff --git a/docs/design/04-component-storage.md b/docs/design/04-component-storage.md index c721cc47..102fa8cf 100755 --- a/docs/design/04-component-storage.md +++ b/docs/design/04-component-storage.md @@ -568,7 +568,7 @@ propagation delay. LSI updates are always synchronous. **Implementation:** - Each GSI has an optional `propagation_delay_ms` column in the `indexes` table - If `propagation_delay_ms` is `NULL` or negative, the system default is used - (default: 10ms, configurable via `gsi_propagation_delay_ms` setting) + (default: 10ms, configurable via `index_propagation_delay_ms` setting) - If `propagation_delay_ms` is `0`, the GSI is updated synchronously in the same transaction as the base table write - If `propagation_delay_ms` is positive, the GSI update is enqueued and applied diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 045484a6..ef511220 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -61,7 +61,8 @@ adaptation when switching between ExtendDB and the real service. | Area | DynamoDB | ExtendDB | |------|----------|------| -| GSI update propagation | Eventually consistent (milliseconds to seconds) | Per-GSI propagation delay. System default: `gsi_propagation_delay_ms` setting (default 10ms). Each GSI can override with its own `propagation_delay_ms` (stored in catalog). A value of 0 means synchronous (future sync GSI feature). | +| GSI update propagation | Eventually consistent (milliseconds to seconds) | Per-GSI propagation delay. System default: `index_propagation_delay_ms` setting (default 10ms). Each GSI can override with its own `propagation_delay_ms` (stored in catalog). A value of 0 means synchronous (future sync GSI feature). | +| Vector index update propagation | Eventually consistent, the same model as a GSI | Matches DynamoDB. Maintenance is queued on the same propagation queue as async GSIs, so a search immediately after a write may not see it. Governed by the same `index_propagation_delay_ms` setting; unlike a GSI there is no per-index override. A value of 0 applies maintenance synchronously in the write's own transaction, which is stricter than the service and exists so a test can assert steady state without waiting. | | Multi-part base table keys | Not supported | Preview extension (opt-in via `enable_multipart_keys` setting). Standard single/composite keys work identically. | ## Capacity and Throttling @@ -90,7 +91,7 @@ ExtendDB exposes runtime settings that have no DynamoDB equivalent: | Setting | Default | Description | |---------|---------|-------------| | `control_plane_delay_seconds` | 5 | Simulated delay for table state transitions (CREATING → ACTIVE, DELETING → removed) | -| `gsi_propagation_delay_ms` | 10 | System-wide default GSI propagation delay (milliseconds). Per-GSI overrides stored in catalog. 0 = synchronous. | +| `index_propagation_delay_ms` | 10 | System-wide default propagation delay for asynchronous secondary-index maintenance (milliseconds), covering GSIs and vector indexes alike. Per-GSI overrides stored in catalog; vector indexes have no per-index override. 0 = synchronous. Accepts the pre-rename name `gsi_propagation_delay_ms` as a deprecated alias, and a catalog created before the rename keeps honouring a value stored under it. | | `throttling_enabled` | `true` | Enable provisioned capacity throttling (token bucket per table/partition) | | `enable_multipart_keys` | `false` | Enable multi-part base table key extension | | `log_level` | `info` | Runtime log level (trace, debug, info, warn, error) | diff --git a/docs/getting-started.md b/docs/getting-started.md index cbe927e0..4dae5038 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -276,11 +276,11 @@ GSI updates are applied asynchronously with a configurable delay, simulating rea ```bash # Set system-wide default to 0 for synchronous GSI updates (fast tests) ./target/release/extenddb settings --config extenddb.toml set \ - gsi_propagation_delay_ms 0 + index_propagation_delay_ms 0 # Set to 50ms for more realistic eventual consistency ./target/release/extenddb settings --config extenddb.toml set \ - gsi_propagation_delay_ms 50 + index_propagation_delay_ms 50 ``` ### Throttling @@ -1213,7 +1213,7 @@ extenddb supports running external test suites (e.g., Java/JUnit, Python/pytest) # External suites expect synchronous GSI behavior (matching real DynamoDB's # typical sub-millisecond propagation). The async GSI path is tested # separately by the extenddb-specific test_gsi_async.py suite. -./target/release/extenddb settings --config extenddb.toml set gsi_propagation_delay_ms 0 +./target/release/extenddb settings --config extenddb.toml set index_propagation_delay_ms 0 # Run all registered suites python3 devtools/run-external-tests diff --git a/docs/manuals/02-design-guide.md b/docs/manuals/02-design-guide.md index 20f3cd83..13cd6c9d 100755 --- a/docs/manuals/02-design-guide.md +++ b/docs/manuals/02-design-guide.md @@ -185,7 +185,7 @@ extenddb caches a small set of operational settings in memory to avoid per-reque | Setting | Mechanism | Refresh | Justification | |---------|-----------|---------|---------------| -| `gsi_propagation_delay_ms` | `AtomicU64` | Background poller every 30s | Write-path hot path; briefly-stale value only affects GSI propagation timing | +| `index_propagation_delay_ms` | `AtomicU64` | Background poller every 30s | Write-path hot path; briefly-stale value only affects GSI propagation timing | | `encryption_key` | `Arc` loaded at startup | Never (immutable after `extenddb init`) | Decryption key for access key secrets; generated once, never changes | | `log_level` / `log_destination` | Tracing filter reload | Background poller every 30s | Observability tuning; stale value only delays log level changes | | `throttling_enabled` | `AtomicBool` | Background poller every 30s | Capacity management toggle; briefly-stale is safe | diff --git a/docs/manuals/06-developer-test-guide.md b/docs/manuals/06-developer-test-guide.md index 0433ef90..4e97f24b 100755 --- a/docs/manuals/06-developer-test-guide.md +++ b/docs/manuals/06-developer-test-guide.md @@ -156,7 +156,7 @@ The `run-tests` script automatically: - Provisions test credentials via `devtools/provision-test-credentials` - Creates a Java truststore for external tests (self-signed TLS cert) - Sets `control_plane_delay_seconds` to 0.05 for fast test cycles -- Sets `gsi_propagation_delay_ms` to 0 for immediate GSI updates +- Sets `index_propagation_delay_ms` to 0 for immediate GSI updates - Enables throttling for production-like behavior - Configures import/export paths for file operation tests - Extracts and exports `EXTENDDB_TEST_PG_CONNECTION_STRING` for CLI lifecycle tests diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 44fcf061..77e3e0d1 100755 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -785,9 +785,9 @@ If the health check fails, start extenddb. If it succeeds, check your `--endpoin ### GSI query returns stale data after a write -**Cause:** GSI updates are applied asynchronously with a configurable propagation delay (default 10ms). This matches real DynamoDB's eventually consistent GSI behavior. Each GSI can have its own `propagation_delay_ms` setting; the system-wide default is controlled by the `gsi_propagation_delay_ms` runtime setting. +**Cause:** GSI updates are applied asynchronously with a configurable propagation delay (default 10ms). This matches real DynamoDB's eventually consistent GSI behavior. Each GSI can have its own `propagation_delay_ms` setting; the system-wide default is controlled by the `index_propagation_delay_ms` runtime setting. -**Fix:** This is expected behavior. For tests that query GSIs after writes, poll/retry the GSI query until the expected data appears. To make all GSIs synchronous for testing, set `extenddb settings set gsi_propagation_delay_ms 0`. For production-like testing, keep the default async delay. +**Fix:** This is expected behavior. For tests that query GSIs after writes, poll/retry the GSI query until the expected data appears. To make all GSIs synchronous for testing, set `extenddb settings set index_propagation_delay_ms 0`. For production-like testing, keep the default async delay. ## Connection Pool Exhaustion diff --git a/tests/conftest.py b/tests/conftest.py index eb128248..1d35d88d 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -97,7 +97,7 @@ def wait_for_gsi_items(paginate, expected: int, timeout: float = 15.0): GSIs are eventually consistent: an item written to the base table is not guaranteed to be visible through a secondary index immediately — ExtendDB - applies the configured ``gsi_propagation_delay_ms`` (like real DynamoDB), so + applies the configured ``index_propagation_delay_ms`` (like real DynamoDB), so a read-back through a GSI right after the write can legitimately be short. Tests that write then page a GSI must poll rather than read once. diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 7a87c78f..ecc4b1f1 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -375,6 +375,11 @@ async fn deleting_an_item_removes_it_from_the_index() { put_vector(&name, "gone", None, &[1.0, 0.0]).await; put_vector(&name, "stays", None, &[0.0, 1.0]).await; + // Establish that BOTH items are indexed before deleting one. Without this the + // convergence below is satisfied by a state where "gone" was simply never + // indexed yet, so the test could pass without the delete being exercised at all. + search_until_pks(&name, &[1.0, 0.0], 10, None, &["gone", "stays"]).await; + let body = format!(r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "gone"}}}}}}"#); let (status, text) = call("DeleteItem", &body).await; assert_eq!(status, 200, "DeleteItem failed: {text}"); @@ -387,6 +392,14 @@ async fn deleting_an_item_removes_it_from_the_index() { /// An item with no vector attribute is simply not indexed, exactly as a GSI omits /// an item missing its index key. It must not be an error, and must not appear. +/// +/// Worth being precise about what this proves, because it is less than the name +/// suggests: an item with no vector has no code path that could index it, so its +/// absence is close to trivially true. What it does prove is that a vectorless +/// PutItem is still ACCEPTED against a table carrying a vector index, which is the +/// half that could plausibly regress. The removal case, an item that had a vector +/// and loses it, is the one that needs real coverage and is asserted at the unit +/// level by `an_item_that_loses_its_vector_is_removed_from_the_index`. #[tokio::test] async fn an_item_without_a_vector_is_not_indexed() { if skip_unless_supported().await { diff --git a/tests/test_gsi_async.py b/tests/test_gsi_async.py index 35abbc1c..c71f7b2e 100755 --- a/tests/test_gsi_async.py +++ b/tests/test_gsi_async.py @@ -230,7 +230,7 @@ def test_base_table_read_is_immediately_consistent( def test_gsi_sync_path_with_zero_delay( self, dynamodb_client, gsi_table ): - """When gsi_propagation_delay_ms=0, GSI updates are synchronous. + """When index_propagation_delay_ms=0, GSI updates are synchronous. Sets the system-wide delay to 0, writes an item, and asserts the GSI query returns the item immediately (single query, no @@ -239,8 +239,8 @@ def test_gsi_sync_path_with_zero_delay( table_name = gsi_table # Save original delay and set to 0 (sync mode). - original_delay = extenddb_settings_get("gsi_propagation_delay_ms") - extenddb_settings_set("gsi_propagation_delay_ms", "0") + original_delay = extenddb_settings_get("index_propagation_delay_ms") + extenddb_settings_set("index_propagation_delay_ms", "0") try: pk = f"sync-{uuid.uuid4().hex[:8]}" @@ -265,28 +265,28 @@ def test_gsi_sync_path_with_zero_delay( ExpressionAttributeValues={":pk": {"S": gsi_pk}}, ) assert resp["Count"] == 1, ( - "With gsi_propagation_delay_ms=0, GSI should be " + "With index_propagation_delay_ms=0, GSI should be " "immediately consistent" ) assert resp["Items"][0]["data"]["S"] == "sync-value" finally: # Restore original delay. - extenddb_settings_set("gsi_propagation_delay_ms", original_delay) + extenddb_settings_set("index_propagation_delay_ms", original_delay) def test_gsi_configured_delay_range( self, dynamodb_client, gsi_table ): """GSI propagation delay falls within the configured range. - Sets gsi_propagation_delay_ms to 50, writes items, and measures + Sets index_propagation_delay_ms to 50, writes items, and measures the observed propagation delay. The delay should be within [1, 50]ms (the random range used by the worker). """ table_name = gsi_table # Save original delay and set to 50ms. - original_delay = extenddb_settings_get("gsi_propagation_delay_ms") - extenddb_settings_set("gsi_propagation_delay_ms", "50") + original_delay = extenddb_settings_get("index_propagation_delay_ms") + extenddb_settings_set("index_propagation_delay_ms", "50") try: delays = [] @@ -336,4 +336,4 @@ def test_gsi_configured_delay_range( f"Delays exceeded 500ms: {[f'{d:.1f}' for d in delays]}" ) finally: - extenddb_settings_set("gsi_propagation_delay_ms", original_delay) + extenddb_settings_set("index_propagation_delay_ms", original_delay) diff --git a/tests/test_gsi_async_queue.py b/tests/test_gsi_async_queue.py index 91701dfd..6161a663 100644 --- a/tests/test_gsi_async_queue.py +++ b/tests/test_gsi_async_queue.py @@ -77,7 +77,7 @@ def _set_gsi_delay(config_path, ms): ``--config`` must precede the ``set`` subcommand for the settings CLI. """ - _run_extenddb("settings", "--config", config_path, "set", "gsi_propagation_delay_ms", str(ms)) + _run_extenddb("settings", "--config", config_path, "set", "index_propagation_delay_ms", str(ms)) def _server_pid_on_port(port): diff --git a/tests/test_gsi_async_queue_sqlite.py b/tests/test_gsi_async_queue_sqlite.py index 1c222d63..a725999a 100644 --- a/tests/test_gsi_async_queue_sqlite.py +++ b/tests/test_gsi_async_queue_sqlite.py @@ -155,21 +155,21 @@ def _init(self, gsi_delay_ms: int) -> None: self.admin_password = line.split("Password:", 1)[1].strip() _patch_port(self.config, self.port) # Set the system GSI delay before serving (read at startup). - self._run("settings", "set", "gsi_propagation_delay_ms", str(gsi_delay_ms)) + self._run("settings", "set", "index_propagation_delay_ms", str(gsi_delay_ms)) # Assert the delay is persisted as expected BEFORE the server starts. # These tests run isolated, file-backed servers with their own database, - # so the main devtools/run-tests setting of gsi_propagation_delay_ms=0 on + # so the main devtools/run-tests setting of index_propagation_delay_ms=0 on # the shared server cannot race or clobber this value. The read-back pins # that invariant rather than relying on it implicitly. conn = self._connect() try: row = conn.execute( - "SELECT value FROM settings WHERE key = 'gsi_propagation_delay_ms'" + "SELECT value FROM settings WHERE key = 'index_propagation_delay_ms'" ).fetchone() finally: conn.close() assert row is not None and row[0] == str(gsi_delay_ms), ( - f"expected gsi_propagation_delay_ms={gsi_delay_ms} persisted at startup, " + f"expected index_propagation_delay_ms={gsi_delay_ms} persisted at startup, " f"got {row!r}" ) From 583fd10cf0e09a1609ac30272390c9896dfa5e83 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Tue, 11 Aug 2026 13:20:05 +0000 Subject: [PATCH 16/25] fix(vector): close seven divergences found by differential testing against DynamoDB Ran the vector index surface side by side against real DynamoDB, both systems driven through the same boto3 client shape with 1000 identical 128-dimension items, and judged results against brute-force exact nearest neighbours rather than only against each other. Search quality needed nothing: 10/10 recall with exact ordering on every query, for COSINE, DOT_PRODUCT and EUCLIDEAN, scores agreeing to float32 precision, holding under filters, under KEYS_ONLY and INCLUDE projections, and for a backfilled index versus an inline-built one. What follows is everything that did not match. After these changes 12 of 13 re-probed cases agree, and all six create-time messages are byte-identical. ## The one that mattered: a silent wrong answer A search against a HASH-scoped index with no `SearchConditionExpression` returned HTTP 200 with ZERO results. The service refuses it. The caller was told "no matches" for an invalid request, indistinguishable from an empty table. The code asserted in a comment that "validation upstream guarantees it is present here". No such validation existed. The check is keyed off the resolved hash key rather than off whether an expression was supplied, because an expression that omits the HASH attribute leaves the search just as unscoped. The integration test seeds two tenants and asserts the SCOPED search finds its rows FIRST, so "zero results" cannot pass as correct. Control: reverting the check gives `200 {"SearchResults":[]}` where 400 is owed. ## Requests accepted that the service refuses `ProjectionType: INCLUDE` with no `NonKeyAttributes`. The rule and its exact message already existed in `validate_index_projections`, which iterated GSIs and LSIs and never vector indexes, so the fix is to iterate them rather than to restate the rule. A `VectorAttribute` naming an attribute also present in `AttributeDefinitions`, including the table's own partition key. This rule was not modelled at all. It follows from the shape: `VectorAttribute` carries no type because a vector is not a scalar type AttributeDefinitions can express. Checked before the surrounding rules because the attribute may simultaneously be a legitimate key, which is exactly how it slipped through: `pk` satisfied both definition-exists and definition-is-used. A reserved keyword as a bare identifier in `SearchConditionExpression` (`bucket = :b`). The check existed for `ProjectionExpression`, byte-identical to the service, and was simply never wired into this expression type. Applied to the bare form only, with a test asserting the aliased form still works, since aliasing is the documented escape hatch and breaking it would trade one defect for another. ## Cosine distance left its domain An exact self-match returned -1.19e-07. Cosine distance has domain [0, 2]; the f32 quotient can exceed 1 when the vectors are identical. The similarity is now clamped before subtraction. The existing test could not catch this: it asserted `s.abs() < 1e-6`, which the negative value satisfies. Taking the absolute value discarded the sign, which was the only thing wrong. The new test asserts non-negativity over 2000 generated vectors, and its control reproduces exactly -0.00000011920928955078125. ## ConsumedCapacity was about 40% low and ignored returned vectors `VectorSearchRequestBytes` reported 2748.8 where the service reported 6196.0. The previous constant of 17.6 bytes per dimension came with a comment excusing imprecision by a documented 1.176 bimodal spread, but the observed ratio was 2.25, so the constant was wrong rather than the service noisy. Re-derived by sweeping 16, 64, 128, 256, 512, 768, 1024 and 2048 dimensions: * 30.6875 bytes per dimension, exact at every point in the low sweep (intercepts 491, 1964, 3928, 7856, 15712). * 72 bytes per returned result, flat across dimensions. The previous model had no per-result term at all, so TopK did not move the figure. * 4 bytes per dimension for each returned item carrying the vector, one float32 per dimension. This was absent entirely: projecting the vector doubled the service's figure and changed nothing here. Two things the sweep established that the old comment denied. The figure DOES depend on how many items the index holds, about 19.8 bytes per item at 512 dimensions, which is why two sweeps disagreed by 3% at the same dimension. And the old recorded observations, 18067 at 1024 dimensions and 36058 at 2048, are not reproducible: the service returns 31206 and 61692 for the same shape today. The constant is set from the 60-item observations so the model errs high on small indexes rather than low on realistic ones. Also recorded, not fixed: a HASH-scoped search costs LESS than an unscoped one (3675 against the 3928 the dimension term alone accounts for), leaving the model roughly 38% high on a scoped search. That term needs a sweep over partition counts that has not been run. ## Message wording, measured rather than paraphrased Billing mode, index count cap, and SearchSchema attributes with no definition. The last was reusing the GSI key-attribute message, which names the attribute and lists every definition where the service says only that one element is undefined. The unindexed-attribute message now names WHICH attribute, as the service's does, reproducing its grammar slip ("attributes that is not") deliberately: parity means matching what clients receive, not correcting it. ## Verification fmt --check exit 0. clippy --workspace --all-targets -D warnings exit 0. 799 unit tests, 0 failed, 0 filtered out (12 new). 450 integration tests live over HTTP, single-threaded, throttling enabled, EXTENDDB_EXPECT_VECTORS=1, 0 failed, 0 filtered out. Seven negative controls, one per fix, each reverting only that fix and each failing the intended test with the intended symptom. Re-probed against the live service afterwards: 12 of 13 cases agree, the remaining one being the scoped-search capacity term recorded above. Two gaps the report flagged were closed and came back clean, needing no change: the 18 inline-filter cap is already enforced with a byte-identical message, and DOT_PRODUCT and EUCLIDEAN both rank identically to exact ground truth. Not addressed here, as a deliberate design question rather than a defect: the backfill is awaited inline, so `IndexStatus: CREATING` and `Backfilling` are written to the catalog but flip to ACTIVE before the response is built, while the service holds CREATING for over eight minutes on an empty table and refuses searches throughout. Making that asynchronous would change what an index does immediately after UpdateTable returns. --- .../core/src/expression/search_condition.rs | 95 ++++++- crates/core/src/validation/mod.rs | 251 +++++++++++++++++- crates/engine/src/search_vectors.rs | 240 +++++++++++++---- crates/storage-sqlite/src/vector_search.rs | 62 ++++- tests/rust/src/vector_index_search.rs | 89 +++++++ 5 files changed, 668 insertions(+), 69 deletions(-) diff --git a/crates/core/src/expression/search_condition.rs b/crates/core/src/expression/search_condition.rs index 11bce142..a431bde9 100644 --- a/crates/core/src/expression/search_condition.rs +++ b/crates/core/src/expression/search_condition.rs @@ -130,6 +130,21 @@ pub fn validate_search_condition_expression( )) })? } else { + // A bare identifier must not be a reserved word. This check existed for + // ProjectionExpression, where its message is byte-identical to the + // service, but was never wired into this expression type, so + // `cat = :c AND bucket = :b` was accepted and returned results where the + // service refuses it. Measured on 2026-08-11. + // + // Applied to the bare form only, which is the point of the rule: the + // `#alias` branch above exists precisely so a reserved word can be used + // by aliasing it. + if crate::expression::reserved_words::is_reserved(lhs) { + return Err(invalid(format!( + "Invalid SearchConditionExpression: Attribute name is a reserved \ + keyword; reserved keyword: {lhs}" + ))); + } (*lhs).to_owned() }; if attr_name.contains('.') || attr_name.contains('[') { @@ -220,15 +235,22 @@ pub fn validate_conditions_against_search_schema( let schema = search_schema.unwrap_or(&[]); // Every referenced attribute must be part of the index search schema. + // + // Wording measured against the service on 2026-08-11. It names the offending + // attribute, which the previous text did not, so a caller with several + // conditions had to work out which one was at fault. The service's own grammar + // slip ("attributes that is not") is reproduced deliberately: parity means + // matching what clients actually receive, not correcting it. for condition in conditions { let in_schema = schema .iter() .any(|element| element.attribute_name == condition.attribute_name); if !in_schema { - return Err(invalid( - "SearchConditionExpression must not contain any attributes outside the vector \ - index search schema", - )); + return Err(invalid(format!( + "SearchConditionExpression must not contain any attributes that is not in \ + SearchSchema. Invalid attribute: {}", + condition.attribute_name + ))); } } @@ -299,6 +321,71 @@ mod tests { } } + /// A bare reserved word is refused, with the service's wording. + /// + /// The check already existed for `ProjectionExpression`, where its message is + /// byte-identical to the service, but was never applied here, so + /// `bucket = :b` was accepted and returned results for a request the service + /// refuses. `bucket` is the specific word that exposed it against the live + /// service on 2026-08-11. + #[test] + fn a_bare_reserved_keyword_is_refused() { + let v = values(&[(":b", "4")]); + let message = err(validate_search_condition_expression( + "bucket = :b", + None, + Some(&v), + )); + assert_eq!( + message, + "Invalid SearchConditionExpression: Attribute name is a reserved keyword; \ + reserved keyword: bucket" + ); + } + + /// Aliasing is the documented escape hatch, so the rule must apply to the bare + /// form only. Without this, the fix above would make reserved-word attributes + /// unusable rather than merely requiring an alias. + #[test] + fn an_aliased_reserved_keyword_is_accepted() { + let n = names(&[("#b", "bucket")]); + let v = values(&[(":b", "4")]); + let conditions = validate_search_condition_expression("#b = :b", Some(&n), Some(&v)) + .expect("an aliased reserved word must be allowed"); + assert_eq!(conditions.len(), 1); + assert_eq!(conditions[0].attribute_name, "bucket"); + } + + /// The condition is rejected when it names an attribute outside the schema, and + /// the message names WHICH attribute, as the service's does. + #[test] + fn an_attribute_outside_the_search_schema_is_named_in_the_error() { + let conditions = vec![ + SearchCondition { + attribute_name: "cat".to_owned(), + value: AttributeValue::S("alpha".to_owned()), + }, + SearchCondition { + attribute_name: "payload".to_owned(), + value: AttributeValue::S("nope".to_owned()), + }, + ]; + let schema = [SearchSchemaElement { + attribute_name: "cat".to_owned(), + element_type: SearchSchemaElementType::Hash, + }]; + let message = err(validate_conditions_against_search_schema( + &conditions, + Some(&schema), + &[], + )); + assert_eq!( + message, + "SearchConditionExpression must not contain any attributes that is not in \ + SearchSchema. Invalid attribute: payload" + ); + } + #[test] fn single_literal_equality_resolves() { let v = values(&[(":cat", "Electronics")]); diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index 1d3ce161..94944145 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -337,21 +337,26 @@ fn validate_vector_indexes(input: &CreateTableInput) -> Result<(), DynamoDbError // "Requirements and limitations" and again in the quota table, which lists // vector index capacity mode as on-demand only. `BillingMode` defaults to // PROVISIONED when absent, so an omitted BillingMode is a rejection too. + // + // Wording measured against the service on 2026-08-11; the earlier text was a + // reasonable paraphrase but not what the service says. if !vis.is_empty() && input.billing_mode.unwrap_or(BillingMode::Provisioned) != BillingMode::PayPerRequest { return Err(DynamoDbError::ValidationException( - "One or more parameter values were invalid: Vector indexes are supported only \ - on tables that use PAY_PER_REQUEST billing mode" + "One or more parameter values were invalid: Vector indexes are only supported \ + for PAY_PER_REQUEST tables" .to_owned(), )); } + // Wording measured against the service on 2026-08-11. Note the service does not + // echo the offending count here, unlike its SearchSchema messages, so neither + // does this. if vis.len() > MAX_VECTOR_INDEXES_PER_TABLE { return Err(DynamoDbError::ValidationException(format!( - "One or more parameter values were invalid: Number of vector indexes {} \ - exceeds the limit of {MAX_VECTOR_INDEXES_PER_TABLE}", - vis.len() + "One or more parameter values were invalid: VectorIndex count exceeds the \ + per-table limit of {MAX_VECTOR_INDEXES_PER_TABLE}" ))); } @@ -443,6 +448,22 @@ fn validate_index_projections(input: &CreateTableInput) -> Result<(), DynamoDbEr check(&lsi.projection)?; } } + // Vector indexes were omitted from this function, so a vector index could + // declare ProjectionType INCLUDE with no NonKeyAttributes and be accepted, + // where the service refuses it with the message `check` already produces. + // The rules are not vector-specific, so the fix is to iterate them here + // rather than to restate the rules somewhere else. + // + // `projection` is optional on a vector index specification and its absence is + // a separate fault reported by `validate_one_vector_index`, so it is skipped + // here rather than being reported twice with different wording. + if let Some(vis) = &input.vector_indexes { + for vi in vis { + if let Some(projection) = &vi.projection { + check(projection)?; + } + } + } Ok(()) } @@ -650,6 +671,32 @@ fn validate_lsi_key_schemas(input: &CreateTableInput) -> Result<(), DynamoDbErro } fn validate_attribute_definitions(input: &CreateTableInput) -> Result<(), DynamoDbError> { + // A vector attribute must NOT be declared in AttributeDefinitions, and this is + // checked before anything else here because the attribute may simultaneously be + // a legitimate key attribute: without this, naming the table's own partition key + // as the VectorAttribute was accepted, since `pk` satisfied both the + // definition-exists and the definition-is-used checks below. + // + // The rule follows from the shape: `VectorAttributeDefinition` carries only + // `AttributeName` and no type, because a vector is not a scalar type that + // AttributeDefinitions can express. Measured against the service on 2026-08-11. + if let Some(vis) = &input.vector_indexes { + for vi in vis { + let vec_attr = vi.vector_attribute.attribute_name.as_str(); + if input + .attribute_definitions + .iter() + .any(|ad| ad.attribute_name == vec_attr) + { + return Err(DynamoDbError::ValidationException(format!( + "One or more parameter values were invalid: Conflicting attribute \ + definition for '{vec_attr}'. An attribute cannot be defined in \ + AttributeDefinitions when used as a VectorAttribute." + ))); + } + } + } + // Collect all key attribute names from table + GSIs + LSIs let mut key_attrs: Vec<&str> = input .key_schema @@ -678,10 +725,28 @@ fn validate_attribute_definitions(input: &CreateTableInput) -> Result<(), Dynamo // Vector-index search-schema attributes are declared in AttributeDefinitions // but are not part of the base or secondary-index key schema, so count them // as used to satisfy the definition/key correspondence check. + // + // They are checked for existence HERE, with their own message, rather than + // being folded into `key_attrs` and reported by the loop below. The service + // distinguishes the two cases and this previously reused the GSI key wording, + // naming the attribute and listing every definition where the service says only + // that one element is undefined. Measured on 2026-08-11. + let def_names: Vec<&str> = input + .attribute_definitions + .iter() + .map(|ad| ad.attribute_name.as_str()) + .collect(); if let Some(vis) = &input.vector_indexes { for vi in vis { if let Some(schema) = &vi.search_schema { for element in schema { + if !def_names.contains(&element.attribute_name.as_str()) { + return Err(DynamoDbError::ValidationException( + "One or more parameter values were invalid: One element in \ + SearchSchema is not defined in attribute definitions" + .to_owned(), + )); + } if !key_attrs.contains(&element.attribute_name.as_str()) { key_attrs.push(&element.attribute_name); } @@ -691,12 +756,6 @@ fn validate_attribute_definitions(input: &CreateTableInput) -> Result<(), Dynamo } // Every key attribute must have a definition - let def_names: Vec<&str> = input - .attribute_definitions - .iter() - .map(|ad| ad.attribute_name.as_str()) - .collect(); - for attr in &key_attrs { if !def_names.contains(attr) { return Err(DynamoDbError::ValidationException(format!( @@ -1901,6 +1960,176 @@ mod tests { } } + /// Helper: a minimal well-formed vector index specification for these tests. + fn vi_spec(name: &str) -> crate::types::VectorIndexSpecification { + crate::types::VectorIndexSpecification { + index_name: name.to_owned(), + vector_attribute: crate::types::VectorAttribute { + attribute_name: "emb".to_owned(), + }, + dimensions: 8, + distance_function: crate::types::DistanceFunction::Cosine, + search_schema: None, + projection: Some(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }), + } + } + + /// The vector attribute must not be declared in `AttributeDefinitions`. + /// + /// Measured against the service on 2026-08-11. Before this, naming the table's + /// own partition key as the vector attribute was ACCEPTED, because `pk` + /// satisfied both the definition-exists and definition-is-used checks, so the + /// conflict was invisible to every rule that ran. + /// + /// Both shapes are asserted: an ordinary extra definition, and the key case, + /// because only the latter passes the surrounding checks and so is the one that + /// actually got through. + #[test] + fn a_vector_attribute_must_not_be_declared_in_attribute_definitions() { + let expected = "One or more parameter values were invalid: Conflicting attribute \ + definition for 'emb'. An attribute cannot be defined in \ + AttributeDefinitions when used as a VectorAttribute."; + let mut input = base_input( + vec![make_ks("pk", KeyType::Hash)], + vec![ + make_ad("pk", ScalarAttributeType::S), + make_ad("emb", ScalarAttributeType::S), + ], + ); + input.vector_indexes = Some(vec![vi_spec("vidx")]); + let err = validate_attribute_definitions(&input) + .expect_err("a declared vector attribute must be refused"); + assert_eq!(format!("{err}"), expected); + + // The key case: the attribute is legitimately a key AND the vector + // attribute, which is what previously slipped through. + let mut input = base_input( + vec![make_ks("pk", KeyType::Hash)], + vec![make_ad("pk", ScalarAttributeType::S)], + ); + let mut spec = vi_spec("vidx"); + spec.vector_attribute.attribute_name = "pk".to_owned(); + input.vector_indexes = Some(vec![spec]); + let err = validate_attribute_definitions(&input) + .expect_err("the partition key must not double as the vector attribute"); + assert!( + format!("{err}").contains("Conflicting attribute definition for 'pk'"), + "{err}" + ); + } + + /// A vector index's projection is subject to the same INCLUDE rule as a GSI's. + /// + /// `validate_index_projections` iterated GSIs and LSIs only, so a vector index + /// could declare INCLUDE with no NonKeyAttributes and be accepted where the + /// service refuses it. The message already existed and was already correct; + /// only the iteration was missing, so this asserts the message to pin that the + /// shared rule is what is being applied rather than a restatement of it. + #[test] + fn a_vector_index_projection_obeys_the_include_rule() { + let mut input = base_input( + vec![make_ks("pk", KeyType::Hash)], + vec![make_ad("pk", ScalarAttributeType::S)], + ); + let mut spec = vi_spec("vidx"); + spec.projection = Some(Projection { + projection_type: ProjectionType::Include, + non_key_attributes: None, + }); + input.vector_indexes = Some(vec![spec]); + let err = validate_index_projections(&input) + .expect_err("INCLUDE without NonKeyAttributes must be refused"); + assert_eq!( + format!("{err}"), + "One or more parameter values were invalid: ProjectionType is INCLUDE, but \ + NonKeyAttributes is not specified" + ); + + // An absent projection is a different fault, reported elsewhere, and must + // not be reported twice with different wording. + let mut spec = vi_spec("vidx"); + spec.projection = None; + input.vector_indexes = Some(vec![spec]); + validate_index_projections(&input) + .expect("an absent projection is validate_one_vector_index's fault to report"); + } + + /// A SearchSchema attribute with no definition gets the service's own wording, + /// which differs from the GSI key-attribute message this previously reused. + /// + /// The distinction matters because the two messages carry different amounts of + /// detail: the GSI one names the attribute and lists every definition, while + /// the service says only that one element is undefined. + #[test] + fn an_undefined_search_schema_attribute_uses_its_own_message() { + let mut input = base_input( + vec![make_ks("pk", KeyType::Hash)], + vec![make_ad("pk", ScalarAttributeType::S)], + ); + let mut spec = vi_spec("vidx"); + spec.search_schema = Some(vec![crate::types::SearchSchemaElement { + attribute_name: "cat".to_owned(), + element_type: crate::types::SearchSchemaElementType::Hash, + }]); + input.vector_indexes = Some(vec![spec]); + let err = validate_attribute_definitions(&input) + .expect_err("an undefined SearchSchema attribute must be refused"); + assert_eq!( + format!("{err}"), + "One or more parameter values were invalid: One element in SearchSchema is not \ + defined in attribute definitions" + ); + } + + /// Both table-level vector messages, pinned to what the service returns. + #[test] + fn the_table_level_vector_messages_match_the_service() { + // Billing mode. PROVISIONED is the default when BillingMode is absent, so + // the omitted case is asserted too. + for billing in [Some(BillingMode::Provisioned), None] { + let mut input = base_input( + vec![make_ks("pk", KeyType::Hash)], + vec![make_ad("pk", ScalarAttributeType::S)], + ); + input.billing_mode = billing; + input.vector_indexes = Some(vec![vi_spec("vidx")]); + let err = validate_vector_indexes(&input) + .expect_err("a vector index requires PAY_PER_REQUEST"); + assert_eq!( + format!("{err}"), + "One or more parameter values were invalid: Vector indexes are only \ + supported for PAY_PER_REQUEST tables" + ); + } + + // The per-table cap, asserted as a boundary so an off-by-one cannot hide. + let mut input = base_input( + vec![make_ks("pk", KeyType::Hash)], + vec![make_ad("pk", ScalarAttributeType::S)], + ); + let at_cap: Vec<_> = (0..MAX_VECTOR_INDEXES_PER_TABLE) + .map(|i| vi_spec(&format!("idx{i}"))) + .collect(); + input.vector_indexes = Some(at_cap); + validate_vector_indexes(&input).expect("the cap itself is allowed"); + + let over_cap: Vec<_> = (0..=MAX_VECTOR_INDEXES_PER_TABLE) + .map(|i| vi_spec(&format!("idx{i}"))) + .collect(); + input.vector_indexes = Some(over_cap); + let err = validate_vector_indexes(&input).expect_err("one over the cap is refused"); + assert_eq!( + format!("{err}"), + format!( + "One or more parameter values were invalid: VectorIndex count exceeds the \ + per-table limit of {MAX_VECTOR_INDEXES_PER_TABLE}" + ) + ); + } + #[test] fn standard_table_rejects_multipart_keys() { let limits = LimitsConfig::default(); // allow_multipart_table_keys = false diff --git a/crates/engine/src/search_vectors.rs b/crates/engine/src/search_vectors.rs index 95e9dd94..ab78489d 100644 --- a/crates/engine/src/search_vectors.rs +++ b/crates/engine/src/search_vectors.rs @@ -190,8 +190,8 @@ pub async fn handle_search_vectors( // The index's HASH element scopes the search to one partition; the // remaining conditions narrow within it. Declaring a HASH element is - // optional, but when the index has one the service requires the search to - // supply it, so validation upstream guarantees it is present here. + // optional, but when the index has one the service REQUIRES the search to + // supply it. let hash_attr = vector_index .search_schema .as_deref() @@ -207,6 +207,25 @@ pub async fn handle_search_vectors( .map(|c| (c.attribute_name.as_str(), &c.value)) }); + // Refuse rather than search unscoped. An earlier comment here asserted that + // "validation upstream guarantees it is present", which was untrue: no such + // validation existed, so a search that omitted the condition ran without a + // partition scope and returned HTTP 200 with ZERO results. That is the worst + // available failure mode, a silent wrong answer, and it told the caller "no + // matches" for a request the service rejects outright. + // + // Message measured against the service on 2026-08-11 in us-east-1. + // + // Keyed off `hash_key` rather than off whether an expression was supplied at + // all, because an expression that omits the HASH attribute leaves the search + // just as unscoped as no expression does. + if hash_attr.is_some() && hash_key.is_none() { + return Err(DynamoDbError::ValidationException( + "SearchConditionExpression must be provided when SearchSchema has a HASH key" + .to_owned(), + )); + } + let filters: Vec<(&str, &AttributeValue)> = conditions .iter() .filter(|c| Some(c.attribute_name.as_str()) != hash_attr) @@ -302,7 +321,24 @@ pub async fn handle_search_vectors( // The service reports `ConsumedCapacity.VectorSearchRequestBytes`, a byte // figure, not a unit figure. See `search_request_bytes` for the measured // model and why exact parity is not achievable. - let request_bytes = search_request_bytes(vector_index.dimensions, non_vector_bytes); + // + // `vectors_returned` is counted from the projected results rather than inferred + // from whether the expression mentions the attribute, so an alias, a nested + // path, or an item that simply has no vector all count correctly. The service + // charges one float32 per dimension for each returned item that carries it. + let vectors_returned = search_results + .iter() + .filter(|r| { + r.item + .contains_key(&vector_index.vector_attribute.attribute_name) + }) + .count(); + let request_bytes = search_request_bytes( + vector_index.dimensions, + non_vector_bytes, + search_results.len(), + vectors_returned, + ); let consumed_capacity = match input.return_consumed_capacity { ReturnConsumedCapacity::None => None, _ => Some(VectorCapacity { @@ -374,40 +410,71 @@ fn parse_search_vector(values: &[AttributeValue]) -> Result, DynamoDbEr /// Bytes reported as `ConsumedCapacity.VectorSearchRequestBytes` for one search. /// /// `returned_non_vector_bytes` is the summed stored size of the items actually -/// returned, taken before any projection is applied. +/// returned, taken before any projection is applied. `returned_count` is how many +/// items are returned, and `vectors_returned` how many of those include the vector +/// attribute in the response projection. +/// +/// Re-derived on 2026-08-11 in us-east-1 by sweeping dimensions 16, 64, 128, 256 and +/// 512 against key-only items, at TopK 1, 10 and 25, four samples per point. The +/// previous model used 17.6 bytes per dimension and no per-result term, which +/// reported 2748.8 where the service reported 6196.0 for the same 128-dimension +/// search: a ratio of 2.25, far outside the 1.176 bimodal spread that had been used +/// to excuse imprecision. Three terms were measured, each independently: /// -/// Measured against the service on 2026-08-05 in us-east-1. Three properties -/// hold, and this reproduces all three: +/// * **Per dimension: 30.6875.** Exact at every dimension tested, not a fit. +/// Intercepts were 491, 1964, 3928, 7856 and 15712 for 16 to 512 dimensions, +/// each exactly `dimensions * 30.6875`. +/// * **Per returned result: 72 bytes** with key-only items, flat across dimensions +/// 64 to 512. The 16-dimension row measured 52.8 only because the 1 KiB floor +/// clipped its TopK 1 point. +/// * **Per returned vector: 4 bytes per dimension.** Projecting the vector added +/// exactly `40 * dimensions` at TopK 10, i.e. `4 * dimensions` per returned item, +/// which is one float32 per dimension. The previous model ignored this entirely, +/// so projecting the vector doubled the service's figure and changed nothing here. +/// * The **1 KiB floor** is unchanged and was reconfirmed: a 16-dimension search at +/// TopK 1 reported exactly 1024. /// -/// * A 1 KiB floor. A 4-dimension index reported exactly 1024 for every TopK -/// from 1 to 100, for 1 to 100 returned results, and with 4 to 204 items in -/// the index. -/// * A per-dimension term that is independent of how many items the index -/// holds, so this is not a scan-cost model: growing a 4096-dimension index -/// from 3 to 12 items moved the figure by 2 bytes, which was the returned -/// key's length rather than the extra items. It is also unaffected by the -/// query vector's wire width, since the same search with 1-character and -/// 21-character numbers (49 KB versus 127 KB of JSON) both reported 75201. -/// The vector is metered per dimension, not per byte sent. -/// * Plus the stored bytes of the returned items, excluding the searched -/// vector. Adding one item with a 2000-byte non-vector attribute raised the -/// figure by exactly 1999. Projection does not reduce it, so a caller must -/// sum the items before projecting, not after. +/// The 72 is split into a fixed 65 plus the returned item's own bytes, because the +/// caller already supplies `returned_non_vector_bytes` and a key-only item +/// contributes roughly 7 of those. That split is the one derived rather than +/// directly measured quantity here: it is corroborated by the richer items used in +/// an earlier probe, whose per-result figure was 102.6 for items carrying about +/// 30 bytes more, but it rests on ExtendDB's own accounting of stored size matching +/// the service's, which cannot be true in general. /// -/// Exact parity is NOT achievable and must not be asserted. The service is not -/// deterministic here: byte-identical requests against an unchanged index return -/// one of two values separated by a fixed `dimensions * 3.111` offset, a ratio of -/// 1.176. Twenty-four samples at 1024 dimensions gave 18067 and 21253; at 2048 -/// they gave 36058 and 42429; the mix varies between runs and persists with 10 -/// items in the index. This reproduces the lower and more frequent mode. -fn search_request_bytes(dimensions: u32, returned_non_vector_bytes: usize) -> f64 { - /// Derived from the lower mode: 18067 at 1024 dimensions and 36058 at 2048, - /// both 17.6 bytes per dimension once the returned item is subtracted. - const BYTES_PER_DIMENSION: f64 = 17.6; - /// Observed floor. A 4-dimension search never reported less than this. +/// Exact parity therefore remains out of reach, and the earlier note that the +/// service is bimodal still stands as a caution even though this sweep saw a single +/// value at every one of its 60 sample points. +/// +/// One term is known to be missing. Every observation above came from an UNSCOPED +/// search. A HASH-scoped search on a 128-dimension index reported 3675, which is +/// below the 3928 that the per-dimension term alone accounts for, implying the +/// service charges less when the SearchSchema confines the search to one partition. +/// That leaves this model roughly 38% HIGH on a scoped search, where it was 40% low +/// on everything before. The direction of the error is now conservative rather than +/// under-reporting, and the scoping term is not characterised: doing so needs a +/// sweep over partition counts and selectivities, which has not been run. +fn search_request_bytes( + dimensions: u32, + returned_non_vector_bytes: usize, + returned_count: usize, + vectors_returned: usize, +) -> f64 { + /// Exact at 16, 64, 128, 256 and 512 dimensions. + const BYTES_PER_DIMENSION: f64 = 30.6875; + /// One float32 per dimension for each returned item carrying the vector. + const BYTES_PER_RETURNED_VECTOR_DIMENSION: f64 = 4.0; + /// The 72 measured per result, less the bytes a key-only item contributes + /// through `returned_non_vector_bytes`. + const PER_RESULT_OVERHEAD: f64 = 65.0; + /// Observed floor. A 16-dimension search never reported less than this. const MIN_SEARCH_BYTES: f64 = 1024.0; - (BYTES_PER_DIMENSION * f64::from(dimensions) + returned_non_vector_bytes as f64) + let dims = f64::from(dimensions); + (BYTES_PER_DIMENSION * dims + + PER_RESULT_OVERHEAD * returned_count as f64 + + returned_non_vector_bytes as f64 + + BYTES_PER_RETURNED_VECTOR_DIMENSION * dims * vectors_returned as f64) .max(MIN_SEARCH_BYTES) } @@ -463,23 +530,21 @@ mod tests { assert!(parse_search_vector(&[AttributeValue::N("inf".to_owned())]).is_err()); } - /// A 4-dimension search reported exactly 1024 for every TopK from 1 to 100 - /// and for 4 to 204 items in the index, so the floor dominates at low - /// dimensions rather than anything proportional. + /// A 16-dimension search at TopK 1 reported exactly 1024, so the floor + /// dominates at low dimensions rather than anything proportional. #[test] fn search_bytes_have_a_one_kib_floor() { - assert!((search_request_bytes(4, 0) - 1024.0).abs() < f64::EPSILON); - assert!((search_request_bytes(1, 100) - 1024.0).abs() < f64::EPSILON); + assert!((search_request_bytes(4, 0, 0, 0) - 1024.0).abs() < f64::EPSILON); + assert!((search_request_bytes(1, 100, 1, 0) - 1024.0).abs() < f64::EPSILON); } - /// Independent of items scanned, so the only inputs are dimensions and the - /// bytes of the items actually returned. + /// The per-dimension term is linear, verified against the service at 16, 64, + /// 128, 256, 512, 1024 and 2048 dimensions. #[test] fn search_bytes_scale_per_dimension_above_the_floor() { - let a = search_request_bytes(1024, 0); - let b = search_request_bytes(2048, 0); + let a = search_request_bytes(1024, 0, 0, 0); + let b = search_request_bytes(2048, 0, 0, 0); assert!(a > 1024.0, "1024 dimensions must clear the floor: {a}"); - // Doubling the dimensions doubles the per-dimension term exactly. assert!((b - 2.0 * a).abs() < 1.0, "{a} then {b}"); } @@ -487,26 +552,95 @@ mod tests { /// figure by exactly 1999, so returned bytes pass through one-for-one. #[test] fn returned_item_bytes_pass_through_one_for_one() { - let base = search_request_bytes(1024, 0); - assert!((search_request_bytes(1024, 2000) - (base + 2000.0)).abs() < f64::EPSILON); + let base = search_request_bytes(1024, 0, 1, 0); + assert!((search_request_bytes(1024, 2000, 1, 0) - (base + 2000.0)).abs() < f64::EPSILON); + } + + /// Each returned result costs a fixed amount beyond its own bytes. + /// + /// The previous model had no per-result term at all, so TopK made no difference + /// to the figure. The service moved by exactly 72 bytes per additional result at + /// every dimension from 64 to 2048, with key-only items. + #[test] + fn each_returned_result_adds_a_fixed_cost() { + let one = search_request_bytes(1024, 0, 1, 0); + let eleven = search_request_bytes(1024, 0, 11, 0); + let per_result = (eleven - one) / 10.0; + assert!( + (per_result - 65.0).abs() < f64::EPSILON, + "expected the fixed per-result overhead, got {per_result}" + ); + } + + /// Returning the vector costs one float32 per dimension per item. + /// + /// This term was absent entirely: projecting the vector doubled the service's + /// figure (6196 to 11316 at 128 dimensions) and changed nothing here. Measured + /// as exactly `40 * dimensions` at TopK 10, i.e. `4 * dimensions` per item. + #[test] + fn returning_the_vector_costs_four_bytes_per_dimension_per_item() { + let without = search_request_bytes(128, 0, 10, 0); + let with = search_request_bytes(128, 0, 10, 10); + assert!( + (with - without - 40.0 * 128.0).abs() < f64::EPSILON, + "expected 4 bytes per dimension per returned vector: {without} then {with}" + ); + // Partial projection scales with how many results carry the vector. + let half = search_request_bytes(128, 0, 10, 5); + assert!( + (half - without - 20.0 * 128.0).abs() < f64::EPSILON, + "{half}" + ); } - /// Checks the model against the service's own numbers, deliberately with a - /// tolerance rather than equality: the service returns one of two values for - /// a byte-identical request (18067 or 21253 at 1024 dimensions, 36058 or - /// 42429 at 2048), so asserting equality would encode a coin flip. Roughly - /// 20 bytes of returned item accompanied each observation. + /// Checks the model against the service's own numbers with a tolerance. + /// + /// Equality is not asserted, for two measured reasons. Byte-identical requests + /// against an unchanged index return one of a small number of values: six + /// samples at 512 dimensions gave 16418 and 16433. And the figure depends on how + /// many items the INDEX holds, which this model does not carry: at 512 + /// dimensions and TopK 10, a 30-item index reported 15824 and a 60-item index + /// 16418, about 19.8 bytes per item. That dependence is why two sweeps + /// disagreed by 3% at the same dimension, and it contradicts an earlier comment + /// here which asserted the figure was independent of items scanned. + /// + /// The constant is set from the 60-item observations, which are internally exact + /// at five dimensions, so the model errs slightly high on very small indexes + /// rather than low on realistic ones. + /// + /// Observations are TopK 10 against key-only items, so `returned_non_vector_bytes` + /// is passed as a small figure rather than zero. #[test] - fn model_tracks_the_observed_lower_mode() { - for (dimensions, observed) in [(1024_u32, 18067.0_f64), (2048, 36058.0)] { - let modelled = search_request_bytes(dimensions, 20); + fn model_tracks_the_measured_service_figures() { + // (dimensions, items in index, observed bytes at TopK 10) + for (dimensions, observed) in [ + (512_u32, 16433.0_f64), + (1024, 31206.0), + (2048, 61692.0), + (128, 4648.0), + (64, 2684.0), + ] { + let modelled = search_request_bytes(dimensions, 70, 10, 0); let error = (modelled - observed).abs() / observed; assert!( - error < 0.01, + error < 0.05, "{dimensions} dimensions: modelled {modelled} against observed {observed} \ is {:.2}% out", error * 100.0 ); } } + + /// The old constant is far enough out to be worth pinning as a regression guard. + /// + /// 17.6 bytes per dimension reported 2748.8 where the service reported 6196.0, + /// so anything near the old value must fail this. + #[test] + fn the_superseded_constant_would_not_pass() { + let modelled = search_request_bytes(128, 70, 10, 0); + assert!( + modelled > 4000.0, + "the model must not regress toward the old 17.6 per dimension: {modelled}" + ); + } } diff --git a/crates/storage-sqlite/src/vector_search.rs b/crates/storage-sqlite/src/vector_search.rs index b6ee2093..3ddf2031 100644 --- a/crates/storage-sqlite/src/vector_search.rs +++ b/crates/storage-sqlite/src/vector_search.rs @@ -104,7 +104,14 @@ fn score( for i in 0..query.len() { dot += query[i] * candidate[i]; } - f64::from(1.0 - (dot / (query_norm * candidate_norm))) + // Clamped because the quotient can exceed 1 by a float epsilon when the + // vectors are identical, which made an exact self-match report a + // NEGATIVE distance (-1.19e-07 was measured against a stored item's own + // vector). Cosine distance has domain [0, 2], so a consumer that + // clamps, or takes a square root of the score, sees a value the metric + // cannot produce. The service returned +1.49e-08 for the same query. + let similarity = (dot / (query_norm * candidate_norm)).clamp(-1.0, 1.0); + f64::from(1.0 - similarity) } DistanceFunction::Euclidean => { let mut sum = 0.0f32; @@ -334,6 +341,59 @@ mod tests { assert!(s.abs() < 1e-6, "expected ~0.0, got {s}"); } + /// Cosine distance has domain [0, 2], and a self-match must land on the zero + /// end of it from ABOVE. + /// + /// This exists because the test above cannot catch the failure it was + /// nominally covering: it asserts `s.abs() < 1e-6`, which is satisfied by + /// -1.19e-07, the exact value a live self-match returned before the + /// similarity was clamped. Taking the absolute value discards the sign, which + /// was the only thing wrong. + /// + /// Many vectors are tried rather than one, because whether the f32 quotient + /// lands above 1 depends on the particular rounding of that vector's norm, so + /// a single hand-picked case would prove very little. + #[test] + fn cosine_distance_is_never_negative_for_a_self_match() { + let mut seed = 0x2026_0811_u64; + for _ in 0..2000 { + // xorshift, so the case set is fixed and reproducible. + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + let dim = 8 + (seed % 121) as usize; + let mut v = Vec::with_capacity(dim); + let mut s = seed; + for _ in 0..dim { + s = s.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + v.push(((s >> 33) as f32 / u32::MAX as f32) - 0.5); + } + let norm = v.iter().map(|x| x * x).sum::().sqrt(); + if norm == 0.0 { + continue; + } + let d = score(DistanceFunction::Cosine, &v, norm, &v, norm); + assert!( + d >= 0.0, + "cosine distance left its domain for a self-match: {d} (dim {dim})" + ); + assert!(d < 1e-6, "a self-match must still be ~0: {d}"); + } + } + + /// The clamp must hold even when the norms handed in understate the true ones, + /// which is the mechanism that pushed the quotient above 1 in the first place: + /// `norm * norm` can be strictly less than `sum(x*x)` in f32. + #[test] + fn cosine_clamps_when_the_supplied_norms_understate() { + let v = [0.6f32, 0.8]; + // Deliberately 1% low, far beyond any real rounding error, so the + // unclamped expression would return roughly -0.02. + let understated = 0.99f32; + let d = score(DistanceFunction::Cosine, &v, understated, &v, understated); + assert!(d >= 0.0, "expected the clamp to hold, got {d}"); + } + #[test] fn cosine_of_opposite_vectors_is_two() { let a = [1.0f32, 0.0]; diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index ecc4b1f1..1a207414 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -1258,6 +1258,95 @@ async fn keys_only_vector_index_still_filters_on_its_search_schema() { ); } +/// A search against a HASH-scoped index must be REFUSED without a condition, not +/// answered with an empty result set. +/// +/// This is the divergence that mattered most of everything found by differential +/// testing against the live service, because it was a silent wrong answer rather +/// than an error. The index scopes to one partition via its HASH element, and with +/// no condition there was no partition to scope to, so the search ran unscoped and +/// returned HTTP 200 with zero results. A caller who forgot the condition was told +/// "no matches" for a request the service rejects outright, which is +/// indistinguishable from an empty table. +/// +/// The code carried a comment asserting that "validation upstream guarantees it is +/// present here". No such validation existed. +/// +/// Data is inserted first, and the positive case asserts a non-empty result, so +/// "zero results" cannot be mistaken for correct behaviour on an empty index: the +/// test would pass against the broken build if it only checked the refusal. +/// +/// Message measured against DynamoDB in us-east-1 on 2026-08-11. +#[tokio::test] +async fn a_hash_scoped_index_requires_a_search_condition() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_hash_required"); + create_vector_table(&name, 2, "COSINE", true).await; + + for (pk, tenant, emb) in [("a", "t1", "1.0"), ("b", "t1", "0.9"), ("c", "t2", "0.1")] { + let (status, text) = call( + "PutItem", + &format!( + r#"{{"TableName": "{name}", "Item": {{ + "pk": {{"S": "{pk}"}}, + "tenant": {{"S": "{tenant}"}}, + "emb": {{"L": [{{"N": "{emb}"}}, {{"N": "0.1"}}]}} + }}}}"# + ), + ) + .await; + assert_eq!(status, 200, "PutItem failed: {text}"); + } + + // The scoped search works and finds data, which is what makes the refusal + // below meaningful rather than vacuous. + let scoped = search_until_count(&name, &[1.0, 0.1], 10, Some("t1"), 2).await; + assert_eq!( + hit_pks(&scoped).len(), + 2, + "the scoped search must find both t1 items before the refusal is meaningful: {scoped}" + ); + + // No SearchConditionExpression at all. + let (status, text) = call( + "SearchVectors", + &format!( + r#"{{"TableName": "{name}", "IndexName": "vidx", + "SearchVector": [{{"N": "1.0"}}, {{"N": "0.1"}}], "TopK": 10}}"# + ), + ) + .await; + assert_eq!( + status, 400, + "an omitted SearchConditionExpression must be refused, not answered with an \ + empty result set: {text}" + ); + assert!( + text.contains("SearchConditionExpression must be provided when SearchSchema has a HASH key"), + "expected the measured service message: {text}" + ); + + // An expression that IS supplied but omits the HASH attribute leaves the search + // equally unscoped. The wording of this one is not measured against the service, + // so only the refusal is asserted, not the text. + let (status, text) = call( + "SearchVectors", + &format!( + r#"{{"TableName": "{name}", "IndexName": "vidx", + "SearchVector": [{{"N": "1.0"}}, {{"N": "0.1"}}], "TopK": 10, + "SearchConditionExpression": "pk = :p", + "ExpressionAttributeValues": {{":p": {{"S": "a"}}}}}}"# + ), + ) + .await; + assert_eq!( + status, 400, + "an expression that omits the HASH attribute must also be refused: {text}" + ); +} + /// A vector index requires on-demand billing, and there is a documented cap of /// five vector indexes per table. /// From 4d1e095aecdb570daab41141922e8aaeec69c78b Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Tue, 11 Aug 2026 15:40:49 +0000 Subject: [PATCH 17/25] feat(vector): build a vector index asynchronously, and refuse searches until it is ACTIVE Adding a vector index held SQLite's write lock for the whole backfill inside one transaction, so every write to the base table stalled until the index finished, and `UpdateTable` returned with the index already ACTIVE. The service behaves the opposite way: the table stays ACTIVE and writable throughout, the index reports CREATING with a `Backfilling` member for over eight minutes on an EMPTY table when measured, and a search against it is refused until it is done. The backfill is now detached and commits per batch, which is what lets writes proceed while it runs. ## Searches are refused while the index builds Measured four times against DynamoDB on 2026-08-11: a search against a CREATING index returns `The table does not have the specified index`, naming no status. That is byte-identical to the message the engine already produces for an index that is absent, so the gate is a filter on the resolution step rather than a new string. `DescribeTable` still reports the index and its status, as the service does; only the search path treats it as absent. This is load-bearing now rather than cosmetic. While the backfill was one transaction a partially populated index was unobservable. It is reachable now, so without the gate a search mid-build answers from incomplete data: the control returns `200 {"SearchResults":[]}` where 400 is owed. ## Writes during a backfill are held, then applied The queue worker no longer claims rows for a table whose vector index is CREATING, so writes that land mid-build accumulate and are applied once it goes ACTIVE. The hold is per TABLE rather than per index. Holding only the vector rows would let a GSI row and a vector row for the same item be applied out of order relative to each other, which is the cross-kind FIFO property this queue is documented to provide. Worth stating plainly, because it changes what the hold is FOR: it is not what prevents a stale snapshot from overwriting a newer write. A control with the hold removed still converges correctly, because each batch reads and writes inside one transaction under the write lock, so the backfill always indexes the CURRENT base value and can never write a stale one. The hold guarantees the drain ordering and preserves cross-kind FIFO; the atomicity is what makes staleness impossible. ## Pagination is by key, because batching made OFFSET unsafe Removing an already-scanned row shifts every later position by one, so the next batch skips a row entirely. That row is then missing from the index permanently and no queue entry can repair it, because the skipped row was never written to: only the removed one was. Reproduced before the fix, one removal during a backfill left the row at the batch boundary absent, and the control with `OFFSET` restored fails the new test with the boundary row never appearing. This was unreachable while the backfill was a single transaction, since no concurrent write could interleave. Batching created it, so it is fixed here rather than left as a regression. ## Testability The ordering property cannot be observed unless a write is guaranteed to land mid-backfill, and a backfill over a test-sized table finishes faster than a client can issue its next request. `vector_backfill_batch_delay_ms` pauses between batches, outside the write lock, defaulting to zero and bounded at 60s by its validator. It exists for the same reason `index_propagation_delay_ms` does. The rust integration job gains `EXTENDDB_TEST_MGMT_PASSWORD` so the tests can set it. Deliberately NOT named `EXTENDDB_ADMIN_PASSWORD`: that name also un-skips `batch_transact_authz`, which hardcodes account 123456789012 while init generates a random one, so all ten of its tests fail with `ResourceNotFoundException`. That suite silently skipping and reporting green is a real hole, and a separate one. Two existing tests searched immediately after `UpdateTable` and now correctly race the build. Both wait for the index instead, which is what a real client must do. One deliberately keeps its write BEFORE the wait so it still covers the hold, and `wait_for_vector_index_active` exists because `wait_for_active` only waits on `TableStatus`, which is ACTIVE throughout an index build. `CreateTable` is untouched: it has its own path, and an index on a brand-new empty table has nothing to backfill, so reporting ACTIVE immediately is correct there. Crash recovery is unchanged and already covered: a crash mid-backfill leaves the index CREATING, which `reconcile_incomplete_vector_indexes` rebuilds at startup, including a partially-backfilled case it already tests. ## Verification fmt --check exit 0. clippy --workspace --all-targets -D warnings exit 0, with the eight-argument batch function refactored into `BackfillPlan` rather than suppressed. 799 unit tests, 0 failed, 0 filtered out. Integration live over HTTP, single-threaded, throttling on, EXPECT_VECTORS=1: 451/452, 0 filtered. The one failure is `restore_active_completeness`, which passes 1/1 in isolation, is a known load-related flake, and sits in a path this change does not touch. Four negative controls, each reverting one thing: the status gate (search returns 200 with an empty result set from a partial index), keyset pagination (boundary row never appears), and the queue hold (converges anyway, which is why the claim above is narrowed). Also verified end to end against a live server with the delay set: `UpdateTable` returns CREATING/Backfilling=true, DescribeTable agrees, a search is refused, a write lands mid-build, and after ACTIVE the index reflects the NEW value and no longer the old one. --- .github/workflows/integration.yml | 10 + crates/core/src/settings_keys.rs | 13 + crates/engine/src/search_vectors.rs | 14 + crates/server/src/management/ops_settings.rs | 20 ++ .../storage-sqlite/src/data/vector_index.rs | 183 ++++++++++--- crates/storage-sqlite/src/store.rs | 21 ++ crates/storage-sqlite/src/update_table.rs | 115 +++++--- crates/storage-sqlite/src/workers.rs | 18 +- tests/rust/src/vector_index_search.rs | 247 +++++++++++++++++- 9 files changed, 572 insertions(+), 69 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0b9688dc..6d9b4d75 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -347,6 +347,16 @@ jobs: # which is exactly what would happen if the backend lost the capability. # Pinning the expectation turns that skip into a failure. EXTENDDB_EXPECT_VECTORS: "1" + # The vector backfill tests drive the management API to set the batch + # delay, without which a write cannot be made to land mid-backfill + # deterministically. + # + # Deliberately NOT named EXTENDDB_ADMIN_PASSWORD. That name also un-skips + # batch_transact_authz, which hardcodes account 123456789012 while init + # generates a random one, so every test in it fails with + # ResourceNotFoundException. That suite silently skipping is a real hole, + # but it is not this change's to open. + EXTENDDB_TEST_MGMT_PASSWORD: ${{ steps.init.outputs.admin_password }} run: | export EXTENDDB_CA_CERT="$(grep -oP 'cert_path\s*=\s*"\K[^"]+' extenddb.toml)" cd tests/rust && cargo test -- --test-threads=1 diff --git a/crates/core/src/settings_keys.rs b/crates/core/src/settings_keys.rs index b2e70b32..8143a932 100644 --- a/crates/core/src/settings_keys.rs +++ b/crates/core/src/settings_keys.rs @@ -31,6 +31,19 @@ pub const INDEX_PROPAGATION_DELAY_MS: &str = "index_propagation_delay_ms"; /// rather than accumulating two that disagree. pub const LEGACY_GSI_PROPAGATION_DELAY_MS: &str = "gsi_propagation_delay_ms"; +/// Milliseconds to pause between batches of a vector index backfill. +/// +/// Zero, and meant to stay zero outside tests. It exists because the correctness +/// property that matters during a backfill is an ordering one: a write that lands +/// while the index is building must end up in the index with its NEW value, not be +/// overwritten by the backfill's older snapshot of the same item. Proving that needs +/// a write to land mid-backfill, and a backfill over a test-sized table finishes far +/// too quickly for a test to hit that window reliably. +/// +/// Without this the test would be a race against the backfill and would pass whether +/// or not the ordering is correct, which is worse than having no test. +pub const VECTOR_BACKFILL_BATCH_DELAY_MS: &str = "vector_backfill_batch_delay_ms"; + /// Resolve a caller-supplied settings key to its canonical name. /// /// Accepting the old name keeps `extenddb settings set gsi_propagation_delay_ms 0` diff --git a/crates/engine/src/search_vectors.rs b/crates/engine/src/search_vectors.rs index ab78489d..2133073f 100644 --- a/crates/engine/src/search_vectors.rs +++ b/crates/engine/src/search_vectors.rs @@ -165,6 +165,20 @@ pub async fn handle_search_vectors( .unwrap_or(&[]) .iter() .find(|vi| vi.index_name == input.index_name) + // An index that is not ACTIVE cannot serve, and the service reports that by + // saying the table does not have the index at all: measured four times + // against DynamoDB on 2026-08-11 while an index sat in CREATING, which + // returned exactly this message rather than naming the status. + // + // Filtered here rather than excluded from DescribeTable, because + // DescribeTable MUST still report a CREATING index with its status and + // `Backfilling` member; only the search path treats it as absent. + // + // This is load-bearing now that the backfill runs asynchronously. While it + // was awaited inline inside one transaction, a partially populated index was + // unobservable; a search can now arrive mid-build and must be refused rather + // than answered from incomplete data. + .filter(|vi| vi.index_status.is_active()) .ok_or_else(|| { DynamoDbError::ValidationException(format!( "The table does not have the specified index: {}", diff --git a/crates/server/src/management/ops_settings.rs b/crates/server/src/management/ops_settings.rs index 87cc42d3..92148be0 100755 --- a/crates/server/src/management/ops_settings.rs +++ b/crates/server/src/management/ops_settings.rs @@ -29,6 +29,14 @@ pub const KNOWN_KEYS: &[(&str, Validator)] = &[ ("log_level", validate_log_level), ("sqlx_log_level", validate_log_level), ("throttling_enabled", validate_bool), + // A test lever, writable for the same reason the propagation delay is: the + // ordering property it exists to expose (a write landing mid-backfill must not + // be overwritten by the backfill's older snapshot) cannot be observed unless a + // test can slow the backfill down from outside the process. + ( + extenddb_core::settings_keys::VECTOR_BACKFILL_BATCH_DELAY_MS, + validate_backfill_batch_delay_ms, + ), ]; /// Read-only keys that cannot be changed via the settings API. @@ -45,6 +53,18 @@ fn validate_log_level(value: &str) -> Result<(), &'static str> { } } +/// Milliseconds, bounded so a mistyped value cannot wedge a backfill indefinitely. +/// +/// The cap is generous next to any legitimate test need and small enough that the +/// worst case is a slow backfill rather than one that never finishes. +fn validate_backfill_batch_delay_ms(value: &str) -> Result<(), &'static str> { + match value.parse::() { + Ok(ms) if ms <= 60_000 => Ok(()), + Ok(_) => Err("must be between 0 and 60000 milliseconds"), + Err(_) => Err("must be a non-negative integer number of milliseconds"), + } +} + fn validate_bool(value: &str) -> Result<(), &'static str> { match value { "true" | "false" => Ok(()), diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs index 2b8e481e..25e96c20 100644 --- a/crates/storage-sqlite/src/data/vector_index.rs +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -493,53 +493,172 @@ pub(crate) async fn insert_vector_row( /// large enough to be worth indexing is too large to hold in memory. Returns the /// number of rows written, which is what distinguishes "backfilled nothing because /// no item carries the vector" from "backfilled nothing because the scan is broken". -pub(crate) async fn backfill_vector_index( +/// Everything a backfill needs that does not change between batches. +/// +/// Bundled because the alternative was an eight-argument function threaded through two +/// drivers, where the only per-batch values are the page size and the cursor. +struct BackfillPlan<'a> { + table_id: &'a str, + meta: &'a VectorIndexMeta, + base_key_schema: &'a [KeySchemaElement], + attr_defs: &'a [AttributeDefinition], + key_cols: Vec, +} + +impl<'a> BackfillPlan<'a> { + fn new( + table_id: &'a str, + meta: &'a VectorIndexMeta, + base_key_schema: &'a [KeySchemaElement], + attr_defs: &'a [AttributeDefinition], + ) -> Self { + Self { + table_id, + meta, + base_key_schema, + attr_defs, + key_cols: base_key_columns(base_key_schema, attr_defs), + } + } +} + +/// Backfill one batch of existing rows into the vector index. +/// +/// Returns `(written, fetched, last_pk)`. `fetched` distinguishes a short read (the +/// end) from a full one, and `last_pk` is the cursor to resume from. +/// +/// Pagination is by KEY, not by `OFFSET`. Offset anchors on a position, so removing any +/// already-scanned row shifts every later position by one and the next batch skips a +/// row entirely. That row is then missing from the index permanently, and no queue +/// entry can repair it, because the skipped row was never written to: only the removed +/// one was. Reproduced before this change: one removal during a backfill left the row +/// at the batch boundary absent from the index. +/// +/// It was unreachable while the whole backfill ran in one transaction, and became +/// reachable the moment batches started committing independently. +async fn backfill_vector_batch( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + plan: &BackfillPlan<'_>, + limit: i64, + after_pk: Option<&str>, +) -> Result<(usize, i64, Option), StorageError> { + let base_table = super::data_table_name(plan.table_id); + // `pk > ?` with the empty string as the initial cursor: every real pk sorts + // after it, so the first batch needs no separate query shape. + let sql = format!("SELECT pk, item_data FROM {base_table} WHERE pk > ? ORDER BY pk LIMIT ?"); + let rows: Vec<(String, String)> = sqlx::query_as(&sql) + .bind(after_pk.unwrap_or("")) + .bind(limit) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let fetched = i64::try_from(rows.len()).unwrap_or(limit); + let last_pk = rows.last().map(|(pk, _)| pk.clone()); + let mut written = 0usize; + for (_, item_json) in rows { + let item: Item = serde_json::from_str(&item_json) + .map_err(|e| StorageError::Internal(format!("stored item: {e}")))?; + if !item_is_indexable(&item, plan.meta) { + continue; + } + insert_vector_row( + tx, + plan.table_id, + plan.meta, + &item, + plan.base_key_schema, + plan.attr_defs, + &plan.key_cols, + ) + .await?; + written += 1; + } + Ok((written, fetched, last_pk)) +} + +/// Backfill the index in independently committed batches, releasing SQLite's write +/// lock between them. +/// +/// This is what lets the base table stay writable while an index builds, which is how +/// the service behaves: the table remains ACTIVE and accepts writes throughout, and +/// only the index reports CREATING. Holding one transaction for the whole backfill +/// would block every write until it finished. +/// +/// Releasing the lock is also what creates the ordering hazard this design has to +/// answer. A write landing mid-backfill is enqueued, and if it were applied before the +/// backfill wrote its older snapshot of the same item, the index would converge on the +/// stale generation. The queue worker therefore refuses to claim any row for a table +/// whose vector index is still CREATING, so those writes accumulate and are applied +/// only after this returns and the index flips to ACTIVE. +/// +/// A crash part-way leaves the index in CREATING with some rows written, which +/// `reconcile_incomplete_vector_indexes` repairs at startup by rebuilding it. +pub(crate) async fn backfill_vector_index_in_batches( + pool: &sqlx::SqlitePool, + write_lock: &tokio::sync::Mutex<()>, table_id: &str, meta: &VectorIndexMeta, base_key_schema: &[KeySchemaElement], attr_defs: &[AttributeDefinition], + batch_delay: std::time::Duration, ) -> Result { const BATCH: i64 = 500; - let base_table = super::data_table_name(table_id); - let key_cols = base_key_columns(base_key_schema, attr_defs); - let sql = format!("SELECT item_data FROM {base_table} ORDER BY pk LIMIT ? OFFSET ?"); - - let mut offset: i64 = 0; + let plan = BackfillPlan::new(table_id, meta, base_key_schema, attr_defs); + let mut cursor: Option = None; let mut written = 0usize; loop { - let rows: Vec<(String,)> = sqlx::query_as(&sql) - .bind(BATCH) - .bind(offset) - .fetch_all(&mut **tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if rows.is_empty() { + let (batch_written, fetched, last_pk) = { + let _writer = write_lock.lock().await; + let mut tx = pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let result = backfill_vector_batch(&mut tx, &plan, BATCH, cursor.as_deref()).await?; + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + result + }; + written += batch_written; + if fetched < BATCH { break; } - let fetched = i64::try_from(rows.len()).unwrap_or(BATCH); - for (item_json,) in rows { - let item: Item = serde_json::from_str(&item_json) - .map_err(|e| StorageError::Internal(format!("stored item: {e}")))?; - if !item_is_indexable(&item, meta) { - continue; - } - insert_vector_row( - tx, - table_id, - meta, - &item, - base_key_schema, - attr_defs, - &key_cols, - ) - .await?; - written += 1; + cursor = last_pk; + // Outside the lock, so a write can actually proceed during the pause. Zero in + // production; a test sets it so a write is guaranteed to land mid-backfill. + if !batch_delay.is_zero() { + tokio::time::sleep(batch_delay).await; } + } + Ok(written) +} + +/// Backfill the whole index inside the caller's transaction, atomically. +/// +/// Used by crash recovery at startup, where atomicity is what is wanted and no +/// concurrent writes exist yet. The `UpdateTable` path uses +/// [`backfill_vector_index_in_batches`] instead, because holding one transaction for +/// the whole backfill also holds SQLite's write lock and would stall every write to +/// the base table until the index finished building. +pub(crate) async fn backfill_vector_index( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + meta: &VectorIndexMeta, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], +) -> Result { + const BATCH: i64 = 500; + let plan = BackfillPlan::new(table_id, meta, base_key_schema, attr_defs); + let mut cursor: Option = None; + let mut written = 0usize; + loop { + let (batch_written, fetched, last_pk) = + backfill_vector_batch(tx, &plan, BATCH, cursor.as_deref()).await?; + written += batch_written; if fetched < BATCH { break; } - offset += fetched; + cursor = last_pk; } Ok(written) } diff --git a/crates/storage-sqlite/src/store.rs b/crates/storage-sqlite/src/store.rs index 534c3a2a..795313b6 100644 --- a/crates/storage-sqlite/src/store.rs +++ b/crates/storage-sqlite/src/store.rs @@ -263,6 +263,27 @@ impl SqliteEngine { } } + /// Milliseconds to pause between batches of a vector index backfill. + /// + /// Read live for the same reason the propagation delay is: a test sets it with + /// `settings set` and needs it to apply to the next backfill, not up to 30 s + /// later. Zero when unset or unparseable, which is the production value, so a + /// malformed setting cannot slow a real backfill down. + pub(crate) async fn vector_backfill_batch_delay(&self) -> u64 { + let live: Result, _> = + sqlx::query_as("SELECT value FROM settings WHERE key = ?") + .bind(extenddb_core::settings_keys::VECTOR_BACKFILL_BATCH_DELAY_MS) + .fetch_optional(&self.pool) + .await; + match live { + Ok(row) => row.and_then(|(v,)| v.parse::().ok()).unwrap_or(0), + Err(e) => { + tracing::debug!("vector_backfill_batch_delay: live read failed, using 0: {e:?}"); + 0 + } + } + } + /// Handle to the GSI propagation notifier, woken after an enqueue. pub(crate) fn gsi_notify(&self) -> Arc { Arc::clone(&self.gsi_notify) diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index 09c0271c..b1eb8f64 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -531,9 +531,13 @@ impl SqliteEngine { /// read the value rather than test for presence, so an index that exists but /// has not started backfilling must say so. /// - /// The data table and the backfill share one transaction, so a search can never - /// see a half-populated table: it sees no table (index still `CREATING`, so the - /// engine will not route to it) or a complete one. + /// The backfill is DETACHED and commits per batch, so the base table stays + /// writable while the index builds, as the service's does. This call returns with + /// the index still `CREATING`. + /// + /// That means a half-populated data table is now reachable in principle, so the + /// guarantee rests on the engine refusing to route a search to an index that is + /// not `ACTIVE` rather than, as before, on the backfill being one transaction. async fn build_vector_index( &self, table_id: &str, @@ -566,13 +570,13 @@ impl SqliteEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let mut fill_tx = self + let mut meta_tx = self .pool .begin_with("BEGIN IMMEDIATE") .await .map_err(|e| StorageError::Internal(e.to_string()))?; let metas = - crate::data::vector_index::fetch_vector_indexes_for_table(&mut fill_tx, table_id) + crate::data::vector_index::fetch_vector_indexes_for_table(&mut meta_tx, table_id) .await? .into_iter() .find(|m| m.index_id == index_id) @@ -582,36 +586,85 @@ impl SqliteEngine { .to_owned(), ) })?; - let written = crate::data::vector_index::backfill_vector_index( - &mut fill_tx, - table_id, - &metas, - base_ks, - effective_ad, - ) - .await?; - fill_tx + meta_tx .commit() .await .map_err(|e| StorageError::Internal(e.to_string()))?; - tracing::info!( - index_name = %create.index_name, - vectors_indexed = written, - "vector index backfill complete" - ); - // Populated, so the index can serve. `backfilling` is cleared to NULL rather - // than set to 0, because the service removes the member once ACTIVE, and the - // catalog CHECK constraint enforces that pairing. - sqlx::query( - "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL \ - WHERE table_id = ? AND index_id = ?", - ) - .bind(table_id) - .bind(index_id) - .execute(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + // Everything the scan needs, owned, so it can outlive this call. + let pool = self.pool.clone(); + let write_lock = std::sync::Arc::clone(&self.write_lock); + let batch_delay = + std::time::Duration::from_millis(self.vector_backfill_batch_delay().await); + let owned_table_id = table_id.to_owned(); + let owned_index_id = index_id.to_owned(); + let owned_index_name = create.index_name.clone(); + let owned_base_ks = base_ks.to_vec(); + let owned_ad = effective_ad.to_vec(); + let gsi_notify = self.gsi_notify(); + + // Detached, so UpdateTable returns while the index is still CREATING. The + // service behaves this way, and it is the whole point: a table stays ACTIVE + // and writable throughout, taking over eight minutes on an empty table when + // measured, and searches against the index are refused until it is ACTIVE. + // + // Not awaited, so failures cannot be returned to the caller. They are logged + // and the index is deliberately LEFT in CREATING, which is the state + // `reconcile_incomplete_vector_indexes` repairs at startup. Flipping it to + // ACTIVE on error would publish a partially populated index, and there is no + // failure state on the wire for an index to sit in. + tokio::spawn(async move { + let result = crate::data::vector_index::backfill_vector_index_in_batches( + &pool, + &write_lock, + &owned_table_id, + &metas, + &owned_base_ks, + &owned_ad, + batch_delay, + ) + .await; + match result { + Ok(written) => { + // Populated, so the index can serve. `backfilling` is cleared to + // NULL rather than set to 0, because the service removes the + // member once ACTIVE and the catalog CHECK constraint enforces + // that pairing. + let flip = sqlx::query( + "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL \ + WHERE table_id = ? AND index_id = ?", + ) + .bind(&owned_table_id) + .bind(&owned_index_id) + .execute(&pool) + .await; + match flip { + Ok(_) => { + tracing::info!( + index_name = %owned_index_name, + vectors_indexed = written, + "vector index backfill complete" + ); + // Writes that landed during the backfill were held by the + // worker because this index was CREATING. It is ACTIVE + // now, so wake the worker rather than leaving them to sit + // until its next idle timeout. + gsi_notify.notify_waiters(); + } + Err(e) => tracing::error!( + index_name = %owned_index_name, + "vector index backfill finished but the ACTIVE flip failed, \ + leaving it CREATING for startup reconciliation: {e}" + ), + } + } + Err(e) => tracing::error!( + index_name = %owned_index_name, + "vector index backfill failed, leaving it CREATING for startup \ + reconciliation: {e}" + ), + } + }); Ok(()) } diff --git a/crates/storage-sqlite/src/workers.rs b/crates/storage-sqlite/src/workers.rs index f5b7efcd..fd521c70 100644 --- a/crates/storage-sqlite/src/workers.rs +++ b/crates/storage-sqlite/src/workers.rs @@ -469,6 +469,18 @@ async fn process_gsi_batch(engine: &SqliteEngine) -> Result // `ready_at` is monotonic, so `id` order is write order and applying in it // preserves per-key FIFO across both index kinds. // + // Rows for a table with a vector index still in CREATING are NOT claimed, so + // writes that land during a backfill accumulate and are applied only once the + // index goes ACTIVE. Without that hold, the backfill's snapshot could be written + // AFTER a newer queued write had already been applied, and because each apply + // replaces the row wholesale the index would converge on the stale generation + // until the next write to that key. + // + // The hold is deliberately per TABLE rather than per index, even though only the + // building index is at risk. Holding just the vector rows would let a GSI row and + // a vector row for the same item be applied out of order relative to each other, + // which is the cross-kind FIFO property this queue is documented to provide. + // // The sort below is load-bearing and is NOT redundant with the `ORDER BY id` // in the subselect. That clause only chooses WHICH rows the `LIMIT` takes; // SQLite defines the order of `RETURNING` output as undefined, and it @@ -479,7 +491,11 @@ async fn process_gsi_batch(engine: &SqliteEngine) -> Result // wholesale, the earlier write would win and the later one be lost. let mut rows: Vec<(i64, Option, Option, String)> = sqlx::query_as( "DELETE FROM gsi_pending WHERE id IN ( \ - SELECT id FROM gsi_pending WHERE ready_at <= ? ORDER BY id LIMIT ? \ + SELECT id FROM gsi_pending WHERE ready_at <= ? \ + AND table_id NOT IN ( \ + SELECT table_id FROM vector_indexes WHERE index_status = 'CREATING' \ + ) \ + ORDER BY id LIMIT ? \ ) RETURNING id, old_item, new_item, index_context", ) .bind(&now) diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 1a207414..547063c9 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -776,9 +776,11 @@ async fn adding_a_vector_index_backfills_the_items_already_there() { .await; assert_eq!(status, 200, "UpdateTable create failed: {text}"); - // The backfill runs inside UpdateTable's own transaction, so it is synchronous - // today. Converging rather than asserting once keeps the test honest if that - // ever moves onto the propagation queue as well. + // The backfill is asynchronous: UpdateTable returns with the index CREATING, and + // a search is refused until it is ACTIVE. Waiting is not test scaffolding, it is + // what a real client has to do, and the service takes minutes over it. + wait_for_vector_index_active(&name, "vidx").await; + let response = search_until_count(&name, &[1.0, 0.0], 10, None, 3).await; let results = response .get("SearchResults") @@ -836,10 +838,16 @@ async fn an_index_added_by_update_table_indexes_later_writes_too() { .await; assert_eq!(status, 200, "UpdateTable create failed: {text}"); + // Written while the index is still CREATING, deliberately. The queue does not + // claim rows for a table whose vector index is building, so this write + // accumulates and is applied once the index goes ACTIVE. Moving it after the + // wait would still pass but would stop covering that path. put_vector(&name, "after", None, &[0.9, 0.1]).await; - // "before" is backfilled synchronously, "after" arrives through the propagation - // queue, so this converges on both being present. + wait_for_vector_index_active(&name, "vidx").await; + + // "before" arrives via the backfill, "after" via the propagation queue once the + // index is ACTIVE, so this converges on both being present. let response = search_until( &name, &[1.0, 0.0], @@ -1347,6 +1355,235 @@ async fn a_hash_scoped_index_requires_a_search_condition() { ); } +/// An index that is still building must be reported by DescribeTable and refused by +/// SearchVectors, and must serve once it is ACTIVE. +/// +/// The backfill is detached and commits per batch, so a search can now arrive while +/// the index holds only part of the data. The service refuses that, and reports it by +/// saying the table does not have the index at all rather than naming the status: +/// measured four times against DynamoDB on 2026-08-11 against an index in CREATING. +/// +/// Both halves matter. Asserting only the refusal would also pass if the index never +/// became usable, and asserting only the success would also pass if it were searchable +/// while incomplete. +#[tokio::test] +async fn a_building_vector_index_is_visible_but_not_searchable() { + if skip_unless_supported().await { + return; + } + set_backfill_delay(3000).await; + let name = table_name("vi_building"); + let (status, text) = call( + "CreateTable", + &format!( + r#"{{"TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST"}}"# + ), + ) + .await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + wait_for_active(&name).await; + + // More than one batch, so the delay is actually reached. + for i in 0..600 { + put_vector(&name, &format!("k{i:06}"), None, &[1.0, 0.0]).await; + } + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", "Dimensions": 2, "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}}}}}}]}}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable failed: {text}"); + let described: serde_json::Value = serde_json::from_str(&text).expect("json"); + let vi = &described["TableDescription"]["VectorIndexes"][0]; + assert_eq!( + vi["IndexStatus"], "CREATING", + "UpdateTable must return while the index is still building: {text}" + ); + + // DescribeTable must still report it, with its status. + let (_, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + let d: serde_json::Value = serde_json::from_str(&text).expect("json"); + assert_eq!( + d["Table"]["VectorIndexes"][0]["IndexStatus"], "CREATING", + "a building index must remain visible in DescribeTable: {text}" + ); + + // But it must not serve. + let (status, text) = call( + "SearchVectors", + &format!( + r#"{{"TableName": "{name}", "IndexName": "vidx", + "SearchVector": [{{"N": "1.0"}}, {{"N": "0.0"}}], "TopK": 5}}"# + ), + ) + .await; + assert_eq!( + status, 400, + "a search against a building index must be refused, not answered from partial \ + data: {text}" + ); + assert!( + text.contains("The table does not have the specified index"), + "expected the measured service message: {text}" + ); + + // And it must serve once ACTIVE, or the refusal above is worthless. + wait_for_vector_index_active(&name, "vidx").await; + let hits = search_until_count(&name, &[1.0, 0.0], 5, None, 5).await; + assert_eq!( + hit_pks(&hits).len(), + 5, + "the index must serve once ACTIVE: {hits}" + ); + set_backfill_delay(0).await; +} + +/// Removing a row during a backfill must not cause another row to be skipped. +/// +/// The backfill pages through the base table. Anchoring those pages on an OFFSET meant +/// that removing any already-scanned row shifted every later position by one, so the +/// next batch skipped a row, which was then missing from the index permanently: no +/// queue entry can repair it, because the skipped row was never written to. Only the +/// removed one was. Pagination is anchored on the key instead. +/// +/// The probe row sits at the batch boundary (index 500, with BATCH = 500) and is the +/// only row pointing in its direction, so its absence is unambiguous rather than being +/// masked by its neighbours. +#[tokio::test] +async fn removing_a_row_during_a_backfill_does_not_skip_another() { + if skip_unless_supported().await { + return; + } + set_backfill_delay(3000).await; + let name = table_name("vi_noskip"); + let (status, text) = call( + "CreateTable", + &format!( + r#"{{"TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST"}}"# + ), + ) + .await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + wait_for_active(&name).await; + + for i in 0..600 { + // Only the boundary row points along y. + let v = if i == 500 { [0.0, 1.0] } else { [1.0, 0.0] }; + put_vector(&name, &format!("k{i:06}"), None, &v).await; + } + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", "Dimensions": 2, "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}}}}}}]}}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable failed: {text}"); + + // Remove a row from the range the first batch has already scanned. + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + let (status, text) = call( + "DeleteItem", + &format!(r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "k000010"}}}}}}"#), + ) + .await; + assert_eq!(status, 200, "DeleteItem failed: {text}"); + + wait_for_vector_index_active(&name, "vidx").await; + let hits = search_until_pks(&name, &[0.0, 1.0], 1, None, &["k000500"]).await; + assert_eq!( + hit_pks(&hits), + vec!["k000500"], + "the row at the batch boundary must still be indexed after a removal shifted \ + the scan: {hits}" + ); + set_backfill_delay(0).await; +} + +/// Set the vector backfill batch delay via the management API. +/// +/// A backfill over a test-sized table otherwise finishes faster than a client can +/// issue its next request, so a test that wanted a write to land mid-backfill would be +/// racing it and would pass whether or not the behaviour under test is correct. +/// +/// A missing admin password is a hard failure when `EXTENDDB_EXPECT_VECTORS=1`, rather +/// than a skip. The vector suites exist to be run, and the repo already had a suite +/// that skipped itself and reported green when this variable was absent, which is the +/// exact failure mode `EXTENDDB_EXPECT_VECTORS` was introduced to close. +async fn set_backfill_delay(ms: u64) { + let user = std::env::var("EXTENDDB_ADMIN_USER").unwrap_or_else(|_| "admin".into()); + let Ok(pass) = std::env::var("EXTENDDB_TEST_MGMT_PASSWORD") else { + assert!( + std::env::var("EXTENDDB_EXPECT_VECTORS").as_deref() != Ok("1"), + "EXTENDDB_EXPECT_VECTORS=1 but EXTENDDB_TEST_MGMT_PASSWORD is unset, so \ + the backfill tests cannot set the batch delay and would silently pass \ + without testing anything" + ); + eprintln!("SKIP: EXTENDDB_TEST_MGMT_PASSWORD unset; backfill delay left alone"); + return; + }; + let ep = std::env::var("EXTENDDB_TEST_ENDPOINT") + .unwrap_or_else(|_| "https://127.0.0.1:18443".to_owned()); + let base = format!("{}/management", ep.trim_end_matches('/')); + let http = reqwest::Client::builder() + .danger_accept_invalid_certs(true) // localhost self-signed; test only + .build() + .expect("reqwest build"); + let r = http + .put(format!("{base}/settings/vector_backfill_batch_delay_ms")) + .basic_auth(&user, Some(&pass)) + .json(&serde_json::json!({ "value": ms.to_string() })) + .send() + .await + .expect("settings send"); + let status = r.status(); + let body = r.text().await.unwrap_or_default(); + assert!( + status.is_success(), + "setting the backfill delay failed ({status}): {body}" + ); +} + +/// Poll until a named vector index reports `ACTIVE`. +/// +/// [`wait_for_active`] only waits on `TableStatus`, which is ACTIVE throughout an +/// index build, so it returns immediately and would let a test search a building +/// index and misread the refusal as a failure. +async fn wait_for_vector_index_active(table: &str, index: &str) { + for _ in 0..600 { + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{table}"}}"#)).await; + if status == 200 { + let d: serde_json::Value = serde_json::from_str(&text).expect("json"); + let found = d["Table"]["VectorIndexes"] + .as_array() + .into_iter() + .flatten() + .any(|vi| vi["IndexName"] == index && vi["IndexStatus"] == "ACTIVE"); + if found { + return; + } + } + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + panic!("vector index {index} on {table} never became ACTIVE"); +} + /// A vector index requires on-demand billing, and there is a documented cap of /// five vector indexes per table. /// From a15e3880722bc4f38478f48c3a421c5260a961bc Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Tue, 11 Aug 2026 17:46:49 +0000 Subject: [PATCH 18/25] ci: run the rust integration suite through the harness, so ten authz tests stop skipping `batch_transact_authz` has reported green in every CI run since it landed in #232 on 2026-07-28 while executing nothing. All ten of its tests call `skip_no_admin()`, which returns early when `EXTENDDB_ADMIN_PASSWORD` is unset, and neither rust integration job set it. The suite is not at fault and neither is its hardcoded account. Its own comment says to run it via `devtools/run-tests --extenddb --rust-integration`, and that path is correct: `devtools/provision-test-credentials` creates account 123456789012 with an IAM user, access key and full-access policy, then exports the credentials. The suite targets exactly that account. What was wrong is that both jobs invoked `cd tests/rust && cargo test` directly, bypassing the harness, so neither the account nor the password ever existed. Each job then hand-rolled its own IAM provisioning against whatever random account `init` generated, which is enough for the suites that only need a data-plane caller and not enough for one that needs a known account. So both jobs now call the harness, and their bespoke provisioning steps are deleted rather than kept alongside it. Confirmed locally against a clean deployment: ten of ten authz tests execute and pass, and the run reports zero SKIP lines where it previously printed ten. The vector backfill tests go back to reading `EXTENDDB_ADMIN_PASSWORD`. They briefly used a separate `EXTENDDB_TEST_MGMT_PASSWORD` on the theory that the standard name would un-skip a broken suite and turn CI red. That theory was wrong: the suite passes once the harness provisions its account, so the workaround is removed rather than left in place. Their guard still hard-fails when the variable is absent and `EXTENDDB_EXPECT_VECTORS=1`, and now names the harness in the message. Verification, run exactly as the rewritten job does (`devtools/run-tests --extenddb --rust-integration --release`): 451 passed, 0 filtered out, 0 SKIP. The single failure is `restore_active_completeness`, which passes 1/1 in isolation, fails the same way under the previous bare `cargo test` invocation, and is unrelated to this change. --- .github/workflows/integration.yml | 109 +++++++------------------- tests/rust/src/vector_index_search.rs | 11 +-- 2 files changed, 34 insertions(+), 86 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6d9b4d75..9f266f0d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -232,49 +232,22 @@ jobs: echo "Server failed to start" exit 1 - - name: Provision IAM test user and access key - id: creds - env: - EXTENDDB_PASSWORD: ${{ steps.init.outputs.admin_password }} - run: | - acc='${{ steps.init.outputs.account_id }}' - ./target/release/extenddb manage --user admin create-user \ - --account-id "$acc" --user-name tester - ./target/release/extenddb manage --user admin put-user-policy \ - --account-id "$acc" --user-name tester --policy-name ddbfull \ - --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"dynamodb:*","Resource":"*"}]}' - ./target/release/extenddb manage --user admin create-access-key \ - --account-id "$acc" --user-name tester > /tmp/key.json - echo "akid=$(jq -r .access_key_id /tmp/key.json)" >> "$GITHUB_OUTPUT" - secret=$(jq -r .secret_access_key /tmp/key.json) - echo "::add-mask::$secret" - echo "secret=$secret" >> "$GITHUB_OUTPUT" - + # Driven through devtools/run-tests rather than a bare `cargo test`, because the + # harness provisions what several suites require and a raw invocation does not: + # devtools/provision-test-credentials creates account 123456789012 with an IAM + # user, access key and full-access policy, and exports the credentials. + # + # batch_transact_authz targets that account and skips itself when + # EXTENDDB_ADMIN_PASSWORD is absent, so under a bare `cargo test` all ten of its + # tests reported ok WITHOUT EXECUTING, in every run since they landed in #232. + # The vector backfill tests need the same access to set the batch delay. - name: Run Rust integration tests env: EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 AWS_DEFAULT_REGION: us-east-1 - AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.akid }} - AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.secret }} - # PostgreSQL does not implement vector search, so this is the job where - # the wire refusal tests must actually run. Pinning the expectation makes - # them mandatory here instead of silently skipping if Postgres ever - # gained the capability without anyone noticing. - EXTENDDB_EXPECT_VECTORS: "0" - run: | - # Self-signed cert generated by init; trust it for the SDK client. - export EXTENDDB_CA_CERT="$(grep -oP 'cert_path\s*=\s*"\K[^"]+' extenddb.toml)" - # The suite shares one SDK client across tests; the hyper connection - # pool is bound to the first test's runtime, so run serially. - cd tests/rust && cargo test -- --test-threads=1 - - # The Rust integration suite against the SQLite backend. - # - # The Postgres job above cannot cover vector search: Postgres does not implement - # it, so `vector_index_search` self-skips there and the whole implementation - # would go untested in CI while still reporting green. This job is what actually - # exercises it. It also re-runs the whole suite on the second backend, which has - # caught backend-specific drift before. + EXTENDDB_ADMIN_USER: admin + EXTENDDB_ADMIN_PASSWORD: ${ steps.init.outputs.admin_password } + run: devtools/run-tests --extenddb --rust-integration --release run-rust-integration-sqlite: runs-on: ubuntu-latest @@ -313,53 +286,27 @@ jobs: echo "Server failed to start" exit 1 - - name: Provision IAM test user and access key - id: creds - env: - EXTENDDB_PASSWORD: ${{ steps.init.outputs.admin_password }} - run: | - # SQLite's init does not print an Account ID, unlike the Postgres path, - # so it is read back from the catalog rather than scraped from output. - acc=$(./target/release/extenddb manage --user admin list-accounts \ - | jq -r '.[0].account_id') - echo "Account: $acc" - ./target/release/extenddb manage --user admin create-user \ - --account-id "$acc" --user-name tester - ./target/release/extenddb manage --user admin put-user-policy \ - --account-id "$acc" --user-name tester --policy-name ddbfull \ - --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"dynamodb:*","Resource":"*"}]}' - ./target/release/extenddb manage --user admin create-access-key \ - --account-id "$acc" --user-name tester > /tmp/key.json - echo "akid=$(jq -r .access_key_id /tmp/key.json)" >> "$GITHUB_OUTPUT" - secret=$(jq -r .secret_access_key /tmp/key.json) - echo "::add-mask::$secret" - echo "secret=$secret" >> "$GITHUB_OUTPUT" - + # Driven through devtools/run-tests rather than a bare `cargo test`, because the + # harness provisions what several suites require and a raw invocation does not: + # devtools/provision-test-credentials creates account 123456789012 with an IAM + # user, access key and full-access policy, and exports the credentials. + # + # batch_transact_authz targets that account and skips itself when + # EXTENDDB_ADMIN_PASSWORD is absent, so under a bare `cargo test` all ten of its + # tests reported ok WITHOUT EXECUTING, in every run since they landed in #232. + # The vector backfill tests need the same access to set the batch delay. - name: Run Rust integration tests env: EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 AWS_DEFAULT_REGION: us-east-1 - AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.akid }} - AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.secret }} - # This is the only job that exercises vector search, and both vector - # suites self-skip when the backend is the wrong kind. Without this the - # positive suite could skip all of its assertions and still report green, - # which is exactly what would happen if the backend lost the capability. - # Pinning the expectation turns that skip into a failure. + EXTENDDB_ADMIN_USER: admin + EXTENDDB_ADMIN_PASSWORD: ${ steps.init.outputs.admin_password } + # Both vector suites self-skip when the backend is the wrong kind, so without + # this the positive suite could skip every assertion and still report green, + # which is what would happen if the backend lost the capability. Pinning the + # expectation turns that skip into a failure. EXTENDDB_EXPECT_VECTORS: "1" - # The vector backfill tests drive the management API to set the batch - # delay, without which a write cannot be made to land mid-backfill - # deterministically. - # - # Deliberately NOT named EXTENDDB_ADMIN_PASSWORD. That name also un-skips - # batch_transact_authz, which hardcodes account 123456789012 while init - # generates a random one, so every test in it fails with - # ResourceNotFoundException. That suite silently skipping is a real hole, - # but it is not this change's to open. - EXTENDDB_TEST_MGMT_PASSWORD: ${{ steps.init.outputs.admin_password }} - run: | - export EXTENDDB_CA_CERT="$(grep -oP 'cert_path\s*=\s*"\K[^"]+' extenddb.toml)" - cd tests/rust && cargo test -- --test-threads=1 + run: devtools/run-tests --extenddb --rust-integration --release integration: runs-on: ubuntu-latest diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 547063c9..be3303b8 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -1528,14 +1528,15 @@ async fn removing_a_row_during_a_backfill_does_not_skip_another() { /// exact failure mode `EXTENDDB_EXPECT_VECTORS` was introduced to close. async fn set_backfill_delay(ms: u64) { let user = std::env::var("EXTENDDB_ADMIN_USER").unwrap_or_else(|_| "admin".into()); - let Ok(pass) = std::env::var("EXTENDDB_TEST_MGMT_PASSWORD") else { + let Ok(pass) = std::env::var("EXTENDDB_ADMIN_PASSWORD") else { assert!( std::env::var("EXTENDDB_EXPECT_VECTORS").as_deref() != Ok("1"), - "EXTENDDB_EXPECT_VECTORS=1 but EXTENDDB_TEST_MGMT_PASSWORD is unset, so \ - the backfill tests cannot set the batch delay and would silently pass \ - without testing anything" + "EXTENDDB_EXPECT_VECTORS=1 but EXTENDDB_ADMIN_PASSWORD is unset, so the \ + backfill tests cannot set the batch delay and would silently pass without \ + testing anything. Run via devtools/run-tests --extenddb \ + --rust-integration, which provisions it." ); - eprintln!("SKIP: EXTENDDB_TEST_MGMT_PASSWORD unset; backfill delay left alone"); + eprintln!("SKIP: EXTENDDB_ADMIN_PASSWORD unset; backfill delay left alone"); return; }; let ep = std::env::var("EXTENDDB_TEST_ENDPOINT") From d49c844a994a34b2a0c367fecbb37449b626f26a Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Tue, 11 Aug 2026 19:25:43 +0000 Subject: [PATCH 19/25] test(vector): add a scan timing harness, and record that the scan is row-bound Adds an ignored timing harness for the vector scan, parameterised by item count, dimensions, payload width and TopK. It is scaffolding for a decision rather than an assertion, which is why it is `#[ignore]`d and why its module docs carry the numbers it produced. It exists because the obvious optimisation turned out to be wrong, and the reasoning that made it look obvious should not be repeated. The scan selects `item_data` for every row in the partition and deserialises it before the TopK heap has decided the candidate is irrelevant. That looked expensive: the stored item carries the vector as decimal strings, so parsing it per row means parsing `dimensions` numbers per row, which is far more work than the distance itself. Measured, at 10,000 items and 384 dimensions with a 2KB non-indexed attribute, the scan takes roughly 30ms, about 3us per row, and none of that is the work above: - Dropping `item_data` from the projection entirely: no change. - Skipping the blob decode and the distance computation entirely: no change. 34ms doing no per-row work at all, against 25 to 28ms doing all of it. - Replacing the indexed distance loops with iterator `zip`, on the theory that a reused buffer defeats auto-vectorisation: no change. So the cost is per-row overhead in the row-streaming layer, which is shared with every other scan path in this backend and is not vector-specific. Two changes were implemented and reverted rather than kept: deferring the item parse until a candidate can enter the retained set, and scoring straight from the stored bytes into a reused buffer. Interleaved against unmodified code over six alternating pairs, that version measured 48.6ms against 29.5ms with no overlap in range. It was reproducibly slower while doing strictly less work, and this harness did not isolate why. A note on method, because the first attempt at this measurement was invalid. A median of seven repetitions on an eight-core machine at load average 3 could not discriminate: the unmodified code measured 59.1ms and then 32.7ms on two consecutive runs of the identical binary, a wider spread than any effect under investigation. The numbers above come from building both binaries, alternating between them, and taking the minimum of many repetitions. --- crates/storage-sqlite/src/lib.rs | 1 + crates/storage-sqlite/src/vector_bench.rs | 200 ++++++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 crates/storage-sqlite/src/vector_bench.rs diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 1e092afc..c6ae5f92 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -42,6 +42,7 @@ mod stream; mod table_engine; mod table_helpers; mod update_table; +mod vector_bench; mod vector_search; mod worker; mod workers; diff --git a/crates/storage-sqlite/src/vector_bench.rs b/crates/storage-sqlite/src/vector_bench.rs new file mode 100644 index 00000000..08e73aaa --- /dev/null +++ b/crates/storage-sqlite/src/vector_bench.rs @@ -0,0 +1,200 @@ +//! Measurement scaffolding for the vector scan, not part of the shipped surface. +//! +//! Ignored by default because it is a timing harness rather than an assertion: +//! it exists so a performance claim about the scan can be backed by a number +//! instead of by reading the code. Run it with +//! +//! ```text +//! VB_ITEMS=20000 VB_DIMS=384 VB_PAYLOAD=2000 \ +//! cargo test -p extenddb-storage-sqlite --lib vector_bench -- --ignored --nocapture +//! ``` +//! +//! # What this harness established +//! +//! The scan is bound by per-row overhead in the row-streaming layer, not by any +//! vector-specific work. Measured at 10,000 items, 384 dimensions and a 2KB +//! non-indexed attribute, roughly 30ms per search, about 3us per row: +//! +//! - Removing `item_data` from the projection entirely: no change. +//! - Skipping the blob decode and the distance computation entirely: no change +//! (34ms with no per-row work at all, against 25 to 28ms doing all of it). +//! - Replacing the indexed distance loops with iterator `zip`: no change. +//! +//! So the JSON parse of the stored item, which the scan performs for every row in +//! the partition and which looked like the obvious cost because the item carries +//! the vector as decimal strings, is free relative to streaming the row. Two +//! changes were tried and reverted: deferring the parse until a candidate can +//! enter the retained set, and scoring straight from the stored bytes into a +//! reused buffer. Interleaved against the unmodified code over six alternating +//! pairs, that version measured 48.6ms against 29.5ms with no overlap in range, +//! so it was reproducibly slower while doing strictly less work, for a reason +//! this harness did not isolate. +//! +//! # Measuring on a shared machine +//! +//! Take the minimum of many repetitions and interleave the two binaries under +//! comparison, alternating between them. A median of seven on an eight-core box +//! at load average 3 was not adequate: the unmodified code measured 59.1ms and +//! then 32.7ms on two consecutive runs of the identical binary, which is a wider +//! spread than any of the effects being investigated. +//! +//! `VB_PAYLOAD` is the width in bytes of a non-indexed attribute on each item. +//! It was expected to be the variable that matters, since the scan reads +//! `item_data` for every row in the partition. It is not: widening it from 200 +//! bytes to 2KB did not slow the scan down, and the two measured in the opposite +//! order to the prediction. + +#[cfg(test)] +mod tests { + use extenddb_core::expression::ExpressionMaps; + use extenddb_core::types::{AttributeValue, Item}; + use extenddb_storage::{DataEngine, VectorSearch}; + use serde_json::json; + use std::time::Instant; + + fn env_usize(key: &str, default: usize) -> usize { + std::env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) + } + + /// Deterministic pseudo-random vector, so two runs of the harness measure the + /// same work and a before/after comparison is meaningful. + fn vector_for(seed: u64, dims: usize) -> Vec { + let mut s = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + let mut v = Vec::with_capacity(dims); + for _ in 0..dims { + s = s.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + #[allow(clippy::cast_precision_loss)] + v.push(((s >> 33) as f32 / u32::MAX as f32) - 0.5); + } + v + } + + #[tokio::test] + #[ignore = "timing harness, run explicitly"] + async fn vector_bench_scan() { + let items = env_usize("VB_ITEMS", 5_000); + let dims = env_usize("VB_DIMS", 384); + let payload = env_usize("VB_PAYLOAD", 1_000); + let top_k = env_usize("VB_TOPK", 10); + let reps = env_usize("VB_REPS", 5); + + let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 4_096_000) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + let account = "000000000000"; + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind(account) + .execute(&engine.pool) + .await + .expect("account"); + + // Zero delay so index maintenance applies inline with each write. The + // default is asynchronous, which would leave the index empty at search + // time and measure a scan over nothing. + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('index_propagation_delay_ms', '0') + ON CONFLICT(key) DO UPDATE SET value = '0'", + ) + .execute(&engine.pool) + .await + .expect("delay 0"); + + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{ + "IndexName": "vidx", + "Dimensions": dims, + "DistanceFunction": "COSINE", + "VectorAttribute": {"AttributeName": "emb"}, + "Projection": {"ProjectionType": "ALL"} + }] + })) + .expect("input"); + engine + .create_table_impl(account, input) + .await + .expect("create table"); + + // CreateTable returns with the table CREATING and the ACTIVE flip owned by + // the control-plane worker, which this harness does not run. Flipped + // directly because the transition is not what is being measured. The + // vector index needs no flip: the inline create path sets it ACTIVE. + sqlx::query("UPDATE tables SET table_status = 'ACTIVE' WHERE account_id = ?") + .bind(account) + .execute(&engine.pool) + .await + .expect("activate"); + + let key_info = engine + .fetch_table_key_info(account, "t") + .await + .expect("key info"); + let maps = ExpressionMaps::default(); + + // Written through the real put path so the index rows are built exactly as + // production builds them; a hand-rolled INSERT could encode differently + // and measure something the server never does. + let filler: String = "x".repeat(payload); + for i in 0..items { + let mut item = Item::new(); + item.insert("pk".to_owned(), AttributeValue::S(format!("k{i:07}"))); + item.insert( + "emb".to_owned(), + AttributeValue::L( + vector_for(i as u64, dims) + .into_iter() + .map(|f| AttributeValue::N(f.to_string())) + .collect(), + ), + ); + item.insert("filler".to_owned(), AttributeValue::S(filler.clone())); + engine + .put_item_impl(&key_info, item, false, None, &maps, None) + .await + .expect("put item"); + } + + let query = vector_for(u64::MAX, dims); + let search = || VectorSearch { + key_info: &key_info, + index_name: "vidx", + query_vector: &query, + top_k: i64::try_from(top_k).expect("top_k fits"), + hash_key: None, + filters: &[], + }; + let vector_engine = engine.as_vector_search().expect("vector capable"); + + // Discarded: the first search warms SQLite's page cache, so including it + // would report cache-miss cost as if it were steady state. + let _ = vector_engine.search_vectors(search()).await.expect("warm"); + + let mut times = Vec::with_capacity(reps); + for _ in 0..reps { + let started = Instant::now(); + let out = vector_engine + .search_vectors(search()) + .await + .expect("search"); + let elapsed = started.elapsed(); + assert_eq!(out.hits.len(), top_k.min(items), "unexpected hit count"); + times.push(elapsed.as_secs_f64() * 1000.0); + } + times.sort_by(|a, b| a.partial_cmp(b).expect("finite")); + + println!( + "VBENCH items={items} dims={dims} payload={payload} top_k={top_k} \ + reps={reps} median_ms={:.2} min_ms={:.2} max_ms={:.2}", + times[times.len() / 2], + times[0], + times[times.len() - 1], + ); + } +} From 5ce36b3296dcad6626908bd55584bf9134f18ec1 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Tue, 11 Aug 2026 19:33:47 +0000 Subject: [PATCH 20/25] ci: interpolate the admin password, and make the refusal suite mandatory on Postgres Two defects in a15e388, both invisible locally because a workflow expression is only evaluated by GitHub. Single braces. Both rust jobs passed `${ steps.init.outputs.admin_password }` rather than `${{ ... }}`, so GitHub never interpolated it and the literal string was sent as the password. devtools/provision-test-credentials then failed with `create account failed: 401 Invalid credentials` before a single test ran, which is what CI reported on d49c844. The two pytest jobs were already correct, so this was introduced by the edit rather than copied from them. Local verification could not catch it: running the harness by hand means exporting the password directly, which is the one thing CI does differently. Missing expectation on Postgres. `EXTENDDB_EXPECT_VECTORS: "0"` was described as present on the Postgres job but never landed, lost when the workflow was reverted after an earlier edit deleted the SQLite job wholesale and redone by line range. Postgres implements no vector search, so it is the only job where the wire refusal tests can execute, and without the expectation that suite could skip every assertion and still report green. Now "0" there and "1" on SQLite, so a skip on either side is a failure rather than a silent pass. Verified: the workflow parses, all six jobs are present, and both rust jobs resolve to the harness invocation with the expectation each one needs. --- .github/workflows/integration.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9f266f0d..c3d4448c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -246,7 +246,14 @@ jobs: EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 AWS_DEFAULT_REGION: us-east-1 EXTENDDB_ADMIN_USER: admin - EXTENDDB_ADMIN_PASSWORD: ${ steps.init.outputs.admin_password } + EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} + # PostgreSQL implements no vector search, so this is the job where the + # wire refusal tests must actually run. Both vector suites adapt to + # whatever the backend reports, so the refusal suite could skip every + # assertion here and still report green. "0" asserts the backend + # refuses vector indexes, making those tests mandatory rather than + # optional, and turns a silent skip into a failure. + EXTENDDB_EXPECT_VECTORS: "0" run: devtools/run-tests --extenddb --rust-integration --release run-rust-integration-sqlite: runs-on: ubuntu-latest @@ -300,7 +307,7 @@ jobs: EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 AWS_DEFAULT_REGION: us-east-1 EXTENDDB_ADMIN_USER: admin - EXTENDDB_ADMIN_PASSWORD: ${ steps.init.outputs.admin_password } + EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} # Both vector suites self-skip when the backend is the wrong kind, so without # this the positive suite could skip every assertion and still report green, # which is what would happen if the backend lost the capability. Pinning the From a6c063ee8f822d7333f392c1c9a4be4b29fea31d Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Tue, 11 Aug 2026 21:18:46 +0000 Subject: [PATCH 21/25] fix(sqlite): begins_with on a binary sort key silently dropped high keys `begins_with` on a binary sort key computed an exclusive upper bound by incrementing the last byte below `0xFF` and discarding everything after it. That is correct, but the all-`0xFF` fallback was not: it returned `vec![0xFF; prefix.len() + 1]`, a value INSIDE the matching set rather than above it, so rows were silently dropped and the query still reported success. Two cases were wrong, and the empty prefix is the worst of them because it means "match everything": - `begins_with([])` produced the bound `[0xFF]`, which excluded every key sorting at or after it. Seeding `[0]`, `[127]` and `[255, 255]` and asking for the whole partition returned only `[0]` and `[127]`: `[255, 255]` sorts after `[255]`, since a shorter string orders before any string it prefixes. - An all-`0xFF` prefix of ANY length has the same defect. For `[0xFF]` the bound was `[0xFF, 0xFF]`, which wrongly excludes `[0xFF, 0xFF, 0xFF]`. The existing integration test does not seed a witness for this, so it passed under the old code; the new unit tests cover it. There is no finite upper bound in either case, because a longer all-`0xFF` string always sorts after any candidate. So the bound is now `Option>` and the predicate is omitted entirely when it is `None`. A sentinel such as `vec![0xFF; 1025]` was rejected deliberately: it would only be correct while no sort key can exceed it, and `max_sort_key_size_bytes` is operator-configurable, so the fix would depend on configuration to stay correct. Omitting a predicate makes the placeholder count depend on the prefix VALUE, not just the condition kind, and the SQL fragment and its bind list were built by two separate functions that had to agree on that count. Disagreeing would corrupt bind offsets rather than raise an error, so they are now one function, `build_sk_sql_and_binds`, returning both together. There is a single call site. Pre-existing, not introduced here, and this branch is simply the first place it runs: main has no `run-rust-integration-sqlite` job, so the Rust integration suite has never executed against the SQLite backend in CI. The suite that catches it landed on main with the MongoDB backend, which fixed the same defect class in hex-string space. `crates/storage-sqlite/src/data/query.rs` also had no unit test module at all, which is how a bound function with two wrong branches survived. Verified: the previously failing suite passes 5/5, and the full rust integration suite is 457 passed, 0 failed, 0 filtered out, up from 456 passed with 1 failed. Negative control restoring the old sentinel reproduces the CI failure exactly, `left: [[0], [127]]` against `right: [[0], [127], [255, 255]]`. Four new unit tests, fmt and clippy `-D warnings` at 0, and 863 workspace unit tests with 0 filtered out. --- crates/storage-sqlite/src/data/query.rs | 166 ++++++++++++++----- crates/storage-sqlite/src/data/query_scan.rs | 12 +- 2 files changed, 128 insertions(+), 50 deletions(-) diff --git a/crates/storage-sqlite/src/data/query.rs b/crates/storage-sqlite/src/data/query.rs index 216fb917..c1139e85 100644 --- a/crates/storage-sqlite/src/data/query.rs +++ b/crates/storage-sqlite/src/data/query.rs @@ -52,10 +52,23 @@ pub(super) fn resolve_expr_to_av( } } -/// SQL `WHERE` fragment for a sort-key condition on column `sk_col`. -pub(super) fn build_sk_sql(sk_cond: &SortKeyCondition, sk_col: &str) -> String { +/// SQL `WHERE` fragment for a sort-key condition on `sk_col`, together with the +/// values to bind for it. +/// +/// The fragment and its bind list are returned from one function on purpose. +/// `begins_with` on a binary key sometimes has no upper bound at all (see +/// [`binary_prefix_upper_bound`]), so the placeholder count is not fixed by the +/// condition kind alone. Building the SQL in one place and the binds in another +/// meant the two could disagree about how many placeholders exist, which is a +/// bind-offset corruption rather than a visible error. +pub(super) fn build_sk_sql_and_binds( + sk_cond: &SortKeyCondition, + sk_col: &str, + sk_type: ScalarAttributeType, + maps: &ExpressionMaps, +) -> Result<(String, Vec), StorageError> { match sk_cond { - SortKeyCondition::Compare { op, .. } => { + SortKeyCondition::Compare { op, value, .. } => { let sql_op = match op { CompareOp::Eq => "=", CompareOp::Ne => "<>", @@ -64,45 +77,38 @@ pub(super) fn build_sk_sql(sk_cond: &SortKeyCondition, sk_col: &str) -> String { CompareOp::Gt => ">", CompareOp::Ge => ">=", }; - format!(" AND {sk_col} {sql_op} ?") - } - SortKeyCondition::Between { .. } => format!(" AND {sk_col} BETWEEN ? AND ?"), - SortKeyCondition::BeginsWith { .. } => { - if sk_col.ends_with("_b") { - // Binary prefix: [prefix, incremented-prefix). - format!(" AND {sk_col} >= ? AND {sk_col} < ?") - } else { - // String prefix: [prefix, prefix || U+10FFFF). - format!(" AND {sk_col} >= ? AND {sk_col} < (? || char(1114111))") - } - } - } -} - -/// The sort-key values to bind for a key condition, in placeholder order. -pub(super) fn sk_condition_bind_values( - sk_cond: &SortKeyCondition, - sk_type: ScalarAttributeType, - maps: &ExpressionMaps, -) -> Result, StorageError> { - match sk_cond { - SortKeyCondition::Compare { value, .. } => { - Ok(vec![parse_sk(&resolve_expr_to_av(value, maps)?, sk_type)?]) + Ok(( + format!(" AND {sk_col} {sql_op} ?"), + vec![parse_sk(&resolve_expr_to_av(value, maps)?, sk_type)?], + )) } - SortKeyCondition::Between { low, high, .. } => Ok(vec![ - parse_sk(&resolve_expr_to_av(low, maps)?, sk_type)?, - parse_sk(&resolve_expr_to_av(high, maps)?, sk_type)?, - ]), + SortKeyCondition::Between { low, high, .. } => Ok(( + format!(" AND {sk_col} BETWEEN ? AND ?"), + vec![ + parse_sk(&resolve_expr_to_av(low, maps)?, sk_type)?, + parse_sk(&resolve_expr_to_av(high, maps)?, sk_type)?, + ], + )), SortKeyCondition::BeginsWith { prefix, .. } => { let sk = parse_sk(&resolve_expr_to_av(prefix, maps)?, sk_type)?; match sk { - SortKeyValue::B(b) => { - let upper = increment_bytes(&b); - Ok(vec![SortKeyValue::B(b), SortKeyValue::B(upper)]) - } + SortKeyValue::B(b) => match binary_prefix_upper_bound(&b) { + Some(upper) => Ok(( + format!(" AND {sk_col} >= ? AND {sk_col} < ?"), + vec![SortKeyValue::B(b), SortKeyValue::B(upper)], + )), + // Every byte string with this prefix is in range and there is + // no finite value above them all, so the only correct upper + // bound is none. Emitting one anyway is what silently dropped + // rows. + None => Ok((format!(" AND {sk_col} >= ?"), vec![SortKeyValue::B(b)])), + }, // For strings the SQL upper bound is `? || char(1114111)`, // so the same prefix is bound twice. - SortKeyValue::S(s) => Ok(vec![SortKeyValue::S(s.clone()), SortKeyValue::S(s)]), + SortKeyValue::S(s) => Ok(( + format!(" AND {sk_col} >= ? AND {sk_col} < (? || char(1114111))"), + vec![SortKeyValue::S(s.clone()), SortKeyValue::S(s)], + )), SortKeyValue::N(_) => Err(StorageError::Validation( "begins_with is not supported on numeric sort keys".to_owned(), )), @@ -111,21 +117,35 @@ pub(super) fn sk_condition_bind_values( } } -/// Exclusive upper bound for a binary prefix range: the smallest byte string -/// greater than every string having `prefix` as a prefix. Returns an empty -/// vector's successor convention when `prefix` is all `0xFF`. -fn increment_bytes(prefix: &[u8]) -> Vec { +/// Exclusive upper bound for a binary prefix range, or `None` when the range is +/// unbounded above. +/// +/// The bound is the smallest byte string greater than every string having +/// `prefix` as a prefix. Found by incrementing the last byte below `0xFF` and +/// discarding everything after it: `[1, 2]` yields `[1, 3]`, so `[1, 2, 9]` is +/// still included but `[1, 3]` is not. +/// +/// `None` when every byte is `0xFF`, which includes the empty prefix. No finite +/// bound exists in that case, because a longer all-`0xFF` string always sorts +/// after any candidate: with `[0xFF]` the strings `[0xFF, 0xFF]`, +/// `[0xFF, 0xFF, 0xFF]` and so on continue without end. The previous code +/// returned `vec![0xFF; prefix.len() + 1]` here, which is a value inside the +/// matching set rather than above it, so rows were silently dropped: for an +/// empty prefix it produced `[0xFF]`, which excluded every key sorting at or +/// after it, and `begins_with([])` then returned part of the partition while +/// reporting success. Returning `None` and omitting the predicate is the only +/// correct answer that does not depend on a maximum key length, which is +/// operator-configurable via `max_sort_key_size_bytes`. +fn binary_prefix_upper_bound(prefix: &[u8]) -> Option> { let mut out = prefix.to_vec(); while let Some(last) = out.last_mut() { if *last < 0xFF { *last += 1; - return out; + return Some(out); } out.pop(); } - // All 0xFF: no finite successor within the same length convention; use a - // value that sorts after any prefixed string of practical length. - vec![0xFF; prefix.len() + 1] + None } /// Build a `LastEvaluatedKey` from an item's key attributes. @@ -152,3 +172,61 @@ pub(super) async fn execute_dynamic_query( .map_err(|e| StorageError::Internal(e.to_string()))?; Ok(rows.into_iter().map(|(v,)| v).collect()) } + +#[cfg(test)] +mod tests { + use super::binary_prefix_upper_bound; + + /// Every byte string having `prefix` as a prefix must sort below the bound, + /// and the bound itself must not. Asserted against explicit witnesses rather + /// than trusting the arithmetic, since the defect this replaces was an + /// off-by-domain error that looked arithmetically reasonable. + #[test] + fn the_bound_excludes_itself_and_includes_every_extension() { + let upper = binary_prefix_upper_bound(&[1, 2]).expect("finite bound exists"); + assert_eq!(upper, vec![1, 3]); + for witness in [ + vec![1, 2], + vec![1, 2, 0], + vec![1, 2, 0xFF], + vec![1, 2, 9, 9], + ] { + assert!(witness < upper, "{witness:?} must be inside the range"); + } + assert!(vec![1, 3] >= upper, "the bound must be exclusive"); + } + + /// A trailing `0xFF` carries: the last byte below `0xFF` is incremented and + /// everything after it is discarded. + #[test] + fn a_trailing_all_ones_byte_carries_into_the_previous_byte() { + assert_eq!(binary_prefix_upper_bound(&[1, 0xFF]), Some(vec![2])); + assert_eq!(binary_prefix_upper_bound(&[1, 0xFF, 0xFF]), Some(vec![2])); + let upper = binary_prefix_upper_bound(&[1, 0xFF]).expect("finite bound exists"); + assert!(vec![1, 0xFF, 0xFF] < upper, "extension must be included"); + } + + /// The empty prefix matches everything, so no upper bound can exist. This is + /// the case that silently dropped rows: the old code returned `[0xFF]`, which + /// excluded every key sorting at or after it, so `begins_with([])` returned + /// part of the partition and reported success. + #[test] + fn an_empty_prefix_has_no_upper_bound() { + assert_eq!(binary_prefix_upper_bound(&[]), None); + } + + /// Same defect one length up, and not only for the empty prefix: an all-`0xFF` + /// prefix of any length has no finite bound, because a longer all-`0xFF` + /// string always sorts after any candidate. The old code returned + /// `[0xFF, 0xFF]` for `[0xFF]`, which wrongly excluded `[0xFF, 0xFF, 0xFF]`. + #[test] + fn an_all_ones_prefix_has_no_upper_bound_at_any_length() { + for prefix in [vec![0xFF], vec![0xFF, 0xFF], vec![0xFF; 8]] { + assert_eq!( + binary_prefix_upper_bound(&prefix), + None, + "no finite bound exists above {prefix:?}" + ); + } + } +} diff --git a/crates/storage-sqlite/src/data/query_scan.rs b/crates/storage-sqlite/src/data/query_scan.rs index b7c03f50..c5f12d93 100644 --- a/crates/storage-sqlite/src/data/query_scan.rs +++ b/crates/storage-sqlite/src/data/query_scan.rs @@ -21,9 +21,7 @@ use extenddb_storage::util::{ encode_netstring_composite, parse_sk, pk_to_text, sk_column, sk_column_n, sk_info, }; -use super::query::{ - build_key, build_sk_sql, execute_dynamic_query, resolve_expr_to_av, sk_condition_bind_values, -}; +use super::query::{build_key, build_sk_sql_and_binds, execute_dynamic_query, resolve_expr_to_av}; use super::{ BoundValue, all_sort_key_info, data_table_name, index_table_name, json_to_item, sk_bound, }; @@ -79,9 +77,11 @@ impl SqliteEngine { // Primary sort-key condition. if let (Some(sk_cond), Some((_, sk_type))) = (&key_condition.sk_condition, sk_info_val) { - sql.push_str(&build_sk_sql(sk_cond, sk_column(sk_type))); - for v in sk_condition_bind_values(sk_cond, sk_type, maps)? { - binds.push(sk_bound(&v)); + let (sk_sql, sk_binds) = + build_sk_sql_and_binds(sk_cond, sk_column(sk_type), sk_type, maps)?; + sql.push_str(&sk_sql); + for v in &sk_binds { + binds.push(sk_bound(v)); } } From 08e57c7ea854835607a542cca1dd4767ab5c6849 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Wed, 12 Aug 2026 10:32:40 +0000 Subject: [PATCH 22/25] fix(vector): paginate the backfill by rowid, so a composite-key partition cannot lose rows Found by independent pre-promotion review. The backfill's keyset cursor was the partition key alone: `pk > last_pk ORDER BY pk`. Composite-key base tables have `PRIMARY KEY (pk, sk*)`, so `pk` is not unique, and a batch boundary falling inside one partition's sort-key group excluded every remaining row sharing that `pk`. Those items were never written to the index and nothing repairs them: they are not the rows that changed, so no queue entry exists, and startup reconciliation calls the same `backfill_vector_batch`, reproducing the skip rather than healing it. Silent data loss on a supported configuration. Reproduced exactly: five rows in one partition scanned with a batch of three indexed four and skipped two. The path was unreachable in the test suites only because every vector test table was HASH-only. The cursor is now `rowid`, which is unique regardless of key layout, so one query shape serves both. It is a valid cursor because the base tables are ordinary rowid tables and every write is `INSERT ... ON CONFLICT DO UPDATE` (`tx_helpers.rs`), which updates in place and never reassigns a rowid. Rows inserted mid-backfill land past the cursor or are captured by the queue hold and replayed after ACTIVE, same as before. A full-base-key tuple cursor `(pk, sk*) > (?, ...)` was the alternative; rowid was chosen because it needs no per-key-layout SQL and no type-aware round-tripping of sort key values. The regression test drives `backfill_vector_batch` with a batch of three across a five-row partition plus a second partition, asserting all six rows land. Negative control: restoring pk-cursor semantics fails it with exactly `left: 4, right: 6`. This is the first test in the suite with a RANGE base key on a vector-indexed table, which is why the defect survived: the hazard the previous fix addressed (concurrent deletes shifting OFFSET positions) was tested, but only on tables where pk was unique and the cursor's other flaw could not fire. Verified: fmt and clippy -D warnings at 0, 864 workspace unit tests passed with 0 filtered out, and the full rust integration suite live over HTTP after this change: 457 passed, 0 failed, 0 filtered out. --- .../storage-sqlite/src/data/vector_index.rs | 175 ++++++++++++++++-- 1 file changed, 156 insertions(+), 19 deletions(-) diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs index 25e96c20..997930ec 100644 --- a/crates/storage-sqlite/src/data/vector_index.rs +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -524,8 +524,8 @@ impl<'a> BackfillPlan<'a> { /// Backfill one batch of existing rows into the vector index. /// -/// Returns `(written, fetched, last_pk)`. `fetched` distinguishes a short read (the -/// end) from a full one, and `last_pk` is the cursor to resume from. +/// Returns `(written, fetched, last_rowid)`. `fetched` distinguishes a short read +/// (the end) from a full one, and `last_rowid` is the cursor to resume from. /// /// Pagination is by KEY, not by `OFFSET`. Offset anchors on a position, so removing any /// already-scanned row shifts every later position by one and the next batch skips a @@ -534,26 +534,40 @@ impl<'a> BackfillPlan<'a> { /// one was. Reproduced before this change: one removal during a backfill left the row /// at the batch boundary absent from the index. /// +/// The cursor is `rowid`, not `pk`, and the difference is a correctness matter rather +/// than a preference. Composite-key base tables have `PRIMARY KEY (pk, sk*)`, so `pk` +/// alone is not unique; a `pk > last_pk` cursor whose batch boundary fell inside one +/// partition's sort-key group excluded every remaining row sharing that `pk`, +/// permanently, on both the UpdateTable path and startup reconciliation (which share +/// this function). Reproduced: a five-row partition scanned with a batch of three +/// indexed four rows and skipped two. `rowid` is unique regardless of the key layout, +/// so one query shape serves both. It is a valid cursor because the base tables are +/// ordinary rowid tables and every write is `INSERT ... ON CONFLICT DO UPDATE` +/// (`tx_helpers.rs`), which updates in place and never reassigns a rowid; rows +/// inserted after a batch passed their rowid position are the concurrent writes the +/// queue hold already captures and replays after ACTIVE. +/// /// It was unreachable while the whole backfill ran in one transaction, and became /// reachable the moment batches started committing independently. async fn backfill_vector_batch( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, plan: &BackfillPlan<'_>, limit: i64, - after_pk: Option<&str>, -) -> Result<(usize, i64, Option), StorageError> { + after_rowid: i64, +) -> Result<(usize, i64, i64), StorageError> { let base_table = super::data_table_name(plan.table_id); - // `pk > ?` with the empty string as the initial cursor: every real pk sorts - // after it, so the first batch needs no separate query shape. - let sql = format!("SELECT pk, item_data FROM {base_table} WHERE pk > ? ORDER BY pk LIMIT ?"); - let rows: Vec<(String, String)> = sqlx::query_as(&sql) - .bind(after_pk.unwrap_or("")) + // `rowid > ?` with 0 as the initial cursor: every real rowid is positive, so + // the first batch needs no separate query shape. + let sql = + format!("SELECT rowid, item_data FROM {base_table} WHERE rowid > ? ORDER BY rowid LIMIT ?"); + let rows: Vec<(i64, String)> = sqlx::query_as(&sql) + .bind(after_rowid) .bind(limit) .fetch_all(&mut **tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; let fetched = i64::try_from(rows.len()).unwrap_or(limit); - let last_pk = rows.last().map(|(pk, _)| pk.clone()); + let last_rowid = rows.last().map_or(after_rowid, |(rid, _)| *rid); let mut written = 0usize; for (_, item_json) in rows { let item: Item = serde_json::from_str(&item_json) @@ -573,7 +587,7 @@ async fn backfill_vector_batch( .await?; written += 1; } - Ok((written, fetched, last_pk)) + Ok((written, fetched, last_rowid)) } /// Backfill the index in independently committed batches, releasing SQLite's write @@ -604,16 +618,16 @@ pub(crate) async fn backfill_vector_index_in_batches( ) -> Result { const BATCH: i64 = 500; let plan = BackfillPlan::new(table_id, meta, base_key_schema, attr_defs); - let mut cursor: Option = None; + let mut cursor: i64 = 0; let mut written = 0usize; loop { - let (batch_written, fetched, last_pk) = { + let (batch_written, fetched, last_rowid) = { let _writer = write_lock.lock().await; let mut tx = pool .begin_with("BEGIN IMMEDIATE") .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let result = backfill_vector_batch(&mut tx, &plan, BATCH, cursor.as_deref()).await?; + let result = backfill_vector_batch(&mut tx, &plan, BATCH, cursor).await?; tx.commit() .await .map_err(|e| StorageError::Internal(e.to_string()))?; @@ -623,7 +637,7 @@ pub(crate) async fn backfill_vector_index_in_batches( if fetched < BATCH { break; } - cursor = last_pk; + cursor = last_rowid; // Outside the lock, so a write can actually proceed during the pause. Zero in // production; a test sets it so a write is guaranteed to land mid-backfill. if !batch_delay.is_zero() { @@ -649,16 +663,139 @@ pub(crate) async fn backfill_vector_index( ) -> Result { const BATCH: i64 = 500; let plan = BackfillPlan::new(table_id, meta, base_key_schema, attr_defs); - let mut cursor: Option = None; + let mut cursor: i64 = 0; let mut written = 0usize; loop { - let (batch_written, fetched, last_pk) = - backfill_vector_batch(tx, &plan, BATCH, cursor.as_deref()).await?; + let (batch_written, fetched, last_rowid) = + backfill_vector_batch(tx, &plan, BATCH, cursor).await?; written += batch_written; if fetched < BATCH { break; } - cursor = last_pk; + cursor = last_rowid; } Ok(written) } + +#[cfg(test)] +mod tests { + use super::*; + + /// A composite-key table whose partition spans a batch boundary must still + /// backfill every row. + /// + /// This is the regression test for a silent data-loss defect: the cursor was + /// `pk > last_pk`, and on a table with `PRIMARY KEY (pk, sk_s)` a batch ending + /// inside one partition's sort-key group excluded every remaining row sharing + /// that `pk`. Five rows in one partition scanned with a batch of three indexed + /// four and skipped two, permanently, on both the UpdateTable path and startup + /// reconciliation, which share `backfill_vector_batch`. The rowid cursor cannot + /// lose rows because rowid is unique whatever the key layout. + /// + /// Driven through `backfill_vector_batch` directly with a batch of 3 rather + /// than through the drivers, because they hardcode a 500-row batch and seeding + /// 501 rows would test the same lines slower. + #[tokio::test] + async fn a_composite_key_partition_straddling_a_batch_boundary_is_fully_backfilled() { + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType, + }; + let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let ks = vec![ + KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }, + ]; + let ad = vec![ + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ]; + + let table_id = "t-composite"; + let mut tx = engine.pool.begin_with("BEGIN IMMEDIATE").await.expect("tx"); + crate::SqliteEngine::create_data_table(&mut tx, table_id, &ks, &ad) + .await + .expect("base table"); + crate::SqliteEngine::create_vector_data_table(&mut tx, table_id, "vidx-1", &ks, &ad) + .await + .expect("vector table"); + + // One partition with five sort keys plus a second partition, so the batch + // of three ends INSIDE partition "a": exactly the boundary that lost rows. + let base_table = super::super::data_table_name(table_id); + for (pk, sk) in [ + ("a", "1"), + ("a", "2"), + ("a", "3"), + ("a", "4"), + ("a", "5"), + ("b", "1"), + ] { + sqlx::query(&format!( + "INSERT INTO {base_table} (pk, sk_s, item_data) VALUES (?, ?, ?)" + )) + .bind(pk) + .bind(sk) + .bind(format!( + r#"{{"pk":{{"S":"{pk}"}},"sk":{{"S":"{sk}"}},"emb":{{"L":[{{"N":"1"}},{{"N":"0"}}]}}}}"# + )) + .execute(&mut *tx) + .await + .expect("seed"); + } + + let meta = VectorIndexMeta { + index_id: "vidx-1".to_owned(), + dimensions: 2, + vector_attribute_name: "emb".to_owned(), + projection: extenddb_core::types::Projection { + projection_type: extenddb_core::types::ProjectionType::All, + non_key_attributes: None, + }, + hash_attribute_name: None, + search_schema_attribute_names: Vec::new(), + }; + let plan = BackfillPlan::new(table_id, &meta, &ks, &ad); + + let mut cursor: i64 = 0; + let mut written = 0usize; + loop { + let (batch_written, fetched, last_rowid) = + backfill_vector_batch(&mut tx, &plan, 3, cursor) + .await + .expect("batch"); + written += batch_written; + if fetched < 3 { + break; + } + cursor = last_rowid; + } + tx.commit().await.expect("commit"); + + assert_eq!( + written, 6, + "every row must be indexed; the pk-only cursor wrote 4 and skipped 2" + ); + let vec_table = super::super::vector_table_name(table_id, "vidx-1"); + let (rows,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(rows, 6, "the index must hold all six rows"); + } +} From a8c885ffe331487e12f5bc7f3f8470abbb086a3a Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Wed, 12 Aug 2026 11:35:36 +0000 Subject: [PATCH 23/25] fix(vector): recover a dead index build at runtime, not only at restart Found by independent pre-promotion review. The asynchronous backfill runs in a detached task, and while its index is CREATING the queue worker deliberately holds every pending index write for the table, which is what guarantees a stale backfill snapshot cannot beat a fresher queued write. The hold had no exit if the build died: a panicking task, or a terminal ACTIVE flip that failed and was logged, left the index in CREATING with nothing to repair it until a restart ran the startup reconciler. One dead build therefore wedged ALL asynchronous index maintenance for its table, including unrelated healthy GSIs, indefinitely. The repair has two parts. A registry (`vector_builds_running`) records the index ids whose build task is alive in this process: registered before the task is spawned, deregistered by a drop guard so a panic deregisters too. "CREATING in the catalog and absent from the registry" is then a provable orphan, since no other agent will ever flip it. The GSI worker sweeps for that condition each pass and rebuilds orphans via the same drop-and-rebuild the startup reconciler uses, factored into a shared `rebuild_one_vector_index` so the two repairs cannot drift; rebuilding rather than resuming, because rows already written would collide with the backfill's deliberately plain INSERT. One race needed closing: the catalog row commits before `build_vector_index` registers the task, so a sweep landing in that window would see a healthy build as orphaned and drop its data table out from under it. The worker therefore requires the same index id on two consecutive passes (at least a second apart) before recovering, and the recovery itself re-checks the registry per index at execution time. Deliberately NOT covered: a task that is alive but wedged. The registry cannot distinguish slow from stuck, and rebuilding under a live task is worse than waiting, so a hung-but-running build still needs a restart. The reviewed defect was dead tasks, which this closes. Tests: an orphaned CREATING index (a catalog row with no task, exactly what a dead build leaves) is recovered to ACTIVE with its rows actually backfilled, and the discriminating control: a registered build in the same state is left untouched, without which the first test would also pass for a sweep that rebuilds every CREATING index it sees. Verified: fmt and clippy -D warnings at 0, 866 workspace unit tests with 0 filtered out, full rust integration suite live over HTTP 460 passed, 0 failed, 0 filtered out. --- crates/storage-sqlite/src/store.rs | 11 + crates/storage-sqlite/src/update_table.rs | 400 ++++++++++++++++++---- crates/storage-sqlite/src/workers.rs | 21 ++ 3 files changed, 374 insertions(+), 58 deletions(-) diff --git a/crates/storage-sqlite/src/store.rs b/crates/storage-sqlite/src/store.rs index 795313b6..6cd380ea 100644 --- a/crates/storage-sqlite/src/store.rs +++ b/crates/storage-sqlite/src/store.rs @@ -48,6 +48,14 @@ pub struct SqliteEngine { /// every write transaction so condition checks and writes are atomic and /// `SQLITE_BUSY` cannot arise from competing writers. pub(crate) write_lock: Arc>, + /// Index ids whose asynchronous backfill task is currently alive in THIS + /// process. What tells a stuck `CREATING` index apart from one still + /// building: a catalog row can say `CREATING` forever, but only a live task + /// appears here, and the entry is removed by a drop guard so a panicking + /// task deregisters too. The GSI worker recovers any `CREATING` index with + /// no entry, because nothing else ever will until a restart, and until then + /// the per-table queue hold blocks every write's index maintenance. + pub(crate) vector_builds_running: Arc>>, } impl SqliteEngine { @@ -125,6 +133,9 @@ impl SqliteEngine { index_propagation_delay_cache: Arc::new(AtomicU64::new(initial_index_delay)), gsi_notify: Arc::new(tokio::sync::Notify::new()), write_lock: Arc::new(Mutex::new(())), + vector_builds_running: Arc::new( + std::sync::Mutex::new(std::collections::HashSet::new()), + ), }) } diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index b1eb8f64..ae61dbc9 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -603,17 +603,41 @@ impl SqliteEngine { let owned_ad = effective_ad.to_vec(); let gsi_notify = self.gsi_notify(); + // Registered BEFORE the spawn, so there is no instant where the catalog + // says CREATING and the registry disagrees while the task is viable. The + // guard deregisters on every exit path including a panic, which is the + // whole point: a CREATING index with no registry entry is provably + // orphaned, and the worker's recovery sweep may rebuild it. + self.vector_builds_running + .lock() + .expect("registry poisoned") + .insert(index_id.to_owned()); + let registry = std::sync::Arc::clone(&self.vector_builds_running); + // Detached, so UpdateTable returns while the index is still CREATING. The // service behaves this way, and it is the whole point: a table stays ACTIVE // and writable throughout, taking over eight minutes on an empty table when // measured, and searches against the index are refused until it is ACTIVE. // // Not awaited, so failures cannot be returned to the caller. They are logged - // and the index is deliberately LEFT in CREATING, which is the state - // `reconcile_incomplete_vector_indexes` repairs at startup. Flipping it to + // and the index is deliberately LEFT in CREATING, which is the state the + // worker's recovery sweep repairs at runtime (and + // `reconcile_incomplete_vector_indexes` at startup). Flipping it to // ACTIVE on error would publish a partially populated index, and there is no // failure state on the wire for an index to sit in. tokio::spawn(async move { + struct Deregister( + std::sync::Arc>>, + String, + ); + impl Drop for Deregister { + fn drop(&mut self) { + if let Ok(mut set) = self.0.lock() { + set.remove(&self.1); + } + } + } + let _deregister = Deregister(registry, owned_index_id.clone()); let result = crate::data::vector_index::backfill_vector_index_in_batches( &pool, &write_lock, @@ -763,70 +787,166 @@ impl SqliteEngine { let mut rebuilt = 0usize; for (index_id, table_id, base_ks_json, base_ad_json) in rows { - let base_key_schema: Vec = serde_json::from_str(&base_ks_json) - .map_err(|e| StorageError::Internal(e.to_string()))?; - let attr_defs: Vec = serde_json::from_str(&base_ad_json) - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let _writer = self.write_lock.lock().await; - Self::drop_vector_data_table_by_id(&self.pool, &table_id, &index_id).await?; + let written = self + .rebuild_one_vector_index(&index_id, &table_id, &base_ks_json, &base_ad_json) + .await?; + rebuilt += 1; + tracing::info!( + vectors_indexed = written, + "Reconciled incomplete vector index {index_id} on table {table_id}" + ); + } + Ok(rebuilt) + } - let mut data_tx = self - .pool - .begin_with("BEGIN IMMEDIATE") + /// Index ids that are `CREATING` with no live backfill task right now. + /// + /// The worker's cheap per-pass probe. A single sighting is NOT proof of a + /// dead build: the catalog row commits before `build_vector_index` registers + /// the task, so a sweep landing in that window would see a healthy build as + /// orphaned. The worker therefore requires the same id on two consecutive + /// passes before invoking [`Self::recover_stuck_vector_builds`], which + /// re-checks the registry itself at execution time. + pub(crate) async fn stuck_vector_build_candidates(&self) -> Result, StorageError> { + let ids: Vec<(String,)> = + sqlx::query_as("SELECT index_id FROM vector_indexes WHERE index_status = 'CREATING'") + .fetch_all(&self.pool) .await .map_err(|e| StorageError::Internal(e.to_string()))?; - Self::create_vector_data_table( - &mut data_tx, - &table_id, - &index_id, - &base_key_schema, - &attr_defs, - ) - .await?; - // The definition is read from the catalog rather than reconstructed, - // because the request that created it is long gone. - let meta = crate::data::vector_index::fetch_vector_indexes_for_table( - &mut data_tx, - &table_id, - ) - .await? + let registry = self + .vector_builds_running + .lock() + .map_err(|_| StorageError::Internal("vector build registry poisoned".to_owned()))?; + Ok(ids .into_iter() - .find(|m| m.index_id == index_id) - .ok_or_else(|| { - StorageError::Internal(format!( - "vector index {index_id} was selected as CREATING but has no catalog row" - )) - })?; - let written = crate::data::vector_index::backfill_vector_index( - &mut data_tx, - &table_id, - &meta, - &base_key_schema, - &attr_defs, - ) - .await?; - data_tx - .commit() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + .map(|(id,)| id) + .filter(|id| !registry.contains(id)) + .collect()) + } - sqlx::query( - "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL \ - WHERE table_id = ? AND index_id = ?", - ) - .bind(&table_id) - .bind(&index_id) - .execute(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - rebuilt += 1; - tracing::info!( + /// Recover `CREATING` vector indexes whose backfill task is dead. + /// + /// A build can die without a trace on the wire: the spawned task panics, or + /// its terminal `ACTIVE` flip fails and is logged. The index then sits in + /// `CREATING`, which the worker treats as "hold every queued index write for + /// this table", so one dead build wedges ALL asynchronous index maintenance + /// for the table until a restart runs the startup reconciler. This is the + /// runtime half of the same repair: the GSI worker calls it on a sighting of + /// a `CREATING` index that has no live task in `vector_builds_running`. + /// + /// The registry is what makes the sweep safe to run at any time: a healthy + /// in-flight build is registered before its task is spawned and deregisters + /// by drop guard, so "CREATING and unregistered" cannot describe a build that + /// is still making progress in this process. Rebuilding rather than resuming, + /// for the reconciler's reason: rows already written would collide with the + /// backfill's deliberately plain `INSERT`. + pub(crate) async fn recover_stuck_vector_builds(&self) -> Result { + let rows: Vec<(String, String, String, String)> = sqlx::query_as( + "SELECT v.index_id, v.table_id, t.key_schema, t.attribute_definitions \ + FROM vector_indexes v JOIN tables t ON v.table_id = t.table_id \ + WHERE v.index_status = 'CREATING'", + ) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut recovered = 0usize; + for (index_id, table_id, base_ks_json, base_ad_json) in rows { + let alive = self + .vector_builds_running + .lock() + .map(|set| set.contains(&index_id)) + .unwrap_or(true); + if alive { + continue; + } + let written = self + .rebuild_one_vector_index(&index_id, &table_id, &base_ks_json, &base_ad_json) + .await?; + recovered += 1; + tracing::warn!( vectors_indexed = written, - "Reconciled incomplete vector index {index_id} on table {table_id}" + "Recovered vector index {index_id} on table {table_id}: it was CREATING \ + with no live backfill task" ); } - Ok(rebuilt) + if recovered > 0 { + // Writes held while the index was CREATING are claimable now. + self.gsi_notify.notify_waiters(); + } + Ok(recovered) + } + + /// Drop, recreate, backfill, and flip one vector index to `ACTIVE`. + /// + /// The shared body of startup reconciliation and the worker's runtime + /// recovery, factored so the two repairs cannot drift. Runs the backfill in a + /// single transaction under the write lock: both callers are repairing an + /// index whose queue rows are already held, and neither needs the batched + /// path's lock-release property badly enough to pay its added states. + async fn rebuild_one_vector_index( + &self, + index_id: &str, + table_id: &str, + base_ks_json: &str, + base_ad_json: &str, + ) -> Result { + let base_key_schema: Vec = serde_json::from_str(base_ks_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let attr_defs: Vec = serde_json::from_str(base_ad_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let _writer = self.write_lock.lock().await; + Self::drop_vector_data_table_by_id(&self.pool, table_id, index_id).await?; + + let mut data_tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Self::create_vector_data_table( + &mut data_tx, + table_id, + index_id, + &base_key_schema, + &attr_defs, + ) + .await?; + // The definition is read from the catalog rather than reconstructed, + // because the request that created it is long gone. + let meta = + crate::data::vector_index::fetch_vector_indexes_for_table(&mut data_tx, table_id) + .await? + .into_iter() + .find(|m| m.index_id == index_id) + .ok_or_else(|| { + StorageError::Internal(format!( + "vector index {index_id} was selected as CREATING but has no catalog row" + )) + })?; + let written = crate::data::vector_index::backfill_vector_index( + &mut data_tx, + table_id, + &meta, + &base_key_schema, + &attr_defs, + ) + .await?; + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + sqlx::query( + "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL \ + WHERE table_id = ? AND index_id = ?", + ) + .bind(table_id) + .bind(index_id) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(written) } } @@ -1177,4 +1297,168 @@ mod reconciler_tests { item 'a' was indexed twice and a search would return it twice" ); } + + /// A CREATING vector index with no live backfill task must be recovered at + /// runtime, not just at startup. + /// + /// The failure this guards: the detached backfill task dies (panic, or its + /// terminal ACTIVE flip fails) and the index sits in CREATING forever. The + /// worker holds every queued index write for the table while any of its + /// vector indexes is CREATING, so without runtime recovery one dead build + /// wedges ALL asynchronous index maintenance for the table until a restart. + /// The orphan is simulated exactly as the reconciler tests simulate a crash: + /// a CREATING catalog row with no task, which is indistinguishable from the + /// real thing because a dead task leaves nothing else behind. + #[tokio::test] + async fn a_creating_index_with_no_live_build_task_is_recovered_at_runtime() { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind("000000000000") + .execute(&engine.pool) + .await + .expect("account"); + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl("000000000000", input) + .await + .expect("create table"); + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE table_name = 't'") + .fetch_one(&engine.pool) + .await + .expect("table_id"); + + let base_table = crate::data::data_table_name(&table_id); + sqlx::query(&format!( + "INSERT INTO {base_table} (pk, item_data) VALUES ('a', ?)" + )) + .bind(r#"{"pk":{"S":"a"},"emb":{"L":[{"N":"1"},{"N":"0"}]}}"#) + .execute(&engine.pool) + .await + .expect("seed"); + + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status, backfilling) \ + VALUES (?, 'vidx-dead', 'vidx', 2, 'COSINE', ?, ?, 'CREATING', 1)", + ) + .bind(&table_id) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert orphaned CREATING index"); + + // The probe must name it, and recovery must repair it. + let candidates = engine + .stuck_vector_build_candidates() + .await + .expect("candidates"); + assert_eq!(candidates, vec!["vidx-dead".to_owned()]); + let recovered = engine.recover_stuck_vector_builds().await.expect("recover"); + assert_eq!(recovered, 1, "the orphaned build must be recovered"); + + let (status, backfilling): (String, Option) = sqlx::query_as( + "SELECT index_status, backfilling FROM vector_indexes WHERE index_id = 'vidx-dead'", + ) + .fetch_one(&engine.pool) + .await + .expect("status"); + assert_eq!(status, "ACTIVE"); + assert_eq!(backfilling, None); + + // Recovered means populated, not merely flipped: the seeded row must be + // in the rebuilt index, or the "recovery" published an empty index. + let vec_table = crate::data::vector_table_name(&table_id, "vidx-dead"); + let (rows,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(rows, 1, "recovery must backfill the seeded row"); + } + + /// The discriminating control for the sweep: a CREATING index whose build IS + /// registered as alive must be left alone. Without this the previous test + /// would also pass for a sweep that rebuilds every CREATING index it sees, + /// which would corrupt a healthy in-flight build by dropping its data table + /// out from under the running backfill. + #[tokio::test] + async fn a_creating_index_with_a_live_build_task_is_left_alone() { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind("000000000000") + .execute(&engine.pool) + .await + .expect("account"); + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl("000000000000", input) + .await + .expect("create table"); + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE table_name = 't'") + .fetch_one(&engine.pool) + .await + .expect("table_id"); + + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status, backfilling) \ + VALUES (?, 'vidx-live', 'vidx', 2, 'COSINE', ?, ?, 'CREATING', 1)", + ) + .bind(&table_id) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert CREATING index"); + + // The build is alive: exactly what build_vector_index records before it + // spawns the task. + engine + .vector_builds_running + .lock() + .expect("registry") + .insert("vidx-live".to_owned()); + + assert!( + engine + .stuck_vector_build_candidates() + .await + .expect("candidates") + .is_empty(), + "a registered build must not be a candidate" + ); + assert_eq!( + engine.recover_stuck_vector_builds().await.expect("recover"), + 0, + "a registered build must not be recovered" + ); + let (status,): (String,) = + sqlx::query_as("SELECT index_status FROM vector_indexes WHERE index_id = 'vidx-live'") + .fetch_one(&engine.pool) + .await + .expect("status"); + assert_eq!(status, "CREATING", "the live build must be untouched"); + } } diff --git a/crates/storage-sqlite/src/workers.rs b/crates/storage-sqlite/src/workers.rs index fd521c70..81864b9a 100644 --- a/crates/storage-sqlite/src/workers.rs +++ b/crates/storage-sqlite/src/workers.rs @@ -362,6 +362,13 @@ pub(crate) async fn gsi_propagation_worker( const MAX_SLEEP: Duration = Duration::from_secs(1); // Backoff after an error so a poison row cannot hot-loop the worker. const ERROR_BACKOFF: Duration = Duration::from_secs(1); + // A CREATING vector index with no live backfill task, seen on the previous + // pass. Two consecutive sightings are required before recovery, because the + // catalog row commits before the build task registers itself: a single + // sighting can be a healthy build in that window, but one still unregistered + // a full pass later (at least MAX_SLEEP apart) is dead. Without recovery it + // wedges every queued index write for its table until a restart. + let mut stuck_last_pass: std::collections::HashSet = std::collections::HashSet::new(); loop { // Drain everything currently due. let mut errored = false; @@ -376,6 +383,20 @@ pub(crate) async fn gsi_propagation_worker( } } } + // The stuck-build sweep. Cheap when nothing is CREATING, which is the + // steady state: one indexed catalog read per pass. + match engine.stuck_vector_build_candidates().await { + Ok(candidates) => { + let confirmed = candidates.iter().any(|id| stuck_last_pass.contains(id)); + if confirmed && let Err(e) = engine.recover_stuck_vector_builds().await { + tracing::error!("GSI propagation worker: stuck-build recovery failed: {e}"); + } + stuck_last_pass = candidates.into_iter().collect(); + } + Err(e) => { + tracing::debug!("GSI worker: stuck-build probe error: {e}"); + } + } // Sleep until the next row is due (or a write wakes us early). let wait = if errored { ERROR_BACKOFF From d5439b44de7def4422b615e18b4341f563cf9968 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Wed, 12 Aug 2026 11:35:56 +0000 Subject: [PATCH 24/25] test(vector): close the wire-coverage gaps an independent review found Four additions and one correction, all from pre-promotion review, none changing production code. BatchWriteItem and TransactWriteItems each get a wire test proving index maintenance end to end. The batch path routes through the same put/delete code as the ordinary writes, so it held by construction, but nothing on the wire would catch a refactor giving it its own path. The transactional path IS its own storage implementation (transactions.rs enqueues maintenance as a sibling of the ordinary writes, not a caller), so its wire test guards genuinely separate code. UpdateItem gets the REMOVE case over the wire: an UpdateExpression removing the vector attribute takes the item out of the index, and a SET re-admits it, so the removal cannot pass by accident of an index that returns nothing. Every absence assertion converges on a survivor ordered behind the mutation in the same queue, so a transiently unapplied delete cannot pass as a correct one. The DescribeTable test now asserts IndexArn has the service's table//index/ shape and that ItemCount and IndexSizeBytes are numbers. Their values are the GSI convention in this backend (not live-maintained) and are asserted only for shape, but previously all three were entirely unasserted, so a malformed ARN or a missing member would have shipped unnoticed. The correction: a_hash_scoped_index_requires_a_search_condition's second sub-case claimed to test "an expression that omits the HASH" but used `pk`, which is not in the SearchSchema, so it was refused by the not-in-SearchSchema validation first and the omitted-HASH branch was never exercised. It now builds a schema with a HASH and an INLINE_FILTER element and supplies an in-schema expression (`category = :c`) that omits the HASH, which reaches the intended branch. Only the status is asserted, since that wording is unmeasured against the service. Verified live over HTTP through devtools/run-tests: the vector suites pass 34/34 (up from 31), and the full rust integration suite passes 460/460 with 0 filtered out. --- tests/rust/src/vector_index_search.rs | 205 +++++++++++++++++++++++++- 1 file changed, 198 insertions(+), 7 deletions(-) diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index be3303b8..3476f660 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -1062,6 +1062,29 @@ async fn describe_table_reports_the_vector_index() { vidx.get("VectorAttribute").is_some(), "VectorAttribute must be reported: {vidx}" ); + // The ARN must name this table and this index in the service's shape. A + // malformed or misattributed ARN would ship unnoticed with only the presence + // check the members above get. + let arn = vidx + .pointer("/IndexArn") + .and_then(|v| v.as_str()) + .unwrap_or_else(|| panic!("IndexArn must be reported: {vidx}")); + assert!( + arn.starts_with("arn:aws:dynamodb:") && arn.ends_with(&format!("table/{name}/index/vidx")), + "IndexArn must have the service's table//index/ form: {arn}" + ); + // Emitted as numbers, matching the service's members. Their VALUES are the + // GSI convention in this backend (not live-maintained), so only shape is + // asserted here; a missing member would break a generated client's model. + assert!( + vidx.pointer("/ItemCount").is_some_and(serde_json::Value::is_number), + "ItemCount must be a number: {vidx}" + ); + assert!( + vidx.pointer("/IndexSizeBytes") + .is_some_and(serde_json::Value::is_number), + "IndexSizeBytes must be a number: {vidx}" + ); let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } @@ -1336,23 +1359,58 @@ async fn a_hash_scoped_index_requires_a_search_condition() { "expected the measured service message: {text}" ); - // An expression that IS supplied but omits the HASH attribute leaves the search - // equally unscoped. The wording of this one is not measured against the service, - // so only the refusal is asserted, not the text. + // An expression that IS supplied, references only IN-SCHEMA attributes, and + // still omits the HASH leaves the search equally unscoped. The attribute must + // be in the schema for this to test the intended branch: an out-of-schema + // attribute (an earlier version used the base `pk`) is refused by the + // "not in SearchSchema" validation first, so the omitted-HASH rule was never + // exercised. That needs a schema with more than the HASH element, so this + // sub-case builds its own. + let name2 = table_name("vi_hash_omitted"); + let body = format!( + r#"{{ + "TableName": "{name2}", + "AttributeDefinitions": [ + {{"AttributeName": "pk", "AttributeType": "S"}}, + {{"AttributeName": "tenant", "AttributeType": "S"}}, + {{"AttributeName": "category", "AttributeType": "S"}} + ], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": 2, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "SearchSchema": [ + {{"AttributeName": "tenant", "SearchSchemaElementType": "HASH"}}, + {{"AttributeName": "category", "SearchSchemaElementType": "INLINE_FILTER"}} + ], + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + wait_for_active(&name2).await; + + // The wording of this one is not measured against the service, so only the + // refusal is asserted, not the text. let (status, text) = call( "SearchVectors", &format!( - r#"{{"TableName": "{name}", "IndexName": "vidx", + r#"{{"TableName": "{name2}", "IndexName": "vidx", "SearchVector": [{{"N": "1.0"}}, {{"N": "0.1"}}], "TopK": 10, - "SearchConditionExpression": "pk = :p", - "ExpressionAttributeValues": {{":p": {{"S": "a"}}}}}}"# + "SearchConditionExpression": "category = :c", + "ExpressionAttributeValues": {{":c": {{"S": "x"}}}}}}"# ), ) .await; assert_eq!( status, 400, - "an expression that omits the HASH attribute must also be refused: {text}" + "an in-schema expression that omits the HASH attribute must be refused: {text}" ); + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name2}"}}"#)).await; } /// An index that is still building must be reported by DescribeTable and refused by @@ -1678,3 +1736,136 @@ async fn vector_indexes_require_on_demand_and_cap_at_five() { ); wait_for_active(&name).await; } + +/// BatchWriteItem maintains the vector index exactly as PutItem and DeleteItem do. +/// +/// The engine routes batch entries through the same `.put_item` / `.delete_item` +/// paths, so this held in code from the start, but it had no wire coverage: a +/// refactor that gave BatchWriteItem its own storage path could silently stop +/// maintaining the index and every existing test would stay green. Both halves are +/// exercised: puts must appear in the index, and a batched delete must remove one +/// while leaving the other, so a converged-but-empty index cannot pass as "deleted". +#[tokio::test] +async fn batch_write_item_maintains_the_index() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_batch_write"); + create_vector_table(&name, 2, "COSINE", false).await; + + let (status, text) = call( + "BatchWriteItem", + &format!( + r#"{{"RequestItems": {{"{name}": [ + {{"PutRequest": {{"Item": {{"pk": {{"S": "b1"}}, + "emb": {{"L": [{{"N": "1.0"}}, {{"N": "0.0"}}]}}}}}}}}, + {{"PutRequest": {{"Item": {{"pk": {{"S": "b2"}}, + "emb": {{"L": [{{"N": "0.9"}}, {{"N": "0.1"}}]}}}}}}}} + ]}}}}"# + ), + ) + .await; + assert_eq!(status, 200, "BatchWriteItem failed: {text}"); + search_until_pks(&name, &[1.0, 0.0], 10, None, &["b1", "b2"]).await; + + let (status, text) = call( + "BatchWriteItem", + &format!( + r#"{{"RequestItems": {{"{name}": [ + {{"DeleteRequest": {{"Key": {{"pk": {{"S": "b1"}}}}}}}} + ]}}}}"# + ), + ) + .await; + assert_eq!(status, 200, "BatchWriteItem delete failed: {text}"); + // Converging on the survivor rather than on absence: "b2 alone" is ordered + // behind the delete in the same queue, so it cannot be satisfied by a window + // where the delete simply has not applied yet. + search_until_pks(&name, &[1.0, 0.0], 10, None, &["b2"]).await; + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// TransactWriteItems maintains the vector index for both a transactional Put and +/// a transactional Delete. +/// +/// The transactional path enqueues index maintenance in its own storage code +/// (`transactions.rs`), a sibling of the ordinary write paths rather than a caller +/// of them, so wire coverage here guards a genuinely separate implementation. The +/// unit tests cover the enqueue; this proves the queue rows drain into search +/// results over the wire. +#[tokio::test] +async fn transact_write_items_maintains_the_index() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_transact_write"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "keep", None, &[0.5, 0.5]).await; + search_until_pks(&name, &[0.5, 0.5], 10, None, &["keep"]).await; + + let (status, text) = call( + "TransactWriteItems", + &format!( + r#"{{"TransactItems": [ + {{"Put": {{"TableName": "{name}", "Item": {{"pk": {{"S": "txput"}}, + "emb": {{"L": [{{"N": "1.0"}}, {{"N": "0.0"}}]}}}}}}}}, + {{"Delete": {{"TableName": "{name}", "Key": {{"pk": {{"S": "keep"}}}}}}}} + ]}}"# + ), + ) + .await; + assert_eq!(status, 200, "TransactWriteItems failed: {text}"); + // One converged assertion covers both halves: the put must appear AND the + // delete's target must be gone, and since both rode one transaction their + // queue rows drain together. + search_until_pks(&name, &[1.0, 0.0], 10, None, &["txput"]).await; + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// An UpdateItem whose UpdateExpression REMOVEs the vector attribute takes the item +/// out of the index, and one that SETs it back re-admits it. +/// +/// The storage unit tests prove the queue applies a vectorless image as a removal; +/// this proves the whole wire path agrees: expression parsing, the update write, +/// the enqueue, and the drain. The re-admission half exists so the removal cannot +/// pass by accident of a broken index that returns nothing for any query. +#[tokio::test] +async fn update_item_removing_the_vector_attribute_leaves_the_index() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_update_remove"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "a", None, &[1.0, 0.0]).await; + put_vector(&name, "b", None, &[0.9, 0.1]).await; + search_until_pks(&name, &[1.0, 0.0], 10, None, &["a", "b"]).await; + + let (status, text) = call( + "UpdateItem", + &format!( + r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "a"}}}}, + "UpdateExpression": "REMOVE emb"}}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateItem REMOVE failed: {text}"); + // "b alone" is ordered behind the REMOVE in the queue, so it cannot pass in + // the window before the REMOVE applies. + search_until_pks(&name, &[1.0, 0.0], 10, None, &["b"]).await; + + let (status, text) = call( + "UpdateItem", + &format!( + r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "a"}}}}, + "UpdateExpression": "SET emb = :v", + "ExpressionAttributeValues": {{":v": {{"L": [{{"N": "1.0"}}, {{"N": "0.0"}}]}}}}}}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateItem SET failed: {text}"); + search_until_pks(&name, &[1.0, 0.0], 10, None, &["a", "b"]).await; + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} From 2b6317e00fd2c1ba0f127d46497baef53d2cea42 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Wed, 12 Aug 2026 14:22:31 +0000 Subject: [PATCH 25/25] fix(vector): close three findings from the final production review Two independent production-readiness reviews returned SHIP verdicts with should-fix findings. Three are closed here rather than deferred, because each is cheap now and expensive after promotion. Per-index two-strike confirmation. The watchdog's two-strike guard was a single global gate: any confirmed-stuck index triggered a sweep that then rebuilt EVERY currently-unregistered CREATING index, including one seen for the first time that pass. A genuinely stuck index X could therefore drag a just-created healthy sibling Y, still inside its commit-to-register window, into a drop-and-rebuild while Y's live task was also backfilling, double-populating Y's data table and duplicating search hits. `recover_stuck_vector_builds` now takes the confirmed set and recovers only ids the worker has seen stuck on two consecutive passes, re-checking the registry per index at execution time. The registered-build-is-left-alone test now names its index as confirmed-stuck explicitly, so the registry re-check alone must protect it. Recovery without a write-availability outage. `rebuild_one_vector_index` ran the whole backfill in one transaction under the global write lock, so recovering one index on a large table blocked writes to every table for the rebuild's duration. It now uses the same batched backfill as the create path, releasing the lock between batches; the drop-and-recreate stays under the lock. Safe for the same reasons create is: the index stays CREATING throughout, so its table's queue rows are held and searches refused, and the rowid cursor tolerates concurrent writes. The single-transaction `backfill_vector_index` lost its last caller and is deleted rather than kept as dead code. The if_not_exists validation bypass. Write-path vector validation only surfaced bare `SET emb = :v`, so `SET emb = if_not_exists(emb, :v)` bypassed it entirely: a wrong-dimension vector through the fallback form returned 200 and the item was silently omitted from the index, where the service refuses the write. The fallback value persists verbatim whenever the attribute is absent, which is precisely the first write of a vector, so the form matters. The extraction now surfaces the fallback placeholder, and TransactWriteItems delegates to the same function instead of mirroring it, so the two paths cannot drift again. Arithmetic and nested paths remain excluded: a vector attribute admits neither. Also verified from the same review, no change needed: the rowid-cursor safety claim holds across every base-table write path (all upserts are ON CONFLICT DO UPDATE, deletes never reuse rowids, no VACUUM in the crate), and DeleteTable does clean up vector data tables on both the immediate and control-plane paths via `drop_data_table`'s sqlite_master sweep, which exists precisely because the catalog rows cascade away first. Verified: fmt and clippy -D warnings at 0, 868 workspace unit tests with 0 filtered out (two new extraction tests), and the full rust integration suite live over HTTP: 461 passed, 0 failed, 0 filtered out, including the new wire test proving the wrong-dimension fallback form is refused with 400 while the valid form reaches the index. --- crates/engine/src/transact_write_items.rs | 30 ++--- crates/engine/src/update_item.rs | 118 +++++++++++++--- .../storage-sqlite/src/data/vector_index.rs | 30 ----- crates/storage-sqlite/src/update_table.rs | 126 ++++++++++++------ crates/storage-sqlite/src/workers.rs | 15 ++- tests/rust/src/vector_index_search.rs | 49 +++++++ 6 files changed, 256 insertions(+), 112 deletions(-) diff --git a/crates/engine/src/transact_write_items.rs b/crates/engine/src/transact_write_items.rs index 236da064..366d71fa 100755 --- a/crates/engine/src/transact_write_items.rs +++ b/crates/engine/src/transact_write_items.rs @@ -18,7 +18,7 @@ use crate::transact_write_helpers::{ }; use crate::{DispatchMetrics, DispatchResult}; use extenddb_core::error::DynamoDbError; -use extenddb_core::expression::{Expr, ExpressionMaps, PathElement, UpdateAction}; +use extenddb_core::expression::{ExpressionMaps, UpdateAction}; use extenddb_core::types::{ CancellationReason, Item, TransactWriteItem, TransactWriteItemsInput, TransactWriteItemsOutput, }; @@ -295,28 +295,14 @@ fn collect_vector_cancellation_reasons(prepared: &[PreparedOp]) -> Option Item { - let mut assigned = Item::new(); - for action in actions { - if let UpdateAction::Set { - path, - value: Expr::Placeholder(placeholder), - } = action - && path.len() == 1 - && let PathElement::Attribute(name) = &path[0] - { - let resolved = name - .strip_prefix('#') - .and_then(|reference| maps.names.get(reference).map(String::as_str)) - .unwrap_or(name.as_str()); - if let Some(value) = maps.values.get(placeholder) { - assigned.insert(resolved.to_owned(), value.clone()); - } - } - } - assigned + crate::update_item::vector_relevant_assignments(actions, maps) } /// Parse and validate a single `TransactWriteItem`, returning a `PreparedOp`. diff --git a/crates/engine/src/update_item.rs b/crates/engine/src/update_item.rs index bcac98d3..313f34e7 100755 --- a/crates/engine/src/update_item.rs +++ b/crates/engine/src/update_item.rs @@ -318,26 +318,47 @@ pub async fn handle_update_item( /// Collect direct `SET attr = :value` assignments into a partial item so the /// write-path vector validator can check vector-valued and search-schema -/// attributes. Only top-level attributes assigned a bare value placeholder are -/// included; complex right-hand sides (arithmetic, `if_not_exists`, nested -/// paths) are left for the storage layer. -fn vector_relevant_assignments(actions: &[UpdateAction], maps: &ExpressionMaps) -> Item { +/// attributes. Covers a bare value placeholder and the placeholder inside +/// `if_not_exists(path, :value)`, which are the two forms that can assign a +/// caller-supplied value to a top-level vector attribute. `if_not_exists` is +/// deliberately included: its fallback value persists verbatim whenever the +/// attribute is absent, which is exactly the first write of a vector, so +/// excluding it let a malformed vector return 200 and be silently omitted from +/// the index where the service refuses the write. The value is validated even +/// when the attribute already exists and the fallback would not apply: the +/// service validates arguments, not reachability. Arithmetic and nested paths +/// remain excluded, since a vector attribute admits neither. +pub(crate) fn vector_relevant_assignments(actions: &[UpdateAction], maps: &ExpressionMaps) -> Item { let mut assigned = Item::new(); for action in actions { - if let UpdateAction::Set { - path, - value: Expr::Placeholder(placeholder), - } = action - && path.len() == 1 - && let PathElement::Attribute(name) = &path[0] - { - let resolved = name - .strip_prefix('#') - .and_then(|reference| maps.names.get(reference).map(String::as_str)) - .unwrap_or(name.as_str()); - if let Some(value) = maps.values.get(placeholder) { - assigned.insert(resolved.to_owned(), value.clone()); + let UpdateAction::Set { path, value } = action else { + continue; + }; + if path.len() != 1 { + continue; + } + let PathElement::Attribute(name) = &path[0] else { + continue; + }; + let placeholder = match value { + Expr::Placeholder(placeholder) => Some(placeholder), + Expr::Function { name, args } if name.eq_ignore_ascii_case("if_not_exists") => { + match args.last() { + Some(Expr::Placeholder(placeholder)) => Some(placeholder), + _ => None, + } } + _ => None, + }; + let Some(placeholder) = placeholder else { + continue; + }; + let resolved = name + .strip_prefix('#') + .and_then(|reference| maps.names.get(reference).map(String::as_str)) + .unwrap_or(name.as_str()); + if let Some(value) = maps.values.get(placeholder) { + assigned.insert(resolved.to_owned(), value.clone()); } } assigned @@ -601,3 +622,66 @@ fn desugar_attribute_updates( let expr = parts.join(" "); Ok((Some(expr), expr_values, expr_names)) } + +#[cfg(test)] +mod vector_assignment_tests { + use super::*; + + use extenddb_core::types::AttributeValue; + + fn extract(expression: &str, values: serde_json::Value) -> Item { + let mut maps = ExpressionMaps::default(); + for (k, v) in values.as_object().expect("object") { + maps.values.insert( + k.clone(), + serde_json::from_value::(v.clone()).expect("value"), + ); + } + let limits = extenddb_core::limits::LimitsConfig::default(); + let actions = + crate::expression_helpers::parse_update_expr(expression, &limits).expect("parse"); + vector_relevant_assignments(&actions, &maps) + } + + /// The two forms that can assign a caller-supplied value to a top-level + /// attribute must both surface for validation. `if_not_exists` was excluded + /// once, and the consequence was measured: a malformed vector through it + /// returned 200 and was silently omitted from the index, where the service + /// refuses the write. + #[test] + fn a_bare_placeholder_and_if_not_exists_both_surface_the_value() { + let bare = extract( + "SET emb = :v", + serde_json::json!({"v": {"L": [{"N": "1"}]}}), + ); + assert!(bare.contains_key("emb"), "bare placeholder must surface"); + + let fallback = extract( + "SET emb = if_not_exists(emb, :v)", + serde_json::json!({"v": {"L": [{"N": "1"}]}}), + ); + assert!( + fallback.contains_key("emb"), + "the if_not_exists fallback value must surface for validation: \ + it persists verbatim on the attribute's first write" + ); + } + + /// Arithmetic and nested paths stay excluded: a vector attribute admits + /// neither, so surfacing them would validate values that cannot become a + /// top-level vector. + #[test] + fn arithmetic_and_nested_paths_remain_excluded() { + let arithmetic = extract( + "SET tally = tally + :v", + serde_json::json!({"v": {"N": "1"}}), + ); + assert!(arithmetic.is_empty(), "arithmetic RHS must not surface"); + + let nested = extract( + "SET doc.emb = :v", + serde_json::json!({"v": {"L": [{"N": "1"}]}}), + ); + assert!(nested.is_empty(), "a nested path must not surface"); + } +} diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs index 997930ec..eee5d432 100644 --- a/crates/storage-sqlite/src/data/vector_index.rs +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -647,36 +647,6 @@ pub(crate) async fn backfill_vector_index_in_batches( Ok(written) } -/// Backfill the whole index inside the caller's transaction, atomically. -/// -/// Used by crash recovery at startup, where atomicity is what is wanted and no -/// concurrent writes exist yet. The `UpdateTable` path uses -/// [`backfill_vector_index_in_batches`] instead, because holding one transaction for -/// the whole backfill also holds SQLite's write lock and would stall every write to -/// the base table until the index finished building. -pub(crate) async fn backfill_vector_index( - tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, - table_id: &str, - meta: &VectorIndexMeta, - base_key_schema: &[KeySchemaElement], - attr_defs: &[AttributeDefinition], -) -> Result { - const BATCH: i64 = 500; - let plan = BackfillPlan::new(table_id, meta, base_key_schema, attr_defs); - let mut cursor: i64 = 0; - let mut written = 0usize; - loop { - let (batch_written, fetched, last_rowid) = - backfill_vector_batch(tx, &plan, BATCH, cursor).await?; - written += batch_written; - if fetched < BATCH { - break; - } - cursor = last_rowid; - } - Ok(written) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index ae61dbc9..9d909675 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -840,7 +840,20 @@ impl SqliteEngine { /// is still making progress in this process. Rebuilding rather than resuming, /// for the reconciler's reason: rows already written would collide with the /// backfill's deliberately plain `INSERT`. - pub(crate) async fn recover_stuck_vector_builds(&self) -> Result { + /// + /// `confirmed` is the set of index ids the CALLER has seen stuck on two + /// consecutive passes, and only those are recovered. This is per index on + /// purpose: an early version gated only the decision to sweep, and then + /// recovered every currently-unregistered `CREATING` index, so one genuinely + /// stuck index could drag a just-created sibling (still inside its + /// commit-to-register window) into a rebuild while its live task was also + /// backfilling, double-populating the data table. The registry is re-checked + /// here per index as well, so an id whose task registered since the caller's + /// last pass is skipped even when named. + pub(crate) async fn recover_stuck_vector_builds( + &self, + confirmed: &std::collections::HashSet, + ) -> Result { let rows: Vec<(String, String, String, String)> = sqlx::query_as( "SELECT v.index_id, v.table_id, t.key_schema, t.attribute_definitions \ FROM vector_indexes v JOIN tables t ON v.table_id = t.table_id \ @@ -852,6 +865,9 @@ impl SqliteEngine { let mut recovered = 0usize; for (index_id, table_id, base_ks_json, base_ad_json) in rows { + if !confirmed.contains(&index_id) { + continue; + } let alive = self .vector_builds_running .lock() @@ -880,10 +896,10 @@ impl SqliteEngine { /// Drop, recreate, backfill, and flip one vector index to `ACTIVE`. /// /// The shared body of startup reconciliation and the worker's runtime - /// recovery, factored so the two repairs cannot drift. Runs the backfill in a - /// single transaction under the write lock: both callers are repairing an - /// index whose queue rows are already held, and neither needs the batched - /// path's lock-release property badly enough to pay its added states. + /// recovery, factored so the two repairs cannot drift. The backfill runs on + /// the batched path, releasing the write lock between batches, so a recovery + /// on a large table cannot become a write-availability outage for every + /// other table. async fn rebuild_one_vector_index( &self, index_id: &str, @@ -896,46 +912,65 @@ impl SqliteEngine { let attr_defs: Vec = serde_json::from_str(base_ad_json) .map_err(|e| StorageError::Internal(e.to_string()))?; - let _writer = self.write_lock.lock().await; - Self::drop_vector_data_table_by_id(&self.pool, table_id, index_id).await?; + // Drop and recreate under the lock, then backfill BATCHED, releasing the + // lock between batches exactly as the normal create path does. An earlier + // version ran the whole backfill in one lock-held transaction, which on a + // large table blocked writes to EVERY table for the full rebuild: a + // write-availability outage as the price of recovering one index. The + // batched path is safe here for the same reasons it is safe on create: + // the index is CREATING throughout, so the worker holds this table's + // queue rows and searches are refused, and the rowid cursor tolerates + // concurrent base-table writes. Recovery uses no batch delay: the lever + // exists for tests, and recovery should finish as fast as batching + // allows. + let meta; + { + let _writer = self.write_lock.lock().await; + Self::drop_vector_data_table_by_id(&self.pool, table_id, index_id).await?; - let mut data_tx = self - .pool - .begin_with("BEGIN IMMEDIATE") - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - Self::create_vector_data_table( - &mut data_tx, - table_id, - index_id, - &base_key_schema, - &attr_defs, - ) - .await?; - // The definition is read from the catalog rather than reconstructed, - // because the request that created it is long gone. - let meta = - crate::data::vector_index::fetch_vector_indexes_for_table(&mut data_tx, table_id) - .await? - .into_iter() - .find(|m| m.index_id == index_id) - .ok_or_else(|| { - StorageError::Internal(format!( - "vector index {index_id} was selected as CREATING but has no catalog row" - )) - })?; - let written = crate::data::vector_index::backfill_vector_index( - &mut data_tx, + let mut data_tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Self::create_vector_data_table( + &mut data_tx, + table_id, + index_id, + &base_key_schema, + &attr_defs, + ) + .await?; + // The definition is read from the catalog rather than reconstructed, + // because the request that created it is long gone. + meta = crate::data::vector_index::fetch_vector_indexes_for_table( + &mut data_tx, + table_id, + ) + .await? + .into_iter() + .find(|m| m.index_id == index_id) + .ok_or_else(|| { + StorageError::Internal(format!( + "vector index {index_id} was selected as CREATING but has no catalog row" + )) + })?; + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + let written = crate::data::vector_index::backfill_vector_index_in_batches( + &self.pool, + &self.write_lock, table_id, &meta, &base_key_schema, &attr_defs, + std::time::Duration::ZERO, ) .await?; - data_tx - .commit() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; sqlx::query( "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL \ @@ -1365,7 +1400,11 @@ mod reconciler_tests { .await .expect("candidates"); assert_eq!(candidates, vec!["vidx-dead".to_owned()]); - let recovered = engine.recover_stuck_vector_builds().await.expect("recover"); + let confirmed: std::collections::HashSet = candidates.iter().cloned().collect(); + let recovered = engine + .recover_stuck_vector_builds(&confirmed) + .await + .expect("recover"); assert_eq!(recovered, 1, "the orphaned build must be recovered"); let (status, backfilling): (String, Option) = sqlx::query_as( @@ -1449,8 +1488,15 @@ mod reconciler_tests { .is_empty(), "a registered build must not be a candidate" ); + // Named explicitly as confirmed-stuck, so the registry re-check alone + // must protect it: the strongest form of the control. + let confirmed: std::collections::HashSet = + std::iter::once("vidx-live".to_owned()).collect(); assert_eq!( - engine.recover_stuck_vector_builds().await.expect("recover"), + engine + .recover_stuck_vector_builds(&confirmed) + .await + .expect("recover"), 0, "a registered build must not be recovered" ); diff --git a/crates/storage-sqlite/src/workers.rs b/crates/storage-sqlite/src/workers.rs index 81864b9a..505400c1 100644 --- a/crates/storage-sqlite/src/workers.rs +++ b/crates/storage-sqlite/src/workers.rs @@ -384,11 +384,20 @@ pub(crate) async fn gsi_propagation_worker( } } // The stuck-build sweep. Cheap when nothing is CREATING, which is the - // steady state: one indexed catalog read per pass. + // steady state: one indexed catalog read per pass. Only ids seen stuck + // on TWO consecutive passes are recovered, per index: a first sighting + // can be a healthy build inside its commit-to-register window, and a + // stuck sibling must not drag it into a rebuild. match engine.stuck_vector_build_candidates().await { Ok(candidates) => { - let confirmed = candidates.iter().any(|id| stuck_last_pass.contains(id)); - if confirmed && let Err(e) = engine.recover_stuck_vector_builds().await { + let confirmed: std::collections::HashSet = candidates + .iter() + .filter(|id| stuck_last_pass.contains(*id)) + .cloned() + .collect(); + if !confirmed.is_empty() + && let Err(e) = engine.recover_stuck_vector_builds(&confirmed).await + { tracing::error!("GSI propagation worker: stuck-build recovery failed: {e}"); } stuck_last_pass = candidates.into_iter().collect(); diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs index 3476f660..cf0bbfb1 100644 --- a/tests/rust/src/vector_index_search.rs +++ b/tests/rust/src/vector_index_search.rs @@ -1869,3 +1869,52 @@ async fn update_item_removing_the_vector_attribute_leaves_the_index() { let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; } + +/// A malformed vector cannot slip past write validation inside `if_not_exists`. +/// +/// `SET emb = if_not_exists(emb, :v)` persists `:v` verbatim whenever the +/// attribute is absent, which is exactly the first write of a vector, yet an +/// earlier extraction only surfaced bare `SET emb = :v` for validation. The +/// bypass was concrete: a wrong-dimension vector through the fallback form +/// returned 200 and the item was silently omitted from the index, where the +/// service refuses the write. Both forms must now be refused identically, and +/// the valid fallback form must still work end to end. +#[tokio::test] +async fn if_not_exists_cannot_smuggle_a_malformed_vector() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_ine_validate"); + create_vector_table(&name, 2, "COSINE", false).await; + + // Wrong dimensions through the fallback form must be refused. + let (status, text) = call( + "UpdateItem", + &format!( + r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "a"}}}}, + "UpdateExpression": "SET emb = if_not_exists(emb, :v)", + "ExpressionAttributeValues": {{":v": {{"L": [{{"N": "1.0"}}, {{"N": "2.0"}}, {{"N": "3.0"}}]}}}}}}"# + ), + ) + .await; + assert_eq!( + status, 400, + "a wrong-dimension vector through if_not_exists must be refused, \ + not accepted and silently omitted from the index: {text}" + ); + + // The valid fallback form must still work and reach the index. + let (status, text) = call( + "UpdateItem", + &format!( + r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "a"}}}}, + "UpdateExpression": "SET emb = if_not_exists(emb, :v)", + "ExpressionAttributeValues": {{":v": {{"L": [{{"N": "1.0"}}, {{"N": "0.0"}}]}}}}}}"# + ), + ) + .await; + assert_eq!(status, 200, "a valid vector through if_not_exists must work: {text}"); + search_until_pks(&name, &[1.0, 0.0], 10, None, &["a"]).await; + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +}