diff --git a/.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md b/.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md index 6e335db5..86c85875 100644 --- a/.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md +++ b/.agents/skills/extenddb/references/troubleshooting/07-runtime-symptoms.md @@ -94,8 +94,8 @@ Stream cleanup worker: GSI query returns stale data after a write ``` -**Cause:** GSI updates are applied asynchronously with a configurable propagation delay (default 10ms). This matches real DynamoDB's eventually consistent GSI behavior. Each GSI can have its own `propagation_delay_ms` setting; the system-wide default is controlled by the `gsi_propagation_delay_ms` runtime setting. +**Cause:** GSI updates are applied asynchronously with a configurable propagation delay (default 10ms). This matches real DynamoDB's eventually consistent GSI behavior. Each GSI can have its own `propagation_delay_ms` setting; the system-wide default is controlled by the `index_propagation_delay_ms` runtime setting. -**Fix:** This is expected behavior. For tests that query GSIs after writes, poll/retry the GSI query until the expected data appears. To make all GSIs synchronous for testing, set `extenddb settings set gsi_propagation_delay_ms 0`. For production-like testing, keep the default async delay. +**Fix:** This is expected behavior. For tests that query GSIs after writes, poll/retry the GSI query until the expected data appears. To make all GSIs synchronous for testing, set `extenddb settings set index_propagation_delay_ms 0`. For production-like testing, keep the default async delay. **Source:** `docs/troubleshooting.md`, section "GSI Async Update Behavior", last synced 2026-05-12. diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index aec1d8cc..c3d4448c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -232,44 +232,88 @@ jobs: echo "Server failed to start" exit 1 - - name: Provision IAM test user and access key - id: creds + # Driven through devtools/run-tests rather than a bare `cargo test`, because the + # harness provisions what several suites require and a raw invocation does not: + # devtools/provision-test-credentials creates account 123456789012 with an IAM + # user, access key and full-access policy, and exports the credentials. + # + # batch_transact_authz targets that account and skips itself when + # EXTENDDB_ADMIN_PASSWORD is absent, so under a bare `cargo test` all ten of its + # tests reported ok WITHOUT EXECUTING, in every run since they landed in #232. + # The vector backfill tests need the same access to set the batch delay. + - name: Run Rust integration tests env: - EXTENDDB_PASSWORD: ${{ steps.init.outputs.admin_password }} + EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 + AWS_DEFAULT_REGION: us-east-1 + EXTENDDB_ADMIN_USER: admin + EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} + # PostgreSQL implements no vector search, so this is the job where the + # wire refusal tests must actually run. Both vector suites adapt to + # whatever the backend reports, so the refusal suite could skip every + # assertion here and still report green. "0" asserts the backend + # refuses vector indexes, making those tests mandatory rather than + # optional, and turns a silent skip into a failure. + EXTENDDB_EXPECT_VECTORS: "0" + run: devtools/run-tests --extenddb --rust-integration --release + run-rust-integration-sqlite: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - name: Build release (SQLite backend) + run: cargo build --release -p extenddb --no-default-features --features sqlite + + - name: Initialize ExtendDB + id: init run: | - acc='${{ steps.init.outputs.account_id }}' - ./target/release/extenddb manage --user admin create-user \ - --account-id "$acc" --user-name tester - ./target/release/extenddb manage --user admin put-user-policy \ - --account-id "$acc" --user-name tester --policy-name ddbfull \ - --policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"dynamodb:*","Resource":"*"}]}' - ./target/release/extenddb manage --user admin create-access-key \ - --account-id "$acc" --user-name tester > /tmp/key.json - echo "akid=$(jq -r .access_key_id /tmp/key.json)" >> "$GITHUB_OUTPUT" - secret=$(jq -r .secret_access_key /tmp/key.json) - echo "::add-mask::$secret" - echo "secret=$secret" >> "$GITHUB_OUTPUT" + output=$(./target/release/extenddb init --backend sqlite --config extenddb.toml 2>&1) + echo "$output" + echo "admin_password=$(echo "$output" | grep -oP 'Password: \K\S+')" >> "$GITHUB_OUTPUT" + + - name: Enable provisioned-capacity throttling enforcement + run: ./target/release/extenddb settings set throttling_enabled true + + - name: Start ExtendDB + run: | + ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file & + for i in $(seq 1 30); do + if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then + echo "Server ready" + exit 0 + fi + sleep 1 + done + echo "Server failed to start" + exit 1 + # Driven through devtools/run-tests rather than a bare `cargo test`, because the + # harness provisions what several suites require and a raw invocation does not: + # devtools/provision-test-credentials creates account 123456789012 with an IAM + # user, access key and full-access policy, and exports the credentials. + # + # batch_transact_authz targets that account and skips itself when + # EXTENDDB_ADMIN_PASSWORD is absent, so under a bare `cargo test` all ten of its + # tests reported ok WITHOUT EXECUTING, in every run since they landed in #232. + # The vector backfill tests need the same access to set the batch delay. - name: Run Rust integration tests env: EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 AWS_DEFAULT_REGION: us-east-1 - AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.akid }} - AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.secret }} - # No in-tree backend implements vector search, so every vector request - # must be refused and the refusal suite must actually run. Without this - # the suite self-skips the moment a backend gains vector support, and the - # contract that non-participating backends refuse would stop being - # checked anywhere while still reporting green. Pinning the expectation - # turns that skip into a failure. The first backend to implement vector - # search sets this to 1 in its own job. - EXTENDDB_EXPECT_VECTORS: "0" - run: | - # Self-signed cert generated by init; trust it for the SDK client. - export EXTENDDB_CA_CERT="$(grep -oP 'cert_path\s*=\s*"\K[^"]+' extenddb.toml)" - # The suite shares one SDK client across tests; the hyper connection - # pool is bound to the first test's runtime, so run serially. - cd tests/rust && cargo test -- --test-threads=1 + EXTENDDB_ADMIN_USER: admin + EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} + # Both vector suites self-skip when the backend is the wrong kind, so without + # this the positive suite could skip every assertion and still report green, + # which is what would happen if the backend lost the capability. Pinning the + # expectation turns that skip into a failure. + EXTENDDB_EXPECT_VECTORS: "1" + run: devtools/run-tests --extenddb --rust-integration --release integration: runs-on: ubuntu-latest @@ -279,6 +323,7 @@ jobs: run-integration-sqlite, run-integration-dev-mode, run-rust-integration, + run-rust-integration-sqlite, ] if: always() steps: @@ -286,6 +331,7 @@ jobs: if [ "${{ needs.run-integration.result }}" != "success" ] || \ [ "${{ needs.run-integration-sqlite.result }}" != "success" ] || \ [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \ - [ "${{ needs.run-rust-integration.result }}" != "success" ]; then + [ "${{ needs.run-rust-integration.result }}" != "success" ] || \ + [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then exit 1 fi diff --git a/crates/app/src/cmd_settings.rs b/crates/app/src/cmd_settings.rs index 88f3e775..02210c86 100755 --- a/crates/app/src/cmd_settings.rs +++ b/crates/app/src/cmd_settings.rs @@ -112,6 +112,9 @@ async fn set(store: &dyn SettingsStore, key: &str, value: &str) -> anyhow::Resul ); } + // Write under the canonical name so the deprecated alias updates the row the read + // path consults, rather than adding a second one that is silently ignored. + let key = extenddb_core::settings_keys::canonical_key(key); store .set_setting(key, value) .await diff --git a/crates/core/src/expression/search_condition.rs b/crates/core/src/expression/search_condition.rs index 11bce142..a431bde9 100644 --- a/crates/core/src/expression/search_condition.rs +++ b/crates/core/src/expression/search_condition.rs @@ -130,6 +130,21 @@ pub fn validate_search_condition_expression( )) })? } else { + // A bare identifier must not be a reserved word. This check existed for + // ProjectionExpression, where its message is byte-identical to the + // service, but was never wired into this expression type, so + // `cat = :c AND bucket = :b` was accepted and returned results where the + // service refuses it. Measured on 2026-08-11. + // + // Applied to the bare form only, which is the point of the rule: the + // `#alias` branch above exists precisely so a reserved word can be used + // by aliasing it. + if crate::expression::reserved_words::is_reserved(lhs) { + return Err(invalid(format!( + "Invalid SearchConditionExpression: Attribute name is a reserved \ + keyword; reserved keyword: {lhs}" + ))); + } (*lhs).to_owned() }; if attr_name.contains('.') || attr_name.contains('[') { @@ -220,15 +235,22 @@ pub fn validate_conditions_against_search_schema( let schema = search_schema.unwrap_or(&[]); // Every referenced attribute must be part of the index search schema. + // + // Wording measured against the service on 2026-08-11. It names the offending + // attribute, which the previous text did not, so a caller with several + // conditions had to work out which one was at fault. The service's own grammar + // slip ("attributes that is not") is reproduced deliberately: parity means + // matching what clients actually receive, not correcting it. for condition in conditions { let in_schema = schema .iter() .any(|element| element.attribute_name == condition.attribute_name); if !in_schema { - return Err(invalid( - "SearchConditionExpression must not contain any attributes outside the vector \ - index search schema", - )); + return Err(invalid(format!( + "SearchConditionExpression must not contain any attributes that is not in \ + SearchSchema. Invalid attribute: {}", + condition.attribute_name + ))); } } @@ -299,6 +321,71 @@ mod tests { } } + /// A bare reserved word is refused, with the service's wording. + /// + /// The check already existed for `ProjectionExpression`, where its message is + /// byte-identical to the service, but was never applied here, so + /// `bucket = :b` was accepted and returned results for a request the service + /// refuses. `bucket` is the specific word that exposed it against the live + /// service on 2026-08-11. + #[test] + fn a_bare_reserved_keyword_is_refused() { + let v = values(&[(":b", "4")]); + let message = err(validate_search_condition_expression( + "bucket = :b", + None, + Some(&v), + )); + assert_eq!( + message, + "Invalid SearchConditionExpression: Attribute name is a reserved keyword; \ + reserved keyword: bucket" + ); + } + + /// Aliasing is the documented escape hatch, so the rule must apply to the bare + /// form only. Without this, the fix above would make reserved-word attributes + /// unusable rather than merely requiring an alias. + #[test] + fn an_aliased_reserved_keyword_is_accepted() { + let n = names(&[("#b", "bucket")]); + let v = values(&[(":b", "4")]); + let conditions = validate_search_condition_expression("#b = :b", Some(&n), Some(&v)) + .expect("an aliased reserved word must be allowed"); + assert_eq!(conditions.len(), 1); + assert_eq!(conditions[0].attribute_name, "bucket"); + } + + /// The condition is rejected when it names an attribute outside the schema, and + /// the message names WHICH attribute, as the service's does. + #[test] + fn an_attribute_outside_the_search_schema_is_named_in_the_error() { + let conditions = vec![ + SearchCondition { + attribute_name: "cat".to_owned(), + value: AttributeValue::S("alpha".to_owned()), + }, + SearchCondition { + attribute_name: "payload".to_owned(), + value: AttributeValue::S("nope".to_owned()), + }, + ]; + let schema = [SearchSchemaElement { + attribute_name: "cat".to_owned(), + element_type: SearchSchemaElementType::Hash, + }]; + let message = err(validate_conditions_against_search_schema( + &conditions, + Some(&schema), + &[], + )); + assert_eq!( + message, + "SearchConditionExpression must not contain any attributes that is not in \ + SearchSchema. Invalid attribute: payload" + ); + } + #[test] fn single_literal_equality_resolves() { let v = values(&[(":cat", "Electronics")]); diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 6d1ec5d3..8b0d30c6 100755 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -12,6 +12,7 @@ pub mod expression; pub mod limits; pub mod metrics; pub mod serde_helpers; +pub mod settings_keys; pub mod throttle; pub mod types; pub mod validation; diff --git a/crates/core/src/settings_keys.rs b/crates/core/src/settings_keys.rs new file mode 100644 index 00000000..8143a932 --- /dev/null +++ b/crates/core/src/settings_keys.rs @@ -0,0 +1,81 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Canonical names for runtime settings keys. +//! +//! These strings live in one place because they are read by both storage backends, +//! written by the management API, seeded by both schemas, and documented. Scattering +//! the literal is what let one of them drift out of step with its own meaning. + +/// Propagation delay applied to asynchronous secondary-index maintenance, in +/// milliseconds. `0` means maintenance is applied synchronously in the write's own +/// transaction. +/// +/// Governs GSIs and vector indexes alike, which is why it is not named for either. +/// Real DynamoDB exposes no such knob: this exists so a test can choose between +/// asserting eventual-consistency behaviour and asserting steady state without +/// waiting. +pub const INDEX_PROPAGATION_DELAY_MS: &str = "index_propagation_delay_ms"; + +/// The pre-rename name of [`INDEX_PROPAGATION_DELAY_MS`], still honoured. +/// +/// Two reasons this cannot simply be deleted. A catalog created before the rename +/// holds the operator's value under the old name, and the server refuses to start on +/// a catalog-version mismatch rather than migrating, so there is no upgrade step in +/// which the row could be rewritten. Silently reading past that row would reset a +/// deliberately configured delay to the default, and a delay of 0 means synchronous, +/// so the silent change would be from strict to eventually consistent. +/// +/// Reads therefore prefer the canonical key and fall back to this one; writes to this +/// name are redirected to the canonical key so a deployment converges on one row +/// rather than accumulating two that disagree. +pub const LEGACY_GSI_PROPAGATION_DELAY_MS: &str = "gsi_propagation_delay_ms"; + +/// Milliseconds to pause between batches of a vector index backfill. +/// +/// Zero, and meant to stay zero outside tests. It exists because the correctness +/// property that matters during a backfill is an ordering one: a write that lands +/// while the index is building must end up in the index with its NEW value, not be +/// overwritten by the backfill's older snapshot of the same item. Proving that needs +/// a write to land mid-backfill, and a backfill over a test-sized table finishes far +/// too quickly for a test to hit that window reliably. +/// +/// Without this the test would be a race against the backfill and would pass whether +/// or not the ordering is correct, which is worse than having no test. +pub const VECTOR_BACKFILL_BATCH_DELAY_MS: &str = "vector_backfill_batch_delay_ms"; + +/// Resolve a caller-supplied settings key to its canonical name. +/// +/// Accepting the old name keeps `extenddb settings set gsi_propagation_delay_ms 0` +/// working for anyone with it in a script or runbook, while ensuring the value lands +/// where the read path looks first. +#[must_use] +pub fn canonical_key(key: &str) -> &str { + if key == LEGACY_GSI_PROPAGATION_DELAY_MS { + INDEX_PROPAGATION_DELAY_MS + } else { + key + } +} + +#[cfg(test)] +mod tests { + use super::{INDEX_PROPAGATION_DELAY_MS, LEGACY_GSI_PROPAGATION_DELAY_MS, canonical_key}; + + #[test] + fn the_legacy_name_resolves_to_the_canonical_one() { + assert_eq!( + canonical_key(LEGACY_GSI_PROPAGATION_DELAY_MS), + INDEX_PROPAGATION_DELAY_MS + ); + } + + #[test] + fn an_unrelated_key_is_returned_unchanged() { + assert_eq!(canonical_key("throttling_enabled"), "throttling_enabled"); + assert_eq!( + canonical_key(INDEX_PROPAGATION_DELAY_MS), + INDEX_PROPAGATION_DELAY_MS + ); + } +} diff --git a/crates/core/src/types/table.rs b/crates/core/src/types/table.rs index 806f13eb..3f7d65bd 100755 --- a/crates/core/src/types/table.rs +++ b/crates/core/src/types/table.rs @@ -89,9 +89,21 @@ impl<'de> serde::Deserialize<'de> for DistanceFunction { "COSINE" => Ok(Self::Cosine), "EUCLIDEAN" => Ok(Self::Euclidean), "DOT_PRODUCT" => Ok(Self::DotProduct), + // Enum order measured against the live service 2026-08-06, and it is + // neither alphabetical nor this enum's declaration order: + // [DOT_PRODUCT, COSINE, EUCLIDEAN] + // An earlier version guessed alphabetical and was wrong. + // + // KNOWN DIVERGENCE: the service reports the positional path + // 'vectorIndexes.1.member.distanceFunction'; this says + // 'distanceFunction'. A serde deserializer for the enum cannot know its + // index within the request, so closing this means deserialising the + // field permissively and validating it in `validate_one_vector_index`, + // which already does exactly that for the required `Projection` and + // knows the 1-based position. Deliberately left as a separate change. other => Err(serde::de::Error::custom(format!( "1 validation error detected: Value '{other}' at 'distanceFunction' \ - failed to satisfy constraint: Member must satisfy enum value set: [COSINE, DOT_PRODUCT, EUCLIDEAN]" + failed to satisfy constraint: Member must satisfy enum value set: [DOT_PRODUCT, COSINE, EUCLIDEAN]" ))), } } @@ -622,10 +634,7 @@ pub struct CreateTableInput { pub key_schema: Vec, #[serde(rename = "AttributeDefinitions")] pub attribute_definitions: Vec, - // Some clients send the billing/throughput mode under the field name - // TableThroughputMode instead of BillingMode; accept it as an alias so such - // requests are not rejected for a missing billing mode. - #[serde(rename = "BillingMode", alias = "TableThroughputMode")] + #[serde(rename = "BillingMode")] pub billing_mode: Option, #[serde(rename = "ProvisionedThroughput")] pub provisioned_throughput: Option, @@ -756,9 +765,7 @@ pub struct UpdateGsiAction { pub struct UpdateTableInput { #[serde(rename = "TableName")] pub table_name: String, - // Accept TableThroughputMode as an alias for BillingMode (see - // CreateTableInput): some clients send the billing mode under that name. - #[serde(rename = "BillingMode", alias = "TableThroughputMode")] + #[serde(rename = "BillingMode")] pub billing_mode: Option, #[serde(rename = "ProvisionedThroughput")] pub provisioned_throughput: Option, @@ -914,10 +921,16 @@ pub struct DescribeLimitsOutput { mod tests { use super::*; + /// `TableThroughputMode` is not a member of DynamoDB's CreateTable request: + /// the model has `BillingMode` only (verified against aws-sdk-dynamodb 1.119.0, + /// where the field does not appear at all). An earlier version of this type + /// accepted it as an alias, which meant a request that produced a + /// PAY_PER_REQUEST table here would be ignored by AWS and produce a + /// PROVISIONED table there: an accept-direction divergence, where code written + /// against ExtendDB breaks against the real service. Under AWS JSON 1.0 an + /// unknown member is ignored, which is what must happen here. #[test] - fn create_table_accepts_table_throughput_mode_alias() { - // A client that sends the billing mode under `TableThroughputMode` - // (instead of `BillingMode`) must populate billing_mode all the same. + fn create_table_ignores_the_unknown_table_throughput_mode_member() { let json = r#"{ "TableName": "t", "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], @@ -925,7 +938,10 @@ mod tests { "TableThroughputMode": "PAY_PER_REQUEST" }"#; let input: CreateTableInput = serde_json::from_str(json).unwrap(); - assert_eq!(input.billing_mode, Some(BillingMode::PayPerRequest)); + assert_eq!( + input.billing_mode, None, + "an unknown member must be ignored, not treated as BillingMode" + ); } #[test] @@ -941,10 +957,13 @@ mod tests { } #[test] - fn update_table_accepts_table_throughput_mode_alias() { + fn update_table_ignores_the_unknown_table_throughput_mode_member() { let json = r#"{"TableName": "t", "TableThroughputMode": "PROVISIONED"}"#; let input: UpdateTableInput = serde_json::from_str(json).unwrap(); - assert_eq!(input.billing_mode, Some(BillingMode::Provisioned)); + assert_eq!( + input.billing_mode, None, + "an unknown member must be ignored" + ); } #[test] diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index aed07de9..94944145 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -3,7 +3,10 @@ pub mod number; pub mod vector_item; -pub use vector_item::{MAX_HASH_KEY_SIZE, MAX_INLINE_FILTER_SIZE, validate_vector_write}; +pub use vector_item::{ + MAX_HASH_KEY_SIZE, MAX_INLINE_FILTER_SIZE, validate_vector_write, vector_components, + vector_norm, +}; use crate::error::{DynamoDbError, ErrorMessageKey, error_message}; use crate::limits::LimitsConfig; @@ -110,13 +113,168 @@ pub fn validate_create_table( Ok(()) } -/// Validate the `VectorIndexes` on a `CreateTable` request. +#[cfg(test)] +mod search_schema_shape_tests { + use super::{MAX_SEARCH_SCHEMA_INLINE_FILTERS, validate_search_schema_shape}; + use crate::types::{SearchSchemaElement, SearchSchemaElementType}; + + fn element(name: &str, element_type: SearchSchemaElementType) -> SearchSchemaElement { + SearchSchemaElement { + attribute_name: name.to_owned(), + element_type, + } + } + + fn hash(name: &str) -> SearchSchemaElement { + element(name, SearchSchemaElementType::Hash) + } + + fn filter(name: &str) -> SearchSchemaElement { + element(name, SearchSchemaElementType::InlineFilter) + } + + #[test] + fn no_search_schema_is_allowed() { + // The HASH element is optional: an index without one searches the table. + validate_search_schema_shape(None).expect("absent schema is valid"); + } + + #[test] + fn one_hash_is_allowed() { + validate_search_schema_shape(Some(&[hash("t")])).expect("one HASH is valid"); + } + + /// The case that mattered: a two-HASH schema was accepted and then could not be + /// honoured, because the query side requires a condition for every HASH while a + /// backend resolves the scope from the first and demotes the rest. + #[test] + fn two_hash_elements_are_rejected_with_the_measured_message() { + let err = validate_search_schema_shape(Some(&[hash("a"), hash("b")])) + .expect_err("two HASH elements must be rejected"); + assert_eq!( + format!("{err}"), + "One or more parameter values were invalid: Value '2' at 'SearchSchema' \ + failed to satisfy constraint: Member must have HASH count less than or \ + equal to 1" + ); + } + + #[test] + fn the_inline_filter_cap_is_a_boundary_not_a_range() { + let at_cap: Vec<_> = (0..MAX_SEARCH_SCHEMA_INLINE_FILTERS) + .map(|i| filter(&format!("f{i}"))) + .collect(); + validate_search_schema_shape(Some(&at_cap)).expect("the cap itself is allowed"); + + let over_cap: Vec<_> = (0..=MAX_SEARCH_SCHEMA_INLINE_FILTERS) + .map(|i| filter(&format!("f{i}"))) + .collect(); + let err = validate_search_schema_shape(Some(&over_cap)) + .expect_err("one over the cap must be rejected"); + assert_eq!( + format!("{err}"), + "One or more parameter values were invalid: Value '19' at 'SearchSchema' \ + failed to satisfy constraint: Member must have INLINE_FILTER count less \ + than or equal to 18" + ); + } + + /// Pins the measured number, because the obvious inference from the query-side + /// cap gives twenty and is wrong. A future edit "tidying" this to match the + /// query cap would break parity silently. + #[test] + fn the_inline_filter_cap_is_eighteen_as_measured() { + assert_eq!(MAX_SEARCH_SCHEMA_INLINE_FILTERS, 18); + } + + #[test] + fn a_hash_plus_filters_is_allowed() { + let mut elements = vec![hash("t")]; + elements.extend((0..MAX_SEARCH_SCHEMA_INLINE_FILTERS).map(|i| filter(&format!("f{i}")))); + validate_search_schema_shape(Some(&elements)) + .expect("one HASH plus the filter cap is valid"); + } +} + +/// Maximum `HASH` elements in a vector index search schema. +/// +/// Measured against the live service 2026-08-06. Without this check the contract +/// accepted a schema it then could not honour: `validate_conditions_against_search_schema` +/// requires a condition for EVERY declared HASH, while a backend resolving the scope +/// takes the first HASH and demotes the rest to filters. So a two-HASH schema was +/// internally contradictory rather than merely unvalidated. +const MAX_SEARCH_SCHEMA_HASH: usize = 1; + +/// Maximum `INLINE_FILTER` elements in a vector index search schema. +/// +/// Measured against the live service 2026-08-06 as **18**, which is deliberately +/// recorded rather than derived: the obvious inference from the query-side cap +/// (`MAX_SEARCH_CONDITIONS`, one HASH plus twenty filters) gives twenty, and is +/// wrong. The schema cap and the per-query cap are different numbers. +const MAX_SEARCH_SCHEMA_INLINE_FILTERS: usize = 18; + +/// Vector indexes per table. Documented default quota, adjustable only by AWS +/// Support, so the emulator enforces the default. +const MAX_VECTOR_INDEXES_PER_TABLE: usize = 5; + +/// Validate the shape of a vector index search schema. /// -/// POC scope: each index name must be well-formed, `Dimensions` must be in -/// `1..=4096`, and the vector attribute name must be non-empty. The distance -/// function is enforced by the type system (enum deserialization). +/// Messages measured against the live service 2026-08-06 by signing raw requests, +/// since no published SDK models vector indexes: +/// +/// ```text +/// One or more parameter values were invalid: Value '2' at 'SearchSchema' failed to +/// satisfy constraint: Member must have HASH count less than or equal to 1 +/// ``` +/// +/// Note the field is `SearchSchema` in its request-shape capitalisation, not the +/// lower-camel positional path used by the projection message: the service is not +/// internally consistent here, so each message is reproduced as observed rather +/// than normalised. +fn validate_search_schema_shape( + search_schema: Option<&[crate::types::SearchSchemaElement]>, +) -> Result<(), DynamoDbError> { + let Some(elements) = search_schema else { + return Ok(()); + }; + let hash_count = elements + .iter() + .filter(|e| e.element_type == crate::types::SearchSchemaElementType::Hash) + .count(); + if hash_count > MAX_SEARCH_SCHEMA_HASH { + return Err(DynamoDbError::ValidationException(format!( + "One or more parameter values were invalid: Value '{hash_count}' at \ + 'SearchSchema' failed to satisfy constraint: Member must have HASH count \ + less than or equal to {MAX_SEARCH_SCHEMA_HASH}" + ))); + } + let filter_count = elements + .iter() + .filter(|e| e.element_type == crate::types::SearchSchemaElementType::InlineFilter) + .count(); + if filter_count > MAX_SEARCH_SCHEMA_INLINE_FILTERS { + return Err(DynamoDbError::ValidationException(format!( + "One or more parameter values were invalid: Value '{filter_count}' at \ + 'SearchSchema' failed to satisfy constraint: Member must have \ + INLINE_FILTER count less than or equal to \ + {MAX_SEARCH_SCHEMA_INLINE_FILTERS}" + ))); + } + Ok(()) +} + /// Rules that apply to one vector index specification, wherever it arrives from. /// +/// Each index needs a `Projection`, a well-formed name, `Dimensions` in `1..=4096`, +/// a non-empty vector attribute name, and a search schema within the measured HASH +/// and `INLINE_FILTER` caps. The distance function is enforced by the type system +/// through enum deserialization. +/// +/// Multi-fault parity is deliberately not attempted: the service aggregates faults +/// (`"N validation errors detected"`) with its own field ordering, whereas this +/// returns the first fault it finds and hardcodes a count of one. Single-fault +/// wording is measured and exact, which is what a client parsing one error sees. +/// /// `CreateTable` and `UpdateTable`'s create action carry the same shape, so they /// get the same rules: a malformed index is rejected identically whichever path /// it arrives by, rather than each handler enforcing its own subset. @@ -142,6 +300,7 @@ fn validate_one_vector_index( ))); } validate_index_name(&vi.index_name)?; + validate_search_schema_shape(vi.search_schema.as_deref())?; if vi.dimensions < 1 || vi.dimensions > 4096 { // Verified against the service 2026-08-05: Dimensions=4097 and 8192 // both return exactly this message. The lower bound is not observable @@ -165,9 +324,42 @@ fn validate_vector_indexes(input: &CreateTableInput) -> Result<(), DynamoDbError let Some(vis) = input.vector_indexes.as_ref() else { return Ok(()); }; + + // Per-index shape first, then the table-level constraints. Which the service + // reports first is unobservable from outside, because botocore rejects a + // malformed index client-side before the request is sent, so the order that + // preserves the already-measured per-index messages is the right one to keep. for (position, vi) in vis.iter().enumerate() { validate_one_vector_index(vi, position + 1, "vectorIndexes")?; } + + // Vector indexes are supported only on on-demand tables. Documented under + // "Requirements and limitations" and again in the quota table, which lists + // vector index capacity mode as on-demand only. `BillingMode` defaults to + // PROVISIONED when absent, so an omitted BillingMode is a rejection too. + // + // Wording measured against the service on 2026-08-11; the earlier text was a + // reasonable paraphrase but not what the service says. + if !vis.is_empty() + && input.billing_mode.unwrap_or(BillingMode::Provisioned) != BillingMode::PayPerRequest + { + return Err(DynamoDbError::ValidationException( + "One or more parameter values were invalid: Vector indexes are only supported \ + for PAY_PER_REQUEST tables" + .to_owned(), + )); + } + + // Wording measured against the service on 2026-08-11. Note the service does not + // echo the offending count here, unlike its SearchSchema messages, so neither + // does this. + if vis.len() > MAX_VECTOR_INDEXES_PER_TABLE { + return Err(DynamoDbError::ValidationException(format!( + "One or more parameter values were invalid: VectorIndex count exceeds the \ + per-table limit of {MAX_VECTOR_INDEXES_PER_TABLE}" + ))); + } + Ok(()) } @@ -185,8 +377,8 @@ fn validate_vector_indexes(input: &CreateTableInput) -> Result<(), DynamoDbError /// so both backends produce identical text. /// /// # Errors -/// Returns [`DynamoDbError::ValidationException`] if vector search is disabled for -/// the deployment, or if a create action carries a malformed index. +/// Returns [`DynamoDbError::ValidationException`] if a create action carries a +/// malformed index. pub fn validate_vector_index_updates( updates: Option<&Vec>, ) -> Result<(), DynamoDbError> { @@ -256,6 +448,22 @@ fn validate_index_projections(input: &CreateTableInput) -> Result<(), DynamoDbEr check(&lsi.projection)?; } } + // Vector indexes were omitted from this function, so a vector index could + // declare ProjectionType INCLUDE with no NonKeyAttributes and be accepted, + // where the service refuses it with the message `check` already produces. + // The rules are not vector-specific, so the fix is to iterate them here + // rather than to restate the rules somewhere else. + // + // `projection` is optional on a vector index specification and its absence is + // a separate fault reported by `validate_one_vector_index`, so it is skipped + // here rather than being reported twice with different wording. + if let Some(vis) = &input.vector_indexes { + for vi in vis { + if let Some(projection) = &vi.projection { + check(projection)?; + } + } + } Ok(()) } @@ -463,6 +671,32 @@ fn validate_lsi_key_schemas(input: &CreateTableInput) -> Result<(), DynamoDbErro } fn validate_attribute_definitions(input: &CreateTableInput) -> Result<(), DynamoDbError> { + // A vector attribute must NOT be declared in AttributeDefinitions, and this is + // checked before anything else here because the attribute may simultaneously be + // a legitimate key attribute: without this, naming the table's own partition key + // as the VectorAttribute was accepted, since `pk` satisfied both the + // definition-exists and the definition-is-used checks below. + // + // The rule follows from the shape: `VectorAttributeDefinition` carries only + // `AttributeName` and no type, because a vector is not a scalar type that + // AttributeDefinitions can express. Measured against the service on 2026-08-11. + if let Some(vis) = &input.vector_indexes { + for vi in vis { + let vec_attr = vi.vector_attribute.attribute_name.as_str(); + if input + .attribute_definitions + .iter() + .any(|ad| ad.attribute_name == vec_attr) + { + return Err(DynamoDbError::ValidationException(format!( + "One or more parameter values were invalid: Conflicting attribute \ + definition for '{vec_attr}'. An attribute cannot be defined in \ + AttributeDefinitions when used as a VectorAttribute." + ))); + } + } + } + // Collect all key attribute names from table + GSIs + LSIs let mut key_attrs: Vec<&str> = input .key_schema @@ -491,10 +725,28 @@ fn validate_attribute_definitions(input: &CreateTableInput) -> Result<(), Dynamo // Vector-index search-schema attributes are declared in AttributeDefinitions // but are not part of the base or secondary-index key schema, so count them // as used to satisfy the definition/key correspondence check. + // + // They are checked for existence HERE, with their own message, rather than + // being folded into `key_attrs` and reported by the loop below. The service + // distinguishes the two cases and this previously reused the GSI key wording, + // naming the attribute and listing every definition where the service says only + // that one element is undefined. Measured on 2026-08-11. + let def_names: Vec<&str> = input + .attribute_definitions + .iter() + .map(|ad| ad.attribute_name.as_str()) + .collect(); if let Some(vis) = &input.vector_indexes { for vi in vis { if let Some(schema) = &vi.search_schema { for element in schema { + if !def_names.contains(&element.attribute_name.as_str()) { + return Err(DynamoDbError::ValidationException( + "One or more parameter values were invalid: One element in \ + SearchSchema is not defined in attribute definitions" + .to_owned(), + )); + } if !key_attrs.contains(&element.attribute_name.as_str()) { key_attrs.push(&element.attribute_name); } @@ -504,12 +756,6 @@ fn validate_attribute_definitions(input: &CreateTableInput) -> Result<(), Dynamo } // Every key attribute must have a definition - let def_names: Vec<&str> = input - .attribute_definitions - .iter() - .map(|ad| ad.attribute_name.as_str()) - .collect(); - for attr in &key_attrs { if !def_names.contains(attr) { return Err(DynamoDbError::ValidationException(format!( @@ -1664,9 +1910,12 @@ mod tests { other => panic!("expected ValidationException, got {other:?}"), } - // Present on every element: accepted. + // Present on every element: accepted. PAY_PER_REQUEST is required because a + // vector index is only valid on an on-demand table, which this function now + // enforces; the projection is what this case is about. let input = CreateTableInput { table_name: "t".to_owned(), + billing_mode: Some(BillingMode::PayPerRequest), vector_indexes: Some(vec![spec(all()), spec(all())]), ..Default::default() }; @@ -1711,6 +1960,176 @@ mod tests { } } + /// Helper: a minimal well-formed vector index specification for these tests. + fn vi_spec(name: &str) -> crate::types::VectorIndexSpecification { + crate::types::VectorIndexSpecification { + index_name: name.to_owned(), + vector_attribute: crate::types::VectorAttribute { + attribute_name: "emb".to_owned(), + }, + dimensions: 8, + distance_function: crate::types::DistanceFunction::Cosine, + search_schema: None, + projection: Some(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }), + } + } + + /// The vector attribute must not be declared in `AttributeDefinitions`. + /// + /// Measured against the service on 2026-08-11. Before this, naming the table's + /// own partition key as the vector attribute was ACCEPTED, because `pk` + /// satisfied both the definition-exists and definition-is-used checks, so the + /// conflict was invisible to every rule that ran. + /// + /// Both shapes are asserted: an ordinary extra definition, and the key case, + /// because only the latter passes the surrounding checks and so is the one that + /// actually got through. + #[test] + fn a_vector_attribute_must_not_be_declared_in_attribute_definitions() { + let expected = "One or more parameter values were invalid: Conflicting attribute \ + definition for 'emb'. An attribute cannot be defined in \ + AttributeDefinitions when used as a VectorAttribute."; + let mut input = base_input( + vec![make_ks("pk", KeyType::Hash)], + vec![ + make_ad("pk", ScalarAttributeType::S), + make_ad("emb", ScalarAttributeType::S), + ], + ); + input.vector_indexes = Some(vec![vi_spec("vidx")]); + let err = validate_attribute_definitions(&input) + .expect_err("a declared vector attribute must be refused"); + assert_eq!(format!("{err}"), expected); + + // The key case: the attribute is legitimately a key AND the vector + // attribute, which is what previously slipped through. + let mut input = base_input( + vec![make_ks("pk", KeyType::Hash)], + vec![make_ad("pk", ScalarAttributeType::S)], + ); + let mut spec = vi_spec("vidx"); + spec.vector_attribute.attribute_name = "pk".to_owned(); + input.vector_indexes = Some(vec![spec]); + let err = validate_attribute_definitions(&input) + .expect_err("the partition key must not double as the vector attribute"); + assert!( + format!("{err}").contains("Conflicting attribute definition for 'pk'"), + "{err}" + ); + } + + /// A vector index's projection is subject to the same INCLUDE rule as a GSI's. + /// + /// `validate_index_projections` iterated GSIs and LSIs only, so a vector index + /// could declare INCLUDE with no NonKeyAttributes and be accepted where the + /// service refuses it. The message already existed and was already correct; + /// only the iteration was missing, so this asserts the message to pin that the + /// shared rule is what is being applied rather than a restatement of it. + #[test] + fn a_vector_index_projection_obeys_the_include_rule() { + let mut input = base_input( + vec![make_ks("pk", KeyType::Hash)], + vec![make_ad("pk", ScalarAttributeType::S)], + ); + let mut spec = vi_spec("vidx"); + spec.projection = Some(Projection { + projection_type: ProjectionType::Include, + non_key_attributes: None, + }); + input.vector_indexes = Some(vec![spec]); + let err = validate_index_projections(&input) + .expect_err("INCLUDE without NonKeyAttributes must be refused"); + assert_eq!( + format!("{err}"), + "One or more parameter values were invalid: ProjectionType is INCLUDE, but \ + NonKeyAttributes is not specified" + ); + + // An absent projection is a different fault, reported elsewhere, and must + // not be reported twice with different wording. + let mut spec = vi_spec("vidx"); + spec.projection = None; + input.vector_indexes = Some(vec![spec]); + validate_index_projections(&input) + .expect("an absent projection is validate_one_vector_index's fault to report"); + } + + /// A SearchSchema attribute with no definition gets the service's own wording, + /// which differs from the GSI key-attribute message this previously reused. + /// + /// The distinction matters because the two messages carry different amounts of + /// detail: the GSI one names the attribute and lists every definition, while + /// the service says only that one element is undefined. + #[test] + fn an_undefined_search_schema_attribute_uses_its_own_message() { + let mut input = base_input( + vec![make_ks("pk", KeyType::Hash)], + vec![make_ad("pk", ScalarAttributeType::S)], + ); + let mut spec = vi_spec("vidx"); + spec.search_schema = Some(vec![crate::types::SearchSchemaElement { + attribute_name: "cat".to_owned(), + element_type: crate::types::SearchSchemaElementType::Hash, + }]); + input.vector_indexes = Some(vec![spec]); + let err = validate_attribute_definitions(&input) + .expect_err("an undefined SearchSchema attribute must be refused"); + assert_eq!( + format!("{err}"), + "One or more parameter values were invalid: One element in SearchSchema is not \ + defined in attribute definitions" + ); + } + + /// Both table-level vector messages, pinned to what the service returns. + #[test] + fn the_table_level_vector_messages_match_the_service() { + // Billing mode. PROVISIONED is the default when BillingMode is absent, so + // the omitted case is asserted too. + for billing in [Some(BillingMode::Provisioned), None] { + let mut input = base_input( + vec![make_ks("pk", KeyType::Hash)], + vec![make_ad("pk", ScalarAttributeType::S)], + ); + input.billing_mode = billing; + input.vector_indexes = Some(vec![vi_spec("vidx")]); + let err = validate_vector_indexes(&input) + .expect_err("a vector index requires PAY_PER_REQUEST"); + assert_eq!( + format!("{err}"), + "One or more parameter values were invalid: Vector indexes are only \ + supported for PAY_PER_REQUEST tables" + ); + } + + // The per-table cap, asserted as a boundary so an off-by-one cannot hide. + let mut input = base_input( + vec![make_ks("pk", KeyType::Hash)], + vec![make_ad("pk", ScalarAttributeType::S)], + ); + let at_cap: Vec<_> = (0..MAX_VECTOR_INDEXES_PER_TABLE) + .map(|i| vi_spec(&format!("idx{i}"))) + .collect(); + input.vector_indexes = Some(at_cap); + validate_vector_indexes(&input).expect("the cap itself is allowed"); + + let over_cap: Vec<_> = (0..=MAX_VECTOR_INDEXES_PER_TABLE) + .map(|i| vi_spec(&format!("idx{i}"))) + .collect(); + input.vector_indexes = Some(over_cap); + let err = validate_vector_indexes(&input).expect_err("one over the cap is refused"); + assert_eq!( + format!("{err}"), + format!( + "One or more parameter values were invalid: VectorIndex count exceeds the \ + per-table limit of {MAX_VECTOR_INDEXES_PER_TABLE}" + ) + ); + } + #[test] fn standard_table_rejects_multipart_keys() { let limits = LimitsConfig::default(); // allow_multipart_table_keys = false diff --git a/crates/core/src/validation/vector_item.rs b/crates/core/src/validation/vector_item.rs index c877f244..a460efa7 100644 --- a/crates/core/src/validation/vector_item.rs +++ b/crates/core/src/validation/vector_item.rs @@ -57,6 +57,97 @@ pub fn validate_vector_write( Ok(()) } +/// Extract the components of a vector attribute as `f32`s. +/// +/// Lives beside [`validate_vector_write`] deliberately: the two must agree on what +/// a vector attribute is, and separating them invites a backend that stores +/// something the validator would have rejected. Every backend indexing a vector +/// should use this rather than reading the attribute itself. +/// +/// Returns `None` when the value is not a list of numbers each parsing to a finite +/// `f32`, which is exactly the condition `validate_vector_write` rejects. A caller +/// that has already validated can treat `None` as an internal inconsistency rather +/// than as bad input. +/// +/// The narrowing to `f32` is deliberate and lossy for a caller supplying more +/// precision: the wire type is arbitrary-precision decimal, embedding models emit +/// single precision, and the service's declared dimensionality is in `f32` terms. +#[must_use] +pub fn vector_components(value: &AttributeValue) -> Option> { + let AttributeValue::L(elements) = value else { + return None; + }; + let mut out = Vec::with_capacity(elements.len()); + for element in elements { + let AttributeValue::N(number) = element else { + return None; + }; + let parsed = number.parse::().ok()?; + if !parsed.is_finite() { + return None; + } + out.push(parsed); + } + Some(out) +} + +/// Rebuild a vector attribute from the `f32` components a backend stored. +/// +/// The inverse of [`vector_components`], and the only way a backend should return a +/// stored vector. An index holds 32-bit floats, which is the width the service +/// validates against: it rejects a component outside +/// `[-3.4028235E38, 3.4028235E38]`, exactly `f32::MAX`, and names the expected type +/// "32-bit floating point number". So a client that writes more precision than an +/// `f32` carries reads back the narrowed value rather than the decimal it sent, and +/// reconstructing from the stored bits is what reproduces that. Retaining the +/// client's original text would be a divergence dressed up as fidelity. +/// +/// Living here rather than in a backend keeps every backend returning identical +/// text for identical stored bits. +#[must_use] +pub fn vector_attribute(components: &[f32]) -> AttributeValue { + AttributeValue::L( + components + .iter() + .map(|component| AttributeValue::N(format_component(*component))) + .collect(), + ) +} + +/// Canonical `N` text for one stored component. +/// +/// Plain decimal, always, which is `f32`'s `Display`. Measured against the live +/// service on 2026-08-07: DynamoDB never returns an exponent, whatever it was sent. +/// `3.4028235E+38` reads back as `340282350000000000000000000000000000000` and +/// `1E-40` as `0.0000000000000000000000000000000000000001`, both of which are +/// exactly what `Display` produces for the same `f32`. +/// +/// This corrects an earlier reading of the 38-digit limit as a limit on characters. +/// It bounds *significant* digits, and an `f32`'s shortest round-tripping form +/// carries at most 9 of them, so no stored component can approach it however long +/// the expansion runs. A 38-significant-digit value was accepted and returned +/// verbatim in the same probe. +/// +/// The one departure from `Display` is negative zero, which the service also +/// normalises: `-0` reads back as `0`. +fn format_component(value: f32) -> String { + if value == 0.0 { + // Covers -0.0, which compares equal to 0.0. + return "0".to_owned(); + } + format!("{value}") +} + +/// The L2 norm of a vector, precomputed at write time. +/// +/// Stored alongside the vector so a cosine search costs one dot product per +/// candidate instead of also summing squares. Computed here rather than in a +/// backend so every backend stores the same value for the same vector. +#[must_use] +pub fn vector_norm(components: &[f32]) -> f32 { + components.iter().map(|x| x * x).sum::().sqrt() +} + /// Validate a single vector-valued attribute against an index definition. fn validate_vector_attribute( value: &AttributeValue, @@ -67,9 +158,13 @@ fn validate_vector_attribute( let dimensions = index.dimensions as usize; let AttributeValue::L(elements) = value else { + // Measured 2026-08-07 against N, S and NS in the vector position, all three + // of which produce this exact text. Note "32-bit floating point number + // list" rather than any phrasing of "list of numbers", and no full stop + // before IndexName, matching the size message below. return Err(invalid(format!( - "One or more parameter values were invalid: Invalid type for parameter {attr}, \ - Expected: a list of numbers. IndexName: {index_name}" + "One or more parameter values were invalid. Invalid type for parameter {attr}, \ + Expected: 32-bit floating point number list IndexName: {index_name}" ))); }; @@ -101,14 +196,15 @@ fn validate_vector_attribute( .map(format_scientific) .unwrap_or_else(|_| number.clone()); return Err(invalid(format!( - "Invalid value for parameter {attr}[{position}], Value: {display} is \ - outside valid range [-3.4028235E38, 3.4028235E38]. IndexName: {index_name}" + "One or more parameter values were invalid. Invalid value for parameter \ + {attr}[{position}], Value: {display} is outside valid range \ + [-3.4028235E38, 3.4028235E38]. IndexName: {index_name}" ))); } } other => { return Err(invalid(format!( - "One or more parameter values were invalid: Invalid type for parameter \ + "One or more parameter values were invalid. Invalid type for parameter \ {attr}[{position}], Expected: 32-bit floating point number, Actual: {}. \ IndexName: {index_name}", attribute_type_token(other) @@ -183,7 +279,11 @@ fn attribute_type_token(value: &AttributeValue) -> &'static str { /// Format a float in upper-case scientific notation with an explicit exponent /// sign, e.g. `1.3E+40` or `-1.3E+40`. fn format_scientific(value: f64) -> String { - let formatted = format!("{value:E}"); + with_exponent_sign(format!("{value:E}")) +} + +/// Rust omits the `+` on a positive exponent; the service includes it. +fn with_exponent_sign(formatted: String) -> String { if let Some(exponent_pos) = formatted.find('E') { let (mantissa, exponent) = formatted.split_at(exponent_pos); let digits = &exponent[1..]; @@ -304,10 +404,19 @@ mod tests { assert!(message.contains("Expected: 5, Actual: 0")); } + /// Asserted whole rather than by fragment. Every message below was measured + /// against the live service on 2026-08-07, and the previous fragment assertions + /// excluded the "One or more parameter values were invalid" prefix, which is + /// precisely where three of the four had drifted: two used a colon where the + /// service uses a full stop, and one omitted the prefix entirely. #[test] fn rejects_non_list_vector() { let message = err(&item_with_vector(AttributeValue::N("0.1".to_owned()))); - assert!(message.contains("Invalid type for parameter ProductEmbedding")); + assert_eq!( + message, + "One or more parameter values were invalid. Invalid type for parameter \ + ProductEmbedding, Expected: 32-bit floating point number list IndexName: ProductIndex" + ); } #[test] @@ -320,10 +429,16 @@ mod tests { AttributeValue::N("0.5".to_owned()), ]); let message = err(&item_with_vector(value)); - assert!(message.contains("Invalid type for parameter ProductEmbedding[2]")); - assert!(message.contains("Expected: 32-bit floating point number, Actual: S")); + assert_eq!( + message, + "One or more parameter values were invalid. Invalid type for parameter \ + ProductEmbedding[2], Expected: 32-bit floating point number, Actual: S. \ + IndexName: ProductIndex" + ); } + /// The offending value is echoed in scientific notation whatever form it was + /// sent in: the service answered `Value: 4E+38` to a plain 39-digit integer. #[test] fn rejects_value_out_of_range() { let value = AttributeValue::L(vec![ @@ -334,10 +449,12 @@ mod tests { AttributeValue::N("0.5".to_owned()), ]); let message = err(&item_with_vector(value)); - assert!(message.contains( - "Invalid value for parameter ProductEmbedding[1], Value: 1.3E+40 is outside \ - valid range [-3.4028235E38, 3.4028235E38]. IndexName: ProductIndex" - )); + assert_eq!( + message, + "One or more parameter values were invalid. Invalid value for parameter \ + ProductEmbedding[1], Value: 1.3E+40 is outside valid range \ + [-3.4028235E38, 3.4028235E38]. IndexName: ProductIndex" + ); } #[test] @@ -392,4 +509,129 @@ mod tests { assert_eq!(format_scientific(1.3e40), "1.3E+40"); assert_eq!(format_scientific(-1.3e40), "-1.3E+40"); } + + /// The invariant that makes reconstruction safe: whatever text is produced for a + /// stored component must parse back to the identical bits. If this fails, a + /// search returns a vector that is not the one indexed. + /// + /// Negative zero is the one deliberate exception and is asserted separately. + #[test] + fn rebuilding_a_vector_round_trips_the_stored_bits() { + let components = [ + 0.0f32, + 1.0, + -1.0, + 0.1, + 0.123_456_79, + f32::MAX, + f32::MIN, + f32::MIN_POSITIVE, + 1e-40, // subnormal + 3.4e38, + -1.5e-30, + ]; + let rebuilt = vector_attribute(&components); + let parsed = vector_components(&rebuilt).expect("rebuilt vector must revalidate"); + assert_eq!(parsed.len(), components.len()); + for (original, back) in components.iter().zip(parsed.iter()) { + assert_eq!( + original.to_bits(), + back.to_bits(), + "component {original} did not round-trip" + ); + } + } + + /// `N` carries one zero, so a stored `-0.0` returns as `0`. The sign of zero is + /// unobservable through any of the three distance functions, so normalising it + /// cannot change a score or an ordering. + #[test] + fn negative_zero_is_normalised() { + assert_eq!(format_component(-0.0), "0"); + assert_eq!(format_component(0.0), "0"); + let parsed = vector_components(&vector_attribute(&[-0.0f32])).expect("valid"); + assert_eq!(parsed[0].to_bits(), 0.0f32.to_bits()); + } + + /// Ordinary embedding magnitudes must not be dressed up in exponents: the plain + /// form is what a client wrote and what it expects to read. + #[test] + fn ordinary_magnitudes_stay_in_plain_decimal() { + assert_eq!(format_component(0.5), "0.5"); + assert_eq!(format_component(-1.25), "-1.25"); + assert_eq!(format_component(1.0), "1"); + assert_eq!(format_component(0.000_123), "0.000123"); + } + + /// `Display` never uses an exponent, and neither does the service. These exact + /// strings were read back from real DynamoDB on 2026-08-07 after sending + /// `3.4028235E+38` and `1E-40`, so matching them is measured parity rather than + /// a formatting preference. + #[test] + fn extreme_magnitudes_match_the_plain_form_the_service_returns() { + assert_eq!( + format_component(f32::MAX), + "340282350000000000000000000000000000000" + ); + assert_eq!( + format_component(1e-40), + "0.0000000000000000000000000000000000000001" + ); + assert!( + !format_component(f32::MAX).contains('E'), + "an exponent is never returned by the service" + ); + } + + /// A client writing more precision than an `f32` carries reads back the narrowed + /// value, which is what the service does, rather than its own decimal string. + #[test] + fn excess_client_precision_is_narrowed_not_preserved() { + let written = + AttributeValue::L(vec![AttributeValue::N("0.12345678901234567890".to_owned())]); + let stored = vector_components(&written).expect("valid vector"); + let returned = vector_attribute(&stored); + assert_eq!( + returned, + AttributeValue::L(vec![AttributeValue::N("0.12345679".to_owned())]) + ); + } + + /// The extractor must accept exactly what the validator accepts. If these + /// drift, a backend stores a vector the validator would have rejected, or + /// refuses one it accepted. + #[test] + fn the_extractor_accepts_what_the_validator_accepts() { + let good = AttributeValue::L(vec![ + AttributeValue::N("0.5".to_owned()), + AttributeValue::N("-1".to_owned()), + AttributeValue::N("0".to_owned()), + ]); + assert_eq!(vector_components(&good), Some(vec![0.5, -1.0, 0.0])); + } + + #[test] + fn the_extractor_rejects_a_non_list() { + assert!(vector_components(&AttributeValue::S("nope".to_owned())).is_none()); + } + + #[test] + fn the_extractor_rejects_a_non_numeric_element() { + let bad = AttributeValue::L(vec![AttributeValue::S("1".to_owned())]); + assert!(vector_components(&bad).is_none()); + } + + /// A component that overflows f32 is rejected rather than becoming infinity, + /// which would poison every distance computed against it. + #[test] + fn the_extractor_rejects_a_component_that_is_not_finite_in_f32() { + let bad = AttributeValue::L(vec![AttributeValue::N("1e40".to_owned())]); + assert!(vector_components(&bad).is_none()); + } + + #[test] + fn the_norm_is_the_euclidean_length() { + assert!((vector_norm(&[3.0, 4.0]) - 5.0).abs() < 1e-6); + assert!(vector_norm(&[0.0, 0.0]).abs() < 1e-6); + } } diff --git a/crates/engine/src/backup.rs b/crates/engine/src/backup.rs index 34f7a0e5..dc57dde6 100755 --- a/crates/engine/src/backup.rs +++ b/crates/engine/src/backup.rs @@ -161,6 +161,13 @@ pub(crate) async fn handle_restore_table_from_backup( .invalidate_table_key_info(&ctx.account_id, target_table_name) .await; + // The readiness invariant is applied on every path that hands a description to + // a client, and restore was the one that omitted it. Currently harmless (a + // restored index is CREATING, and a non-vector backend could never hold a + // vector-index backup because create is gated), but the invariant claims to + // cover exactly this class of path, so the omission was a latent inconsistency + // rather than a deliberate exception. + desc.validate_vector_index_readiness()?; serialize_output(&json!({ "TableDescription": desc })) } diff --git a/crates/engine/src/search_vectors.rs b/crates/engine/src/search_vectors.rs index c8acb7b3..2133073f 100644 --- a/crates/engine/src/search_vectors.rs +++ b/crates/engine/src/search_vectors.rs @@ -165,6 +165,20 @@ pub async fn handle_search_vectors( .unwrap_or(&[]) .iter() .find(|vi| vi.index_name == input.index_name) + // An index that is not ACTIVE cannot serve, and the service reports that by + // saying the table does not have the index at all: measured four times + // against DynamoDB on 2026-08-11 while an index sat in CREATING, which + // returned exactly this message rather than naming the status. + // + // Filtered here rather than excluded from DescribeTable, because + // DescribeTable MUST still report a CREATING index with its status and + // `Backfilling` member; only the search path treats it as absent. + // + // This is load-bearing now that the backfill runs asynchronously. While it + // was awaited inline inside one transaction, a partially populated index was + // unobservable; a search can now arrive mid-build and must be refused rather + // than answered from incomplete data. + .filter(|vi| vi.index_status.is_active()) .ok_or_else(|| { DynamoDbError::ValidationException(format!( "The table does not have the specified index: {}", @@ -190,8 +204,8 @@ pub async fn handle_search_vectors( // The index's HASH element scopes the search to one partition; the // remaining conditions narrow within it. Declaring a HASH element is - // optional, but when the index has one the service requires the search to - // supply it, so validation upstream guarantees it is present here. + // optional, but when the index has one the service REQUIRES the search to + // supply it. let hash_attr = vector_index .search_schema .as_deref() @@ -207,6 +221,25 @@ pub async fn handle_search_vectors( .map(|c| (c.attribute_name.as_str(), &c.value)) }); + // Refuse rather than search unscoped. An earlier comment here asserted that + // "validation upstream guarantees it is present", which was untrue: no such + // validation existed, so a search that omitted the condition ran without a + // partition scope and returned HTTP 200 with ZERO results. That is the worst + // available failure mode, a silent wrong answer, and it told the caller "no + // matches" for a request the service rejects outright. + // + // Message measured against the service on 2026-08-11 in us-east-1. + // + // Keyed off `hash_key` rather than off whether an expression was supplied at + // all, because an expression that omits the HASH attribute leaves the search + // just as unscoped as no expression does. + if hash_attr.is_some() && hash_key.is_none() { + return Err(DynamoDbError::ValidationException( + "SearchConditionExpression must be provided when SearchSchema has a HASH key" + .to_owned(), + )); + } + let filters: Vec<(&str, &AttributeValue)> = conditions .iter() .filter(|c| Some(c.attribute_name.as_str()) != hash_attr) @@ -227,8 +260,28 @@ pub async fn handle_search_vectors( let hits = search_output.hits; // Bytes read from the index for the returned items, excluding the vector - // component (the stored item already omits the vector attribute). - let non_vector_bytes: usize = hits.iter().map(|h| item_size_bytes(&h.item)).sum(); + // component, which is metered separately as the query's own dimension cost. + // + // Subtracted explicitly rather than assumed absent, because this is computed + // from what the BACKEND returned, before the response projection runs. A + // backend is free to hand over the vector (the SQLite path does, since it holds + // the projected item verbatim as the GSI path does), so the figure must not rest + // on an assumption about how a backend stores its rows. An earlier version of + // this comment asserted the vector was already absent and was wrong, inflating + // the billed figure by roughly 10 to 15 KB per hit at 1024 dimensions. + let non_vector_bytes: usize = hits + .iter() + .map(|h| { + let total = item_size_bytes(&h.item); + let vector = key_info + .vector_indexes + .iter() + .find(|vi| vi.index_name == input.index_name) + .and_then(|vi| h.item.get(&vi.vector_attribute_name)) + .map_or(0, extenddb_core::types::attribute_value_size); + total.saturating_sub(vector) + }) + .sum(); // Compile the projection once, if supplied. let compiled_projection = if let Some(ref proj_str) = input.projection_expression { @@ -250,8 +303,20 @@ pub async fn handle_search_vectors( .into_iter() .map(|hit| { let item = match compiled_projection.as_ref() { + // A supplied ProjectionExpression already restricts the item to the + // paths it names, so naming the vector attribute keeps it and not + // naming it drops it. Nothing extra to do. Some(proj) => proj.apply(&hit.item), - None => hit.item, + // With no ProjectionExpression the vector attribute is withheld. + // The service returns it only when it is explicitly asked for, even + // under a Projection of ALL, so returning it by default would be + // both a parity divergence and a large one: a 1024-dimension vector + // serialises to roughly 10 to 15 KB per hit. + None => { + let mut item = hit.item; + item.remove(&vector_index.vector_attribute.attribute_name); + item + } }; SearchResult { item, @@ -270,7 +335,24 @@ pub async fn handle_search_vectors( // The service reports `ConsumedCapacity.VectorSearchRequestBytes`, a byte // figure, not a unit figure. See `search_request_bytes` for the measured // model and why exact parity is not achievable. - let request_bytes = search_request_bytes(vector_index.dimensions, non_vector_bytes); + // + // `vectors_returned` is counted from the projected results rather than inferred + // from whether the expression mentions the attribute, so an alias, a nested + // path, or an item that simply has no vector all count correctly. The service + // charges one float32 per dimension for each returned item that carries it. + let vectors_returned = search_results + .iter() + .filter(|r| { + r.item + .contains_key(&vector_index.vector_attribute.attribute_name) + }) + .count(); + let request_bytes = search_request_bytes( + vector_index.dimensions, + non_vector_bytes, + search_results.len(), + vectors_returned, + ); let consumed_capacity = match input.return_consumed_capacity { ReturnConsumedCapacity::None => None, _ => Some(VectorCapacity { @@ -342,40 +424,71 @@ fn parse_search_vector(values: &[AttributeValue]) -> Result, DynamoDbEr /// Bytes reported as `ConsumedCapacity.VectorSearchRequestBytes` for one search. /// /// `returned_non_vector_bytes` is the summed stored size of the items actually -/// returned, taken before any projection is applied. +/// returned, taken before any projection is applied. `returned_count` is how many +/// items are returned, and `vectors_returned` how many of those include the vector +/// attribute in the response projection. +/// +/// Re-derived on 2026-08-11 in us-east-1 by sweeping dimensions 16, 64, 128, 256 and +/// 512 against key-only items, at TopK 1, 10 and 25, four samples per point. The +/// previous model used 17.6 bytes per dimension and no per-result term, which +/// reported 2748.8 where the service reported 6196.0 for the same 128-dimension +/// search: a ratio of 2.25, far outside the 1.176 bimodal spread that had been used +/// to excuse imprecision. Three terms were measured, each independently: /// -/// Measured against the service on 2026-08-05 in us-east-1. Three properties -/// hold, and this reproduces all three: +/// * **Per dimension: 30.6875.** Exact at every dimension tested, not a fit. +/// Intercepts were 491, 1964, 3928, 7856 and 15712 for 16 to 512 dimensions, +/// each exactly `dimensions * 30.6875`. +/// * **Per returned result: 72 bytes** with key-only items, flat across dimensions +/// 64 to 512. The 16-dimension row measured 52.8 only because the 1 KiB floor +/// clipped its TopK 1 point. +/// * **Per returned vector: 4 bytes per dimension.** Projecting the vector added +/// exactly `40 * dimensions` at TopK 10, i.e. `4 * dimensions` per returned item, +/// which is one float32 per dimension. The previous model ignored this entirely, +/// so projecting the vector doubled the service's figure and changed nothing here. +/// * The **1 KiB floor** is unchanged and was reconfirmed: a 16-dimension search at +/// TopK 1 reported exactly 1024. /// -/// * A 1 KiB floor. A 4-dimension index reported exactly 1024 for every TopK -/// from 1 to 100, for 1 to 100 returned results, and with 4 to 204 items in -/// the index. -/// * A per-dimension term that is independent of how many items the index -/// holds, so this is not a scan-cost model: growing a 4096-dimension index -/// from 3 to 12 items moved the figure by 2 bytes, which was the returned -/// key's length rather than the extra items. It is also unaffected by the -/// query vector's wire width, since the same search with 1-character and -/// 21-character numbers (49 KB versus 127 KB of JSON) both reported 75201. -/// The vector is metered per dimension, not per byte sent. -/// * Plus the stored bytes of the returned items, excluding the searched -/// vector. Adding one item with a 2000-byte non-vector attribute raised the -/// figure by exactly 1999. Projection does not reduce it, so a caller must -/// sum the items before projecting, not after. +/// The 72 is split into a fixed 65 plus the returned item's own bytes, because the +/// caller already supplies `returned_non_vector_bytes` and a key-only item +/// contributes roughly 7 of those. That split is the one derived rather than +/// directly measured quantity here: it is corroborated by the richer items used in +/// an earlier probe, whose per-result figure was 102.6 for items carrying about +/// 30 bytes more, but it rests on ExtendDB's own accounting of stored size matching +/// the service's, which cannot be true in general. /// -/// Exact parity is NOT achievable and must not be asserted. The service is not -/// deterministic here: byte-identical requests against an unchanged index return -/// one of two values separated by a fixed `dimensions * 3.111` offset, a ratio of -/// 1.176. Twenty-four samples at 1024 dimensions gave 18067 and 21253; at 2048 -/// they gave 36058 and 42429; the mix varies between runs and persists with 10 -/// items in the index. This reproduces the lower and more frequent mode. -fn search_request_bytes(dimensions: u32, returned_non_vector_bytes: usize) -> f64 { - /// Derived from the lower mode: 18067 at 1024 dimensions and 36058 at 2048, - /// both 17.6 bytes per dimension once the returned item is subtracted. - const BYTES_PER_DIMENSION: f64 = 17.6; - /// Observed floor. A 4-dimension search never reported less than this. +/// Exact parity therefore remains out of reach, and the earlier note that the +/// service is bimodal still stands as a caution even though this sweep saw a single +/// value at every one of its 60 sample points. +/// +/// One term is known to be missing. Every observation above came from an UNSCOPED +/// search. A HASH-scoped search on a 128-dimension index reported 3675, which is +/// below the 3928 that the per-dimension term alone accounts for, implying the +/// service charges less when the SearchSchema confines the search to one partition. +/// That leaves this model roughly 38% HIGH on a scoped search, where it was 40% low +/// on everything before. The direction of the error is now conservative rather than +/// under-reporting, and the scoping term is not characterised: doing so needs a +/// sweep over partition counts and selectivities, which has not been run. +fn search_request_bytes( + dimensions: u32, + returned_non_vector_bytes: usize, + returned_count: usize, + vectors_returned: usize, +) -> f64 { + /// Exact at 16, 64, 128, 256 and 512 dimensions. + const BYTES_PER_DIMENSION: f64 = 30.6875; + /// One float32 per dimension for each returned item carrying the vector. + const BYTES_PER_RETURNED_VECTOR_DIMENSION: f64 = 4.0; + /// The 72 measured per result, less the bytes a key-only item contributes + /// through `returned_non_vector_bytes`. + const PER_RESULT_OVERHEAD: f64 = 65.0; + /// Observed floor. A 16-dimension search never reported less than this. const MIN_SEARCH_BYTES: f64 = 1024.0; - (BYTES_PER_DIMENSION * f64::from(dimensions) + returned_non_vector_bytes as f64) + let dims = f64::from(dimensions); + (BYTES_PER_DIMENSION * dims + + PER_RESULT_OVERHEAD * returned_count as f64 + + returned_non_vector_bytes as f64 + + BYTES_PER_RETURNED_VECTOR_DIMENSION * dims * vectors_returned as f64) .max(MIN_SEARCH_BYTES) } @@ -431,23 +544,21 @@ mod tests { assert!(parse_search_vector(&[AttributeValue::N("inf".to_owned())]).is_err()); } - /// A 4-dimension search reported exactly 1024 for every TopK from 1 to 100 - /// and for 4 to 204 items in the index, so the floor dominates at low - /// dimensions rather than anything proportional. + /// A 16-dimension search at TopK 1 reported exactly 1024, so the floor + /// dominates at low dimensions rather than anything proportional. #[test] fn search_bytes_have_a_one_kib_floor() { - assert!((search_request_bytes(4, 0) - 1024.0).abs() < f64::EPSILON); - assert!((search_request_bytes(1, 100) - 1024.0).abs() < f64::EPSILON); + assert!((search_request_bytes(4, 0, 0, 0) - 1024.0).abs() < f64::EPSILON); + assert!((search_request_bytes(1, 100, 1, 0) - 1024.0).abs() < f64::EPSILON); } - /// Independent of items scanned, so the only inputs are dimensions and the - /// bytes of the items actually returned. + /// The per-dimension term is linear, verified against the service at 16, 64, + /// 128, 256, 512, 1024 and 2048 dimensions. #[test] fn search_bytes_scale_per_dimension_above_the_floor() { - let a = search_request_bytes(1024, 0); - let b = search_request_bytes(2048, 0); + let a = search_request_bytes(1024, 0, 0, 0); + let b = search_request_bytes(2048, 0, 0, 0); assert!(a > 1024.0, "1024 dimensions must clear the floor: {a}"); - // Doubling the dimensions doubles the per-dimension term exactly. assert!((b - 2.0 * a).abs() < 1.0, "{a} then {b}"); } @@ -455,26 +566,95 @@ mod tests { /// figure by exactly 1999, so returned bytes pass through one-for-one. #[test] fn returned_item_bytes_pass_through_one_for_one() { - let base = search_request_bytes(1024, 0); - assert!((search_request_bytes(1024, 2000) - (base + 2000.0)).abs() < f64::EPSILON); + let base = search_request_bytes(1024, 0, 1, 0); + assert!((search_request_bytes(1024, 2000, 1, 0) - (base + 2000.0)).abs() < f64::EPSILON); + } + + /// Each returned result costs a fixed amount beyond its own bytes. + /// + /// The previous model had no per-result term at all, so TopK made no difference + /// to the figure. The service moved by exactly 72 bytes per additional result at + /// every dimension from 64 to 2048, with key-only items. + #[test] + fn each_returned_result_adds_a_fixed_cost() { + let one = search_request_bytes(1024, 0, 1, 0); + let eleven = search_request_bytes(1024, 0, 11, 0); + let per_result = (eleven - one) / 10.0; + assert!( + (per_result - 65.0).abs() < f64::EPSILON, + "expected the fixed per-result overhead, got {per_result}" + ); + } + + /// Returning the vector costs one float32 per dimension per item. + /// + /// This term was absent entirely: projecting the vector doubled the service's + /// figure (6196 to 11316 at 128 dimensions) and changed nothing here. Measured + /// as exactly `40 * dimensions` at TopK 10, i.e. `4 * dimensions` per item. + #[test] + fn returning_the_vector_costs_four_bytes_per_dimension_per_item() { + let without = search_request_bytes(128, 0, 10, 0); + let with = search_request_bytes(128, 0, 10, 10); + assert!( + (with - without - 40.0 * 128.0).abs() < f64::EPSILON, + "expected 4 bytes per dimension per returned vector: {without} then {with}" + ); + // Partial projection scales with how many results carry the vector. + let half = search_request_bytes(128, 0, 10, 5); + assert!( + (half - without - 20.0 * 128.0).abs() < f64::EPSILON, + "{half}" + ); } - /// Checks the model against the service's own numbers, deliberately with a - /// tolerance rather than equality: the service returns one of two values for - /// a byte-identical request (18067 or 21253 at 1024 dimensions, 36058 or - /// 42429 at 2048), so asserting equality would encode a coin flip. Roughly - /// 20 bytes of returned item accompanied each observation. + /// Checks the model against the service's own numbers with a tolerance. + /// + /// Equality is not asserted, for two measured reasons. Byte-identical requests + /// against an unchanged index return one of a small number of values: six + /// samples at 512 dimensions gave 16418 and 16433. And the figure depends on how + /// many items the INDEX holds, which this model does not carry: at 512 + /// dimensions and TopK 10, a 30-item index reported 15824 and a 60-item index + /// 16418, about 19.8 bytes per item. That dependence is why two sweeps + /// disagreed by 3% at the same dimension, and it contradicts an earlier comment + /// here which asserted the figure was independent of items scanned. + /// + /// The constant is set from the 60-item observations, which are internally exact + /// at five dimensions, so the model errs slightly high on very small indexes + /// rather than low on realistic ones. + /// + /// Observations are TopK 10 against key-only items, so `returned_non_vector_bytes` + /// is passed as a small figure rather than zero. #[test] - fn model_tracks_the_observed_lower_mode() { - for (dimensions, observed) in [(1024_u32, 18067.0_f64), (2048, 36058.0)] { - let modelled = search_request_bytes(dimensions, 20); + fn model_tracks_the_measured_service_figures() { + // (dimensions, items in index, observed bytes at TopK 10) + for (dimensions, observed) in [ + (512_u32, 16433.0_f64), + (1024, 31206.0), + (2048, 61692.0), + (128, 4648.0), + (64, 2684.0), + ] { + let modelled = search_request_bytes(dimensions, 70, 10, 0); let error = (modelled - observed).abs() / observed; assert!( - error < 0.01, + error < 0.05, "{dimensions} dimensions: modelled {modelled} against observed {observed} \ is {:.2}% out", error * 100.0 ); } } + + /// The old constant is far enough out to be worth pinning as a regression guard. + /// + /// 17.6 bytes per dimension reported 2748.8 where the service reported 6196.0, + /// so anything near the old value must fail this. + #[test] + fn the_superseded_constant_would_not_pass() { + let modelled = search_request_bytes(128, 70, 10, 0); + assert!( + modelled > 4000.0, + "the model must not regress toward the old 17.6 per dimension: {modelled}" + ); + } } diff --git a/crates/engine/src/transact_write_items.rs b/crates/engine/src/transact_write_items.rs index 236da064..366d71fa 100755 --- a/crates/engine/src/transact_write_items.rs +++ b/crates/engine/src/transact_write_items.rs @@ -18,7 +18,7 @@ use crate::transact_write_helpers::{ }; use crate::{DispatchMetrics, DispatchResult}; use extenddb_core::error::DynamoDbError; -use extenddb_core::expression::{Expr, ExpressionMaps, PathElement, UpdateAction}; +use extenddb_core::expression::{ExpressionMaps, UpdateAction}; use extenddb_core::types::{ CancellationReason, Item, TransactWriteItem, TransactWriteItemsInput, TransactWriteItemsOutput, }; @@ -295,28 +295,14 @@ fn collect_vector_cancellation_reasons(prepared: &[PreparedOp]) -> Option Item { - let mut assigned = Item::new(); - for action in actions { - if let UpdateAction::Set { - path, - value: Expr::Placeholder(placeholder), - } = action - && path.len() == 1 - && let PathElement::Attribute(name) = &path[0] - { - let resolved = name - .strip_prefix('#') - .and_then(|reference| maps.names.get(reference).map(String::as_str)) - .unwrap_or(name.as_str()); - if let Some(value) = maps.values.get(placeholder) { - assigned.insert(resolved.to_owned(), value.clone()); - } - } - } - assigned + crate::update_item::vector_relevant_assignments(actions, maps) } /// Parse and validate a single `TransactWriteItem`, returning a `PreparedOp`. diff --git a/crates/engine/src/update_item.rs b/crates/engine/src/update_item.rs index bcac98d3..313f34e7 100755 --- a/crates/engine/src/update_item.rs +++ b/crates/engine/src/update_item.rs @@ -318,26 +318,47 @@ pub async fn handle_update_item( /// Collect direct `SET attr = :value` assignments into a partial item so the /// write-path vector validator can check vector-valued and search-schema -/// attributes. Only top-level attributes assigned a bare value placeholder are -/// included; complex right-hand sides (arithmetic, `if_not_exists`, nested -/// paths) are left for the storage layer. -fn vector_relevant_assignments(actions: &[UpdateAction], maps: &ExpressionMaps) -> Item { +/// attributes. Covers a bare value placeholder and the placeholder inside +/// `if_not_exists(path, :value)`, which are the two forms that can assign a +/// caller-supplied value to a top-level vector attribute. `if_not_exists` is +/// deliberately included: its fallback value persists verbatim whenever the +/// attribute is absent, which is exactly the first write of a vector, so +/// excluding it let a malformed vector return 200 and be silently omitted from +/// the index where the service refuses the write. The value is validated even +/// when the attribute already exists and the fallback would not apply: the +/// service validates arguments, not reachability. Arithmetic and nested paths +/// remain excluded, since a vector attribute admits neither. +pub(crate) fn vector_relevant_assignments(actions: &[UpdateAction], maps: &ExpressionMaps) -> Item { let mut assigned = Item::new(); for action in actions { - if let UpdateAction::Set { - path, - value: Expr::Placeholder(placeholder), - } = action - && path.len() == 1 - && let PathElement::Attribute(name) = &path[0] - { - let resolved = name - .strip_prefix('#') - .and_then(|reference| maps.names.get(reference).map(String::as_str)) - .unwrap_or(name.as_str()); - if let Some(value) = maps.values.get(placeholder) { - assigned.insert(resolved.to_owned(), value.clone()); + let UpdateAction::Set { path, value } = action else { + continue; + }; + if path.len() != 1 { + continue; + } + let PathElement::Attribute(name) = &path[0] else { + continue; + }; + let placeholder = match value { + Expr::Placeholder(placeholder) => Some(placeholder), + Expr::Function { name, args } if name.eq_ignore_ascii_case("if_not_exists") => { + match args.last() { + Some(Expr::Placeholder(placeholder)) => Some(placeholder), + _ => None, + } } + _ => None, + }; + let Some(placeholder) = placeholder else { + continue; + }; + let resolved = name + .strip_prefix('#') + .and_then(|reference| maps.names.get(reference).map(String::as_str)) + .unwrap_or(name.as_str()); + if let Some(value) = maps.values.get(placeholder) { + assigned.insert(resolved.to_owned(), value.clone()); } } assigned @@ -601,3 +622,66 @@ fn desugar_attribute_updates( let expr = parts.join(" "); Ok((Some(expr), expr_values, expr_names)) } + +#[cfg(test)] +mod vector_assignment_tests { + use super::*; + + use extenddb_core::types::AttributeValue; + + fn extract(expression: &str, values: serde_json::Value) -> Item { + let mut maps = ExpressionMaps::default(); + for (k, v) in values.as_object().expect("object") { + maps.values.insert( + k.clone(), + serde_json::from_value::(v.clone()).expect("value"), + ); + } + let limits = extenddb_core::limits::LimitsConfig::default(); + let actions = + crate::expression_helpers::parse_update_expr(expression, &limits).expect("parse"); + vector_relevant_assignments(&actions, &maps) + } + + /// The two forms that can assign a caller-supplied value to a top-level + /// attribute must both surface for validation. `if_not_exists` was excluded + /// once, and the consequence was measured: a malformed vector through it + /// returned 200 and was silently omitted from the index, where the service + /// refuses the write. + #[test] + fn a_bare_placeholder_and_if_not_exists_both_surface_the_value() { + let bare = extract( + "SET emb = :v", + serde_json::json!({"v": {"L": [{"N": "1"}]}}), + ); + assert!(bare.contains_key("emb"), "bare placeholder must surface"); + + let fallback = extract( + "SET emb = if_not_exists(emb, :v)", + serde_json::json!({"v": {"L": [{"N": "1"}]}}), + ); + assert!( + fallback.contains_key("emb"), + "the if_not_exists fallback value must surface for validation: \ + it persists verbatim on the attribute's first write" + ); + } + + /// Arithmetic and nested paths stay excluded: a vector attribute admits + /// neither, so surfacing them would validate values that cannot become a + /// top-level vector. + #[test] + fn arithmetic_and_nested_paths_remain_excluded() { + let arithmetic = extract( + "SET tally = tally + :v", + serde_json::json!({"v": {"N": "1"}}), + ); + assert!(arithmetic.is_empty(), "arithmetic RHS must not surface"); + + let nested = extract( + "SET doc.emb = :v", + serde_json::json!({"v": {"L": [{"N": "1"}]}}), + ); + assert!(nested.is_empty(), "a nested path must not surface"); + } +} diff --git a/crates/server/src/console/pages/settings_pages.rs b/crates/server/src/console/pages/settings_pages.rs index e2efa500..a41940af 100755 --- a/crates/server/src/console/pages/settings_pages.rs +++ b/crates/server/src/console/pages/settings_pages.rs @@ -30,7 +30,7 @@ const RUNTIME_DEFAULTS: &[(&str, &str)] = &[ ("control_plane_delay_seconds", "0.25"), ("data_database_connection_string", "—"), ("data_database_name", "—"), - ("gsi_propagation_delay_ms", "500"), + ("index_propagation_delay_ms", "500"), ("log_level", "info"), ("sqlx_log_level", "warn"), ("throttling_enabled", "false"), diff --git a/crates/server/src/management/ops_settings.rs b/crates/server/src/management/ops_settings.rs index 86e6ed54..92148be0 100755 --- a/crates/server/src/management/ops_settings.rs +++ b/crates/server/src/management/ops_settings.rs @@ -15,10 +15,28 @@ pub type Validator = fn(&str) -> Result<(), &'static str>; pub const KNOWN_KEYS: &[(&str, Validator)] = &[ ("allow_credential_import", validate_bool), ("control_plane_delay_seconds", validate_delay_seconds), - ("gsi_propagation_delay_ms", validate_gsi_delay_ms), + ( + extenddb_core::settings_keys::INDEX_PROPAGATION_DELAY_MS, + validate_index_propagation_delay_ms, + ), + // Deprecated alias, still writable so an existing script or runbook keeps + // working. `set_setting` canonicalises it, so it updates the same row rather + // than creating a second one that the read path would ignore. + ( + extenddb_core::settings_keys::LEGACY_GSI_PROPAGATION_DELAY_MS, + validate_index_propagation_delay_ms, + ), ("log_level", validate_log_level), ("sqlx_log_level", validate_log_level), ("throttling_enabled", validate_bool), + // A test lever, writable for the same reason the propagation delay is: the + // ordering property it exists to expose (a write landing mid-backfill must not + // be overwritten by the backfill's older snapshot) cannot be observed unless a + // test can slow the backfill down from outside the process. + ( + extenddb_core::settings_keys::VECTOR_BACKFILL_BATCH_DELAY_MS, + validate_backfill_batch_delay_ms, + ), ]; /// Read-only keys that cannot be changed via the settings API. @@ -35,6 +53,18 @@ fn validate_log_level(value: &str) -> Result<(), &'static str> { } } +/// Milliseconds, bounded so a mistyped value cannot wedge a backfill indefinitely. +/// +/// The cap is generous next to any legitimate test need and small enough that the +/// worst case is a slow backfill rather than one that never finishes. +fn validate_backfill_batch_delay_ms(value: &str) -> Result<(), &'static str> { + match value.parse::() { + Ok(ms) if ms <= 60_000 => Ok(()), + Ok(_) => Err("must be between 0 and 60000 milliseconds"), + Err(_) => Err("must be a non-negative integer number of milliseconds"), + } +} + fn validate_bool(value: &str) -> Result<(), &'static str> { match value { "true" | "false" => Ok(()), @@ -50,7 +80,7 @@ fn validate_delay_seconds(value: &str) -> Result<(), &'static str> { } } -fn validate_gsi_delay_ms(value: &str) -> Result<(), &'static str> { +fn validate_index_propagation_delay_ms(value: &str) -> Result<(), &'static str> { match value.parse::() { Ok(0..=10000) => Ok(()), Ok(_) => Err("must be between 0 and 10000"), @@ -93,6 +123,9 @@ pub async fn set_setting( ))); } + // Write under the canonical name, so setting the deprecated alias updates the row + // the read path actually consults instead of adding a second, ignored one. + let key = extenddb_core::settings_keys::canonical_key(key); store.set_setting(key, value).await?; tracing::warn!( diff --git a/crates/storage-postgres/migrations/001_schema.sql b/crates/storage-postgres/migrations/001_schema.sql index 3a684189..8ad5fea5 100644 --- a/crates/storage-postgres/migrations/001_schema.sql +++ b/crates/storage-postgres/migrations/001_schema.sql @@ -315,7 +315,7 @@ INSERT INTO settings (key, value) VALUES ('catalog_version', '0.0.2') ON CONFLICT (key) DO NOTHING; INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0.25') ON CONFLICT (key) DO NOTHING; -INSERT INTO settings (key, value) VALUES ('gsi_propagation_delay_ms', '10') +INSERT INTO settings (key, value) VALUES ('index_propagation_delay_ms', '10') ON CONFLICT (key) DO NOTHING; COMMIT; diff --git a/crates/storage-postgres/src/data/delete_item.rs b/crates/storage-postgres/src/data/delete_item.rs index 6765bae1..7e53254a 100755 --- a/crates/storage-postgres/src/data/delete_item.rs +++ b/crates/storage-postgres/src/data/delete_item.rs @@ -39,7 +39,7 @@ impl PostgresEngine { let sys_delay = if indexes.is_empty() { 0 } else { - self.gsi_default_delay().await + self.index_propagation_delay().await }; let needs_tx = condition.is_some() || return_old || !indexes.is_empty() || stream.is_some(); diff --git a/crates/storage-postgres/src/data/put_item.rs b/crates/storage-postgres/src/data/put_item.rs index b8b0920a..9ca9d985 100755 --- a/crates/storage-postgres/src/data/put_item.rs +++ b/crates/storage-postgres/src/data/put_item.rs @@ -59,7 +59,7 @@ impl PostgresEngine { let sys_delay = if indexes.is_empty() { 0 } else { - self.gsi_default_delay().await + self.index_propagation_delay().await }; // When there's a condition, return_old, indexes, or stream capture, we need a transaction diff --git a/crates/storage-postgres/src/data/transactions.rs b/crates/storage-postgres/src/data/transactions.rs index 5e098046..c7bebebb 100644 --- a/crates/storage-postgres/src/data/transactions.rs +++ b/crates/storage-postgres/src/data/transactions.rs @@ -85,7 +85,7 @@ impl PostgresEngine { // D-4: Read the system default delay live (P119), so a runtime change // applies to this transaction rather than up to 30 s later. - let sys_delay = self.gsi_default_delay().await; + let sys_delay = self.index_propagation_delay().await; let mut tx = self .data_pool diff --git a/crates/storage-postgres/src/data/update_item.rs b/crates/storage-postgres/src/data/update_item.rs index a2c668af..39fd336c 100755 --- a/crates/storage-postgres/src/data/update_item.rs +++ b/crates/storage-postgres/src/data/update_item.rs @@ -50,7 +50,7 @@ impl PostgresEngine { let sys_delay = if indexes.is_empty() { 0 } else { - self.gsi_default_delay().await + self.index_propagation_delay().await }; // Fetch existing item diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index dcdb4507..4a30ee33 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -135,10 +135,23 @@ pub struct PostgresConfig { /// (`_ddb_*` tables, GSI tables). This separation allows the catalog and /// data to live in different `PostgreSQL` databases (Bug 1, P54). /// Default GSI propagation delay (milliseconds) when the -/// `gsi_propagation_delay_ms` setting is absent. Mirrors the value seeded by +/// `index_propagation_delay_ms` setting is absent. Mirrors the value seeded by /// the catalog schema, and is the single definition used by both the live read /// on the write path and the background refresh worker. -pub(crate) const DEFAULT_GSI_PROPAGATION_DELAY_MS: u64 = 10; +pub(crate) const DEFAULT_INDEX_PROPAGATION_DELAY_MS: u64 = 10; + +/// Read the propagation-delay setting, preferring the canonical key and falling back +/// to the pre-rename one. +/// +/// A catalog created before the rename holds the operator's value under the old name, +/// and the server refuses to start on a catalog-version mismatch rather than migrating, +/// so no upgrade step ever rewrites that row. Reading past it would silently reset a +/// configured delay to the default; since 0 means synchronous, the silent change would +/// be from strict to eventually consistent. `ORDER BY ... DESC` makes the preference +/// deterministic when both rows exist. +pub(crate) const INDEX_PROPAGATION_DELAY_QUERY: &str = "SELECT value FROM settings \ + WHERE key IN ('index_propagation_delay_ms', 'gsi_propagation_delay_ms') \ + ORDER BY key = 'index_propagation_delay_ms' DESC LIMIT 1"; pub struct PostgresEngine { pub(crate) pool: PgPool, @@ -152,10 +165,10 @@ pub struct PostgresEngine { /// D-4: Async GSI update queue. `None` until `start_gsi_workers()` is called. pub(crate) gsi_queue: Option>, /// P119: Cached GSI default propagation delay (milliseconds). Refreshed by - /// the background poller every 30s and re-warmed by `gsi_default_delay`. + /// the background poller every 30s and re-warmed by `index_propagation_delay`. /// This is only a fallback for when the live read fails; the write path /// reads the setting live so a runtime change applies to the next write. - pub gsi_default_delay_ms: Arc, + pub index_propagation_delay_cache: Arc, } impl PostgresEngine { @@ -208,15 +221,13 @@ impl PostgresEngine { }; // P119: Read initial GSI propagation delay from settings table. - let initial_gsi_delay: u64 = sqlx::query_as::<_, (String,)>( - "SELECT value FROM settings WHERE key = 'gsi_propagation_delay_ms'", - ) - .fetch_optional(&pool) - .await - .ok() - .flatten() - .and_then(|(v,)| v.parse::().ok()) - .unwrap_or(DEFAULT_GSI_PROPAGATION_DELAY_MS); + let initial_gsi_delay: u64 = sqlx::query_as::<_, (String,)>(INDEX_PROPAGATION_DELAY_QUERY) + .fetch_optional(&pool) + .await + .ok() + .flatten() + .and_then(|(v,)| v.parse::().ok()) + .unwrap_or(DEFAULT_INDEX_PROPAGATION_DELAY_MS); Ok(Self { pool, @@ -225,7 +236,9 @@ impl PostgresEngine { max_item_size_bytes: config.max_item_size_bytes, control_plane_notify: Arc::new(tokio::sync::Notify::new()), gsi_queue: None, - gsi_default_delay_ms: Arc::new(std::sync::atomic::AtomicU64::new(initial_gsi_delay)), + index_propagation_delay_cache: Arc::new(std::sync::atomic::AtomicU64::new( + initial_gsi_delay, + )), }) } @@ -236,7 +249,7 @@ impl PostgresEngine { #[must_use] /// Current GSI propagation delay (ms); `0` means synchronous. /// - /// Reads the `gsi_propagation_delay_ms` setting live from the catalog so an + /// Reads the `index_propagation_delay_ms` setting live from the catalog so an /// out-of-process change (`extenddb settings set`) applies to the next write /// rather than up to 30 s later when the poll worker refreshes the cache. /// Callers skip this entirely for tables with no secondary indexes, so a @@ -245,24 +258,23 @@ impl PostgresEngine { /// On a read error the cached value is used and the error is logged, so a /// degraded catalog serves a stale delay loudly rather than silently. On /// success the cache is re-warmed, keeping the fallback fresh. - pub(crate) async fn gsi_default_delay(&self) -> u64 { + pub(crate) async fn index_propagation_delay(&self) -> u64 { use std::sync::atomic::Ordering; - let live = sqlx::query_as::<_, (String,)>( - "SELECT value FROM settings WHERE key = 'gsi_propagation_delay_ms'", - ) - .fetch_optional(&self.pool) - .await; + let live = sqlx::query_as::<_, (String,)>(INDEX_PROPAGATION_DELAY_QUERY) + .fetch_optional(&self.pool) + .await; match live { Ok(row) => { let ms = row .and_then(|(v,)| v.parse::().ok()) - .unwrap_or(DEFAULT_GSI_PROPAGATION_DELAY_MS); - self.gsi_default_delay_ms.store(ms, Ordering::Relaxed); + .unwrap_or(DEFAULT_INDEX_PROPAGATION_DELAY_MS); + self.index_propagation_delay_cache + .store(ms, Ordering::Relaxed); ms } Err(e) => { - tracing::debug!("gsi_default_delay: live read failed, using cache: {e:?}"); - self.gsi_default_delay_ms.load(Ordering::Relaxed) + tracing::debug!("index_propagation_delay: live read failed, using cache: {e:?}"); + self.index_propagation_delay_cache.load(Ordering::Relaxed) } } } @@ -376,7 +388,7 @@ use extenddb_storage::server_components::{BackendError, ServerComponents}; struct PostgresRuntimeHooks { engine: Arc, control_plane_notify: Arc, - gsi_default_delay_ms: Arc, + index_propagation_delay_cache: Arc, data_db_name: String, } @@ -444,7 +456,7 @@ impl ServerRuntimeHooks for PostgresRuntimeHooks { // 7. GSI delay poller let catalog_store_for_gsi = ctx.catalog_store.clone(); - let gsi_delay = self.gsi_default_delay_ms.clone(); + let gsi_delay = self.index_propagation_delay_cache.clone(); let token = ctx.shutdown.clone(); let gsi_poller = tokio::spawn(async move { workers::poll_gsi_delay(catalog_store_for_gsi, gsi_delay, token).await; @@ -527,7 +539,7 @@ fn server_components_factory( // Get references to fields we need before wrapping let control_plane_notify = engine.control_plane_notify.clone(); - let gsi_default_delay_ms = engine.gsi_default_delay_ms.clone(); + let index_propagation_delay_cache = engine.index_propagation_delay_cache.clone(); // Wrap engine in Arc let engine = Arc::new(engine); @@ -584,7 +596,7 @@ fn server_components_factory( let runtime_hooks = Box::new(PostgresRuntimeHooks { engine: engine.clone(), control_plane_notify, - gsi_default_delay_ms, + index_propagation_delay_cache, data_db_name, }); diff --git a/crates/storage-postgres/src/workers.rs b/crates/storage-postgres/src/workers.rs index 7e1d144a..467262f3 100644 --- a/crates/storage-postgres/src/workers.rs +++ b/crates/storage-postgres/src/workers.rs @@ -14,6 +14,22 @@ use extenddb_storage::{CancellationToken, DataEngine, MetadataEngine, StreamEngi use sqlx::PgPool; use crate::PostgresEngine; +/// Read the propagation-delay setting through the settings store, preferring the +/// canonical key and falling back to the pre-rename one. See +/// `INDEX_PROPAGATION_DELAY_QUERY` for why the old name is still honoured. +async fn read_index_propagation_delay( + settings: &S, +) -> extenddb_storage::management_store::OpResult> { + if let Some(v) = settings + .get_setting(extenddb_core::settings_keys::INDEX_PROPAGATION_DELAY_MS) + .await? + { + return Ok(Some(v)); + } + settings + .get_setting(extenddb_core::settings_keys::LEGACY_GSI_PROPAGATION_DELAY_MS) + .await +} pub(crate) async fn poll_control_plane_transitions( storage: Arc, @@ -175,7 +191,7 @@ pub(crate) async fn poll_gsi_delay( const POLL_INTERVAL: Duration = Duration::from_secs(30); while tick(&token, POLL_INTERVAL).await { - match store.get_setting("gsi_propagation_delay_ms").await { + match read_index_propagation_delay(store.as_ref()).await { Ok(Some(val)) => { if let Ok(ms) = val.parse::() { gsi_delay.store(ms, std::sync::atomic::Ordering::Relaxed); @@ -184,12 +200,12 @@ pub(crate) async fn poll_gsi_delay( Ok(None) => { // Setting removed - revert to default gsi_delay.store( - crate::DEFAULT_GSI_PROPAGATION_DELAY_MS, + crate::DEFAULT_INDEX_PROPAGATION_DELAY_MS, std::sync::atomic::Ordering::Relaxed, ); } Err(e) => { - tracing::debug!("Failed to query gsi_propagation_delay_ms: {e:?}"); + tracing::debug!("Failed to query index_propagation_delay_ms: {e:?}"); } } } diff --git a/crates/storage-sqlite/docs/design-decisions.md b/crates/storage-sqlite/docs/design-decisions.md index 8630df3d..b34f1b01 100644 --- a/crates/storage-sqlite/docs/design-decisions.md +++ b/crates/storage-sqlite/docs/design-decisions.md @@ -148,7 +148,7 @@ only) — identical to Postgres, no change needed. Postgres backend's persistent-queue design rather than its `FOR UPDATE SKIP LOCKED` mechanics, which are unnecessary under the single-writer model. The effective delay is the per-GSI `propagation_delay_ms` override when set, - otherwise the `gsi_propagation_delay_ms` runtime setting (cached on the engine + otherwise the `index_propagation_delay_ms` runtime setting (cached on the engine and refreshed by a poller). Delay 0 ⇒ fully synchronous GSI maintenance. This **supersedes the earlier "synchronous because local I/O is fast" rationale**: that approach ignored the configured propagation delay and so did diff --git a/crates/storage-sqlite/src/create_table.rs b/crates/storage-sqlite/src/create_table.rs index 8c4e2202..9485b681 100644 --- a/crates/storage-sqlite/src/create_table.rs +++ b/crates/storage-sqlite/src/create_table.rs @@ -194,6 +194,61 @@ impl SqliteEngine { } } + // Vector indexes. A CreateTable's table is empty, so there is nothing to + // backfill: the index goes straight to ACTIVE with no `backfilling` + // member, which is the state the service reports for an index created + // this way. The UpdateTable path is the one that drives a real lifecycle. + let mut vector_ids: Vec = Vec::new(); + if let Some(vis) = &input.vector_indexes { + for vi in vis { + let vec_attr = serde_json::to_string(&vi.vector_attribute) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let search_schema = vi + .search_schema + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let proj = vi + .projection + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + // Core validation requires Projection, so reaching here + // means the request bypassed validation rather than that + // the caller omitted it. + StorageError::Internal( + "vector index reached storage without a projection".to_owned(), + ) + })?; + let distance = serde_json::to_string(&vi.distance_function) + .map_err(|e| StorageError::Internal(e.to_string()))? + .trim_matches('"') + .to_owned(); + let index_id = uuid::Uuid::new_v4().to_string(); + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_name, index_id, dimensions, distance_function, \ + vector_attribute, search_schema, projection, index_status, backfilling) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'ACTIVE', NULL)", + ) + .bind(&table_id) + .bind(&vi.index_name) + .bind(&index_id) + .bind(i64::from(vi.dimensions)) + .bind(&distance) + .bind(&vec_attr) + .bind(&search_schema) + .bind(&proj) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + vector_ids.push(index_id); + } + } + // Tags. if let Some(tags) = &input.tags { for tag in tags { @@ -262,6 +317,18 @@ impl SqliteEngine { .await?; } } + if input.vector_indexes.is_some() { + for index_id in &vector_ids { + Self::create_vector_data_table( + &mut data_tx, + &table_id, + index_id, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + } + } data_tx .commit() .await @@ -368,6 +435,37 @@ impl SqliteEngine { }) }); + // Echo the vector indexes we just created. Built from the request plus the + // ids assigned above rather than re-read from the catalog, which would add + // a round trip to say something already known. A CreateTable's table is + // empty, so each index is ACTIVE with no `backfilling` member. + let vector_index_descs: Option> = input + .vector_indexes + .as_ref() + .map(|vis| { + vis.iter() + .map(|vi| extenddb_core::types::VectorIndexDescription { + index_name: vi.index_name.clone(), + vector_attribute: vi.vector_attribute.clone(), + dimensions: vi.dimensions, + search_schema: vi.search_schema.clone(), + distance_function: vi.distance_function, + index_status: extenddb_core::types::IndexStatus::Active, + backfilling: None, + index_size_bytes: 0, + item_count: 0, + index_arn: extenddb_storage::util::index_arn( + &self.region, + account_id, + &input.table_name, + &vi.index_name, + ), + projection: vi.projection.clone(), + }) + .collect() + }) + .filter(|v: &Vec<_>| !v.is_empty()); + Ok(TableDescription { table_name: input.table_name, key_schema: input.key_schema, @@ -398,10 +496,12 @@ impl SqliteEngine { .as_ref() .map(|tc| serde_json::json!({ "TableClass": tc })), on_demand_throughput: input.on_demand_throughput, - // Fields for features this backend does not implement, vector - // indexes today, take their defaults, so adding one to this type - // does not break this build. - ..Default::default() + // Every field is populated deliberately, with no `..Default::default()` + // spread. This response is the complete description of what was just + // created, so a new core field should break this site and force a + // decision about whether create must report it, rather than silently + // defaulting. Sites that legitimately opt out still use the spread. + vector_indexes: vector_index_descs, }) } } diff --git a/crates/storage-sqlite/src/data/data_engine.rs b/crates/storage-sqlite/src/data/data_engine.rs index 72071bd7..6228a564 100644 --- a/crates/storage-sqlite/src/data/data_engine.rs +++ b/crates/storage-sqlite/src/data/data_engine.rs @@ -17,6 +17,13 @@ use futures::future::BoxFuture; use crate::store::SqliteEngine; impl DataEngine for SqliteEngine { + /// Declares vector support by handing over the implementation. `Some(self)` + /// only compiles because `SqliteEngine` implements `VectorSearchEngine`, so + /// this cannot claim a capability the backend does not have. + fn as_vector_search(&self) -> Option<&dyn extenddb_storage::VectorSearchEngine> { + Some(self) + } + fn put_item( &self, key_info: &TableKeyInfo, diff --git a/crates/storage-sqlite/src/data/ddl.rs b/crates/storage-sqlite/src/data/ddl.rs index 49c9880a..f1c12f96 100644 --- a/crates/storage-sqlite/src/data/ddl.rs +++ b/crates/storage-sqlite/src/data/ddl.rs @@ -15,7 +15,10 @@ use extenddb_core::types::{ use extenddb_storage::error::StorageError; use extenddb_storage::util::sk_column_n; -use super::{all_sort_key_info, data_table_name, index_table_name}; +use super::{ + all_sort_key_info, data_table_name, index_table_name, vector_table_glob_pattern, + vector_table_name, +}; use crate::store::SqliteEngine; /// SQLite column type for the Nth sort-key position and scalar type (D2). @@ -95,6 +98,113 @@ impl SqliteEngine { Ok(()) } + /// Create a vector-index data table: one row per indexed vector. + /// + /// `part` is the search-schema HASH value when one is declared, and a single + /// constant otherwise, so an unscoped index is one partition rather than a + /// separate code path. `nrm` is the vector's precomputed L2 norm, so cosine + /// costs one dot product at query time instead of two passes. + /// + /// # Safety (SQL injection) + /// `index_id` is a server-generated UUID and column names are constants, so + /// no user input reaches the DDL. Vector attribute names are stored as data, + /// never as identifiers. + pub(crate) async fn create_vector_data_table( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + index_id: &str, + base_key_schema: &[KeySchemaElement], + base_attr_defs: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let vec_table = vector_table_name(table_id, index_id); + let base_sks = all_sort_key_info(base_key_schema, base_attr_defs); + + let mut col_defs = vec![ + "part TEXT NOT NULL".to_owned(), + "base_pk TEXT NOT NULL".to_owned(), + ]; + for i in 0..base_sks.len() { + col_defs.extend(base_sk_col_defs(i)); + } + col_defs.push("vec BLOB NOT NULL".to_owned()); + col_defs.push("nrm REAL NOT NULL".to_owned()); + col_defs.push("item_data TEXT NOT NULL".to_owned()); + + // Keyed by the base item, not by the partition, so one base item yields + // at most one vector row and a re-put replaces rather than duplicates. + let mut pk_cols = vec!["base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + pk_cols.push(format!("base_{}", sk_column_n(i, sk_type))); + } + + let ddl = format!( + "CREATE TABLE {vec_table} (\n {},\n PRIMARY KEY ({})\n)", + col_defs.join(",\n "), + pk_cols.join(", ") + ); + sqlx::query(&ddl) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // The scan is always partition-scoped, so this index is what keeps a + // search off the full table when a HASH element is declared. + let part_idx = + format!("CREATE INDEX \"_vidx_part_{table_id}_{index_id}\" ON {vec_table} (part)"); + sqlx::query(&part_idx) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + /// Drop every vector-index data table belonging to one DynamoDB table. + /// + /// Discovered from `sqlite_master` rather than from the catalog, because the + /// caller runs this after the catalog rows have been cascade-deleted in the + /// same transaction, so `vector_indexes` is already empty for this table. + /// Without this, dropping a table would leave its vector data tables behind + /// forever with nothing left pointing at them. + /// Drop one vector index's data table. + /// + /// The table-drop path sweeps `sqlite_master` instead, because by the time it + /// runs the catalog rows have already cascade-deleted and the index ids are + /// unreadable. Here the id is known, so the name is derived directly rather + /// than matched by pattern. + pub(crate) async fn drop_vector_data_table_by_id( + pool: &sqlx::SqlitePool, + table_id: &str, + index_id: &str, + ) -> Result<(), StorageError> { + let vec_table = vector_table_name(table_id, index_id); + sqlx::query(&format!("DROP TABLE IF EXISTS {vec_table}")) + .execute(pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn drop_all_vector_data_tables( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + ) -> Result<(), StorageError> { + let names: Vec = sqlx::query_scalar( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name GLOB ?", + ) + .bind(vector_table_glob_pattern(table_id)) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + for name in names { + // Names come from sqlite_master, not from user input, and are quoted. + sqlx::query(&format!("DROP TABLE IF EXISTS \"{name}\"")) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + } + /// Drop the per-DynamoDB-table data table. pub(crate) async fn drop_data_table( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, @@ -105,6 +215,9 @@ impl SqliteEngine { .execute(&mut **tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; + // Vector data tables are keyed by index, not by table, so they are not + // reached by dropping the item table or by the catalog cascade. + Self::drop_all_vector_data_tables(tx, table_id).await?; Ok(()) } @@ -229,6 +342,10 @@ impl SqliteEngine { let (global_secondary_indexes, local_secondary_indexes) = self.fetch_all_index_info(&table_id).await?; let has_lsi = !local_secondary_indexes.is_empty(); + // Vector indexes ride on the cached key info too, so the write path can + // decide whether any maintenance is needed without a query, and so the + // engine can validate vector attributes on writes. + let vector_indexes = self.fetch_vector_index_key_info(&table_id).await?; Ok(TableKeyInfo { table_name: table_name.to_owned(), @@ -241,13 +358,53 @@ impl SqliteEngine { global_secondary_indexes, local_secondary_indexes, stream_specification, - // Fields for features this backend does not implement, vector - // indexes today, take their defaults, so adding one to this type - // does not break this build. - ..Default::default() + // Every field is populated, with no `..Default::default()` spread: a + // new core field should break this site and force a decision about + // whether the write path needs it, rather than silently defaulting. + vector_indexes, }) } + /// Fetch the vector indexes of a table in the shape the engine caches. + /// + /// Note what this cannot carry: `VectorIndexKeyInfo` has no distance function + /// and no projection, so a search still reads the catalog for those. Widening + /// that type would remove the last per-search catalog read. + async fn fetch_vector_index_key_info( + &self, + table_id: &str, + ) -> Result, StorageError> { + let rows: Vec<(String, i64, String, Option)> = sqlx::query_as( + "SELECT index_name, dimensions, vector_attribute, search_schema \ + FROM vector_indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut out = Vec::with_capacity(rows.len()); + for (index_name, dimensions, vector_attribute, search_schema) in rows { + let attr: extenddb_core::types::VectorAttribute = + serde_json::from_str(&vector_attribute) + .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))?; + let search_schema = match search_schema.as_deref() { + Some(json) => serde_json::from_str(json) + .map_err(|e| StorageError::Internal(format!("search_schema: {e}")))?, + None => Vec::new(), + }; + out.push(extenddb_core::types::VectorIndexKeyInfo { + index_name, + dimensions: u32::try_from(dimensions).map_err(|_| { + StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) + })?, + vector_attribute_name: attr.attribute_name, + search_schema, + }); + } + Ok(out) + } + /// Fetch every secondary index defined on a table, split into /// `(global_secondary_indexes, local_secondary_indexes)`. async fn fetch_all_index_info( diff --git a/crates/storage-sqlite/src/data/delete_item.rs b/crates/storage-sqlite/src/data/delete_item.rs index a807045e..91c0d8ee 100644 --- a/crates/storage-sqlite/src/data/delete_item.rs +++ b/crates/storage-sqlite/src/data/delete_item.rs @@ -27,7 +27,7 @@ impl SqliteEngine { // runtime setting, not an invariant of this write, so it does not need // to be read under the lock, and the lock serialises every write in the // process: work done inside it is the backend's throughput bottleneck. - let system_delay = self.gsi_default_delay().await; + let system_delay = self.index_propagation_delay().await; let _writer = self.write_lock.lock().await; // Read the index set after acquiring the write lock so a concurrently // added GSI (UpdateTable holds the same lock) is not missed. @@ -70,6 +70,24 @@ impl SqliteEngine { ) .await?; } + // Vector rows for this base item are removed too. `new_item` is None, + // so this is a pure removal, applied in this transaction when the + // propagation delay is 0 and enqueued otherwise. + if !key_info.vector_indexes.is_empty() + && crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + None, + system_delay, + ) + .await? + > 0 + { + enqueued = true; + } if enqueue_async_indexes( &mut tx, key_info, diff --git a/crates/storage-sqlite/src/data/index.rs b/crates/storage-sqlite/src/data/index.rs index edd6705d..13ca76a1 100644 --- a/crates/storage-sqlite/src/data/index.rs +++ b/crates/storage-sqlite/src/data/index.rs @@ -77,6 +77,65 @@ pub(crate) struct GsiApplyContext { pub(crate) index: GsiIndexDef, } +/// What one `gsi_pending` row asks the worker to do: maintain a GSI, or maintain a +/// vector index. One queue serves both so that updates to a single base item stay +/// ordered across index kinds, which two queues drained independently could not +/// guarantee. +/// +/// `untagged` is load-bearing, not stylistic. A GSI context serializes to exactly +/// the bytes it did before this enum existed, so rows already on disk from an +/// earlier version still deserialize, and rows written now are still readable by +/// one. That matters more than it looks: an unparseable `index_context` is treated +/// as a poison row and *dropped*, so a tagged representation would silently discard +/// every in-flight GSI update across an upgrade. +/// +/// The variants are unambiguous by shape rather than by tag: a GSI context requires +/// `index` and a vector context requires `vector`, and no writer emits the other's +/// field, so for any context this code produces exactly one variant matches. +/// `pending_context_tests` pins both directions, including a verbatim legacy payload. +/// +/// To be exact rather than reassuring: a hand-corrupted payload carrying BOTH fields +/// would match `Gsi`, because untagged tries variants in declaration order and +/// ignores unknown fields. That is unreachable from any serializer here, and a +/// genuinely malformed context is already dropped as a poison row, so it is a +/// property of the representation worth knowing rather than a case to defend. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub(crate) enum PendingApplyContext { + Gsi(GsiApplyContext), + Vector(super::vector_index::VectorApplyContext), +} + +impl PendingApplyContext { + /// The base table's key schema, which both kinds carry and the queue needs in + /// order to place the row in its base key's partition. + fn base_key_schema(&self) -> &[KeySchemaElement] { + match self { + Self::Gsi(c) => &c.base_key_schema, + Self::Vector(c) => &c.base_key_schema, + } + } +} + +/// Apply one claimed row to whichever index kind it describes. +/// +/// Both arms tolerate a data table that no longer exists, so a base table or index +/// dropped while a row was in flight is applied-as-skip rather than logged and +/// dropped as unprocessable. +pub(crate) async fn apply_pending_context( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + old_item: Option<&Item>, + new_item: Option<&Item>, + context: &PendingApplyContext, +) -> Result<(), StorageError> { + match context { + PendingApplyContext::Gsi(c) => apply_claimed_row(tx, old_item, new_item, c).await, + PendingApplyContext::Vector(c) => { + super::vector_index::apply_vector_context(tx, old_item, new_item, c).await + } + } +} + /// Metadata for a single index, used on the write path and by the GSI worker. pub(crate) struct IndexMeta { pub(super) index_id: String, @@ -161,27 +220,36 @@ pub(crate) async fn enqueue_async_indexes( projection: idx.projection.clone(), }, }; - enqueue_gsi_pending(tx, &key_info.table_id, old_item, new_item, delay, &context).await?; + enqueue_pending_row( + tx, + &key_info.table_id, + old_item, + new_item, + delay, + &PendingApplyContext::Gsi(context), + ) + .await?; enqueued += 1; } Ok(enqueued) } -/// Insert one self-describing `gsi_pending` row for a single index inside the -/// base write transaction (zero crash window). +/// Insert one self-describing `gsi_pending` row inside the base write transaction +/// (zero crash window). Shared by the GSI and vector write paths, which is what puts +/// both index kinds in one totally ordered queue. /// -/// `delay_ms` is the index's effective delay; a jitter in `[delay/2, delay]` -/// is applied. `ready_at` is clamped to `max(now + jitter, MAX(ready_at) in the -/// base key's partition)` so a later write that draws a smaller jitter can never -/// become eligible before an earlier one — preserving per-key FIFO when the -/// worker drains the partition in `id` order. -pub(crate) async fn enqueue_gsi_pending( +/// `delay_ms` is the effective delay; a jitter in `[delay/2, delay]` is applied. +/// `ready_at` is clamped to `max(now + jitter, MAX(ready_at) in the base key's +/// partition)` so a later write that draws a smaller jitter can never become +/// eligible before an earlier one — preserving per-key FIFO when the worker drains +/// the partition in `id` order. +pub(crate) async fn enqueue_pending_row( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, table_id: &str, old_item: Option<&Item>, new_item: Option<&Item>, delay_ms: u64, - context: &GsiApplyContext, + context: &PendingApplyContext, ) -> Result<(), StorageError> { let old_json = old_item .map(serde_json::to_string) @@ -196,9 +264,11 @@ pub(crate) async fn enqueue_gsi_pending( // Route all updates for one base item to a single partition (per-key FIFO). // The base key is immutable: `new_item` carries it for puts/updates, - // `old_item` for deletes. + // `old_item` for deletes. Both context kinds describe the same base table, so + // a GSI row and a vector row for one item hash to the same partition and stay + // mutually ordered. let worker_partition = match new_item.or(old_item) { - Some(item) => partition_for(&composite_pk_to_text(item, &context.base_key_schema)?), + Some(item) => partition_for(&composite_pk_to_text(item, context.base_key_schema())?), None => 0, }; @@ -434,9 +504,9 @@ pub(crate) fn index_sk_column(index: usize, sk_type: ScalarAttributeType) -> Str } /// True if a storage error is a "missing table" error (the `_ddb_*` index table -/// was dropped, e.g. the base table was deleted while a `gsi_pending` row was -/// in flight). Such rows are benignly skipped. -fn is_no_such_table(e: &StorageError) -> bool { +/// or a `_vidx_*` vector table was dropped, e.g. the base table was deleted +/// while a `gsi_pending` row was in flight). Such rows are benignly skipped. +pub(crate) fn is_no_such_table(e: &StorageError) -> bool { matches!(e, StorageError::Internal(msg) if msg.contains("no such table")) } @@ -486,3 +556,256 @@ pub(crate) async fn apply_claimed_row( } Ok(()) } + +#[cfg(test)] +mod pending_context_tests { + use super::{GsiApplyContext, GsiIndexDef, PendingApplyContext}; + use crate::data::vector_index::{VectorApplyContext, VectorIndexMeta}; + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, Projection, ProjectionType, + ScalarAttributeType, + }; + + /// A context written by a build that predates vector rows, verbatim. It must + /// still deserialize. + /// + /// This is the test that protects an upgrade. A row whose `index_context` fails + /// to parse is treated as poison and DROPPED, so if the representation had + /// changed incompatibly, every GSI update in flight at the moment of the upgrade + /// would have been discarded silently, with the item written and its index never + /// catching up. + const LEGACY_GSI_CONTEXT: &str = r#"{ + "base_key_schema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "attribute_definitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "index": { + "index_id": "idx-1", + "key_schema": [{"AttributeName": "gsipk", "KeyType": "HASH"}], + "projection": {"ProjectionType": "ALL"} + } + }"#; + + fn base_ks() -> Vec { + vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }] + } + + fn base_ad() -> Vec { + vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }] + } + + fn gsi_context() -> GsiApplyContext { + GsiApplyContext { + base_key_schema: base_ks(), + attribute_definitions: base_ad(), + index: GsiIndexDef { + index_id: "idx-1".to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "gsipk".to_owned(), + key_type: KeyType::Hash, + }], + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + }, + } + } + + fn vector_context() -> VectorApplyContext { + VectorApplyContext { + base_key_schema: base_ks(), + attribute_definitions: base_ad(), + table_id: "t-1".to_owned(), + vector: VectorIndexMeta { + index_id: "vidx-1".to_owned(), + dimensions: 2, + vector_attribute_name: "emb".to_owned(), + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + hash_attribute_name: Some("tenant".to_owned()), + search_schema_attribute_names: vec!["tenant".to_owned()], + }, + } + } + + #[test] + fn a_legacy_gsi_context_still_deserializes_as_a_gsi_row() { + let parsed: PendingApplyContext = + serde_json::from_str(LEGACY_GSI_CONTEXT).expect("legacy context must still parse"); + match parsed { + PendingApplyContext::Gsi(c) => assert_eq!(c.index.index_id, "idx-1"), + PendingApplyContext::Vector(_) => { + panic!("a legacy GSI context must not be read as a vector row") + } + } + } + + /// The bytes a GSI row writes today are the bytes it wrote before the enum + /// existed, so a row written by this build is still readable by the previous + /// one. `untagged` is what buys this, and this test is what keeps it. + #[test] + fn wrapping_a_gsi_context_does_not_change_its_serialized_form() { + let bare = serde_json::to_string(&gsi_context()).expect("bare"); + let wrapped = + serde_json::to_string(&PendingApplyContext::Gsi(gsi_context())).expect("wrapped"); + assert_eq!( + bare, wrapped, + "the queue's on-disk format must not change for GSI rows" + ); + } + + /// A vector context round-trips and carries no GSI discriminant field. + /// + /// Note what this does NOT guard: it still passes if `untagged` is removed, so it + /// is not the protection for on-disk compatibility. The two tests above are, and + /// both fail without `untagged`. This one pins the shape contract that makes the + /// discrimination possible in the first place. + #[test] + fn a_vector_context_round_trips_and_carries_no_gsi_discriminant() { + let json = serde_json::to_string(&PendingApplyContext::Vector(vector_context())) + .expect("serialize"); + assert!( + !json.contains("\"index\":"), + "a vector context must not carry the GSI discriminant field: {json}" + ); + let parsed: PendingApplyContext = serde_json::from_str(&json).expect("deserialize"); + match parsed { + PendingApplyContext::Vector(c) => { + assert_eq!(c.vector.index_id, "vidx-1"); + assert_eq!(c.table_id, "t-1"); + assert_eq!(c.vector.dimensions, 2); + assert_eq!(c.vector.hash_attribute_name.as_deref(), Some("tenant")); + assert_eq!(c.vector.search_schema_attribute_names, ["tenant"]); + assert_eq!( + c.vector.projection.projection_type, + ProjectionType::KeysOnly + ); + } + PendingApplyContext::Gsi(_) => panic!("a vector context must not be read as a GSI row"), + } + } + + /// A GSI row and a vector row for the SAME base item must land in the same queue + /// partition, which is what makes them mutually ordered. + /// + /// This is the claim that reusing one queue buys ordering ACROSS index kinds, and + /// it holds only because both contexts hash the same base key rather than + /// anything index-specific. Asserted on the partition because the partition is the + /// mechanism: two kinds hashing differently would land in separate partitions, + /// each monotonic on its own, leaving the relative order of a GSI and a vector + /// update to one item unconstrained. + /// + /// Driven through the real `enqueue_pending_row` rather than a test shim, so it is + /// the production path being measured. + #[tokio::test] + async fn a_gsi_row_and_a_vector_row_for_one_item_share_a_partition() { + let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let the_item: extenddb_core::types::Item = + serde_json::from_str(r#"{"pk":{"S":"shared-key"},"emb":{"L":[{"N":"1"}]}}"#) + .expect("item"); + let mut tx = engine.pool.begin_with("BEGIN IMMEDIATE").await.expect("tx"); + for context in [ + PendingApplyContext::Gsi(gsi_context()), + PendingApplyContext::Vector(vector_context()), + ] { + super::enqueue_pending_row(&mut tx, "t-1", None, Some(&the_item), 60_000, &context) + .await + .expect("enqueue"); + } + tx.commit().await.expect("commit"); + + let partitions: Vec<(i64,)> = + sqlx::query_as("SELECT DISTINCT worker_partition FROM gsi_pending") + .fetch_all(&engine.pool) + .await + .expect("partitions"); + assert_eq!( + partitions.len(), + 1, + "a GSI row and a vector row for one base item must share a partition, or \ + their relative order is unconstrained" + ); + let (depth,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM gsi_pending") + .fetch_one(&engine.pool) + .await + .expect("depth"); + assert_eq!( + depth, 2, + "both kinds must have enqueued, so one partition is not an artefact of a \ + missing row" + ); + } + + /// A catalog created before the rename must keep honouring its operator's value. + /// + /// This is the compatibility property the whole fallback exists for, and the cost + /// of getting it wrong is not cosmetic: the server refuses to start on a + /// catalog-version mismatch rather than migrating, so nothing ever rewrites the old + /// row. Reading past it would silently reset a configured delay to the default, and + /// since 0 means synchronous, the silent change would be from strict to eventually + /// consistent, which is exactly the direction that turns a passing test suite into + /// a flaky one somewhere else. + #[tokio::test] + async fn a_pre_rename_catalog_still_honours_its_configured_delay() { + let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + // Reshape the catalog to look as it did before the rename: the legacy key + // only, carrying a deliberately non-default value. + sqlx::query("DELETE FROM settings WHERE key = 'index_propagation_delay_ms'") + .execute(&engine.pool) + .await + .expect("drop canonical row"); + sqlx::query("INSERT INTO settings (key, value) VALUES ('gsi_propagation_delay_ms', '0')") + .execute(&engine.pool) + .await + .expect("seed legacy row"); + + assert_eq!( + engine.index_propagation_delay().await, + 0, + "a value set under the pre-rename key must still be honoured, or an \ + operator's synchronous setting silently becomes asynchronous" + ); + } + + /// With both rows present the canonical one wins, deterministically. + /// + /// Reachable if an operator sets the legacy key on a build that predates the + /// canonicalising write path and then upgrades. Without the explicit ordering the + /// winner would be whichever row SQLite happened to return first. + #[tokio::test] + async fn the_canonical_key_wins_when_both_are_present() { + let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + sqlx::query( + "INSERT OR REPLACE INTO settings (key, value) \ + VALUES ('index_propagation_delay_ms', '7'), ('gsi_propagation_delay_ms', '999')", + ) + .execute(&engine.pool) + .await + .expect("seed both rows"); + + assert_eq!( + engine.index_propagation_delay().await, + 7, + "the canonical key must take precedence over the deprecated alias" + ); + } +} diff --git a/crates/storage-sqlite/src/data/mod.rs b/crates/storage-sqlite/src/data/mod.rs index 931ee50d..20d5db83 100644 --- a/crates/storage-sqlite/src/data/mod.rs +++ b/crates/storage-sqlite/src/data/mod.rs @@ -39,9 +39,10 @@ mod query_scan; mod transactions; mod tx_helpers; mod update_item; +pub(crate) mod vector_index; pub(crate) use index::{ - GsiApplyContext, apply_claimed_row, insert_index_row_multi, project_item_for_index, + PendingApplyContext, apply_pending_context, insert_index_row_multi, project_item_for_index, }; pub(crate) use tx_helpers::upsert_item_in_tx; @@ -55,6 +56,39 @@ pub(crate) fn index_table_name(index_id: &str) -> String { format!("\"_ddb_{index_id}\"") } +/// Quoted SQL identifier for a vector-index data table. +/// +/// The name carries the base `table_id` as well as the `index_id` so every vector +/// table belonging to a DynamoDB table is discoverable from the table alone. That +/// is what lets `drop_data_table` clean them up: by the time it runs, the catalog +/// rows have already been cascade-deleted inside the same transaction, so the +/// index ids can no longer be read from `vector_indexes`. +/// +/// One row per vector rather than a packed blob per partition. Measured +/// 2026-08-06: a packed blob reads 2 to 4x faster but makes every write +/// O(partition), since inserting one vector rewrites the whole blob (390 MB for +/// a 100k-vector partition at 1024 dimensions). Vector indexes are maintained on +/// every write to an indexed attribute, so that trade is not available. +pub(crate) fn vector_table_name(table_id: &str, index_id: &str) -> String { + format!("\"_vidx_{table_id}_{index_id}\"") +} + +/// `GLOB` pattern matching every vector data table of one DynamoDB table. +/// +/// `GLOB` rather than `LIKE` because the pattern itself is full of underscores, and +/// in `LIKE` an underscore is a single-character wildcard: `_vidx__%` would match +/// far more than it appears to. It can only ever over-match, since a wildcard also +/// matches a literal underscore, and no other table could share the UUID, so `LIKE` +/// was safe in practice. It was not safe for the stated reason, though, and a comment +/// that justifies the wrong half is worse than none. +/// +/// `GLOB` treats `_` literally and uses `*` for the wildcard, so the pattern means +/// what it reads. `table_id` is a server-generated UUID, which contains no `GLOB` +/// metacharacters either. +pub(crate) fn vector_table_glob_pattern(table_id: &str) -> String { + format!("_vidx_{table_id}_*") +} + /// All RANGE key attributes in key-schema order, paired with their scalar type. pub(crate) fn all_sort_key_info<'a>( key_schema: &'a [KeySchemaElement], diff --git a/crates/storage-sqlite/src/data/put_item.rs b/crates/storage-sqlite/src/data/put_item.rs index de43d830..4efbaeb8 100644 --- a/crates/storage-sqlite/src/data/put_item.rs +++ b/crates/storage-sqlite/src/data/put_item.rs @@ -37,7 +37,7 @@ impl SqliteEngine { // runtime setting, not an invariant of this write, so it does not need // to be read under the lock, and the lock serialises every write in the // process: work done inside it is the backend's throughput bottleneck. - let system_delay = self.gsi_default_delay().await; + let system_delay = self.index_propagation_delay().await; let _writer = self.write_lock.lock().await; let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; @@ -102,7 +102,7 @@ impl SqliteEngine { ) .await?; } - let enqueued = enqueue_async_indexes( + let enqueued_gsi = enqueue_async_indexes( &mut tx, key_info, &indexes, @@ -112,6 +112,27 @@ impl SqliteEngine { ) .await? > 0; + // Vector indexes: applied in this transaction when the propagation delay is + // 0, otherwise enqueued alongside the async GSI work. Gated on the cached + // key info so a table without them costs no extra query, and deliberately + // outside the `indexes` guard above: a table may have a vector index and no + // GSI or LSI at all. + let enqueued_vector = if key_info.vector_indexes.is_empty() { + false + } else { + crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + Some(&item), + system_delay, + ) + .await? + > 0 + }; + let enqueued = enqueued_gsi || enqueued_vector; if let Some(capture) = stream { write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), Some(&item)) diff --git a/crates/storage-sqlite/src/data/query.rs b/crates/storage-sqlite/src/data/query.rs index 216fb917..c1139e85 100644 --- a/crates/storage-sqlite/src/data/query.rs +++ b/crates/storage-sqlite/src/data/query.rs @@ -52,10 +52,23 @@ pub(super) fn resolve_expr_to_av( } } -/// SQL `WHERE` fragment for a sort-key condition on column `sk_col`. -pub(super) fn build_sk_sql(sk_cond: &SortKeyCondition, sk_col: &str) -> String { +/// SQL `WHERE` fragment for a sort-key condition on `sk_col`, together with the +/// values to bind for it. +/// +/// The fragment and its bind list are returned from one function on purpose. +/// `begins_with` on a binary key sometimes has no upper bound at all (see +/// [`binary_prefix_upper_bound`]), so the placeholder count is not fixed by the +/// condition kind alone. Building the SQL in one place and the binds in another +/// meant the two could disagree about how many placeholders exist, which is a +/// bind-offset corruption rather than a visible error. +pub(super) fn build_sk_sql_and_binds( + sk_cond: &SortKeyCondition, + sk_col: &str, + sk_type: ScalarAttributeType, + maps: &ExpressionMaps, +) -> Result<(String, Vec), StorageError> { match sk_cond { - SortKeyCondition::Compare { op, .. } => { + SortKeyCondition::Compare { op, value, .. } => { let sql_op = match op { CompareOp::Eq => "=", CompareOp::Ne => "<>", @@ -64,45 +77,38 @@ pub(super) fn build_sk_sql(sk_cond: &SortKeyCondition, sk_col: &str) -> String { CompareOp::Gt => ">", CompareOp::Ge => ">=", }; - format!(" AND {sk_col} {sql_op} ?") - } - SortKeyCondition::Between { .. } => format!(" AND {sk_col} BETWEEN ? AND ?"), - SortKeyCondition::BeginsWith { .. } => { - if sk_col.ends_with("_b") { - // Binary prefix: [prefix, incremented-prefix). - format!(" AND {sk_col} >= ? AND {sk_col} < ?") - } else { - // String prefix: [prefix, prefix || U+10FFFF). - format!(" AND {sk_col} >= ? AND {sk_col} < (? || char(1114111))") - } - } - } -} - -/// The sort-key values to bind for a key condition, in placeholder order. -pub(super) fn sk_condition_bind_values( - sk_cond: &SortKeyCondition, - sk_type: ScalarAttributeType, - maps: &ExpressionMaps, -) -> Result, StorageError> { - match sk_cond { - SortKeyCondition::Compare { value, .. } => { - Ok(vec![parse_sk(&resolve_expr_to_av(value, maps)?, sk_type)?]) + Ok(( + format!(" AND {sk_col} {sql_op} ?"), + vec![parse_sk(&resolve_expr_to_av(value, maps)?, sk_type)?], + )) } - SortKeyCondition::Between { low, high, .. } => Ok(vec![ - parse_sk(&resolve_expr_to_av(low, maps)?, sk_type)?, - parse_sk(&resolve_expr_to_av(high, maps)?, sk_type)?, - ]), + SortKeyCondition::Between { low, high, .. } => Ok(( + format!(" AND {sk_col} BETWEEN ? AND ?"), + vec![ + parse_sk(&resolve_expr_to_av(low, maps)?, sk_type)?, + parse_sk(&resolve_expr_to_av(high, maps)?, sk_type)?, + ], + )), SortKeyCondition::BeginsWith { prefix, .. } => { let sk = parse_sk(&resolve_expr_to_av(prefix, maps)?, sk_type)?; match sk { - SortKeyValue::B(b) => { - let upper = increment_bytes(&b); - Ok(vec![SortKeyValue::B(b), SortKeyValue::B(upper)]) - } + SortKeyValue::B(b) => match binary_prefix_upper_bound(&b) { + Some(upper) => Ok(( + format!(" AND {sk_col} >= ? AND {sk_col} < ?"), + vec![SortKeyValue::B(b), SortKeyValue::B(upper)], + )), + // Every byte string with this prefix is in range and there is + // no finite value above them all, so the only correct upper + // bound is none. Emitting one anyway is what silently dropped + // rows. + None => Ok((format!(" AND {sk_col} >= ?"), vec![SortKeyValue::B(b)])), + }, // For strings the SQL upper bound is `? || char(1114111)`, // so the same prefix is bound twice. - SortKeyValue::S(s) => Ok(vec![SortKeyValue::S(s.clone()), SortKeyValue::S(s)]), + SortKeyValue::S(s) => Ok(( + format!(" AND {sk_col} >= ? AND {sk_col} < (? || char(1114111))"), + vec![SortKeyValue::S(s.clone()), SortKeyValue::S(s)], + )), SortKeyValue::N(_) => Err(StorageError::Validation( "begins_with is not supported on numeric sort keys".to_owned(), )), @@ -111,21 +117,35 @@ pub(super) fn sk_condition_bind_values( } } -/// Exclusive upper bound for a binary prefix range: the smallest byte string -/// greater than every string having `prefix` as a prefix. Returns an empty -/// vector's successor convention when `prefix` is all `0xFF`. -fn increment_bytes(prefix: &[u8]) -> Vec { +/// Exclusive upper bound for a binary prefix range, or `None` when the range is +/// unbounded above. +/// +/// The bound is the smallest byte string greater than every string having +/// `prefix` as a prefix. Found by incrementing the last byte below `0xFF` and +/// discarding everything after it: `[1, 2]` yields `[1, 3]`, so `[1, 2, 9]` is +/// still included but `[1, 3]` is not. +/// +/// `None` when every byte is `0xFF`, which includes the empty prefix. No finite +/// bound exists in that case, because a longer all-`0xFF` string always sorts +/// after any candidate: with `[0xFF]` the strings `[0xFF, 0xFF]`, +/// `[0xFF, 0xFF, 0xFF]` and so on continue without end. The previous code +/// returned `vec![0xFF; prefix.len() + 1]` here, which is a value inside the +/// matching set rather than above it, so rows were silently dropped: for an +/// empty prefix it produced `[0xFF]`, which excluded every key sorting at or +/// after it, and `begins_with([])` then returned part of the partition while +/// reporting success. Returning `None` and omitting the predicate is the only +/// correct answer that does not depend on a maximum key length, which is +/// operator-configurable via `max_sort_key_size_bytes`. +fn binary_prefix_upper_bound(prefix: &[u8]) -> Option> { let mut out = prefix.to_vec(); while let Some(last) = out.last_mut() { if *last < 0xFF { *last += 1; - return out; + return Some(out); } out.pop(); } - // All 0xFF: no finite successor within the same length convention; use a - // value that sorts after any prefixed string of practical length. - vec![0xFF; prefix.len() + 1] + None } /// Build a `LastEvaluatedKey` from an item's key attributes. @@ -152,3 +172,61 @@ pub(super) async fn execute_dynamic_query( .map_err(|e| StorageError::Internal(e.to_string()))?; Ok(rows.into_iter().map(|(v,)| v).collect()) } + +#[cfg(test)] +mod tests { + use super::binary_prefix_upper_bound; + + /// Every byte string having `prefix` as a prefix must sort below the bound, + /// and the bound itself must not. Asserted against explicit witnesses rather + /// than trusting the arithmetic, since the defect this replaces was an + /// off-by-domain error that looked arithmetically reasonable. + #[test] + fn the_bound_excludes_itself_and_includes_every_extension() { + let upper = binary_prefix_upper_bound(&[1, 2]).expect("finite bound exists"); + assert_eq!(upper, vec![1, 3]); + for witness in [ + vec![1, 2], + vec![1, 2, 0], + vec![1, 2, 0xFF], + vec![1, 2, 9, 9], + ] { + assert!(witness < upper, "{witness:?} must be inside the range"); + } + assert!(vec![1, 3] >= upper, "the bound must be exclusive"); + } + + /// A trailing `0xFF` carries: the last byte below `0xFF` is incremented and + /// everything after it is discarded. + #[test] + fn a_trailing_all_ones_byte_carries_into_the_previous_byte() { + assert_eq!(binary_prefix_upper_bound(&[1, 0xFF]), Some(vec![2])); + assert_eq!(binary_prefix_upper_bound(&[1, 0xFF, 0xFF]), Some(vec![2])); + let upper = binary_prefix_upper_bound(&[1, 0xFF]).expect("finite bound exists"); + assert!(vec![1, 0xFF, 0xFF] < upper, "extension must be included"); + } + + /// The empty prefix matches everything, so no upper bound can exist. This is + /// the case that silently dropped rows: the old code returned `[0xFF]`, which + /// excluded every key sorting at or after it, so `begins_with([])` returned + /// part of the partition and reported success. + #[test] + fn an_empty_prefix_has_no_upper_bound() { + assert_eq!(binary_prefix_upper_bound(&[]), None); + } + + /// Same defect one length up, and not only for the empty prefix: an all-`0xFF` + /// prefix of any length has no finite bound, because a longer all-`0xFF` + /// string always sorts after any candidate. The old code returned + /// `[0xFF, 0xFF]` for `[0xFF]`, which wrongly excluded `[0xFF, 0xFF, 0xFF]`. + #[test] + fn an_all_ones_prefix_has_no_upper_bound_at_any_length() { + for prefix in [vec![0xFF], vec![0xFF, 0xFF], vec![0xFF; 8]] { + assert_eq!( + binary_prefix_upper_bound(&prefix), + None, + "no finite bound exists above {prefix:?}" + ); + } + } +} diff --git a/crates/storage-sqlite/src/data/query_scan.rs b/crates/storage-sqlite/src/data/query_scan.rs index b7c03f50..c5f12d93 100644 --- a/crates/storage-sqlite/src/data/query_scan.rs +++ b/crates/storage-sqlite/src/data/query_scan.rs @@ -21,9 +21,7 @@ use extenddb_storage::util::{ encode_netstring_composite, parse_sk, pk_to_text, sk_column, sk_column_n, sk_info, }; -use super::query::{ - build_key, build_sk_sql, execute_dynamic_query, resolve_expr_to_av, sk_condition_bind_values, -}; +use super::query::{build_key, build_sk_sql_and_binds, execute_dynamic_query, resolve_expr_to_av}; use super::{ BoundValue, all_sort_key_info, data_table_name, index_table_name, json_to_item, sk_bound, }; @@ -79,9 +77,11 @@ impl SqliteEngine { // Primary sort-key condition. if let (Some(sk_cond), Some((_, sk_type))) = (&key_condition.sk_condition, sk_info_val) { - sql.push_str(&build_sk_sql(sk_cond, sk_column(sk_type))); - for v in sk_condition_bind_values(sk_cond, sk_type, maps)? { - binds.push(sk_bound(&v)); + let (sk_sql, sk_binds) = + build_sk_sql_and_binds(sk_cond, sk_column(sk_type), sk_type, maps)?; + sql.push_str(&sk_sql); + for v in &sk_binds { + binds.push(sk_bound(v)); } } diff --git a/crates/storage-sqlite/src/data/transactions.rs b/crates/storage-sqlite/src/data/transactions.rs index f3ae8dec..ff8adfe9 100644 --- a/crates/storage-sqlite/src/data/transactions.rs +++ b/crates/storage-sqlite/src/data/transactions.rs @@ -72,7 +72,7 @@ impl SqliteEngine { // runtime setting, not an invariant of this write, so it does not need // to be read under the lock, and the lock serialises every write in the // process: work done inside it is the backend's throughput bottleneck. - let system_delay = self.gsi_default_delay().await; + let system_delay = self.index_propagation_delay().await; let _writer = self.write_lock.lock().await; // Fetch index metadata per distinct table AFTER acquiring the write lock, // so a GSI added by a concurrent UpdateTable (same lock) is not missed and @@ -159,7 +159,17 @@ impl SqliteEngine { } } - // Persist async GSI work for each op inside the same transaction. + // Persist index work for each op inside the same transaction: async GSI + // rows, and vector maintenance which is applied here when the propagation + // delay is 0 and enqueued otherwise. + // + // Deliberately after every op is staged rather than inside each op, so + // there is one call site instead of three. Both paths stay atomic with the + // item writes either way: the synchronous apply runs in this transaction, + // and a pending row is inserted into it, so a cancelled transaction leaves + // neither behind. The vector apply reads only the op's own before/after + // items, never the base table, so its position relative to the other ops' + // staged writes cannot change its result. let mut needs_notify = false; for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { let indexes = &table_indexes[op_table_name(op)]; @@ -176,6 +186,22 @@ impl SqliteEngine { if n > 0 { needs_notify = true; } + let key_info = op_key_info(op); + if !key_info.vector_indexes.is_empty() { + let n = crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old_item.as_ref(), + new_item.as_ref(), + system_delay, + ) + .await?; + if n > 0 { + needs_notify = true; + } + } } } diff --git a/crates/storage-sqlite/src/data/update_item.rs b/crates/storage-sqlite/src/data/update_item.rs index 92ac84a5..147c3032 100644 --- a/crates/storage-sqlite/src/data/update_item.rs +++ b/crates/storage-sqlite/src/data/update_item.rs @@ -37,7 +37,7 @@ impl SqliteEngine { // runtime setting, not an invariant of this write, so it does not need // to be read under the lock, and the lock serialises every write in the // process: work done inside it is the backend's throughput bottleneck. - let system_delay = self.gsi_default_delay().await; + let system_delay = self.index_propagation_delay().await; let _writer = self.write_lock.lock().await; // Read the index set after acquiring the write lock so a concurrently // added GSI (UpdateTable holds the same lock) is not missed. @@ -98,7 +98,7 @@ impl SqliteEngine { ) .await?; } - let enqueued = enqueue_async_indexes( + let enqueued_gsi = enqueue_async_indexes( &mut tx, key_info, &indexes, @@ -108,6 +108,26 @@ impl SqliteEngine { ) .await? > 0; + // Vector indexes: applied in this transaction when the propagation delay is + // 0, otherwise enqueued alongside the async GSI work. Gated on the cached + // key info so a table without them costs no extra query, and kept outside + // the `indexes` guard because a table may have a vector index and no GSI. + let enqueued_vector = if key_info.vector_indexes.is_empty() { + false + } else { + crate::data::vector_index::maintain_vector_indexes( + &mut tx, + &key_info.table_id, + &key_info.key_schema, + &key_info.attribute_definitions, + old.as_ref(), + Some(&item), + system_delay, + ) + .await? + > 0 + }; + let enqueued = enqueued_gsi || enqueued_vector; if let Some(capture) = stream { write_stream_record_in_tx(&mut tx, key_info, capture, old.as_ref(), Some(&item)) diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs new file mode 100644 index 00000000..eee5d432 --- /dev/null +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -0,0 +1,771 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Vector index maintenance on the write path. +//! +//! Vector indexes are **eventually consistent**, the same model the service gives +//! them and the same model a GSI has here, so maintenance runs on the existing +//! `gsi_pending` queue rather than in the base write transaction. [`maintain_vector_indexes`] +//! is the single entry point and owns that choice: a propagation delay of 0 keeps +//! the work in the caller's transaction, any other delay enqueues it. +//! +//! Reusing the GSI queue is what makes the asynchronous path correct rather than +//! merely deferred, and none of these properties would come free from a queue of +//! its own: +//! +//! * **Crash safety.** The pending row is inserted in the base write transaction, +//! so there is no window in which the item is committed and the index work is +//! not yet durable. The worker claims and applies in one transaction, so a crash +//! mid-apply rolls back and the row is retried. At-least-once delivery is safe +//! because applying a row is idempotent: it deletes the base key's row and +//! reinserts it from the snapshotted item. +//! * **Per-key ordering, across index kinds.** The row's partition is a hash of the +//! *base* key, so a vector row and a GSI row for the same item share a partition, +//! `ready_at` is clamped monotonic within it, and the worker drains in `id` order. +//! Two writes to one item therefore reach both index kinds in write order even +//! though the delay is jittered. +//! * **Snapshot semantics.** The row carries its own [`VectorApplyContext`], so the +//! worker needs no catalog read and an index dropped, or redefined, between +//! enqueue and apply cannot make a queued write unapplicable or retroactively +//! change how it was indexed. +//! +//! A write to a table whose items do not carry the vector still enqueues, because +//! the removal is the point: an item that loses its vector attribute must leave the +//! index, and skipping the enqueue would leave the stale row in place forever. + +use serde::{Deserialize, Serialize}; + +use extenddb_core::types::{AttributeDefinition, Item, KeySchemaElement, SearchSchemaElementType}; +use extenddb_core::validation::{vector_components, vector_norm}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::pk_to_text; + +use super::{BoundValue, all_sort_key_info, sk_bound, vector_table_name}; +use crate::vector_search::partition_value; + +/// A vector index as the write path needs it. +/// +/// Serializable because the asynchronous path snapshots it verbatim into the +/// pending row's [`VectorApplyContext`]. The write path and the worker therefore +/// apply from the *same* description of the index, which is the property that stops +/// a queued write from being reinterpreted under a later definition. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct VectorIndexMeta { + pub index_id: String, + pub dimensions: usize, + pub vector_attribute_name: String, + /// The index's projection, applied to the stored row exactly as the GSI path + /// applies its own. Not applying it was an unexplained divergence from the + /// sibling, and it made a search return attributes the index does not project. + pub projection: extenddb_core::types::Projection, + /// The single HASH element's attribute name, when the index declares one. + /// `None` means the index is unscoped and every row shares one partition. + pub hash_attribute_name: Option, + /// Every attribute named by the SearchSchema, HASH and INLINE_FILTER alike. + /// + /// These are projected regardless of `ProjectionType`, which is the documented + /// rule for a vector index and is NOT GSI `KEYS_ONLY` semantics: `KEYS_ONLY` + /// on a vector index projects the base primary key, the vector attribute and + /// the inline filter attributes. Withholding them is not merely a reporting + /// difference, it breaks search: the filter is evaluated against the stored + /// payload, so a missing filter attribute makes every row fail the predicate + /// and a filtered search match nothing. + pub search_schema_attribute_names: Vec, +} + +/// Load the vector indexes of a table. +/// +/// Read inside the write transaction rather than taken from the cached +/// `TableKeyInfo`, because the cache carries the search schema but not the index +/// id, and the id is what names the data table. +pub(crate) async fn fetch_vector_indexes_for_table( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, +) -> Result, StorageError> { + let rows: Vec<(String, i64, String, Option, String)> = sqlx::query_as( + "SELECT index_id, dimensions, vector_attribute, search_schema, projection \ + FROM vector_indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut out = Vec::with_capacity(rows.len()); + for (index_id, dimensions, vector_attribute, search_schema, projection) in rows { + let attr: extenddb_core::types::VectorAttribute = + serde_json::from_str(&vector_attribute) + .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))?; + let (hash_attribute_name, search_schema_attribute_names) = match search_schema.as_deref() { + Some(json) => { + let elements: Vec = + serde_json::from_str(json) + .map_err(|e| StorageError::Internal(format!("search_schema: {e}")))?; + let hash = elements + .iter() + .find(|e| e.element_type == SearchSchemaElementType::Hash) + .map(|e| e.attribute_name.clone()); + let all = elements + .into_iter() + .map(|e| e.attribute_name) + .collect::>(); + (hash, all) + } + None => (None, Vec::new()), + }; + let projection: extenddb_core::types::Projection = serde_json::from_str(&projection) + .map_err(|e| StorageError::Internal(format!("vector projection: {e}")))?; + out.push(VectorIndexMeta { + index_id, + dimensions: usize::try_from(dimensions).map_err(|_| { + StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) + })?, + vector_attribute_name: attr.attribute_name, + hash_attribute_name, + search_schema_attribute_names, + projection, + }); + } + Ok(out) +} + +/// Whether an item belongs in a vector index. +/// +/// It must carry the vector attribute, and the HASH attribute when the index +/// declares one: without the latter the row could not be placed in a partition, +/// and putting it in the unscoped partition would make it visible to searches of +/// every other partition. Not an error, exactly as a GSI silently omits an item +/// missing its index key. +fn item_is_indexable(item: &Item, meta: &VectorIndexMeta) -> bool { + if !item.contains_key(&meta.vector_attribute_name) { + return false; + } + match &meta.hash_attribute_name { + Some(name) => item.contains_key(name), + None => true, + } +} + +/// The partition column value for an item under one index. +fn item_partition(item: &Item, meta: &VectorIndexMeta) -> Result { + match &meta.hash_attribute_name { + Some(name) => { + let value = item.get(name).ok_or_else(|| { + StorageError::Internal( + "indexable check passed but the hash attribute is absent".to_owned(), + ) + })?; + partition_value(Some((name.as_str(), value))) + } + None => partition_value(None), + } +} + +/// Base-key bind values for a row, in key-schema order. +fn base_key_binds( + item: &Item, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], +) -> Result, StorageError> { + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + let pk_attr = &base_key_schema[0].attribute_name; + let pk = item.get(pk_attr).ok_or_else(|| { + StorageError::Internal("item written without its partition key".to_owned()) + })?; + let mut binds = vec![BoundValue::Text(pk_to_text(pk)?.into_owned())]; + for &(name, sk_type) in &base_sks { + // Sort keys use the same storage representation as the GSI/LSI tables: + // order-preserving text for numbers and a BLOB for binary. Encoding them + // any other way would still be self-consistent here but would diverge + // from every other index table for the same item. + match item.get(name) { + Some(value) => binds.push(sk_bound(&extenddb_storage::util::parse_sk(value, sk_type)?)), + None => binds.push(BoundValue::Text(String::new())), + } + } + Ok(binds) +} + +/// Column names for the base key, in key-schema order. +fn base_key_columns( + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], +) -> Vec { + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + let mut cols = vec!["base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + cols.push(format!( + "base_{}", + extenddb_storage::util::sk_column_n(i, sk_type) + )); + } + cols +} + +/// Everything the propagation worker needs to apply one vector index update, +/// serialized into `gsi_pending.index_context`. +/// +/// `table_id` is carried here even though the queue row has a `table_id` column of +/// its own, because a vector data table is named from the table id *and* the index +/// id. Reading it from the context preserves the invariant that the context alone +/// is sufficient, rather than splitting one apply's inputs across a column and a +/// JSON blob. Both are written from the same variable in the same statement, so +/// they cannot disagree. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct VectorApplyContext { + pub(crate) base_key_schema: Vec, + pub(crate) attribute_definitions: Vec, + pub(crate) table_id: String, + /// Deliberately named `vector` rather than `index`: it is the field whose + /// presence lets the untagged `PendingApplyContext` tell a vector row from a + /// GSI row by shape alone. See that type for why the discriminant is a shape + /// and not a tag. + pub(crate) vector: VectorIndexMeta, +} + +/// Maintain every vector index on a table for one item write. +/// +/// The single entry point for the write path, and the one place that decides +/// between synchronous and asynchronous. `delay_ms` of 0 applies in the caller's +/// transaction; anything else enqueues one `gsi_pending` row per index. Returns the +/// number of rows enqueued, so the caller knows whether to wake the worker, and +/// returns 0 for the synchronous path because there is nothing to wake. +/// +/// Keeping the branch here rather than at each call site matters: there are seven +/// write paths, and a single one that enqueued while also applying inline would +/// double-apply, while one that did neither would silently stop indexing. +/// +/// `old_item` and `new_item` follow the same convention as `sync_indexes`: a put +/// supplies both when replacing, a delete supplies only the old. +pub(crate) async fn maintain_vector_indexes( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + old_item: Option<&Item>, + new_item: Option<&Item>, + delay_ms: u64, +) -> Result { + // Read inside the transaction rather than from the cached `TableKeyInfo`: the + // cache carries the search schema but not the index id, and the id is what + // names the data table. + let metas = fetch_vector_indexes_for_table(tx, table_id).await?; + if metas.is_empty() { + return Ok(0); + } + let key_cols = base_key_columns(base_key_schema, attr_defs); + + if delay_ms == 0 { + for meta in &metas { + apply_vector_index( + tx, + table_id, + meta, + base_key_schema, + attr_defs, + &key_cols, + old_item, + new_item, + ) + .await?; + } + return Ok(0); + } + + let mut enqueued = 0usize; + for meta in metas { + // Enqueued even when the new item carries no vector: the removal is the + // work in that case, and skipping it would leave a stale row indexed. + let context = super::index::PendingApplyContext::Vector(VectorApplyContext { + base_key_schema: base_key_schema.to_vec(), + attribute_definitions: attr_defs.to_vec(), + table_id: table_id.to_owned(), + vector: meta, + }); + super::index::enqueue_pending_row(tx, table_id, old_item, new_item, delay_ms, &context) + .await?; + enqueued += 1; + } + Ok(enqueued) +} + +/// Apply one claimed vector pending row, from its self-describing context. +/// +/// A missing data table is skipped rather than treated as a failure: the base table +/// or the index itself can be dropped while a row is in flight, which is a routine +/// race and not a defect. +/// +/// This is log hygiene rather than data safety, and worth being exact about. The +/// batch already guards every row with a savepoint, so without this the row would be +/// rolled back and dropped, reaching the same end state by a noisier route. What the +/// tolerance changes is that an expected race stops emitting an ERROR line, which +/// otherwise trains operators to ignore the one signal that says a row was thrown +/// away. Matches the GSI sibling, so both arms of the dispatcher behave alike. +pub(crate) async fn apply_vector_context( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + old_item: Option<&Item>, + new_item: Option<&Item>, + context: &VectorApplyContext, +) -> Result<(), StorageError> { + let key_cols = base_key_columns(&context.base_key_schema, &context.attribute_definitions); + apply_vector_index( + tx, + &context.table_id, + &context.vector, + &context.base_key_schema, + &context.attribute_definitions, + &key_cols, + old_item, + new_item, + ) + .await + .or_else(|e| { + if super::index::is_no_such_table(&e) { + Ok(()) + } else { + Err(e) + } + }) +} + +/// Apply an item write to a single vector index. +/// +/// The delete-then-insert shape matters. An item can move between partitions when +/// its HASH attribute changes, and the row is keyed by the base item rather than by +/// the partition, so an insert alone would leave the old partition's row in place +/// and the item would be findable in two partitions at once. +/// +/// The delete keys off `old_item.or(new_item)` because the base key is immutable, so +/// either carries it. That is what lets a put whose caller had no reason to read the +/// old item still displace the row it replaces. +#[allow(clippy::too_many_arguments)] +async fn apply_vector_index( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + meta: &VectorIndexMeta, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + key_cols: &[String], + old_item: Option<&Item>, + new_item: Option<&Item>, +) -> Result<(), StorageError> { + let vec_table = vector_table_name(table_id, &meta.index_id); + let where_clause = key_cols + .iter() + .map(|c| format!("{c} = ?")) + .collect::>() + .join(" AND "); + + // Remove any existing row for this base item first, whatever partition it + // was in. + if let Some(source) = old_item.or(new_item) { + let binds = base_key_binds(source, base_key_schema, attr_defs)?; + let sql = format!("DELETE FROM {vec_table} WHERE {where_clause}"); + let mut q = sqlx::query(&sql); + for b in binds { + q = super::bind_bound!(q, b); + } + q.execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + let Some(new_item) = new_item else { + return Ok(()); // A delete: removal above is the whole of the work. + }; + insert_vector_row( + tx, + table_id, + meta, + new_item, + base_key_schema, + attr_defs, + key_cols, + ) + .await +} + +/// Write one item's row into one vector index. +/// +/// Shared by the write path and by backfill deliberately. These are the only two +/// producers of a vector row, and a second copy of this logic would be free to +/// drift: a backfilled row shaped differently from a live-written one would search +/// correctly right up until the difference mattered, with nothing to catch it. +/// +/// A non-indexable item is a no-op rather than an error, which is what makes a +/// backfill over a table where only some items carry the vector work. +pub(crate) async fn insert_vector_row( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + meta: &VectorIndexMeta, + item: &Item, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + key_cols: &[String], +) -> Result<(), StorageError> { + if !item_is_indexable(item, meta) { + return Ok(()); + } + let vec_table = vector_table_name(table_id, &meta.index_id); + + let value = item.get(&meta.vector_attribute_name).ok_or_else(|| { + StorageError::Internal("indexable check passed but the vector is absent".to_owned()) + })?; + let components = vector_components(value).ok_or_else(|| { + // Core validates the write before it reaches storage, so a malformed + // vector here means validation was bypassed rather than that a caller + // sent bad input. + StorageError::Internal( + "vector attribute reached storage without passing validation".to_owned(), + ) + })?; + if components.len() != meta.dimensions { + return Err(StorageError::Internal(format!( + "vector has {} components, index declares {}", + components.len(), + meta.dimensions + ))); + } + + let mut blob = Vec::with_capacity(components.len() * 4); + for x in &components { + blob.extend_from_slice(&x.to_le_bytes()); + } + let norm = vector_norm(&components); + let part = item_partition(item, meta)?; + // Projected exactly as the GSI sibling projects, so a search returns what + // the index declares and no more. + let mut projected = + super::index::project_item_for_index(item, &[], base_key_schema, &meta.projection); + // The SearchSchema attributes are always projected, whatever the + // ProjectionType. See `search_schema_attribute_names` for why: the inline + // filter is evaluated against this payload, so dropping the attribute would + // silently turn every filtered search into a zero-result search. + for name in &meta.search_schema_attribute_names { + if !projected.contains_key(name) + && let Some(v) = item.get(name) + { + projected.insert(name.clone(), v.clone()); + } + } + // The vector itself is not kept in the payload: it is already in the `vec` + // column as `f32`, which is the width the service validates against, and the + // search path rebuilds the attribute from those bits. Keeping a verbatim + // decimal copy here duplicated 10 to 15 KB per row at 1024 dimensions and + // would have returned the client's original precision where the service + // returns the narrowed value. + projected.remove(&meta.vector_attribute_name); + let item_json = serde_json::to_string(&projected) + .map_err(|e| StorageError::Internal(format!("serialize item: {e}")))?; + + // A plain INSERT, deliberately, where the GSI sibling uses INSERT OR REPLACE. + // Every caller reaches this through `apply_vector_index`, which unconditionally + // deletes the base key's row first, so no live row can exist here and a conflict + // is impossible. Keeping it a plain INSERT means that if a future refactor ever + // makes that delete conditional, this fails loudly with a primary key violation + // rather than silently replacing a row and hiding the broken invariant. + let cols = std::iter::once("part".to_owned()) + .chain(key_cols.iter().cloned()) + .chain(["vec".to_owned(), "nrm".to_owned(), "item_data".to_owned()]) + .collect::>(); + let placeholders = vec!["?"; cols.len()].join(", "); + let sql = format!( + "INSERT INTO {vec_table} ({}) VALUES ({placeholders})", + cols.join(", ") + ); + let key_binds = base_key_binds(item, base_key_schema, attr_defs)?; + let mut q = sqlx::query(&sql).bind(part); + for b in key_binds { + q = super::bind_bound!(q, b); + } + q.bind(blob) + .bind(f64::from(norm)) + .bind(item_json) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) +} + +/// Populate a newly created vector index from the base table. +/// +/// Batched by offset exactly as `backfill_gsi` is, and for the same reason: a table +/// large enough to be worth indexing is too large to hold in memory. Returns the +/// number of rows written, which is what distinguishes "backfilled nothing because +/// no item carries the vector" from "backfilled nothing because the scan is broken". +/// Everything a backfill needs that does not change between batches. +/// +/// Bundled because the alternative was an eight-argument function threaded through two +/// drivers, where the only per-batch values are the page size and the cursor. +struct BackfillPlan<'a> { + table_id: &'a str, + meta: &'a VectorIndexMeta, + base_key_schema: &'a [KeySchemaElement], + attr_defs: &'a [AttributeDefinition], + key_cols: Vec, +} + +impl<'a> BackfillPlan<'a> { + fn new( + table_id: &'a str, + meta: &'a VectorIndexMeta, + base_key_schema: &'a [KeySchemaElement], + attr_defs: &'a [AttributeDefinition], + ) -> Self { + Self { + table_id, + meta, + base_key_schema, + attr_defs, + key_cols: base_key_columns(base_key_schema, attr_defs), + } + } +} + +/// Backfill one batch of existing rows into the vector index. +/// +/// Returns `(written, fetched, last_rowid)`. `fetched` distinguishes a short read +/// (the end) from a full one, and `last_rowid` is the cursor to resume from. +/// +/// Pagination is by KEY, not by `OFFSET`. Offset anchors on a position, so removing any +/// already-scanned row shifts every later position by one and the next batch skips a +/// row entirely. That row is then missing from the index permanently, and no queue +/// entry can repair it, because the skipped row was never written to: only the removed +/// one was. Reproduced before this change: one removal during a backfill left the row +/// at the batch boundary absent from the index. +/// +/// The cursor is `rowid`, not `pk`, and the difference is a correctness matter rather +/// than a preference. Composite-key base tables have `PRIMARY KEY (pk, sk*)`, so `pk` +/// alone is not unique; a `pk > last_pk` cursor whose batch boundary fell inside one +/// partition's sort-key group excluded every remaining row sharing that `pk`, +/// permanently, on both the UpdateTable path and startup reconciliation (which share +/// this function). Reproduced: a five-row partition scanned with a batch of three +/// indexed four rows and skipped two. `rowid` is unique regardless of the key layout, +/// so one query shape serves both. It is a valid cursor because the base tables are +/// ordinary rowid tables and every write is `INSERT ... ON CONFLICT DO UPDATE` +/// (`tx_helpers.rs`), which updates in place and never reassigns a rowid; rows +/// inserted after a batch passed their rowid position are the concurrent writes the +/// queue hold already captures and replays after ACTIVE. +/// +/// It was unreachable while the whole backfill ran in one transaction, and became +/// reachable the moment batches started committing independently. +async fn backfill_vector_batch( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + plan: &BackfillPlan<'_>, + limit: i64, + after_rowid: i64, +) -> Result<(usize, i64, i64), StorageError> { + let base_table = super::data_table_name(plan.table_id); + // `rowid > ?` with 0 as the initial cursor: every real rowid is positive, so + // the first batch needs no separate query shape. + let sql = + format!("SELECT rowid, item_data FROM {base_table} WHERE rowid > ? ORDER BY rowid LIMIT ?"); + let rows: Vec<(i64, String)> = sqlx::query_as(&sql) + .bind(after_rowid) + .bind(limit) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let fetched = i64::try_from(rows.len()).unwrap_or(limit); + let last_rowid = rows.last().map_or(after_rowid, |(rid, _)| *rid); + let mut written = 0usize; + for (_, item_json) in rows { + let item: Item = serde_json::from_str(&item_json) + .map_err(|e| StorageError::Internal(format!("stored item: {e}")))?; + if !item_is_indexable(&item, plan.meta) { + continue; + } + insert_vector_row( + tx, + plan.table_id, + plan.meta, + &item, + plan.base_key_schema, + plan.attr_defs, + &plan.key_cols, + ) + .await?; + written += 1; + } + Ok((written, fetched, last_rowid)) +} + +/// Backfill the index in independently committed batches, releasing SQLite's write +/// lock between them. +/// +/// This is what lets the base table stay writable while an index builds, which is how +/// the service behaves: the table remains ACTIVE and accepts writes throughout, and +/// only the index reports CREATING. Holding one transaction for the whole backfill +/// would block every write until it finished. +/// +/// Releasing the lock is also what creates the ordering hazard this design has to +/// answer. A write landing mid-backfill is enqueued, and if it were applied before the +/// backfill wrote its older snapshot of the same item, the index would converge on the +/// stale generation. The queue worker therefore refuses to claim any row for a table +/// whose vector index is still CREATING, so those writes accumulate and are applied +/// only after this returns and the index flips to ACTIVE. +/// +/// A crash part-way leaves the index in CREATING with some rows written, which +/// `reconcile_incomplete_vector_indexes` repairs at startup by rebuilding it. +pub(crate) async fn backfill_vector_index_in_batches( + pool: &sqlx::SqlitePool, + write_lock: &tokio::sync::Mutex<()>, + table_id: &str, + meta: &VectorIndexMeta, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + batch_delay: std::time::Duration, +) -> Result { + const BATCH: i64 = 500; + let plan = BackfillPlan::new(table_id, meta, base_key_schema, attr_defs); + let mut cursor: i64 = 0; + let mut written = 0usize; + loop { + let (batch_written, fetched, last_rowid) = { + let _writer = write_lock.lock().await; + let mut tx = pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let result = backfill_vector_batch(&mut tx, &plan, BATCH, cursor).await?; + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + result + }; + written += batch_written; + if fetched < BATCH { + break; + } + cursor = last_rowid; + // Outside the lock, so a write can actually proceed during the pause. Zero in + // production; a test sets it so a write is guaranteed to land mid-backfill. + if !batch_delay.is_zero() { + tokio::time::sleep(batch_delay).await; + } + } + Ok(written) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A composite-key table whose partition spans a batch boundary must still + /// backfill every row. + /// + /// This is the regression test for a silent data-loss defect: the cursor was + /// `pk > last_pk`, and on a table with `PRIMARY KEY (pk, sk_s)` a batch ending + /// inside one partition's sort-key group excluded every remaining row sharing + /// that `pk`. Five rows in one partition scanned with a batch of three indexed + /// four and skipped two, permanently, on both the UpdateTable path and startup + /// reconciliation, which share `backfill_vector_batch`. The rowid cursor cannot + /// lose rows because rowid is unique whatever the key layout. + /// + /// Driven through `backfill_vector_batch` directly with a batch of 3 rather + /// than through the drivers, because they hardcode a 500-row batch and seeding + /// 501 rows would test the same lines slower. + #[tokio::test] + async fn a_composite_key_partition_straddling_a_batch_boundary_is_fully_backfilled() { + use extenddb_core::types::{ + AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType, + }; + let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let ks = vec![ + KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }, + ]; + let ad = vec![ + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + ]; + + let table_id = "t-composite"; + let mut tx = engine.pool.begin_with("BEGIN IMMEDIATE").await.expect("tx"); + crate::SqliteEngine::create_data_table(&mut tx, table_id, &ks, &ad) + .await + .expect("base table"); + crate::SqliteEngine::create_vector_data_table(&mut tx, table_id, "vidx-1", &ks, &ad) + .await + .expect("vector table"); + + // One partition with five sort keys plus a second partition, so the batch + // of three ends INSIDE partition "a": exactly the boundary that lost rows. + let base_table = super::super::data_table_name(table_id); + for (pk, sk) in [ + ("a", "1"), + ("a", "2"), + ("a", "3"), + ("a", "4"), + ("a", "5"), + ("b", "1"), + ] { + sqlx::query(&format!( + "INSERT INTO {base_table} (pk, sk_s, item_data) VALUES (?, ?, ?)" + )) + .bind(pk) + .bind(sk) + .bind(format!( + r#"{{"pk":{{"S":"{pk}"}},"sk":{{"S":"{sk}"}},"emb":{{"L":[{{"N":"1"}},{{"N":"0"}}]}}}}"# + )) + .execute(&mut *tx) + .await + .expect("seed"); + } + + let meta = VectorIndexMeta { + index_id: "vidx-1".to_owned(), + dimensions: 2, + vector_attribute_name: "emb".to_owned(), + projection: extenddb_core::types::Projection { + projection_type: extenddb_core::types::ProjectionType::All, + non_key_attributes: None, + }, + hash_attribute_name: None, + search_schema_attribute_names: Vec::new(), + }; + let plan = BackfillPlan::new(table_id, &meta, &ks, &ad); + + let mut cursor: i64 = 0; + let mut written = 0usize; + loop { + let (batch_written, fetched, last_rowid) = + backfill_vector_batch(&mut tx, &plan, 3, cursor) + .await + .expect("batch"); + written += batch_written; + if fetched < 3 { + break; + } + cursor = last_rowid; + } + tx.commit().await.expect("commit"); + + assert_eq!( + written, 6, + "every row must be indexed; the pk-only cursor wrote 4 and skipped 2" + ); + let vec_table = super::super::vector_table_name(table_id, "vidx-1"); + let (rows,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(rows, 6, "the index must hold all six rows"); + } +} diff --git a/crates/storage-sqlite/src/hooks.rs b/crates/storage-sqlite/src/hooks.rs index a67d2f78..0b8a634c 100644 --- a/crates/storage-sqlite/src/hooks.rs +++ b/crates/storage-sqlite/src/hooks.rs @@ -66,11 +66,11 @@ impl ServerRuntimeHooks for SqliteRuntimeHooks { }); // Keep the cached GSI propagation delay in sync with the setting. - let gsi_default = self.engine.gsi_default_delay_ms.clone(); + let index_delay_cache = self.engine.index_propagation_delay_cache.clone(); let catalog_store = ctx.catalog_store.clone(); let token = ctx.shutdown.clone(); let gsi_delay = tokio::spawn(async move { - workers::poll_gsi_delay(catalog_store, gsi_default, token).await; + workers::poll_index_propagation_delay(catalog_store, index_delay_cache, token).await; }); vec![ diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 9f7b3c68..c6ae5f92 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -42,14 +42,29 @@ mod stream; mod table_engine; mod table_helpers; mod update_table; +mod vector_bench; +mod vector_search; mod worker; mod workers; -/// Default GSI propagation delay (milliseconds) when the -/// `gsi_propagation_delay_ms` setting is absent. Mirrors the value seeded by +/// Default secondary-index propagation delay (milliseconds) when the +/// `index_propagation_delay_ms` setting is absent. Mirrors the value seeded by /// the catalog schema, and is the single definition used by both the live read /// on the write path and the background refresh worker. -pub(crate) const DEFAULT_GSI_PROPAGATION_DELAY_MS: u64 = 10; +pub(crate) const DEFAULT_INDEX_PROPAGATION_DELAY_MS: u64 = 10; + +/// Read the propagation-delay setting, preferring the canonical key and falling back +/// to the pre-rename one. +/// +/// A catalog created before the rename holds the operator's value under the old name, +/// and the server refuses to start on a catalog-version mismatch rather than migrating, +/// so no upgrade step ever rewrites that row. Reading past it would silently reset a +/// configured delay to the default; since 0 means synchronous, the silent change would +/// be from strict to eventually consistent. `ORDER BY ... DESC` makes the preference +/// deterministic when both rows exist. +pub(crate) const INDEX_PROPAGATION_DELAY_QUERY: &str = "SELECT value FROM settings \ + WHERE key IN ('index_propagation_delay_ms', 'gsi_propagation_delay_ms') \ + ORDER BY key = 'index_propagation_delay_ms' DESC LIMIT 1"; pub use bootstrapper::SqliteBootstrapper; pub use catalog_store::SqliteCatalogStore; @@ -233,6 +248,17 @@ fn sqlite_server_components_factory( Err(e) => tracing::error!("Failed to reconcile incomplete GSIs: {e}"), } + // Same for a vector index. A separate pass rather than a shared one, because + // the two live in different catalog tables and are built by different code; + // a failure to reconcile one must not skip the other. + match engine.reconcile_incomplete_vector_indexes().await { + Ok(n) if n > 0 => { + tracing::info!("Reconciled {n} incomplete vector index(es) at startup"); + } + Ok(_) => {} + Err(e) => tracing::error!("Failed to reconcile incomplete vector indexes: {e}"), + } + let control_plane_notify = engine.control_plane_notify(); let engine = Arc::new(engine); diff --git a/crates/storage-sqlite/src/schema.rs b/crates/storage-sqlite/src/schema.rs index 220e5b6e..530bd994 100644 --- a/crates/storage-sqlite/src/schema.rs +++ b/crates/storage-sqlite/src/schema.rs @@ -27,7 +27,7 @@ use sqlx::SqlitePool; /// Compiled-in catalog version. Single source of truth for the SQLite backend; /// mirrors the PostgreSQL backend's `CATALOG_VERSION`. pub const CATALOG_VERSION: extenddb_core::version::CatalogVersion = - extenddb_core::version::CatalogVersion::new(0, 0, 2); + extenddb_core::version::CatalogVersion::new(0, 0, 3); /// Complete catalog schema, applied once on a fresh database. /// @@ -91,6 +91,46 @@ CREATE TABLE IF NOT EXISTS indexes ( CHECK (propagation_delay_ms IS NULL OR propagation_delay_ms >= 0) ); +-- Vector index metadata. Kept out of `indexes` deliberately: a vector index is +-- not described by a key schema, so reusing that table's `key_schema` column +-- would mean storing something meaningless in a NOT NULL column. The engine +-- supplies index_id, as it does for GSIs. +-- +-- `search_schema` is nullable because the HASH element is optional (measured +-- against the live service): with one the search is partition-scoped and +-- SearchConditionExpression is required, without one it spans the table. +-- +-- `backfilling` mirrors the measured lifecycle: false while CREATING before the +-- scan starts, true while it runs, and the member is absent once ACTIVE. Stored +-- as an integer so the ACTIVE state is representable as NULL rather than as a +-- third boolean value. +CREATE TABLE IF NOT EXISTS vector_indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + dimensions INTEGER NOT NULL, + distance_function TEXT NOT NULL, + vector_attribute TEXT NOT NULL, + search_schema TEXT, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'CREATING', + backfilling INTEGER, + PRIMARY KEY (table_id, index_name), + CONSTRAINT vector_indexes_table_id_fkey + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, + CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), + CONSTRAINT chk_vector_backfilling_bool + CHECK (backfilling IS NULL OR backfilling IN (0, 1)), + -- An ACTIVE index must not carry the member at all, which is what the + -- service does. Enforced here as well as in core, so a bug in the backend + -- cannot persist a state the wire contract forbids. + CONSTRAINT chk_vector_active_has_no_backfilling + CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_vector_indexes_index_id + ON vector_indexes (index_id); + -- Resource tags. CREATE TABLE IF NOT EXISTS tags ( resource_arn TEXT NOT NULL, @@ -375,9 +415,15 @@ INSERT OR IGNORE INTO seq_counters (name, value) VALUES ('stream', CAST(strftime('%s','now') AS INTEGER) * 1000000); -- Seed settings (mirror PostgreSQL defaults). -INSERT OR IGNORE INTO settings (key, value) VALUES ('catalog_version', '0.0.2'); +-- Recorded catalog version. Upserts rather than INSERT OR IGNORE: this schema is +-- re-applied by `extenddb migrate`, and with IGNORE the recorded version would +-- never advance, so a migration could add objects and still leave the server +-- refusing to start on a version mismatch. Must stay in step with +-- `CATALOG_VERSION` above; they are checked against each other in a test. +INSERT INTO settings (key, value) VALUES ('catalog_version', '0.0.3') + ON CONFLICT(key) DO UPDATE SET value = excluded.value; INSERT OR IGNORE INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0.25'); -INSERT OR IGNORE INTO settings (key, value) VALUES ('gsi_propagation_delay_ms', '10'); +INSERT OR IGNORE INTO settings (key, value) VALUES ('index_propagation_delay_ms', '10'); "#; /// Apply the full catalog schema to a fresh (or existing) database. @@ -402,3 +448,36 @@ pub async fn table_exists(pool: &SqlitePool, name: &str) -> OpResult { .map_err(|e| OpError::Internal(format!("table_exists({name}): {e}")))?; Ok(exists) } + +#[cfg(test)] +mod tests { + use super::{CATALOG_VERSION, SCHEMA_SQL}; + + /// The schema seeds the recorded catalog version as a SQL literal, and the + /// server compares that recorded value against `CATALOG_VERSION` at startup. + /// If the two drift, a freshly initialised deployment refuses to serve with a + /// version mismatch, which is a confusing failure a long way from its cause. + #[test] + fn the_seeded_catalog_version_matches_the_compiled_constant() { + let expected = format!( + "INSERT INTO settings (key, value) VALUES ('catalog_version', '{CATALOG_VERSION}')" + ); + assert!( + SCHEMA_SQL.contains(&expected), + "schema must seed catalog_version = {CATALOG_VERSION}; \ + update the literal in SCHEMA_SQL when bumping CATALOG_VERSION" + ); + } + + /// The seed must upsert. With `INSERT OR IGNORE` the recorded version never + /// advances, so `extenddb migrate` would add the new objects and still leave + /// the server refusing to start. + #[test] + fn the_catalog_version_seed_upserts_rather_than_ignoring() { + assert!( + !SCHEMA_SQL + .contains("INSERT OR IGNORE INTO settings (key, value) VALUES ('catalog_version'"), + "catalog_version must not be seeded with INSERT OR IGNORE" + ); + } +} diff --git a/crates/storage-sqlite/src/store.rs b/crates/storage-sqlite/src/store.rs index 39f29370..6cd380ea 100644 --- a/crates/storage-sqlite/src/store.rs +++ b/crates/storage-sqlite/src/store.rs @@ -26,6 +26,7 @@ use sqlx::SqlitePool; use sqlx::sqlite::SqlitePoolOptions; use tokio::sync::Mutex; +use crate::INDEX_PROPAGATION_DELAY_QUERY; use crate::schema::CATALOG_VERSION; use crate::sqlite_util::sqlite_url; @@ -40,13 +41,21 @@ pub struct SqliteEngine { pub(crate) control_plane_notify: Arc, /// Cached default GSI propagation delay (ms); refreshed by a worker and /// read on the write path to decide sync-vs-async index maintenance. - pub(crate) gsi_default_delay_ms: Arc, + pub(crate) index_propagation_delay_cache: Arc, /// Wakes the GSI propagation worker when a write enqueues into `gsi_pending`. pub(crate) gsi_notify: Arc, /// Serializes all writers (design decision D1). Held for the duration of /// every write transaction so condition checks and writes are atomic and /// `SQLITE_BUSY` cannot arise from competing writers. pub(crate) write_lock: Arc>, + /// Index ids whose asynchronous backfill task is currently alive in THIS + /// process. What tells a stuck `CREATING` index apart from one still + /// building: a catalog row can say `CREATING` forever, but only a live task + /// appears here, and the entry is removed by a drop guard so a panicking + /// task deregisters too. The GSI worker recovers any `CREATING` index with + /// no entry, because nothing else ever will until a restart, and until then + /// the per-table queue hold blocks every write's index maintenance. + pub(crate) vector_builds_running: Arc>>, } impl SqliteEngine { @@ -107,24 +116,26 @@ impl SqliteEngine { } } - let initial_gsi_delay: u64 = sqlx::query_as::<_, (String,)>( - "SELECT value FROM settings WHERE key = 'gsi_propagation_delay_ms'", - ) - .fetch_optional(&pool) - .await - .ok() - .flatten() - .and_then(|(v,)| v.parse::().ok()) - .unwrap_or(10); + let initial_index_delay: u64 = + sqlx::query_as::<_, (String,)>(INDEX_PROPAGATION_DELAY_QUERY) + .fetch_optional(&pool) + .await + .ok() + .flatten() + .and_then(|(v,)| v.parse::().ok()) + .unwrap_or(10); Ok(Self { pool, region: region.to_owned(), max_item_size_bytes, control_plane_notify: Arc::new(tokio::sync::Notify::new()), - gsi_default_delay_ms: Arc::new(AtomicU64::new(initial_gsi_delay)), + index_propagation_delay_cache: Arc::new(AtomicU64::new(initial_index_delay)), gsi_notify: Arc::new(tokio::sync::Notify::new()), write_lock: Arc::new(Mutex::new(())), + vector_builds_running: Arc::new( + std::sync::Mutex::new(std::collections::HashSet::new()), + ), }) } @@ -151,7 +162,7 @@ impl SqliteEngine { let internal = |e: String| StorageError::Internal(e); // Schema (creates settings, accounts, admin_users, … and seeds - // catalog_version + gsi_propagation_delay_ms). + // catalog_version + index_propagation_delay_ms). crate::schema::apply(&self.pool) .await .map_err(|e| internal(format!("apply schema: {e:?}")))?; @@ -232,33 +243,54 @@ impl SqliteEngine { Ok(if from_env { None } else { Some(password) }) } - /// Current GSI propagation delay (ms); `0` means synchronous. + /// Current secondary-index propagation delay (ms); `0` means synchronous. /// - /// Reads the `gsi_propagation_delay_ms` setting live from the catalog so + /// Reads the `index_propagation_delay_ms` setting live from the catalog so /// out-of-process changes (`extenddb settings set`) take effect on the /// next write, not up to 30 s later when the poll worker refreshes the /// cache. `SQLite` is a local file, so this is an indexed point lookup with /// negligible cost next to the write it precedes. On a read error the /// cached value (still refreshed by the poll worker) is the fallback; on /// success the cache is re-warmed so fallback reads stay fresh. - pub(crate) async fn gsi_default_delay(&self) -> u64 { + pub(crate) async fn index_propagation_delay(&self) -> u64 { use std::sync::atomic::Ordering; - let live: Result, _> = - sqlx::query_as("SELECT value FROM settings WHERE key = 'gsi_propagation_delay_ms'") - .fetch_optional(&self.pool) - .await; + let live: Result, _> = sqlx::query_as(INDEX_PROPAGATION_DELAY_QUERY) + .fetch_optional(&self.pool) + .await; match live { Ok(row) => { // Missing row means the default, matching poll_gsi_delay. let ms = row .and_then(|(v,)| v.parse::().ok()) - .unwrap_or(crate::DEFAULT_GSI_PROPAGATION_DELAY_MS); - self.gsi_default_delay_ms.store(ms, Ordering::Relaxed); + .unwrap_or(crate::DEFAULT_INDEX_PROPAGATION_DELAY_MS); + self.index_propagation_delay_cache + .store(ms, Ordering::Relaxed); ms } Err(e) => { - tracing::debug!("gsi_default_delay: live read failed, using cache: {e:?}"); - self.gsi_default_delay_ms.load(Ordering::Relaxed) + tracing::debug!("index_propagation_delay: live read failed, using cache: {e:?}"); + self.index_propagation_delay_cache.load(Ordering::Relaxed) + } + } + } + + /// Milliseconds to pause between batches of a vector index backfill. + /// + /// Read live for the same reason the propagation delay is: a test sets it with + /// `settings set` and needs it to apply to the next backfill, not up to 30 s + /// later. Zero when unset or unparseable, which is the production value, so a + /// malformed setting cannot slow a real backfill down. + pub(crate) async fn vector_backfill_batch_delay(&self) -> u64 { + let live: Result, _> = + sqlx::query_as("SELECT value FROM settings WHERE key = ?") + .bind(extenddb_core::settings_keys::VECTOR_BACKFILL_BATCH_DELAY_MS) + .fetch_optional(&self.pool) + .await; + match live { + Ok(row) => row.and_then(|(v,)| v.parse::().ok()).unwrap_or(0), + Err(e) => { + tracing::debug!("vector_backfill_batch_delay: live read failed, using 0: {e:?}"); + 0 } } } diff --git a/crates/storage-sqlite/src/table_helpers.rs b/crates/storage-sqlite/src/table_helpers.rs index 52b64b85..5cd30599 100644 --- a/crates/storage-sqlite/src/table_helpers.rs +++ b/crates/storage-sqlite/src/table_helpers.rs @@ -52,6 +52,25 @@ pub(crate) struct IndexRow { pub provisioned_throughput: Option, } +/// A `vector_indexes` catalog row, in `FromRow` field order. +#[derive(sqlx::FromRow)] +pub(crate) struct VectorIndexRow { + pub index_name: String, + #[allow(dead_code)] + pub index_id: String, + pub dimensions: i64, + pub distance_function: String, + pub vector_attribute: String, + pub search_schema: Option, + pub projection: String, + pub index_status: String, + pub backfilling: Option, +} + +/// Columns selected for a `VectorIndexRow`, in `FromRow` field order. +pub(crate) const VECTOR_INDEX_COLUMNS: &str = "index_name, index_id, dimensions, \ + distance_function, vector_attribute, search_schema, projection, index_status, backfilling"; + /// Columns selected for a `TableRow`, in `FromRow` field order. pub(crate) const TABLE_COLUMNS: &str = "table_name, key_schema, attribute_definitions, \ billing_mode, provisioned_throughput, stream_specification, table_status, \ @@ -93,7 +112,19 @@ impl SqliteEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - self.build_table_description_from_row(account_id, row, index_rows) + let vector_rows: Vec = sqlx::query_as(&format!( + "SELECT {VECTOR_INDEX_COLUMNS} FROM vector_indexes WHERE table_id = ?" + )) + .bind(&row.table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let table_name_owned = row.table_name.clone(); + let mut desc = self.build_table_description_from_row(account_id, row, index_rows)?; + desc.vector_indexes = + self.vector_index_descriptions(account_id, &table_name_owned, vector_rows)?; + Ok(desc) } pub(crate) fn build_table_description_from_row( @@ -240,13 +271,69 @@ impl SqliteEngine { .on_demand_throughput .as_deref() .and_then(|s| serde_json::from_str(s).ok()), - // Fields for features this backend does not implement, vector - // indexes today, take their defaults, so adding one to this type - // does not break this build. + // Any core field this backend does not populate takes its default, so + // adding one does not break this build. ..Default::default() }) } + /// Build the vector index descriptions for a table. + /// + /// Separate from `build_table_description_from_row` so the shared builder + /// keeps one signature for every caller. Applied on the describe path, which + /// is where a client reads an index definition in order to search it. + pub(crate) fn vector_index_descriptions( + &self, + account_id: &str, + table_name: &str, + vector_rows: Vec, + ) -> Result>, StorageError> { + let mut vector_index_descs: Vec = Vec::new(); + for vi in vector_rows { + // Deliberately fails rather than defaulting on a bad parse: a vector + // index we cannot describe faithfully must not be reported as if we + // could, because a client uses the description to build a search. + let vector_attribute = parse_json(&vi.vector_attribute, "vector_attribute")?; + let search_schema = vi + .search_schema + .as_deref() + .map(|s| parse_json(s, "vector search_schema")) + .transpose()?; + let projection = parse_json(&vi.projection, "vector projection")?; + let distance_function = parse_json( + &format!("\"{}\"", vi.distance_function), + "distance_function", + )?; + let index_status = parse_json(&format!("\"{}\"", vi.index_status), "vector status")?; + let desc = extenddb_core::types::VectorIndexDescription { + index_name: vi.index_name.clone(), + vector_attribute, + dimensions: u32::try_from(vi.dimensions).map_err(|_| { + StorageError::Internal(format!( + "vector dimensions out of range: {}", + vi.dimensions + )) + })?, + search_schema, + distance_function, + index_status, + backfilling: vi.backfilling.map(|b| b != 0), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn(&self.region, account_id, table_name, &vi.index_name), + projection: Some(projection), + }; + // The readiness rule is core's, applied here so a backend bug cannot + // emit a description the wire contract forbids. Cheaper to catch on + // the way out than to debug from a client. + desc.validate_readiness() + .map_err(|e| StorageError::Internal(e.to_string()))?; + vector_index_descs.push(desc); + } + + Ok((!vector_index_descs.is_empty()).then_some(vector_index_descs)) + } + /// Backfill existing base-table items into a newly created GSI, batched to /// bound memory. #[allow(clippy::too_many_arguments)] diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index 149a8344..9d909675 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -9,9 +9,13 @@ //! replaces `FOR UPDATE`. GSI data tables are created (and backfilled) or //! dropped after the catalog transaction commits, with catalog cleanup on //! a data-DDL failure. +//! +//! Vector index create/delete follows the same two-phase shape, with the +//! backfill lifecycle the service was measured to report. use extenddb_core::types::{ AttributeDefinition, BillingMode, KeySchemaElement, TableDescription, UpdateTableInput, + VectorIndexSpecification, }; use extenddb_storage::error::StorageError; @@ -293,6 +297,90 @@ impl SqliteEngine { } } + // Vector index create/delete. Structured exactly like the GSI block above + // and for the same reason: the catalog row is committed first at CREATING, + // and the data table plus backfill happen after, so a crash in between + // leaves a CREATING row the startup reconciler rebuilds rather than an + // ACTIVE index with a partial table. + let mut vec_created: Vec<(String, VectorIndexSpecification)> = Vec::new(); + let mut vec_deleted: Vec = Vec::new(); + if let Some(updates) = &input.vector_index_updates { + for update in updates { + if let Some(create) = &update.create { + let dup: Option<(String,)> = sqlx::query_as( + "SELECT index_name FROM vector_indexes \ + WHERE table_id = ? AND index_name = ?", + ) + .bind(&table_id) + .bind(&create.index_name) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if dup.is_some() { + return Err(StorageError::IndexAlreadyExists(create.index_name.clone())); + } + let index_id = uuid::Uuid::new_v4().to_string(); + let vec_attr = serde_json::to_string(&create.vector_attribute) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let search_schema = create + .search_schema + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let projection = serde_json::to_string(&create.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let dimensions = i64::from(create.dimensions); + let distance = serde_json::to_string(&create.distance_function) + .map_err(|e| StorageError::Internal(e.to_string()))? + .trim_matches('"') + .to_owned(); + // `backfilling` starts at false rather than absent or true. + // Measured against the service on 2026-08-06: the member appears + // as false while the index exists but its backfill has not + // started, flips to true during, and is removed once ACTIVE. + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, \ + vector_attribute, search_schema, projection, index_status, backfilling) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'CREATING', 0)", + ) + .bind(&table_id) + .bind(&index_id) + .bind(&create.index_name) + .bind(dimensions) + .bind(&distance) + .bind(&vec_attr) + .bind(&search_schema) + .bind(&projection) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + vec_created.push((index_id, create.clone())); + } + if let Some(delete) = &update.delete { + let existing: Option<(String,)> = sqlx::query_as( + "SELECT index_id FROM vector_indexes \ + WHERE table_id = ? AND index_name = ?", + ) + .bind(&table_id) + .bind(&delete.index_name) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let (del_id,) = existing + .ok_or_else(|| StorageError::IndexNotFound(delete.index_name.clone()))?; + sqlx::query("DELETE FROM vector_indexes WHERE table_id = ? AND index_name = ?") + .bind(&table_id) + .bind(&delete.index_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + vec_deleted.push(del_id); + } + } + } + tx.commit() .await .map_err(|e| StorageError::Internal(e.to_string()))?; @@ -391,10 +479,219 @@ impl SqliteEngine { } } + // Vector index data DDL and backfill, after the catalog commit. + if !vec_created.is_empty() || !vec_deleted.is_empty() { + let base_ks: Vec = serde_json::from_str(&ks_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let base_ad: Vec = serde_json::from_str(&ad_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let effective_ad = input.attribute_definitions.as_deref().unwrap_or(&base_ad); + + for (index_id, create) in &vec_created { + let result = self + .build_vector_index(&table_id, index_id, create, &base_ks, effective_ad) + .await; + if let Err(e) = result { + tracing::error!( + "Failed to build vector index '{}' on '{}', cleaning up catalog: {e}", + create.index_name, + input.table_name + ); + // Same cleanup as the GSI path: leaving the CREATING row behind + // would have the reconciler retry a build that just failed + // deterministically, on every startup. + let _ = sqlx::query( + "DELETE FROM vector_indexes WHERE table_id = ? AND index_name = ?", + ) + .bind(&table_id) + .bind(&create.index_name) + .execute(&self.pool) + .await; + let _ = + Self::drop_vector_data_table_by_id(&self.pool, &table_id, index_id).await; + return Err(e); + } + } + + for index_id in &vec_deleted { + Self::drop_vector_data_table_by_id(&self.pool, &table_id, index_id).await?; + } + } + self.build_table_description(account_id, &input.table_name) .await } + /// Create a vector index's data table and populate it, then mark it ready. + /// + /// The status sequence is the one measured against the service on 2026-08-06: + /// `CREATING` with `Backfilling: false`, then `CREATING` with `true` while the + /// scan runs, then `ACTIVE` with the member absent. Writing `false` first rather + /// than jumping straight to `true` matters because a client is documented to + /// read the value rather than test for presence, so an index that exists but + /// has not started backfilling must say so. + /// + /// The backfill is DETACHED and commits per batch, so the base table stays + /// writable while the index builds, as the service's does. This call returns with + /// the index still `CREATING`. + /// + /// That means a half-populated data table is now reachable in principle, so the + /// guarantee rests on the engine refusing to route a search to an index that is + /// not `ACTIVE` rather than, as before, on the backfill being one transaction. + async fn build_vector_index( + &self, + table_id: &str, + index_id: &str, + create: &VectorIndexSpecification, + base_ks: &[KeySchemaElement], + effective_ad: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let mut data_tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Self::create_vector_data_table(&mut data_tx, table_id, index_id, base_ks, effective_ad) + .await?; + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // The scan is about to start, so the member becomes true. Set outside the + // backfill transaction, otherwise no observer could see it: the whole point + // of the flag is to be readable while the scan is in progress. + sqlx::query( + "UPDATE vector_indexes SET backfilling = 1 WHERE table_id = ? AND index_id = ?", + ) + .bind(table_id) + .bind(index_id) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut meta_tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let metas = + crate::data::vector_index::fetch_vector_indexes_for_table(&mut meta_tx, table_id) + .await? + .into_iter() + .find(|m| m.index_id == index_id) + .ok_or_else(|| { + StorageError::Internal( + "the vector index catalog row vanished between commit and backfill" + .to_owned(), + ) + })?; + meta_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Everything the scan needs, owned, so it can outlive this call. + let pool = self.pool.clone(); + let write_lock = std::sync::Arc::clone(&self.write_lock); + let batch_delay = + std::time::Duration::from_millis(self.vector_backfill_batch_delay().await); + let owned_table_id = table_id.to_owned(); + let owned_index_id = index_id.to_owned(); + let owned_index_name = create.index_name.clone(); + let owned_base_ks = base_ks.to_vec(); + let owned_ad = effective_ad.to_vec(); + let gsi_notify = self.gsi_notify(); + + // Registered BEFORE the spawn, so there is no instant where the catalog + // says CREATING and the registry disagrees while the task is viable. The + // guard deregisters on every exit path including a panic, which is the + // whole point: a CREATING index with no registry entry is provably + // orphaned, and the worker's recovery sweep may rebuild it. + self.vector_builds_running + .lock() + .expect("registry poisoned") + .insert(index_id.to_owned()); + let registry = std::sync::Arc::clone(&self.vector_builds_running); + + // Detached, so UpdateTable returns while the index is still CREATING. The + // service behaves this way, and it is the whole point: a table stays ACTIVE + // and writable throughout, taking over eight minutes on an empty table when + // measured, and searches against the index are refused until it is ACTIVE. + // + // Not awaited, so failures cannot be returned to the caller. They are logged + // and the index is deliberately LEFT in CREATING, which is the state the + // worker's recovery sweep repairs at runtime (and + // `reconcile_incomplete_vector_indexes` at startup). Flipping it to + // ACTIVE on error would publish a partially populated index, and there is no + // failure state on the wire for an index to sit in. + tokio::spawn(async move { + struct Deregister( + std::sync::Arc>>, + String, + ); + impl Drop for Deregister { + fn drop(&mut self) { + if let Ok(mut set) = self.0.lock() { + set.remove(&self.1); + } + } + } + let _deregister = Deregister(registry, owned_index_id.clone()); + let result = crate::data::vector_index::backfill_vector_index_in_batches( + &pool, + &write_lock, + &owned_table_id, + &metas, + &owned_base_ks, + &owned_ad, + batch_delay, + ) + .await; + match result { + Ok(written) => { + // Populated, so the index can serve. `backfilling` is cleared to + // NULL rather than set to 0, because the service removes the + // member once ACTIVE and the catalog CHECK constraint enforces + // that pairing. + let flip = sqlx::query( + "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL \ + WHERE table_id = ? AND index_id = ?", + ) + .bind(&owned_table_id) + .bind(&owned_index_id) + .execute(&pool) + .await; + match flip { + Ok(_) => { + tracing::info!( + index_name = %owned_index_name, + vectors_indexed = written, + "vector index backfill complete" + ); + // Writes that landed during the backfill were held by the + // worker because this index was CREATING. It is ACTIVE + // now, so wake the worker rather than leaving them to sit + // until its next idle timeout. + gsi_notify.notify_waiters(); + } + Err(e) => tracing::error!( + index_name = %owned_index_name, + "vector index backfill finished but the ACTIVE flip failed, \ + leaving it CREATING for startup reconciliation: {e}" + ), + } + } + Err(e) => tracing::error!( + index_name = %owned_index_name, + "vector index backfill failed, leaving it CREATING for startup \ + reconciliation: {e}" + ), + } + }); + Ok(()) + } + /// Rebuild any GSI left in `CREATING` by a crash between the catalog commit /// and the completion of its data-table backfill. Runs once at startup: for /// each such index it drops any partial data table, recreates and backfills @@ -468,6 +765,224 @@ impl SqliteEngine { } Ok(rebuilt) } + + /// Rebuild any vector index left in `CREATING` by a crash between the catalog + /// commit and the end of its backfill. + /// + /// Same contract as [`Self::reconcile_incomplete_gsis`]: an `ACTIVE` vector index + /// with a missing or partial data table can never be observed, and nothing is + /// left permanently stuck in `CREATING`. Idempotent, because the data table is + /// dropped and rebuilt rather than appended to; without the drop, a retry would + /// duplicate every row it had already written before the crash, and a search + /// would return the same item several times. + pub(crate) async fn reconcile_incomplete_vector_indexes(&self) -> Result { + let rows: Vec<(String, String, String, String)> = sqlx::query_as( + "SELECT v.index_id, v.table_id, t.key_schema, t.attribute_definitions \ + FROM vector_indexes v JOIN tables t ON v.table_id = t.table_id \ + WHERE v.index_status = 'CREATING'", + ) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut rebuilt = 0usize; + for (index_id, table_id, base_ks_json, base_ad_json) in rows { + let written = self + .rebuild_one_vector_index(&index_id, &table_id, &base_ks_json, &base_ad_json) + .await?; + rebuilt += 1; + tracing::info!( + vectors_indexed = written, + "Reconciled incomplete vector index {index_id} on table {table_id}" + ); + } + Ok(rebuilt) + } + + /// Index ids that are `CREATING` with no live backfill task right now. + /// + /// The worker's cheap per-pass probe. A single sighting is NOT proof of a + /// dead build: the catalog row commits before `build_vector_index` registers + /// the task, so a sweep landing in that window would see a healthy build as + /// orphaned. The worker therefore requires the same id on two consecutive + /// passes before invoking [`Self::recover_stuck_vector_builds`], which + /// re-checks the registry itself at execution time. + pub(crate) async fn stuck_vector_build_candidates(&self) -> Result, StorageError> { + let ids: Vec<(String,)> = + sqlx::query_as("SELECT index_id FROM vector_indexes WHERE index_status = 'CREATING'") + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let registry = self + .vector_builds_running + .lock() + .map_err(|_| StorageError::Internal("vector build registry poisoned".to_owned()))?; + Ok(ids + .into_iter() + .map(|(id,)| id) + .filter(|id| !registry.contains(id)) + .collect()) + } + + /// Recover `CREATING` vector indexes whose backfill task is dead. + /// + /// A build can die without a trace on the wire: the spawned task panics, or + /// its terminal `ACTIVE` flip fails and is logged. The index then sits in + /// `CREATING`, which the worker treats as "hold every queued index write for + /// this table", so one dead build wedges ALL asynchronous index maintenance + /// for the table until a restart runs the startup reconciler. This is the + /// runtime half of the same repair: the GSI worker calls it on a sighting of + /// a `CREATING` index that has no live task in `vector_builds_running`. + /// + /// The registry is what makes the sweep safe to run at any time: a healthy + /// in-flight build is registered before its task is spawned and deregisters + /// by drop guard, so "CREATING and unregistered" cannot describe a build that + /// is still making progress in this process. Rebuilding rather than resuming, + /// for the reconciler's reason: rows already written would collide with the + /// backfill's deliberately plain `INSERT`. + /// + /// `confirmed` is the set of index ids the CALLER has seen stuck on two + /// consecutive passes, and only those are recovered. This is per index on + /// purpose: an early version gated only the decision to sweep, and then + /// recovered every currently-unregistered `CREATING` index, so one genuinely + /// stuck index could drag a just-created sibling (still inside its + /// commit-to-register window) into a rebuild while its live task was also + /// backfilling, double-populating the data table. The registry is re-checked + /// here per index as well, so an id whose task registered since the caller's + /// last pass is skipped even when named. + pub(crate) async fn recover_stuck_vector_builds( + &self, + confirmed: &std::collections::HashSet, + ) -> Result { + let rows: Vec<(String, String, String, String)> = sqlx::query_as( + "SELECT v.index_id, v.table_id, t.key_schema, t.attribute_definitions \ + FROM vector_indexes v JOIN tables t ON v.table_id = t.table_id \ + WHERE v.index_status = 'CREATING'", + ) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut recovered = 0usize; + for (index_id, table_id, base_ks_json, base_ad_json) in rows { + if !confirmed.contains(&index_id) { + continue; + } + let alive = self + .vector_builds_running + .lock() + .map(|set| set.contains(&index_id)) + .unwrap_or(true); + if alive { + continue; + } + let written = self + .rebuild_one_vector_index(&index_id, &table_id, &base_ks_json, &base_ad_json) + .await?; + recovered += 1; + tracing::warn!( + vectors_indexed = written, + "Recovered vector index {index_id} on table {table_id}: it was CREATING \ + with no live backfill task" + ); + } + if recovered > 0 { + // Writes held while the index was CREATING are claimable now. + self.gsi_notify.notify_waiters(); + } + Ok(recovered) + } + + /// Drop, recreate, backfill, and flip one vector index to `ACTIVE`. + /// + /// The shared body of startup reconciliation and the worker's runtime + /// recovery, factored so the two repairs cannot drift. The backfill runs on + /// the batched path, releasing the write lock between batches, so a recovery + /// on a large table cannot become a write-availability outage for every + /// other table. + async fn rebuild_one_vector_index( + &self, + index_id: &str, + table_id: &str, + base_ks_json: &str, + base_ad_json: &str, + ) -> Result { + let base_key_schema: Vec = serde_json::from_str(base_ks_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let attr_defs: Vec = serde_json::from_str(base_ad_json) + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Drop and recreate under the lock, then backfill BATCHED, releasing the + // lock between batches exactly as the normal create path does. An earlier + // version ran the whole backfill in one lock-held transaction, which on a + // large table blocked writes to EVERY table for the full rebuild: a + // write-availability outage as the price of recovering one index. The + // batched path is safe here for the same reasons it is safe on create: + // the index is CREATING throughout, so the worker holds this table's + // queue rows and searches are refused, and the rowid cursor tolerates + // concurrent base-table writes. Recovery uses no batch delay: the lever + // exists for tests, and recovery should finish as fast as batching + // allows. + let meta; + { + let _writer = self.write_lock.lock().await; + Self::drop_vector_data_table_by_id(&self.pool, table_id, index_id).await?; + + let mut data_tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Self::create_vector_data_table( + &mut data_tx, + table_id, + index_id, + &base_key_schema, + &attr_defs, + ) + .await?; + // The definition is read from the catalog rather than reconstructed, + // because the request that created it is long gone. + meta = crate::data::vector_index::fetch_vector_indexes_for_table( + &mut data_tx, + table_id, + ) + .await? + .into_iter() + .find(|m| m.index_id == index_id) + .ok_or_else(|| { + StorageError::Internal(format!( + "vector index {index_id} was selected as CREATING but has no catalog row" + )) + })?; + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + let written = crate::data::vector_index::backfill_vector_index_in_batches( + &self.pool, + &self.write_lock, + table_id, + &meta, + &base_key_schema, + &attr_defs, + std::time::Duration::ZERO, + ) + .await?; + + sqlx::query( + "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL \ + WHERE table_id = ? AND index_id = ?", + ) + .bind(table_id) + .bind(index_id) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(written) + } } /// Update a single string column on the `tables` row. @@ -578,4 +1093,418 @@ mod reconciler_tests { "reconciler must be idempotent" ); } + + /// A vector index left in `CREATING` by a crash must be rebuilt on startup and + /// flipped to `ACTIVE`, with `Backfilling` cleared. + /// + /// Asserts the rebuilt data table is POPULATED, not merely that the status + /// changed. Flipping the row to `ACTIVE` over an empty or missing data table + /// would satisfy a status-only assertion while every search returned nothing, + /// which is the exact failure the reconciler exists to prevent. + #[tokio::test] + async fn reconcile_rebuilds_a_creating_vector_index_and_populates_it() { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + + let account = "000000000000"; + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind(account) + .execute(&engine.pool) + .await + .expect("account"); + + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl(account, input) + .await + .expect("create table"); + + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE account_id = ? AND table_name = 't'") + .bind(account) + .fetch_one(&engine.pool) + .await + .expect("table_id"); + + // Two items already in the base table, so a rebuild has something to find. + let base_table = crate::data::data_table_name(&table_id); + for (pk, vec) in [("a", "[1,0]"), ("b", "[0,1]")] { + let item = format!( + r#"{{"pk":{{"S":"{pk}"}},"emb":{{"L":[{{"N":"{}"}},{{"N":"{}"}}]}}}}"#, + if vec == "[1,0]" { 1 } else { 0 }, + if vec == "[1,0]" { 0 } else { 1 } + ); + sqlx::query(&format!( + "INSERT INTO {base_table} (pk, item_data) VALUES (?, ?)" + )) + .bind(pk) + .bind(&item) + .execute(&engine.pool) + .await + .expect("seed item"); + } + + // Simulate a crash mid-build: a CREATING row, mid-backfill, whose data table + // was never created. + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status, backfilling) \ + VALUES (?, 'vidx-1', 'vidx', 2, 'COSINE', ?, ?, 'CREATING', 1)", + ) + .bind(&table_id) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert CREATING vector index"); + + let rebuilt = engine + .reconcile_incomplete_vector_indexes() + .await + .expect("reconcile"); + assert_eq!(rebuilt, 1, "one CREATING vector index should be rebuilt"); + + let (status, backfilling): (String, Option) = sqlx::query_as( + "SELECT index_status, backfilling FROM vector_indexes \ + WHERE table_id = ? AND index_id = 'vidx-1'", + ) + .bind(&table_id) + .fetch_one(&engine.pool) + .await + .expect("status"); + assert_eq!(status, "ACTIVE"); + assert_eq!( + backfilling, None, + "an ACTIVE index must not carry the Backfilling member" + ); + + // The rebuilt table must actually hold the rows. + let vec_table = crate::data::vector_table_name(&table_id, "vidx-1"); + let (rows,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(rows, 2, "the rebuild must backfill both seeded items"); + + // Idempotent, and the second run must not duplicate the rows it already + // wrote: the reconciler drops and rebuilds rather than appending. + assert_eq!( + engine + .reconcile_incomplete_vector_indexes() + .await + .expect("second reconcile"), + 0, + "reconciler must be idempotent" + ); + let (rows_after,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count again"); + assert_eq!(rows_after, 2, "a second pass must not duplicate rows"); + } + + /// The crash that actually happens: the data table exists and holds SOME of the + /// rows, because the process died partway through the backfill. + /// + /// The reconciler must drop and rebuild rather than resume, or the rows already + /// written are written again and a search returns the same item twice. The + /// previous test cannot catch this: its simulated crash leaves no data table at + /// all, so the drop is a no-op there. + #[tokio::test] + async fn reconcile_rebuilds_a_partially_backfilled_vector_index_without_duplicating() { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind("000000000000") + .execute(&engine.pool) + .await + .expect("account"); + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl("000000000000", input) + .await + .expect("create table"); + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE table_name = 't'") + .fetch_one(&engine.pool) + .await + .expect("table_id"); + + let base_table = crate::data::data_table_name(&table_id); + for pk in ["a", "b"] { + sqlx::query(&format!( + "INSERT INTO {base_table} (pk, item_data) VALUES (?, ?)" + )) + .bind(pk) + .bind(format!( + r#"{{"pk":{{"S":"{pk}"}},"emb":{{"L":[{{"N":"1"}},{{"N":"0"}}]}}}}"# + )) + .execute(&engine.pool) + .await + .expect("seed"); + } + + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status, backfilling) \ + VALUES (?, 'vidx-2', 'vidx', 2, 'COSINE', ?, ?, 'CREATING', 1)", + ) + .bind(&table_id) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert CREATING"); + + // The partial state: the data table exists and already holds one of the two + // rows, exactly as a crash midway through the scan would leave it. + let ks = vec![extenddb_core::types::KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: extenddb_core::types::KeyType::Hash, + }]; + let ad = vec![extenddb_core::types::AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: extenddb_core::types::ScalarAttributeType::S, + }]; + let mut tx = engine.pool.begin_with("BEGIN IMMEDIATE").await.expect("tx"); + SqliteEngine::create_vector_data_table(&mut tx, &table_id, "vidx-2", &ks, &ad) + .await + .expect("create partial data table"); + let meta = crate::data::vector_index::fetch_vector_indexes_for_table(&mut tx, &table_id) + .await + .expect("metas") + .into_iter() + .find(|m| m.index_id == "vidx-2") + .expect("meta"); + let item: extenddb_core::types::Item = + serde_json::from_str(r#"{"pk":{"S":"a"},"emb":{"L":[{"N":"1"},{"N":"0"}]}}"#) + .expect("item"); + crate::data::vector_index::insert_vector_row( + &mut tx, + &table_id, + &meta, + &item, + &ks, + &ad, + &["base_pk".to_owned()], + ) + .await + .expect("partial row"); + tx.commit().await.expect("commit"); + + let vec_table = crate::data::vector_table_name(&table_id, "vidx-2"); + let (before,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count before"); + assert_eq!(before, 1, "the setup must leave a genuinely partial table"); + + engine + .reconcile_incomplete_vector_indexes() + .await + .expect("reconcile"); + + let (after,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count after"); + assert_eq!( + after, 2, + "the rebuild must drop the partial table, not append to it: 3 rows would mean \ + item 'a' was indexed twice and a search would return it twice" + ); + } + + /// A CREATING vector index with no live backfill task must be recovered at + /// runtime, not just at startup. + /// + /// The failure this guards: the detached backfill task dies (panic, or its + /// terminal ACTIVE flip fails) and the index sits in CREATING forever. The + /// worker holds every queued index write for the table while any of its + /// vector indexes is CREATING, so without runtime recovery one dead build + /// wedges ALL asynchronous index maintenance for the table until a restart. + /// The orphan is simulated exactly as the reconciler tests simulate a crash: + /// a CREATING catalog row with no task, which is indistinguishable from the + /// real thing because a dead task leaves nothing else behind. + #[tokio::test] + async fn a_creating_index_with_no_live_build_task_is_recovered_at_runtime() { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind("000000000000") + .execute(&engine.pool) + .await + .expect("account"); + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl("000000000000", input) + .await + .expect("create table"); + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE table_name = 't'") + .fetch_one(&engine.pool) + .await + .expect("table_id"); + + let base_table = crate::data::data_table_name(&table_id); + sqlx::query(&format!( + "INSERT INTO {base_table} (pk, item_data) VALUES ('a', ?)" + )) + .bind(r#"{"pk":{"S":"a"},"emb":{"L":[{"N":"1"},{"N":"0"}]}}"#) + .execute(&engine.pool) + .await + .expect("seed"); + + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status, backfilling) \ + VALUES (?, 'vidx-dead', 'vidx', 2, 'COSINE', ?, ?, 'CREATING', 1)", + ) + .bind(&table_id) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert orphaned CREATING index"); + + // The probe must name it, and recovery must repair it. + let candidates = engine + .stuck_vector_build_candidates() + .await + .expect("candidates"); + assert_eq!(candidates, vec!["vidx-dead".to_owned()]); + let confirmed: std::collections::HashSet = candidates.iter().cloned().collect(); + let recovered = engine + .recover_stuck_vector_builds(&confirmed) + .await + .expect("recover"); + assert_eq!(recovered, 1, "the orphaned build must be recovered"); + + let (status, backfilling): (String, Option) = sqlx::query_as( + "SELECT index_status, backfilling FROM vector_indexes WHERE index_id = 'vidx-dead'", + ) + .fetch_one(&engine.pool) + .await + .expect("status"); + assert_eq!(status, "ACTIVE"); + assert_eq!(backfilling, None); + + // Recovered means populated, not merely flipped: the seeded row must be + // in the rebuilt index, or the "recovery" published an empty index. + let vec_table = crate::data::vector_table_name(&table_id, "vidx-dead"); + let (rows,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {vec_table}")) + .fetch_one(&engine.pool) + .await + .expect("count"); + assert_eq!(rows, 1, "recovery must backfill the seeded row"); + } + + /// The discriminating control for the sweep: a CREATING index whose build IS + /// registered as alive must be left alone. Without this the previous test + /// would also pass for a sweep that rebuilds every CREATING index it sees, + /// which would corrupt a healthy in-flight build by dropping its data table + /// out from under the running backfill. + #[tokio::test] + async fn a_creating_index_with_a_live_build_task_is_left_alone() { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind("000000000000") + .execute(&engine.pool) + .await + .expect("account"); + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl("000000000000", input) + .await + .expect("create table"); + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE table_name = 't'") + .fetch_one(&engine.pool) + .await + .expect("table_id"); + + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status, backfilling) \ + VALUES (?, 'vidx-live', 'vidx', 2, 'COSINE', ?, ?, 'CREATING', 1)", + ) + .bind(&table_id) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert CREATING index"); + + // The build is alive: exactly what build_vector_index records before it + // spawns the task. + engine + .vector_builds_running + .lock() + .expect("registry") + .insert("vidx-live".to_owned()); + + assert!( + engine + .stuck_vector_build_candidates() + .await + .expect("candidates") + .is_empty(), + "a registered build must not be a candidate" + ); + // Named explicitly as confirmed-stuck, so the registry re-check alone + // must protect it: the strongest form of the control. + let confirmed: std::collections::HashSet = + std::iter::once("vidx-live".to_owned()).collect(); + assert_eq!( + engine + .recover_stuck_vector_builds(&confirmed) + .await + .expect("recover"), + 0, + "a registered build must not be recovered" + ); + let (status,): (String,) = + sqlx::query_as("SELECT index_status FROM vector_indexes WHERE index_id = 'vidx-live'") + .fetch_one(&engine.pool) + .await + .expect("status"); + assert_eq!(status, "CREATING", "the live build must be untouched"); + } } diff --git a/crates/storage-sqlite/src/vector_bench.rs b/crates/storage-sqlite/src/vector_bench.rs new file mode 100644 index 00000000..08e73aaa --- /dev/null +++ b/crates/storage-sqlite/src/vector_bench.rs @@ -0,0 +1,200 @@ +//! Measurement scaffolding for the vector scan, not part of the shipped surface. +//! +//! Ignored by default because it is a timing harness rather than an assertion: +//! it exists so a performance claim about the scan can be backed by a number +//! instead of by reading the code. Run it with +//! +//! ```text +//! VB_ITEMS=20000 VB_DIMS=384 VB_PAYLOAD=2000 \ +//! cargo test -p extenddb-storage-sqlite --lib vector_bench -- --ignored --nocapture +//! ``` +//! +//! # What this harness established +//! +//! The scan is bound by per-row overhead in the row-streaming layer, not by any +//! vector-specific work. Measured at 10,000 items, 384 dimensions and a 2KB +//! non-indexed attribute, roughly 30ms per search, about 3us per row: +//! +//! - Removing `item_data` from the projection entirely: no change. +//! - Skipping the blob decode and the distance computation entirely: no change +//! (34ms with no per-row work at all, against 25 to 28ms doing all of it). +//! - Replacing the indexed distance loops with iterator `zip`: no change. +//! +//! So the JSON parse of the stored item, which the scan performs for every row in +//! the partition and which looked like the obvious cost because the item carries +//! the vector as decimal strings, is free relative to streaming the row. Two +//! changes were tried and reverted: deferring the parse until a candidate can +//! enter the retained set, and scoring straight from the stored bytes into a +//! reused buffer. Interleaved against the unmodified code over six alternating +//! pairs, that version measured 48.6ms against 29.5ms with no overlap in range, +//! so it was reproducibly slower while doing strictly less work, for a reason +//! this harness did not isolate. +//! +//! # Measuring on a shared machine +//! +//! Take the minimum of many repetitions and interleave the two binaries under +//! comparison, alternating between them. A median of seven on an eight-core box +//! at load average 3 was not adequate: the unmodified code measured 59.1ms and +//! then 32.7ms on two consecutive runs of the identical binary, which is a wider +//! spread than any of the effects being investigated. +//! +//! `VB_PAYLOAD` is the width in bytes of a non-indexed attribute on each item. +//! It was expected to be the variable that matters, since the scan reads +//! `item_data` for every row in the partition. It is not: widening it from 200 +//! bytes to 2KB did not slow the scan down, and the two measured in the opposite +//! order to the prediction. + +#[cfg(test)] +mod tests { + use extenddb_core::expression::ExpressionMaps; + use extenddb_core::types::{AttributeValue, Item}; + use extenddb_storage::{DataEngine, VectorSearch}; + use serde_json::json; + use std::time::Instant; + + fn env_usize(key: &str, default: usize) -> usize { + std::env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) + } + + /// Deterministic pseudo-random vector, so two runs of the harness measure the + /// same work and a before/after comparison is meaningful. + fn vector_for(seed: u64, dims: usize) -> Vec { + let mut s = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + let mut v = Vec::with_capacity(dims); + for _ in 0..dims { + s = s.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + #[allow(clippy::cast_precision_loss)] + v.push(((s >> 33) as f32 / u32::MAX as f32) - 0.5); + } + v + } + + #[tokio::test] + #[ignore = "timing harness, run explicitly"] + async fn vector_bench_scan() { + let items = env_usize("VB_ITEMS", 5_000); + let dims = env_usize("VB_DIMS", 384); + let payload = env_usize("VB_PAYLOAD", 1_000); + let top_k = env_usize("VB_TOPK", 10); + let reps = env_usize("VB_REPS", 5); + + let engine = crate::SqliteEngine::new(":memory:", 1, "us-east-1", 4_096_000) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + let account = "000000000000"; + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind(account) + .execute(&engine.pool) + .await + .expect("account"); + + // Zero delay so index maintenance applies inline with each write. The + // default is asynchronous, which would leave the index empty at search + // time and measure a scan over nothing. + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('index_propagation_delay_ms', '0') + ON CONFLICT(key) DO UPDATE SET value = '0'", + ) + .execute(&engine.pool) + .await + .expect("delay 0"); + + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{ + "IndexName": "vidx", + "Dimensions": dims, + "DistanceFunction": "COSINE", + "VectorAttribute": {"AttributeName": "emb"}, + "Projection": {"ProjectionType": "ALL"} + }] + })) + .expect("input"); + engine + .create_table_impl(account, input) + .await + .expect("create table"); + + // CreateTable returns with the table CREATING and the ACTIVE flip owned by + // the control-plane worker, which this harness does not run. Flipped + // directly because the transition is not what is being measured. The + // vector index needs no flip: the inline create path sets it ACTIVE. + sqlx::query("UPDATE tables SET table_status = 'ACTIVE' WHERE account_id = ?") + .bind(account) + .execute(&engine.pool) + .await + .expect("activate"); + + let key_info = engine + .fetch_table_key_info(account, "t") + .await + .expect("key info"); + let maps = ExpressionMaps::default(); + + // Written through the real put path so the index rows are built exactly as + // production builds them; a hand-rolled INSERT could encode differently + // and measure something the server never does. + let filler: String = "x".repeat(payload); + for i in 0..items { + let mut item = Item::new(); + item.insert("pk".to_owned(), AttributeValue::S(format!("k{i:07}"))); + item.insert( + "emb".to_owned(), + AttributeValue::L( + vector_for(i as u64, dims) + .into_iter() + .map(|f| AttributeValue::N(f.to_string())) + .collect(), + ), + ); + item.insert("filler".to_owned(), AttributeValue::S(filler.clone())); + engine + .put_item_impl(&key_info, item, false, None, &maps, None) + .await + .expect("put item"); + } + + let query = vector_for(u64::MAX, dims); + let search = || VectorSearch { + key_info: &key_info, + index_name: "vidx", + query_vector: &query, + top_k: i64::try_from(top_k).expect("top_k fits"), + hash_key: None, + filters: &[], + }; + let vector_engine = engine.as_vector_search().expect("vector capable"); + + // Discarded: the first search warms SQLite's page cache, so including it + // would report cache-miss cost as if it were steady state. + let _ = vector_engine.search_vectors(search()).await.expect("warm"); + + let mut times = Vec::with_capacity(reps); + for _ in 0..reps { + let started = Instant::now(); + let out = vector_engine + .search_vectors(search()) + .await + .expect("search"); + let elapsed = started.elapsed(); + assert_eq!(out.hits.len(), top_k.min(items), "unexpected hit count"); + times.push(elapsed.as_secs_f64() * 1000.0); + } + times.sort_by(|a, b| a.partial_cmp(b).expect("finite")); + + println!( + "VBENCH items={items} dims={dims} payload={payload} top_k={top_k} \ + reps={reps} median_ms={:.2} min_ms={:.2} max_ms={:.2}", + times[times.len() / 2], + times[0], + times[times.len() - 1], + ); + } +} diff --git a/crates/storage-sqlite/src/vector_search.rs b/crates/storage-sqlite/src/vector_search.rs new file mode 100644 index 00000000..3ddf2031 --- /dev/null +++ b/crates/storage-sqlite/src/vector_search.rs @@ -0,0 +1,518 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Vector similarity search: exact scan over one partition. +//! +//! Exact rather than approximate, and that is a measured decision rather than a +//! placeholder. No SQLite vector extension meets this backend's constraints: the +//! static-musl `FROM scratch` build cannot `dlopen` a loadable extension, the one +//! extension with a compatible licence and an in-database index (`sqlite-vec`) is +//! brute force in every stable release anyway, and every option offering a real +//! ANN index either stores it in a sidecar file, forbids transactions, or is not +//! open source. See `docs/adr` for the full elimination. +//! +//! Measured cost on one core, warm cache, row-per-vector layout: roughly 213k to +//! 334k vectors/sec at 256 dimensions, 94k to 103k at 1024, and 39k to 43k at +//! 4096. So a partition stays inside a 10 ms budget up to about 1,000 vectors at +//! 1024 dimensions, and inside 100 ms up to about 10,000. The scan is dominated by +//! getting bytes out of SQLite rather than by the arithmetic, which is why a +//! zero-copy `&[f32]` view of the blob measured no faster than decoding per +//! element. + +use extenddb_core::types::{AttributeValue, DistanceFunction, Item}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::pk_to_text; +use extenddb_storage::{ + BoxedFuture, VectorHit, VectorSearch, VectorSearchEngine, VectorSearchOutput, + VectorSearchResult, +}; + +use crate::data::vector_table_name; +use crate::store::SqliteEngine; + +/// Partition value for an index that declares no HASH element. +/// +/// Such an index searches the whole table, so every row shares one partition +/// rather than the scan needing a second code path. +/// +/// What makes this safe is **not** that the value is unguessable. `pk_to_text` +/// stores an `S` attribute verbatim, so a caller could supply this exact string as +/// a partition key. The guarantee is structural instead: the partition is chosen +/// from the *index's* schema, not from the item, and each index has its own data +/// table. So within one table either every row is keyed by a real hash value (the +/// index declares a HASH element) or every row uses this sentinel (it does not). +/// The two never coexist, so there is nothing for a collision to leak into. +/// +/// The leading NUL is defence in depth for the day that invariant changes, for +/// instance if several indexes ever shared one table. It is deliberately not what +/// correctness rests on, because a reader who believed it was would then feel free +/// to weaken it. +pub(crate) const UNSCOPED_PARTITION: &str = "\u{0}all"; + +/// The partition column value for a vector row. +/// +/// Uses the same `pk_to_text` encoding as item partition keys, so a value written +/// by the write path and a value derived from a search request are byte-identical. +/// Getting this wrong would not fail loudly: it would silently return no hits. +pub(crate) fn partition_value( + hash_key: Option<(&str, &AttributeValue)>, +) -> Result { + match hash_key { + Some((_, value)) => Ok(pk_to_text(value)?.into_owned()), + None => Ok(UNSCOPED_PARTITION.to_owned()), + } +} + +/// Decode a stored vector blob into `f32`s. +/// +/// Rejects a truncated blob rather than reading a short vector, because a +/// dimension mismatch would silently change every distance in the result. +fn decode_vector(bytes: &[u8], dimensions: usize) -> Result, StorageError> { + if bytes.len() != dimensions * 4 { + return Err(StorageError::Internal(format!( + "stored vector is {} bytes, expected {} for {dimensions} dimensions", + bytes.len(), + dimensions * 4 + ))); + } + Ok(bytes + .chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect()) +} + +/// Score one candidate under the index's distance function. +/// +/// Cosine and Euclidean are distances, so smaller is more similar; dot product is +/// a similarity, so larger is. The caller must not compare scores across +/// functions, which is why the output reports which one was used. +fn score( + function: DistanceFunction, + query: &[f32], + query_norm: f32, + candidate: &[f32], + candidate_norm: f32, +) -> f64 { + match function { + DistanceFunction::Cosine => { + if query_norm == 0.0 || candidate_norm == 0.0 { + // Undefined angle. Reported as maximally distant rather than as + // an error, matching how a zero vector is treated elsewhere. + return 1.0; + } + let mut dot = 0.0f32; + for i in 0..query.len() { + dot += query[i] * candidate[i]; + } + // Clamped because the quotient can exceed 1 by a float epsilon when the + // vectors are identical, which made an exact self-match report a + // NEGATIVE distance (-1.19e-07 was measured against a stored item's own + // vector). Cosine distance has domain [0, 2], so a consumer that + // clamps, or takes a square root of the score, sees a value the metric + // cannot produce. The service returned +1.49e-08 for the same query. + let similarity = (dot / (query_norm * candidate_norm)).clamp(-1.0, 1.0); + f64::from(1.0 - similarity) + } + DistanceFunction::Euclidean => { + let mut sum = 0.0f32; + for i in 0..query.len() { + let d = query[i] - candidate[i]; + sum += d * d; + } + f64::from(sum.sqrt()) + } + DistanceFunction::DotProduct => { + let mut dot = 0.0f32; + for i in 0..query.len() { + dot += query[i] * candidate[i]; + } + f64::from(dot) + } + } +} + +/// Keeps the best `k` seen so far, ordered by the index's distance function. +/// +/// A full sort of the partition would dominate the scan for a large partition and +/// is unnecessary: only `k` rows are ever returned. Insertion into a `k`-sized +/// vector is cheap because the common case after the first `k` candidates is a +/// single comparison against the current worst. +struct TopK { + k: usize, + function: DistanceFunction, + /// The decoded components ride along with each retained hit so the returned + /// attribute is rebuilt only for the `k` survivors. Rebuilding during the scan + /// would allocate a decimal string per component per row examined, which at + /// 4096 dimensions over a large partition would cost far more than the scan. + /// Moving the already-decoded vector in is free. + hits: Vec<(f64, Item, Vec)>, +} + +impl TopK { + fn new(k: usize, function: DistanceFunction) -> Self { + Self { + k, + function, + hits: Vec::with_capacity(k.saturating_add(1)), + } + } + + /// True when `a` should rank ahead of `b`. + fn ranks_before(&self, a: f64, b: f64) -> bool { + self.function.ranks_before(a, b) + } + + fn offer(&mut self, candidate_score: f64, item: Item, components: Vec) { + if self.hits.len() < self.k { + let pos = self + .hits + .iter() + .position(|(s, _, _)| self.ranks_before(candidate_score, *s)) + .unwrap_or(self.hits.len()); + self.hits.insert(pos, (candidate_score, item, components)); + return; + } + if self.k == 0 { + return; + } + let worst = self.hits[self.k - 1].0; + if !self.ranks_before(candidate_score, worst) { + return; + } + let pos = self + .hits + .iter() + .position(|(s, _, _)| self.ranks_before(candidate_score, *s)) + .unwrap_or(self.k - 1); + self.hits.insert(pos, (candidate_score, item, components)); + self.hits.truncate(self.k); + } +} + +impl VectorSearchEngine for SqliteEngine { + fn search_vectors(&self, req: VectorSearch<'_>) -> BoxedFuture<'_, VectorSearchResult> { + // The request borrows; own what the async body needs so the future is not + // tied to the caller's frame. + let table_id = req.key_info.table_id.clone(); + let index_name = req.index_name.to_owned(); + let query_vector = req.query_vector.to_vec(); + let top_k = req.top_k; + let partition = partition_value(req.hash_key); + let filters: Vec<(String, AttributeValue)> = req + .filters + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).clone())) + .collect(); + + Box::pin(async move { + let partition = partition?; + + // The index definition comes from the catalog rather than from + // TableKeyInfo, because the cached key info carries dimensions and the + // search schema but not the distance function, without which a score + // cannot be computed or ordered. + let row: Option<(String, i64, String, String)> = sqlx::query_as( + "SELECT index_id, dimensions, distance_function, vector_attribute \ + FROM vector_indexes WHERE table_id = ? AND index_name = ?", + ) + .bind(&table_id) + .bind(&index_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (index_id, dimensions, distance_raw, vector_attribute_json) = + row.ok_or_else(|| StorageError::IndexNotFound(index_name.clone()))?; + // Stored as the serialized `VectorAttribute`, not a bare name, so it is + // deserialized exactly as the write path does. Treating the column as a + // plain string yields the key `{"AttributeName":"emb"}`. + let vector_attribute_name = + serde_json::from_str::( + &vector_attribute_json, + ) + .map_err(|e| StorageError::Internal(format!("vector_attribute: {e}")))? + .attribute_name; + let dimensions = usize::try_from(dimensions).map_err(|_| { + StorageError::Internal(format!("vector dimensions out of range: {dimensions}")) + })?; + let function: DistanceFunction = serde_json::from_str(&format!("\"{distance_raw}\"")) + .map_err(|e| { + StorageError::Internal(format!("unknown distance function: {e}")) + })?; + + if query_vector.len() != dimensions { + // Core validates this against the cached key info, so reaching + // here means the catalog and the cache disagree. + return Err(StorageError::Validation(format!( + "query vector has {} dimensions, index expects {dimensions}", + query_vector.len() + ))); + } + + let vec_table = vector_table_name(&table_id, &index_id); + let sql = format!("SELECT vec, nrm, item_data FROM {vec_table} WHERE part = ?"); + + let mut query_norm = 0.0f32; + for x in &query_vector { + query_norm += x * x; + } + let query_norm = query_norm.sqrt(); + + let k = usize::try_from(top_k.max(0)).unwrap_or(0); + let mut top = TopK::new(k, function); + + // Streamed rather than fetched all at once, so a large partition does + // not allocate proportionally to its size. This is the reason the + // row-per-vector layout was chosen over a packed blob per partition. + use futures::TryStreamExt; + let mut stream = sqlx::query_as::<_, (Vec, f64, String)>(&sql) + .bind(&partition) + .fetch(&self.pool); + + while let Some((blob, norm, item_json)) = stream + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let candidate = decode_vector(&blob, dimensions)?; + let item: Item = serde_json::from_str(&item_json) + .map_err(|e| StorageError::Internal(format!("stored item: {e}")))?; + + // Inline-filter attributes are applied here rather than in SQL, + // because they are item attributes rather than columns. Equality + // only, which is all the wire surface admits today. + if !filters.is_empty() + && !filters + .iter() + .all(|(name, expected)| item.get(name) == Some(expected)) + { + continue; + } + + #[allow(clippy::cast_possible_truncation)] + let candidate_norm = norm as f32; + let candidate_score = score( + function, + &query_vector, + query_norm, + &candidate, + candidate_norm, + ); + top.offer(candidate_score, item, candidate); + } + + Ok(VectorSearchOutput { + hits: top + .hits + .into_iter() + .map(|(score, mut item, components)| { + // Reinstated from the stored `f32`s rather than from a + // second copy in the payload, so what comes back is the + // narrowed value that was actually indexed. The engine drops + // it again unless a `ProjectionExpression` names it, and the + // billed byte count subtracts it, so putting it here does not + // change either the default response or the metric. + item.insert( + vector_attribute_name.clone(), + extenddb_core::validation::vector_item::vector_attribute(&components), + ); + VectorHit { item, score } + }) + .collect(), + distance_function: function, + }) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn item() -> Item { + Item::new() + } + + #[test] + fn cosine_of_identical_vectors_is_zero() { + let v = [1.0f32, 2.0, 3.0]; + let n = (14.0f32).sqrt(); + let s = score(DistanceFunction::Cosine, &v, n, &v, n); + assert!(s.abs() < 1e-6, "expected ~0.0, got {s}"); + } + + /// Cosine distance has domain [0, 2], and a self-match must land on the zero + /// end of it from ABOVE. + /// + /// This exists because the test above cannot catch the failure it was + /// nominally covering: it asserts `s.abs() < 1e-6`, which is satisfied by + /// -1.19e-07, the exact value a live self-match returned before the + /// similarity was clamped. Taking the absolute value discards the sign, which + /// was the only thing wrong. + /// + /// Many vectors are tried rather than one, because whether the f32 quotient + /// lands above 1 depends on the particular rounding of that vector's norm, so + /// a single hand-picked case would prove very little. + #[test] + fn cosine_distance_is_never_negative_for_a_self_match() { + let mut seed = 0x2026_0811_u64; + for _ in 0..2000 { + // xorshift, so the case set is fixed and reproducible. + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + let dim = 8 + (seed % 121) as usize; + let mut v = Vec::with_capacity(dim); + let mut s = seed; + for _ in 0..dim { + s = s.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + v.push(((s >> 33) as f32 / u32::MAX as f32) - 0.5); + } + let norm = v.iter().map(|x| x * x).sum::().sqrt(); + if norm == 0.0 { + continue; + } + let d = score(DistanceFunction::Cosine, &v, norm, &v, norm); + assert!( + d >= 0.0, + "cosine distance left its domain for a self-match: {d} (dim {dim})" + ); + assert!(d < 1e-6, "a self-match must still be ~0: {d}"); + } + } + + /// The clamp must hold even when the norms handed in understate the true ones, + /// which is the mechanism that pushed the quotient above 1 in the first place: + /// `norm * norm` can be strictly less than `sum(x*x)` in f32. + #[test] + fn cosine_clamps_when_the_supplied_norms_understate() { + let v = [0.6f32, 0.8]; + // Deliberately 1% low, far beyond any real rounding error, so the + // unclamped expression would return roughly -0.02. + let understated = 0.99f32; + let d = score(DistanceFunction::Cosine, &v, understated, &v, understated); + assert!(d >= 0.0, "expected the clamp to hold, got {d}"); + } + + #[test] + fn cosine_of_opposite_vectors_is_two() { + let a = [1.0f32, 0.0]; + let b = [-1.0f32, 0.0]; + let s = score(DistanceFunction::Cosine, &a, 1.0, &b, 1.0); + assert!((s - 2.0).abs() < 1e-6, "expected ~2.0, got {s}"); + } + + #[test] + fn a_zero_vector_is_maximally_distant_rather_than_an_error() { + let a = [1.0f32, 0.0]; + let z = [0.0f32, 0.0]; + assert!((score(DistanceFunction::Cosine, &a, 1.0, &z, 0.0) - 1.0).abs() < 1e-6); + } + + #[test] + fn euclidean_is_the_straight_line_distance() { + let a = [0.0f32, 0.0]; + let b = [3.0f32, 4.0]; + let s = score(DistanceFunction::Euclidean, &a, 0.0, &b, 5.0); + assert!((s - 5.0).abs() < 1e-6, "expected 5.0, got {s}"); + } + + #[test] + fn dot_product_is_reported_raw_and_can_be_negative() { + let a = [1.0f32, 0.0]; + let b = [-2.0f32, 0.0]; + let s = score(DistanceFunction::DotProduct, &a, 1.0, &b, 2.0); + assert!((s + 2.0).abs() < 1e-6, "expected -2.0, got {s}"); + } + + /// The direction of "better" is not uniform, so top-k must consult the + /// distance function. A single ordering would silently return the *worst* + /// matches for dot product. + #[test] + fn top_k_orders_distances_ascending_and_similarities_descending() { + let mut cosine = TopK::new(2, DistanceFunction::Cosine); + for s in [0.9, 0.1, 0.5] { + cosine.offer(s, item(), vec![]); + } + assert_eq!( + cosine.hits.iter().map(|(s, _, _)| *s).collect::>(), + vec![0.1, 0.5] + ); + + let mut dot = TopK::new(2, DistanceFunction::DotProduct); + for s in [0.9, 0.1, 0.5] { + dot.offer(s, item(), vec![]); + } + assert_eq!( + dot.hits.iter().map(|(s, _, _)| *s).collect::>(), + vec![0.9, 0.5] + ); + } + + /// Each retained hit must keep its *own* vector. The components are inserted at + /// a computed position alongside the score, so an off-by-one there would return + /// a neighbour's vector against this item's attributes: wrong data, no error. + #[test] + fn a_retained_hit_keeps_its_own_vector() { + let mut t = TopK::new(3, DistanceFunction::Cosine); + // Offered worst-first so every insert lands at the front and the pairing is + // exercised rather than incidentally correct. + for (score, tag) in [(0.9f64, 9.0f32), (0.5, 5.0), (0.1, 1.0)] { + t.offer(score, item(), vec![tag]); + } + let paired: Vec<(f64, f32)> = t + .hits + .iter() + .map(|(s, _, components)| (*s, components[0])) + .collect(); + assert_eq!(paired, vec![(0.1, 1.0), (0.5, 5.0), (0.9, 9.0)]); + } + + #[test] + fn top_k_of_zero_returns_nothing_rather_than_panicking() { + let mut t = TopK::new(0, DistanceFunction::Cosine); + t.offer(0.5, item(), vec![]); + assert!(t.hits.is_empty()); + } + + #[test] + fn a_truncated_stored_vector_is_rejected_rather_than_read_short() { + let err = decode_vector(&[0u8; 8], 3).expect_err("must reject"); + assert!( + format!("{err:?}").contains("expected 12"), + "unexpected: {err:?}" + ); + } + + /// The partition is chosen from the index's schema, never from the item, which + /// is the invariant that makes the sentinel safe. Asserted as the property + /// rather than against one example: the previous version compared the sentinel + /// to `pk_to_text(S("all"))` only, and passed unchanged when the sentinel was + /// weakened to the ordinary string `"unscoped"`. + #[test] + fn the_partition_comes_from_the_index_schema_not_the_item() { + // No HASH element declared: the sentinel, whatever the item holds. + assert_eq!(partition_value(None).unwrap(), UNSCOPED_PARTITION); + + // A HASH element declared: the item's value, verbatim for S. + for value in ["all", "unscoped", UNSCOPED_PARTITION, ""] { + let scoped = + partition_value(Some(("pk", &AttributeValue::S(value.to_owned())))).unwrap(); + assert_eq!( + scoped, value, + "a scoped partition must be the attribute value itself" + ); + } + } + + /// The sentinel keeps a leading NUL as defence in depth. Not what correctness + /// rests on (see the constant's documentation), but weakening it to an ordinary + /// string should break a test rather than pass silently. + #[test] + fn the_unscoped_sentinel_keeps_its_unusual_prefix() { + assert!( + UNSCOPED_PARTITION.starts_with('\0'), + "sentinel must keep its NUL prefix: {UNSCOPED_PARTITION:?}" + ); + } +} diff --git a/crates/storage-sqlite/src/workers.rs b/crates/storage-sqlite/src/workers.rs index 82776713..505400c1 100644 --- a/crates/storage-sqlite/src/workers.rs +++ b/crates/storage-sqlite/src/workers.rs @@ -362,6 +362,13 @@ pub(crate) async fn gsi_propagation_worker( const MAX_SLEEP: Duration = Duration::from_secs(1); // Backoff after an error so a poison row cannot hot-loop the worker. const ERROR_BACKOFF: Duration = Duration::from_secs(1); + // A CREATING vector index with no live backfill task, seen on the previous + // pass. Two consecutive sightings are required before recovery, because the + // catalog row commits before the build task registers itself: a single + // sighting can be a healthy build in that window, but one still unregistered + // a full pass later (at least MAX_SLEEP apart) is dead. Without recovery it + // wedges every queued index write for its table until a restart. + let mut stuck_last_pass: std::collections::HashSet = std::collections::HashSet::new(); loop { // Drain everything currently due. let mut errored = false; @@ -376,6 +383,29 @@ pub(crate) async fn gsi_propagation_worker( } } } + // The stuck-build sweep. Cheap when nothing is CREATING, which is the + // steady state: one indexed catalog read per pass. Only ids seen stuck + // on TWO consecutive passes are recovered, per index: a first sighting + // can be a healthy build inside its commit-to-register window, and a + // stuck sibling must not drag it into a rebuild. + match engine.stuck_vector_build_candidates().await { + Ok(candidates) => { + let confirmed: std::collections::HashSet = candidates + .iter() + .filter(|id| stuck_last_pass.contains(*id)) + .cloned() + .collect(); + if !confirmed.is_empty() + && let Err(e) = engine.recover_stuck_vector_builds(&confirmed).await + { + tracing::error!("GSI propagation worker: stuck-build recovery failed: {e}"); + } + stuck_last_pass = candidates.into_iter().collect(); + } + Err(e) => { + tracing::debug!("GSI worker: stuck-build probe error: {e}"); + } + } // Sleep until the next row is due (or a write wakes us early). let wait = if errored { ERROR_BACKOFF @@ -428,7 +458,8 @@ async fn next_ready_delay(engine: &SqliteEngine) -> Result, Sto /// Parse a claimed `gsi_pending` row and apply its index update within `tx`. /// Any error (malformed context or a non-recoverable apply failure) is returned /// to the caller, which drops the row rather than stalling the queue. A -/// dropped-index race is handled inside `apply_claimed_row` (returns `Ok`). +/// dropped-index race is handled inside the apply, for both index kinds, and +/// returns `Ok`. async fn apply_pending_row( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, old_json: &Option, @@ -445,9 +476,10 @@ async fn apply_pending_row( .map(serde_json::from_str) .transpose() .map_err(|e| StorageError::Internal(e.to_string()))?; - let context: crate::data::GsiApplyContext = + // One queue carries GSI and vector work; the context's shape says which. + let context: crate::data::PendingApplyContext = serde_json::from_str(ctx_json).map_err(|e| StorageError::Internal(e.to_string()))?; - crate::data::apply_claimed_row(tx, old.as_ref(), new.as_ref(), &context).await + crate::data::apply_pending_context(tx, old.as_ref(), new.as_ref(), &context).await } /// Claim and apply one batch of due `gsi_pending` rows in a single transaction @@ -463,18 +495,45 @@ async fn process_gsi_batch(engine: &SqliteEngine) -> Result .map_err(|e| StorageError::Internal(e.to_string()))?; let now = crate::sqlite_util::format_timestamp(time::OffsetDateTime::now_utc()); - // Drain in `id` order; per-partition `ready_at` is monotonic, so this - // preserves per-key FIFO. Each row is self-describing via `index_context`. - let rows: Vec<(Option, Option, String)> = sqlx::query_as( + // Claim the oldest due rows, then apply them in `id` order. Per-partition + // `ready_at` is monotonic, so `id` order is write order and applying in it + // preserves per-key FIFO across both index kinds. + // + // Rows for a table with a vector index still in CREATING are NOT claimed, so + // writes that land during a backfill accumulate and are applied only once the + // index goes ACTIVE. Without that hold, the backfill's snapshot could be written + // AFTER a newer queued write had already been applied, and because each apply + // replaces the row wholesale the index would converge on the stale generation + // until the next write to that key. + // + // The hold is deliberately per TABLE rather than per index, even though only the + // building index is at risk. Holding just the vector rows would let a GSI row and + // a vector row for the same item be applied out of order relative to each other, + // which is the cross-kind FIFO property this queue is documented to provide. + // + // The sort below is load-bearing and is NOT redundant with the `ORDER BY id` + // in the subselect. That clause only chooses WHICH rows the `LIMIT` takes; + // SQLite defines the order of `RETURNING` output as undefined, and it + // demonstrably ignores the subselect's ordering (a `DESC` subselect still + // returns ascending). Relying on it left per-key FIFO resting on an + // implementation detail: two writes to one item claimed in the same batch + // could be applied newest-first, and because each apply overwrites the row + // wholesale, the earlier write would win and the later one be lost. + let mut rows: Vec<(i64, Option, Option, String)> = sqlx::query_as( "DELETE FROM gsi_pending WHERE id IN ( \ - SELECT id FROM gsi_pending WHERE ready_at <= ? ORDER BY id LIMIT ? \ - ) RETURNING old_item, new_item, index_context", + SELECT id FROM gsi_pending WHERE ready_at <= ? \ + AND table_id NOT IN ( \ + SELECT table_id FROM vector_indexes WHERE index_status = 'CREATING' \ + ) \ + ORDER BY id LIMIT ? \ + ) RETURNING id, old_item, new_item, index_context", ) .bind(&now) .bind(GSI_BATCH) .fetch_all(&mut *tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; + rows.sort_unstable_by_key(|(id, ..)| *id); if rows.is_empty() { tx.commit() @@ -484,7 +543,7 @@ async fn process_gsi_batch(engine: &SqliteEngine) -> Result } let count = rows.len(); - for (old_json, new_json, ctx_json) in &rows { + for (id, old_json, new_json, ctx_json) in &rows { // Guard each row with a SAVEPOINT. A row that cannot be applied (bad // context, or a persistent apply error) is undone and DROPPED — it was // already removed from `gsi_pending` by the batch DELETE above, so the @@ -505,7 +564,7 @@ async fn process_gsi_batch(engine: &SqliteEngine) -> Result .map_err(|e| StorageError::Internal(e.to_string()))?; } Err(e) => { - tracing::error!("GSI worker: dropping unprocessable gsi_pending row: {e}"); + tracing::error!("GSI worker: dropping unprocessable gsi_pending row {id}: {e}"); sqlx::query("ROLLBACK TO gsi_row") .execute(&mut *tx) .await @@ -524,25 +583,45 @@ async fn process_gsi_batch(engine: &SqliteEngine) -> Result Ok(count) } -/// Refresh the cached GSI propagation delay from the `gsi_propagation_delay_ms` -/// runtime setting, so changes take effect without restart. -pub(crate) async fn poll_gsi_delay( +/// Read the propagation-delay setting through the settings store, preferring the +/// canonical key and falling back to the pre-rename one. +/// +/// The raw-SQL read on the write path uses one query to express the same preference; +/// this path goes through the `SettingsStore` trait, which fetches by exact key, so the +/// fallback is two lookups instead. It runs once per poll interval, not per write. +async fn read_index_propagation_delay( + settings: &S, +) -> extenddb_storage::management_store::OpResult> { + if let Some(v) = settings + .get_setting(extenddb_core::settings_keys::INDEX_PROPAGATION_DELAY_MS) + .await? + { + return Ok(Some(v)); + } + settings + .get_setting(extenddb_core::settings_keys::LEGACY_GSI_PROPAGATION_DELAY_MS) + .await +} + +/// Refresh the cached secondary-index propagation delay from the +/// `index_propagation_delay_ms` runtime setting, so changes take effect without a +/// restart. Falls back to the pre-rename key for a catalog created before it. +pub(crate) async fn poll_index_propagation_delay( settings: Arc, - gsi_default: Arc, + index_delay_cache: Arc, token: CancellationToken, ) { use std::sync::atomic::Ordering; const POLL: Duration = Duration::from_secs(30); while sleep_or_shutdown(&token, POLL).await { - match settings.get_setting("gsi_propagation_delay_ms").await { + match read_index_propagation_delay(settings.as_ref()).await { Ok(Some(v)) => { if let Ok(ms) = v.parse::() { - gsi_default.store(ms, Ordering::Relaxed); + index_delay_cache.store(ms, Ordering::Relaxed); } } - Ok(None) => { - gsi_default.store(crate::DEFAULT_GSI_PROPAGATION_DELAY_MS, Ordering::Relaxed) - } + Ok(None) => index_delay_cache + .store(crate::DEFAULT_INDEX_PROPAGATION_DELAY_MS, Ordering::Relaxed), Err(e) => tracing::debug!("poll_gsi_delay: {e:?}"), } } @@ -628,3 +707,474 @@ mod poison_row_tests { assert_eq!(remaining.0, 0, "both poison rows dropped"); } } + +#[cfg(test)] +mod vector_propagation_tests { + use super::process_gsi_batch; + use crate::SqliteEngine; + use extenddb_core::types::{ + AttributeDefinition, Item, KeySchemaElement, KeyType, ScalarAttributeType, + }; + use serde_json::json; + + const INDEX_ID: &str = "vidx-async"; + + fn base_ks() -> Vec { + vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }] + } + + fn base_ad() -> Vec { + vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }] + } + + /// A real table with one unscoped vector index and its data table, built + /// through the engine rather than by hand so the catalog rows and the data + /// table are shaped exactly as production makes them. + async fn table_with_vector_index() -> (SqliteEngine, String) { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 409_600) + .await + .expect("engine"); + crate::schema::apply(&engine.pool).await.expect("schema"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind("000000000000") + .execute(&engine.pool) + .await + .expect("account"); + let input: extenddb_core::types::CreateTableInput = serde_json::from_value(json!({ + "TableName": "t", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("input"); + engine + .create_table_impl("000000000000", input) + .await + .expect("create table"); + let (table_id,): (String,) = + sqlx::query_as("SELECT table_id FROM tables WHERE table_name = 't'") + .fetch_one(&engine.pool) + .await + .expect("table_id"); + + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status) \ + VALUES (?, ?, 'vidx', 2, 'COSINE', ?, ?, 'ACTIVE')", + ) + .bind(&table_id) + .bind(INDEX_ID) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert vector index"); + + let mut tx = engine.pool.begin_with("BEGIN IMMEDIATE").await.expect("tx"); + SqliteEngine::create_vector_data_table( + &mut tx, + &table_id, + INDEX_ID, + &base_ks(), + &base_ad(), + ) + .await + .expect("create vector data table"); + tx.commit().await.expect("commit ddl"); + (engine, table_id) + } + + fn item(pk: &str, generation: i32) -> Item { + serde_json::from_value(json!({ + "pk": {"S": pk}, + "gen": {"N": generation.to_string()}, + "emb": {"L": [{"N": "1"}, {"N": "0"}]}, + })) + .expect("item") + } + + /// Drive one write's vector maintenance the way a write path does, committing + /// it. Returns the number of pending rows enqueued. + async fn write( + engine: &SqliteEngine, + table_id: &str, + old: Option<&Item>, + new: Option<&Item>, + delay_ms: u64, + ) -> usize { + let mut tx = engine.pool.begin_with("BEGIN IMMEDIATE").await.expect("tx"); + let n = crate::data::vector_index::maintain_vector_indexes( + &mut tx, + table_id, + &base_ks(), + &base_ad(), + old, + new, + delay_ms, + ) + .await + .expect("maintain"); + tx.commit().await.expect("commit write"); + n + } + + async fn indexed_rows(engine: &SqliteEngine, table_id: &str) -> Vec { + let vec_table = crate::data::vector_table_name(table_id, INDEX_ID); + sqlx::query_as::<_, (String,)>(&format!( + "SELECT item_data FROM {vec_table} ORDER BY base_pk" + )) + .fetch_all(&engine.pool) + .await + .expect("read vector rows") + .into_iter() + .map(|(json,)| json) + .collect() + } + + async fn queue_depth(engine: &SqliteEngine) -> i64 { + sqlx::query_as::<_, (i64,)>("SELECT COUNT(*) FROM gsi_pending") + .fetch_one(&engine.pool) + .await + .expect("count") + .0 + } + + /// Mark every queued row due, so a drain can be tested without sleeping for the + /// propagation delay. Deliberately not a sleep: a test that waits out a real + /// delay is slow and still races a loaded machine. + async fn make_all_rows_due(engine: &SqliteEngine) { + sqlx::query("UPDATE gsi_pending SET ready_at = '2000-01-01T00:00:00.000Z'") + .execute(&engine.pool) + .await + .expect("backdate"); + } + + /// The whole point of the change: with a propagation delay, a write does not + /// touch the vector index at all. It queues, stays unapplied until it is due, + /// and only the worker applies it. + /// + /// All four claims are asserted in one test on purpose. Split apart, each half + /// passes for the wrong reason — "not indexed yet" is indistinguishable from + /// "never indexed", and "indexed after a drain" is indistinguishable from + /// "indexed by the write". + #[tokio::test] + async fn a_write_reaches_the_vector_index_only_through_the_worker() { + let (engine, table_id) = table_with_vector_index().await; + + let enqueued = write(&engine, &table_id, None, Some(&item("a", 1)), 60_000).await; + assert_eq!(enqueued, 1, "the write must enqueue exactly one row"); + assert!( + indexed_rows(&engine, &table_id).await.is_empty(), + "an async write must not index inline" + ); + assert_eq!(queue_depth(&engine).await, 1, "the row must be durable"); + + // Not yet due: draining must leave it alone rather than apply it early. + assert_eq!( + process_gsi_batch(&engine).await.expect("drain"), + 0, + "a row that is not due must not be claimed" + ); + assert!( + indexed_rows(&engine, &table_id).await.is_empty(), + "the delay must be honoured, not merely recorded" + ); + + make_all_rows_due(&engine).await; + assert_eq!( + process_gsi_batch(&engine).await.expect("drain"), + 1, + "the due row must be claimed" + ); + let rows = indexed_rows(&engine, &table_id).await; + assert_eq!(rows.len(), 1, "the worker must index the item"); + assert!( + rows[0].contains("\"gen\""), + "payload projected: {}", + rows[0] + ); + assert_eq!(queue_depth(&engine).await, 0, "the queue must drain"); + } + + /// The other half of the branch. A zero delay is the documented way to ask for + /// synchronous maintenance, and it must apply in the caller's transaction and + /// enqueue nothing at all. + #[tokio::test] + async fn a_zero_delay_write_is_applied_inline_and_queues_nothing() { + let (engine, table_id) = table_with_vector_index().await; + + let enqueued = write(&engine, &table_id, None, Some(&item("a", 1)), 0).await; + assert_eq!(enqueued, 0, "a synchronous write has nothing to enqueue"); + assert_eq!( + indexed_rows(&engine, &table_id).await.len(), + 1, + "a zero-delay write must index in its own transaction" + ); + assert_eq!(queue_depth(&engine).await, 0, "and must not queue"); + } + + /// Two writes to one item must reach the index in write order, whatever jitter + /// each drew. The guarantee comes from the queue: both rows hash to the base + /// key's partition and `ready_at` is clamped monotonic within it, so draining in + /// `id` order cannot invert them. + /// + /// Asserted on the surviving payload rather than on timestamps, because the + /// property that matters is which write won, not what the clamp computed. + #[tokio::test] + async fn successive_writes_to_one_item_are_applied_in_write_order() { + let (engine, table_id) = table_with_vector_index().await; + let first = item("a", 1); + let second = item("a", 2); + + write(&engine, &table_id, None, Some(&first), 60_000).await; + write(&engine, &table_id, Some(&first), Some(&second), 60_000).await; + + let partitions: Vec<(i64,)> = + sqlx::query_as("SELECT DISTINCT worker_partition FROM gsi_pending") + .fetch_all(&engine.pool) + .await + .expect("partitions"); + assert_eq!( + partitions.len(), + 1, + "both writes to one base key must share a partition, or ordering is not enforced" + ); + + make_all_rows_due(&engine).await; + assert_eq!(process_gsi_batch(&engine).await.expect("drain"), 2); + + let rows = indexed_rows(&engine, &table_id).await; + assert_eq!(rows.len(), 1, "one base item indexes to one row"); + assert!( + rows[0].contains("\"2\""), + "the later write must win, found: {}", + rows[0] + ); + } + + /// An item that loses its vector attribute must leave the index. This is why the + /// enqueue is unconditional: the row carries no vector to insert, so a write path + /// that skipped queueing "because there is nothing to index" would leave the + /// stale row searchable forever. + #[tokio::test] + async fn an_item_that_loses_its_vector_is_removed_from_the_index() { + let (engine, table_id) = table_with_vector_index().await; + let with_vector = item("a", 1); + let without_vector: Item = + serde_json::from_value(json!({"pk": {"S": "a"}, "gen": {"N": "2"}})).expect("item"); + + write(&engine, &table_id, None, Some(&with_vector), 60_000).await; + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("drain first"); + assert_eq!(indexed_rows(&engine, &table_id).await.len(), 1); + + let enqueued = write( + &engine, + &table_id, + Some(&with_vector), + Some(&without_vector), + 60_000, + ) + .await; + assert_eq!( + enqueued, 1, + "a write with no vector must still enqueue: the removal is the work" + ); + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("drain second"); + assert!( + indexed_rows(&engine, &table_id).await.is_empty(), + "the row must be removed once the item stops carrying a vector" + ); + } + + /// A delete removes the row, driven only by the old item. + #[tokio::test] + async fn a_delete_removes_the_indexed_row() { + let (engine, table_id) = table_with_vector_index().await; + let existing = item("a", 1); + + write(&engine, &table_id, None, Some(&existing), 60_000).await; + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("drain put"); + assert_eq!(indexed_rows(&engine, &table_id).await.len(), 1); + + write(&engine, &table_id, Some(&existing), None, 60_000).await; + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("drain delete"); + assert!(indexed_rows(&engine, &table_id).await.is_empty()); + } + + /// The index can be dropped while a row is in flight. The batch must still + /// commit and a sibling row in the same batch must still land: the batch shares + /// one transaction, so an unhandled failure would take the sibling down with it. + /// + /// Deliberately not claimed as a test of the missing-table tolerance in + /// `apply_vector_context`. Removing that tolerance leaves this test passing, + /// because the per-row savepoint rolls the row back and drops it, reaching the + /// same end state. The difference between the two paths is an ERROR log line, not + /// data, and this test cannot see it. + #[tokio::test] + async fn a_row_whose_index_was_dropped_does_not_take_the_batch_down() { + let (engine, table_id) = table_with_vector_index().await; + write(&engine, &table_id, None, Some(&item("a", 1)), 60_000).await; + + // A second table and index, standing in for unrelated work in the same batch. + let survivor_index = "vidx-survivor"; + sqlx::query( + "INSERT INTO vector_indexes \ + (table_id, index_id, index_name, dimensions, distance_function, vector_attribute, \ + projection, index_status) \ + VALUES (?, ?, 'vidx2', 2, 'COSINE', ?, ?, 'ACTIVE')", + ) + .bind(&table_id) + .bind(survivor_index) + .bind(json!({"AttributeName": "emb"}).to_string()) + .bind(json!({"ProjectionType": "ALL"}).to_string()) + .execute(&engine.pool) + .await + .expect("insert survivor index"); + let mut tx = engine.pool.begin_with("BEGIN IMMEDIATE").await.expect("tx"); + SqliteEngine::create_vector_data_table( + &mut tx, + &table_id, + survivor_index, + &base_ks(), + &base_ad(), + ) + .await + .expect("create survivor table"); + tx.commit().await.expect("commit ddl"); + + // This write enqueues for both indexes; then the first index's table is + // dropped out from under its queued row. + write(&engine, &table_id, None, Some(&item("b", 7)), 60_000).await; + let dropped = crate::data::vector_table_name(&table_id, INDEX_ID); + sqlx::query(&format!("DROP TABLE {dropped}")) + .execute(&engine.pool) + .await + .expect("drop index data table"); + + make_all_rows_due(&engine).await; + let processed = process_gsi_batch(&engine).await.expect("drain"); + assert_eq!(processed, 3, "every claimed row must be consumed"); + assert_eq!(queue_depth(&engine).await, 0, "no row may be left behind"); + + let survivor = crate::data::vector_table_name(&table_id, survivor_index); + let (rows,): (i64,) = sqlx::query_as(&format!("SELECT COUNT(*) FROM {survivor}")) + .fetch_one(&engine.pool) + .await + .expect("count survivor"); + assert_eq!( + rows, 1, + "the sibling index must still be maintained when another index vanishes" + ); + } + + /// Applying the same claimed context twice must leave the same single row. + /// + /// The batch is at-least-once: claim and apply share a transaction, so a crash + /// mid-apply rolls back and the row is claimed again. That story is only safe if + /// an apply is idempotent, which is asserted here directly rather than by + /// implication. + /// + /// Being exact about its strength: this is a structural regression guard, not a + /// discriminating test. Its assertions cannot fail today without a schema change, + /// because the vector row's primary key is the base item key, so a replay can only + /// ever overwrite. Measured, not assumed: making the delete conditional on + /// `old_item` leaves this test PASSING, since the replayed insert then collides on + /// the primary key, the row is dropped by the savepoint, and the end state is the + /// same single row. The test below is the one that catches that refactor. + #[tokio::test] + async fn applying_the_same_row_twice_is_idempotent() { + let (engine, table_id) = table_with_vector_index().await; + write(&engine, &table_id, None, Some(&item("a", 1)), 60_000).await; + + // Capture the claimed row's payload, then replay it a second time. + let claimed: (Option, Option, String) = + sqlx::query_as("SELECT old_item, new_item, index_context FROM gsi_pending LIMIT 1") + .fetch_one(&engine.pool) + .await + .expect("read the queued row"); + + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("first apply"); + let after_first = indexed_rows(&engine, &table_id).await; + assert_eq!(after_first.len(), 1, "first apply indexes the item"); + + // Re-enqueue the identical context, exactly as a rolled-back batch would + // leave it, and drain again. + sqlx::query( + "INSERT INTO gsi_pending \ + (table_id, worker_partition, old_item, new_item, index_context, ready_at) \ + VALUES (?, 0, ?, ?, ?, '2000-01-01T00:00:00.000Z')", + ) + .bind(&table_id) + .bind(&claimed.0) + .bind(&claimed.1) + .bind(&claimed.2) + .execute(&engine.pool) + .await + .expect("replay the row"); + process_gsi_batch(&engine).await.expect("second apply"); + + let after_second = indexed_rows(&engine, &table_id).await; + assert_eq!( + after_second.len(), + 1, + "a replayed apply must not duplicate the row" + ); + assert_eq!( + after_first, after_second, + "a replayed apply must reach an identical end state" + ); + } + + /// A write that carries NO old image must still replace the indexed row. + /// + /// This is a real production path, not a contrived one: `put_item` only reads the + /// old image when a condition, a stream, `ReturnValues`, or a GSI needs it, so a + /// table whose only index is a vector index writes with `old_item = None` on the + /// common path. The apply therefore cannot rely on the old image to find the row + /// to displace, and keys the delete off `old_item.or(new_item)` because the base + /// key is immutable. + /// + /// This is the test that guards the invariant behind the plain `INSERT` in + /// `insert_vector_row`. Making the delete conditional on `old_item` fails it: the + /// second insert collides on the primary key, the savepoint drops the row, and the + /// STALE payload survives. That failure is silent in production, which is why it + /// is worth a dedicated test rather than leaving it to the idempotency case above. + #[tokio::test] + async fn a_write_with_no_old_image_still_replaces_the_indexed_row() { + let (engine, table_id) = table_with_vector_index().await; + + write(&engine, &table_id, None, Some(&item("a", 1)), 60_000).await; + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("first apply"); + assert!( + indexed_rows(&engine, &table_id).await[0].contains("\"1\""), + "precondition: the first generation is indexed" + ); + + // The second write supplies no old image, exactly as put_item does when + // nothing else needs it. + write(&engine, &table_id, None, Some(&item("a", 2)), 60_000).await; + make_all_rows_due(&engine).await; + process_gsi_batch(&engine).await.expect("second apply"); + + let rows = indexed_rows(&engine, &table_id).await; + assert_eq!(rows.len(), 1, "one base item indexes to one row: {rows:?}"); + assert!( + rows[0].contains("\"2\""), + "the newer write must replace the row even with no old image, found: {}", + rows[0] + ); + } +} diff --git a/devtools/run-tests b/devtools/run-tests index b11ee976..819bf469 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -321,8 +321,8 @@ if $NEEDS_INTEGRATION && [[ "$TARGET" != "real-dynamodb" ]]; then if "$BINARY" settings --config "$CONFIG_FOR_SETTINGS" set control_plane_delay_seconds 0.05 2>/dev/null; then echo " ✓ control_plane_delay_seconds set to 0.05" fi - if "$BINARY" settings --config "$CONFIG_FOR_SETTINGS" set gsi_propagation_delay_ms 0 2>/dev/null; then - echo " ✓ gsi_propagation_delay_ms set to 0" + if "$BINARY" settings --config "$CONFIG_FOR_SETTINGS" set index_propagation_delay_ms 0 2>/dev/null; then + echo " ✓ index_propagation_delay_ms set to 0" fi if "$BINARY" settings --config "$CONFIG_FOR_SETTINGS" set throttling_enabled true 2>/dev/null; then echo " ✓ throttling_enabled set to true" diff --git a/docs/adr/0004-vector-search-exact-scan.md b/docs/adr/0004-vector-search-exact-scan.md new file mode 100644 index 00000000..63e33e77 --- /dev/null +++ b/docs/adr/0004-vector-search-exact-scan.md @@ -0,0 +1,137 @@ +# ADR-0004: Vector search is an exact scan over one row per vector + +- Status: Proposed +- Date: 2026-08-06 +- Deciders: @LeeroyHannigan + +## Context + +DynamoDB vector indexes let a table be searched by vector similarity. Supporting +them means two separate decisions: how a backend finds nearest neighbours, and how +the vectors are stored. + +Constraints that narrow the field, all of them properties of this project rather +than preferences: + +- **The SQLite deployment target is a statically linked musl binary in a + `FROM scratch` container** (`docs/design/01-requirements.md`, REQ-DEPLOY-006). A + static binary cannot `dlopen`, so a runtime-loadable SQLite extension is + unusable there. +- **The project is Apache-2.0.** A dependency under a non-OSI source-available + licence cannot be carried. +- **The index must live in the database file.** The SQLite backend has single-file + persistent and `:memory:` ephemeral modes and an existing backup and restore + path; an index in a sidecar file would not be captured by any of them. +- **Vector search is partition-scoped.** A search supplies an equality on the + index's `HASH` element, so the candidate set is normally one partition rather + than the table. The `HASH` element is optional, though, so an unscoped index + spans everything. +- **Three distance functions are required**: `COSINE`, `EUCLIDEAN` and + `DOT_PRODUCT`, because the service supports all three. + +## Options Considered + +1. **`sqlite-vec`** (asg017) — dual Apache-2.0/MIT, index in in-database shadow + tables, native partition-key columns that pre-filter. The closest fit. +2. **`sqlite-vss`** (asg017) — Faiss-backed, real ANN. +3. **`sqliteai/sqlite-vector`** — SIMD, works on existing table schemas. +4. **`vectorlite`** — real HNSW via hnswlib. +5. **`usearch`'s SQLite extension** — from the usearch project. +6. **`sqlite-muninn`** — HNSW, loadable extension. +7. **libSQL / Turso native vector search** — LM-DiskANN built into a SQLite fork. +8. **A Rust index crate persisted in the database** — `hnsw_rs`, + `instant-distance`, `usearch` as a library, `arroy`, `hannoy`. +9. **An exact scan written in Rust**, with vectors stored as ordinary rows. + +## Decision + +Exact scan in Rust, with one row per vector in a per-index data table. No +third-party vector dependency. + +## Rationale + +- **Every extension is eliminated by a hard constraint, not by preference.** + `sqlite-vss` is deprecated by its own maintainer, ships only as a loadable object + with BLAS/LAPACK/OpenMP, and supports no filtering on a KNN query. + `sqliteai/sqlite-vector` is Elastic License 2.0, which is not OSI-approved and + carries a managed-service restriction, and it has no Rust binding. + `vectorlite` holds its HNSW index **in memory with a sidecar `.bin` file**, does + not support transactions, and filters by rowid only. `usearch`'s SQLite surface + is distance functions with no index at all; its HNSW exists only in the library. + `sqlite-muninn` is four stars with no tagged release. libSQL has a genuine + in-file LM-DiskANN index but would require replacing sqlx entirely, offers no + dot-product metric, cannot pre-filter a partition (its `vector_top_k` takes only + index, vector and k, so scoping is a post-filter that can under-return), is in + maintenance mode, and has an open index-corruption-on-delete bug. + +- **The one viable extension would not have bought an index.** `sqlite-vec` is + brute force in every stable release; its ANN work exists only in a `v0.1.10` + alpha line that had a DELETE data-loss bug. Adopting it meant taking on a C + toolchain, an unverified static-musl build and a pre-1.0 dependency in exchange + for a constant-factor speedup on the same asymptotic scan. It also has no + dot-product metric, and dot-product ranking cannot be recovered from an L2 + top-k, because the ordering depends on each candidate's own norm. + +- **Row per vector, because a packed blob makes writes O(partition).** Measured: + a contiguous blob per partition reads 2 to 4x faster at 256 dimensions and 1.3 + to 2.5x at 1024, and is *slower* at 4096. But inserting or deleting one vector + rewrites the whole blob, which is 390 MB for a 100k-vector partition at 1024 + dimensions, and vector indexes are maintained on every write touching an indexed + attribute. No read gain justifies that. + +- **Row per vector also inherits the existing machinery.** It streams, so a scan + allocates nothing proportional to the partition; it has no per-partition blob + ceiling; and it follows the per-index data table pattern the GSI and LSI paths + already use, so it reuses their transaction, backup and cleanup behaviour. + +## Measured evidence + +Single core, warm page cache, release build, real SQLite via sqlx with WAL. +Harness and raw output recorded with the benchmark; summary: + +| dimensions | vectors/sec | inside 10 ms | inside 100 ms | +|---|---|---|---| +| 256 | 213k to 334k | ~2,100 to 3,300 | ~21k to 33k | +| 1024 | 94k to 103k | ~940 to 1,030 | ~9.4k to 10.3k | +| 4096 | 39k to 43k | ~390 to 430 | ~3.9k to 4.3k | + +Two findings from the benchmark matter more than the headline numbers. + +**An earlier modelled estimate was wrong by 10 to 30x.** It assumed `N x D x 4` +bytes streaming at ~15 GB/s and predicted ~36,000 vectors at 1024 dimensions +inside 10 ms. Measured effective throughput is 0.2 to 1.2 GB/s, so the scan is not +memory-bandwidth bound and the model was unusable. Any figure derived from it is +void. + +**The cost is the SQLite read path, not the arithmetic.** A variant that +reinterpreted the stored blob as `&[f32]`, so the dot product could vectorise +instead of decoding each element, measured **no faster** (1024d/100k: 766 ms +packed versus 850 ms zero-copy). The time goes on overflow-page assembly and the +copy sqlx must make. Optimising the distance loop would be wasted effort until +that changes. + +## Consequences + +**Easier.** No third-party vector dependency, so no licence question, no C or C++ +toolchain, no static-musl uncertainty, and nothing pre-1.0 in the dependency tree. +This was the only option with zero unconfirmed constraints, precisely because +there is nothing external to be uncertain about. All three distance functions cost +the same to support: they are the same loop over the same bytes. + +**Harder.** Search cost is linear in partition size, and the crossover is roughly +an order of magnitude lower than the earlier estimate claimed. A partition stays +inside a 10 ms budget up to about 1,000 vectors at 1024 dimensions and 100 ms up +to about 10,000. + +**The trigger for revisiting, stated as a measurement rather than a judgement.** A +vector index declared with no `HASH` element searches the whole table, so its +corpus is not bounded by a partition. Past roughly the figures above, that leaves +an interactive budget. If that case becomes real, the answer is an index crate +persisted inside the database file, and on the research the candidates are +`usearch` (verified `save_to_buffer`, `remove` and `filtered_search`, but a C++17 +core to check against static musl) with `hnsw_rs` as the pure-Rust fallback. Not +now, and gated on a measured need rather than on anticipation. + +**Not measured.** Cold cache, concurrency beyond one core, and real embedding +distributions. Cold reads can only be worse; the other two do not change the +layout decision. diff --git a/docs/adr/README.md b/docs/adr/README.md index d63c9828..11afd8d3 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -33,3 +33,4 @@ decision, write a new ADR. | [0001](0001-documentation-format.md) | Documentation format — Markdown over LaTeX | Accepted | | [0002](0002-sql-injection-defense.md) | SQL injection defense | Accepted | | [0003](0003-catalog-migration-mechanism.md) | Adopt sqlx::migrate for PostgreSQL catalog and data schema migrations | Proposed | +| [0004](0004-vector-search-exact-scan.md) | Vector search is an exact scan over one row per vector | Proposed | diff --git a/docs/design/04-component-storage.md b/docs/design/04-component-storage.md index c721cc47..102fa8cf 100755 --- a/docs/design/04-component-storage.md +++ b/docs/design/04-component-storage.md @@ -568,7 +568,7 @@ propagation delay. LSI updates are always synchronous. **Implementation:** - Each GSI has an optional `propagation_delay_ms` column in the `indexes` table - If `propagation_delay_ms` is `NULL` or negative, the system default is used - (default: 10ms, configurable via `gsi_propagation_delay_ms` setting) + (default: 10ms, configurable via `index_propagation_delay_ms` setting) - If `propagation_delay_ms` is `0`, the GSI is updated synchronously in the same transaction as the base table write - If `propagation_delay_ms` is positive, the GSI update is enqueued and applied diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 3922260d..f0c07d30 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -63,7 +63,8 @@ adaptation when switching between ExtendDB and the real service. | Area | DynamoDB | ExtendDB | |------|----------|------| -| GSI update propagation | Eventually consistent (milliseconds to seconds) | Per-GSI propagation delay. System default: `gsi_propagation_delay_ms` setting (default 10ms). Each GSI can override with its own `propagation_delay_ms` (stored in catalog). A value of 0 means synchronous (future sync GSI feature). | +| GSI update propagation | Eventually consistent (milliseconds to seconds) | Per-GSI propagation delay. System default: `index_propagation_delay_ms` setting (default 10ms). Each GSI can override with its own `propagation_delay_ms` (stored in catalog). A value of 0 means synchronous (future sync GSI feature). | +| Vector index update propagation | Eventually consistent, the same model as a GSI | Matches DynamoDB. Maintenance is queued on the same propagation queue as async GSIs, so a search immediately after a write may not see it. Governed by the same `index_propagation_delay_ms` setting; unlike a GSI there is no per-index override. A value of 0 applies maintenance synchronously in the write's own transaction, which is stricter than the service and exists so a test can assert steady state without waiting. | | Multi-part base table keys | Not supported | Preview extension (opt-in via `enable_multipart_keys` setting). Standard single/composite keys work identically. | ## Capacity and Throttling @@ -92,7 +93,7 @@ ExtendDB exposes runtime settings that have no DynamoDB equivalent: | Setting | Default | Description | |---------|---------|-------------| | `control_plane_delay_seconds` | 5 | Simulated delay for table state transitions (CREATING → ACTIVE, DELETING → removed) | -| `gsi_propagation_delay_ms` | 10 | System-wide default GSI propagation delay (milliseconds). Per-GSI overrides stored in catalog. 0 = synchronous. | +| `index_propagation_delay_ms` | 10 | System-wide default propagation delay for asynchronous secondary-index maintenance (milliseconds), covering GSIs and vector indexes alike. Per-GSI overrides stored in catalog; vector indexes have no per-index override. 0 = synchronous. Accepts the pre-rename name `gsi_propagation_delay_ms` as a deprecated alias, and a catalog created before the rename keeps honouring a value stored under it. | | `throttling_enabled` | `true` | Enable provisioned capacity throttling (token bucket per table/partition) | | `enable_multipart_keys` | `false` | Enable multi-part base table key extension | | `log_level` | `info` | Runtime log level (trace, debug, info, warn, error) | diff --git a/docs/getting-started.md b/docs/getting-started.md index 4ab53bf5..61d66630 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -290,11 +290,11 @@ GSI updates are applied asynchronously with a configurable delay, simulating rea ```bash # Set system-wide default to 0 for synchronous GSI updates (fast tests) ./target/release/extenddb settings --config extenddb.toml set \ - gsi_propagation_delay_ms 0 + index_propagation_delay_ms 0 # Set to 50ms for more realistic eventual consistency ./target/release/extenddb settings --config extenddb.toml set \ - gsi_propagation_delay_ms 50 + index_propagation_delay_ms 50 ``` ### Throttling @@ -1227,7 +1227,7 @@ extenddb supports running external test suites (e.g., Java/JUnit, Python/pytest) # External suites expect synchronous GSI behavior (matching real DynamoDB's # typical sub-millisecond propagation). The async GSI path is tested # separately by the extenddb-specific test_gsi_async.py suite. -./target/release/extenddb settings --config extenddb.toml set gsi_propagation_delay_ms 0 +./target/release/extenddb settings --config extenddb.toml set index_propagation_delay_ms 0 # Run all registered suites python3 devtools/run-external-tests diff --git a/docs/manuals/02-design-guide.md b/docs/manuals/02-design-guide.md index 20f3cd83..13cd6c9d 100755 --- a/docs/manuals/02-design-guide.md +++ b/docs/manuals/02-design-guide.md @@ -185,7 +185,7 @@ extenddb caches a small set of operational settings in memory to avoid per-reque | Setting | Mechanism | Refresh | Justification | |---------|-----------|---------|---------------| -| `gsi_propagation_delay_ms` | `AtomicU64` | Background poller every 30s | Write-path hot path; briefly-stale value only affects GSI propagation timing | +| `index_propagation_delay_ms` | `AtomicU64` | Background poller every 30s | Write-path hot path; briefly-stale value only affects GSI propagation timing | | `encryption_key` | `Arc` loaded at startup | Never (immutable after `extenddb init`) | Decryption key for access key secrets; generated once, never changes | | `log_level` / `log_destination` | Tracing filter reload | Background poller every 30s | Observability tuning; stale value only delays log level changes | | `throttling_enabled` | `AtomicBool` | Background poller every 30s | Capacity management toggle; briefly-stale is safe | diff --git a/docs/manuals/06-developer-test-guide.md b/docs/manuals/06-developer-test-guide.md index 0433ef90..4e97f24b 100755 --- a/docs/manuals/06-developer-test-guide.md +++ b/docs/manuals/06-developer-test-guide.md @@ -156,7 +156,7 @@ The `run-tests` script automatically: - Provisions test credentials via `devtools/provision-test-credentials` - Creates a Java truststore for external tests (self-signed TLS cert) - Sets `control_plane_delay_seconds` to 0.05 for fast test cycles -- Sets `gsi_propagation_delay_ms` to 0 for immediate GSI updates +- Sets `index_propagation_delay_ms` to 0 for immediate GSI updates - Enables throttling for production-like behavior - Configures import/export paths for file operation tests - Extracts and exports `EXTENDDB_TEST_PG_CONNECTION_STRING` for CLI lifecycle tests diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 44fcf061..77e3e0d1 100755 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -785,9 +785,9 @@ If the health check fails, start extenddb. If it succeeds, check your `--endpoin ### GSI query returns stale data after a write -**Cause:** GSI updates are applied asynchronously with a configurable propagation delay (default 10ms). This matches real DynamoDB's eventually consistent GSI behavior. Each GSI can have its own `propagation_delay_ms` setting; the system-wide default is controlled by the `gsi_propagation_delay_ms` runtime setting. +**Cause:** GSI updates are applied asynchronously with a configurable propagation delay (default 10ms). This matches real DynamoDB's eventually consistent GSI behavior. Each GSI can have its own `propagation_delay_ms` setting; the system-wide default is controlled by the `index_propagation_delay_ms` runtime setting. -**Fix:** This is expected behavior. For tests that query GSIs after writes, poll/retry the GSI query until the expected data appears. To make all GSIs synchronous for testing, set `extenddb settings set gsi_propagation_delay_ms 0`. For production-like testing, keep the default async delay. +**Fix:** This is expected behavior. For tests that query GSIs after writes, poll/retry the GSI query until the expected data appears. To make all GSIs synchronous for testing, set `extenddb settings set index_propagation_delay_ms 0`. For production-like testing, keep the default async delay. ## Connection Pool Exhaustion diff --git a/tests/conftest.py b/tests/conftest.py index eb128248..1d35d88d 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -97,7 +97,7 @@ def wait_for_gsi_items(paginate, expected: int, timeout: float = 15.0): GSIs are eventually consistent: an item written to the base table is not guaranteed to be visible through a secondary index immediately — ExtendDB - applies the configured ``gsi_propagation_delay_ms`` (like real DynamoDB), so + applies the configured ``index_propagation_delay_ms`` (like real DynamoDB), so a read-back through a GSI right after the write can legitimately be short. Tests that write then page a GSI must poll rather than read once. diff --git a/tests/rust/src/main.rs b/tests/rust/src/main.rs index c14d4c69..1e4c016b 100755 --- a/tests/rust/src/main.rs +++ b/tests/rust/src/main.rs @@ -114,6 +114,8 @@ mod update_item_number_validation; #[cfg(test)] mod update_table_billing_validation; #[cfg(test)] +mod vector_index_search; +#[cfg(test)] mod vector_index_unsupported; #[cfg(test)] mod wording_parity_validation; diff --git a/tests/rust/src/vector_index_search.rs b/tests/rust/src/vector_index_search.rs new file mode 100644 index 00000000..cf0bbfb1 --- /dev/null +++ b/tests/rust/src/vector_index_search.rs @@ -0,0 +1,1920 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Wire-level tests for a backend that implements vector indexes. +//! +//! The mirror of `vector_index_unsupported`: that file asserts the refusals a +//! non-participating backend must give, this one asserts the behaviour a +//! participating one must give. Both probe the running backend and skip when it is +//! the wrong kind, so one suite runs everywhere. +//! +//! Hand-built JSON and SigV4 signing for the same reason as the other file: no +//! published `aws-sdk-dynamodb` models vector indexes. + +use crate::vector_index_unsupported::{call, expect_vectors, table_name, vectors_supported}; + +/// Skip guard for this suite, with the anchor that closes the silent-green hole. +/// +/// Both vector suites adapt to whatever the backend reports, so on their own they +/// can never assert *which* backend is under test. If the shipping backend silently +/// lost vector support, this suite would skip all of its assertions and the refusal +/// suite would start passing, and the whole positive contract would evaporate with +/// a green run. `EXTENDDB_EXPECT_VECTORS=1` turns that skip into a failure, so a CI +/// job can state the expectation it is there to check. +async fn skip_unless_supported() -> bool { + let supported = vectors_supported().await; + assert!( + !(!supported && expect_vectors() == Some(true)), + "EXTENDDB_EXPECT_VECTORS=1 but the backend does not support vector \ + indexes, so every assertion in this suite would be skipped: the \ + capability was lost, or the probe is broken" + ); + !supported +} + +/// Create a table with one vector index and wait for it to be usable. +async fn create_vector_table(name: &str, dims: usize, distance: &str, scoped: bool) { + let search_schema = if scoped { + r#""SearchSchema": [{"AttributeName": "tenant", "SearchSchemaElementType": "HASH"}],"# + } else { + "" + }; + let attr_defs = if scoped { + r#"{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "tenant", "AttributeType": "S"}"# + } else { + r#"{"AttributeName": "pk", "AttributeType": "S"}"# + }; + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{attr_defs}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": {dims}, + "DistanceFunction": "{distance}", + "VectorAttribute": {{"AttributeName": "emb"}}, + {search_schema} + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + wait_for_active(name).await; +} + +/// Poll until the table reports ACTIVE. +/// +/// CreateTable returns while the table is still CREATING, and a write against a +/// CREATING table is rejected with ResourceNotFound, so without this every test +/// here fails on its first PutItem for a reason unrelated to what it asserts. +async fn wait_for_active(name: &str) { + for _ in 0..100 { + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + if status == 200 && text.contains(r#""TableStatus":"ACTIVE""#) { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + panic!("table {name} never became ACTIVE"); +} + +fn vector_json(values: &[f32]) -> String { + let parts: Vec = values + .iter() + .map(|v| format!(r#"{{"N": "{v}"}}"#)) + .collect(); + format!("[{}]", parts.join(", ")) +} + +async fn put_vector(table: &str, pk: &str, tenant: Option<&str>, values: &[f32]) { + let tenant_attr = tenant + .map(|t| format!(r#", "tenant": {{"S": "{t}"}}"#)) + .unwrap_or_default(); + let body = format!( + r#"{{ + "TableName": "{table}", + "Item": {{"pk": {{"S": "{pk}"}}, "emb": {{"L": {}}}{tenant_attr}}} + }}"#, + vector_json(values) + ); + let (status, text) = call("PutItem", &body).await; + assert_eq!(status, 200, "PutItem failed: {text}"); +} + +async fn search( + table: &str, + values: &[f32], + top_k: usize, + condition: Option<&str>, +) -> serde_json::Value { + let cond = condition + .map(|c| { + format!( + r#", "SearchConditionExpression": "tenant = :t", "ExpressionAttributeValues": {{":t": {{"S": "{c}"}}}}"# + ) + }) + .unwrap_or_default(); + let body = format!( + r#"{{ + "TableName": "{table}", + "IndexName": "vidx", + "SearchVector": {}, + "TopK": {top_k}{cond} + }}"#, + vector_json(values) + ); + let (status, text) = call("SearchVectors", &body).await; + assert_eq!(status, 200, "SearchVectors failed: {text}"); + serde_json::from_str(&text).expect("search response is JSON") +} + +fn hit_pks(response: &serde_json::Value) -> Vec { + response + .get("SearchResults") + .and_then(|v| v.as_array()) + .unwrap_or_else(|| panic!("no results array in: {response}")) + .iter() + .map(|hit| { + hit.get("Item") + .and_then(|i| i.get("pk")) + .and_then(|p| p.get("S")) + .and_then(|s| s.as_str()) + .unwrap_or_else(|| panic!("hit has no pk: {hit}")) + .to_owned() + }) + .collect() +} + +/// How long to let the index converge before failing. +/// +/// Generous on purpose. The propagation delay itself is milliseconds, but a missed +/// worker wake falls back to a one second backstop, and these tests run in parallel +/// against one server, so the bound has to cover a loaded machine rather than a +/// quiet one. Overshooting costs nothing when propagation is fast, because every +/// helper returns as soon as its condition holds. +const CONVERGE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); +const CONVERGE_POLL: std::time::Duration = std::time::Duration::from_millis(50); + +/// Search until `predicate` accepts the response, then return that response. +/// +/// A vector index is eventually consistent, so a single search after a write proves +/// nothing in either direction: a missing item may simply not have propagated, and +/// an item that should have been removed may not have been removed yet. Every +/// assertion about index contents therefore has to be an assertion about what the +/// index converges to. +/// +/// Polling rather than sleeping is deliberate. A fixed sleep is simultaneously too +/// slow when propagation is immediate and too short when the machine is loaded, +/// which is exactly how index tests become flaky. `what` is folded into the failure +/// message so a timeout says which condition was never reached. +async fn search_until( + table: &str, + values: &[f32], + top_k: usize, + condition: Option<&str>, + what: &str, + predicate: impl Fn(&serde_json::Value) -> bool, +) -> serde_json::Value { + let deadline = std::time::Instant::now() + CONVERGE_TIMEOUT; + let mut last = search(table, values, top_k, condition).await; + loop { + if predicate(&last) { + return last; + } + if std::time::Instant::now() >= deadline { + panic!( + "index never converged: {what}. Last response after {:?}: {last}", + CONVERGE_TIMEOUT + ); + } + tokio::time::sleep(CONVERGE_POLL).await; + last = search(table, values, top_k, condition).await; + } +} + +/// Poll a caller-supplied `SearchVectors` body until `predicate` accepts the parsed +/// response, then return it. +/// +/// For tests whose query the `search` helper cannot express: a different filter +/// attribute, a `ProjectionExpression`, or anything else shaped by hand. Same +/// convergence contract as [`search_until`], and it asserts the status on every +/// attempt so a request that becomes malformed fails loudly instead of timing out. +async fn search_body_until( + body: &str, + what: &str, + predicate: impl Fn(&serde_json::Value) -> bool, +) -> serde_json::Value { + let deadline = std::time::Instant::now() + CONVERGE_TIMEOUT; + loop { + let (status, text) = call("SearchVectors", body).await; + assert_eq!(status, 200, "SearchVectors failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("search response is JSON"); + if predicate(&json) { + return json; + } + assert!( + std::time::Instant::now() < deadline, + "index never converged: {what}. Last response after {CONVERGE_TIMEOUT:?}: {text}" + ); + tokio::time::sleep(CONVERGE_POLL).await; + } +} + +/// Search until the hits are exactly `expected`, in order. +/// +/// Covers presence and absence in one assertion, which matters because they are not +/// separable under eventual consistency: waiting for an item to appear and then +/// asserting a different item is absent would pass whenever the second item is +/// merely late. +async fn search_until_pks( + table: &str, + values: &[f32], + top_k: usize, + condition: Option<&str>, + expected: &[&str], +) -> serde_json::Value { + search_until( + table, + values, + top_k, + condition, + &format!("expected hits {expected:?}"), + |response| hit_pks(response) == expected, + ) + .await +} + +/// Search until the index holds exactly `count` hits, for tests whose subject is the +/// number of rows rather than which rows. +async fn search_until_count( + table: &str, + values: &[f32], + top_k: usize, + condition: Option<&str>, + count: usize, +) -> serde_json::Value { + search_until( + table, + values, + top_k, + condition, + &format!("expected {count} hits"), + |response| hit_pks(response).len() == count, + ) + .await +} + +/// The nearest vector comes back first, and the ordering is by actual distance +/// rather than by insertion order. +#[tokio::test] +async fn search_returns_nearest_first() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_order"); + create_vector_table(&name, 2, "COSINE", false).await; + + // Inserted worst-first, so passing cannot be an artefact of scan order. + put_vector(&name, "opposite", None, &[-1.0, 0.0]).await; + put_vector(&name, "orthogonal", None, &[0.0, 1.0]).await; + put_vector(&name, "exact", None, &[1.0, 0.0]).await; + + let response = + search_until_pks(&name, &[1.0, 0.0], 3, None, &["exact", "orthogonal", "opposite"]).await; + assert_eq!( + hit_pks(&response), + vec!["exact", "orthogonal", "opposite"], + "response: {response}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// TopK bounds the result set. +#[tokio::test] +async fn top_k_limits_the_results() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_topk"); + create_vector_table(&name, 2, "COSINE", false).await; + for i in 0..5 { + put_vector(&name, &format!("i{i}"), None, &[1.0, i as f32]).await; + } + + // Wait for all five to be indexed before bounding, otherwise a result of two + // could just as easily mean three writes had not propagated yet. + search_until_count(&name, &[1.0, 0.0], 10, None, 5).await; + + let response = search(&name, &[1.0, 0.0], 2, None).await; + assert_eq!(hit_pks(&response).len(), 2, "response: {response}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Overwriting an item replaces its vector rather than leaving both, which a +/// row keyed by partition instead of by base item would get wrong. +#[tokio::test] +async fn overwriting_an_item_replaces_its_vector() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_replace"); + create_vector_table(&name, 2, "COSINE", false).await; + + put_vector(&name, "a", None, &[-1.0, 0.0]).await; + put_vector(&name, "a", None, &[1.0, 0.0]).await; + + // Converge on the score rather than the pk set: exactly one row for "a" is + // already true after the FIRST write, so a pk assertion would be satisfied by a + // state where the overwrite has not propagated at all. + let response = search_until( + &name, + &[1.0, 0.0], + 10, + None, + "the overwrite to have propagated (cosine score ~0)", + |r| { + r.pointer("/SearchResults/0/Score") + .and_then(serde_json::Value::as_f64) + .is_some_and(|score| score.abs() < 1e-5) + }, + ) + .await; + let pks = hit_pks(&response); + assert_eq!(pks, vec!["a"], "one row per base item: {response}"); + + // Row count alone is not enough. A delete-then-insert that reinserted the OLD + // image would also leave exactly one row for "a", and the assertion above would + // pass while the stored vector was stale. So check the vector itself, via the + // score: the query is the new vector, so cosine distance must be ~0, whereas + // the old vector was its exact opposite and would score ~2. + let score = response + .pointer("/SearchResults/0/Score") + .and_then(serde_json::Value::as_f64) + .unwrap_or_else(|| panic!("no score in: {response}")); + assert!( + score.abs() < 1e-5, + "the stored vector must be the NEW one (cosine ~0), got score {score}: {response}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Deleting an item removes it from the index. +#[tokio::test] +async fn deleting_an_item_removes_it_from_the_index() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_remove"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "gone", None, &[1.0, 0.0]).await; + put_vector(&name, "stays", None, &[0.0, 1.0]).await; + + // Establish that BOTH items are indexed before deleting one. Without this the + // convergence below is satisfied by a state where "gone" was simply never + // indexed yet, so the test could pass without the delete being exercised at all. + search_until_pks(&name, &[1.0, 0.0], 10, None, &["gone", "stays"]).await; + + let body = format!(r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "gone"}}}}}}"#); + let (status, text) = call("DeleteItem", &body).await; + assert_eq!(status, 200, "DeleteItem failed: {text}"); + + let response = search_until_pks(&name, &[1.0, 0.0], 10, None, &["stays"]).await; + assert_eq!(hit_pks(&response), vec!["stays"], "response: {response}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// An item with no vector attribute is simply not indexed, exactly as a GSI omits +/// an item missing its index key. It must not be an error, and must not appear. +/// +/// Worth being precise about what this proves, because it is less than the name +/// suggests: an item with no vector has no code path that could index it, so its +/// absence is close to trivially true. What it does prove is that a vectorless +/// PutItem is still ACCEPTED against a table carrying a vector index, which is the +/// half that could plausibly regress. The removal case, an item that had a vector +/// and loses it, is the one that needs real coverage and is asserted at the unit +/// level by `an_item_that_loses_its_vector_is_removed_from_the_index`. +#[tokio::test] +async fn an_item_without_a_vector_is_not_indexed() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_novec"); + create_vector_table(&name, 2, "COSINE", false).await; + + let body = format!(r#"{{"TableName": "{name}", "Item": {{"pk": {{"S": "novec"}}}}}}"#); + let (status, text) = call("PutItem", &body).await; + assert_eq!( + status, 200, + "a vectorless item must still be writable: {text}" + ); + put_vector(&name, "hasvec", None, &[1.0, 0.0]).await; + + let response = search_until_pks(&name, &[1.0, 0.0], 10, None, &["hasvec"]).await; + assert_eq!(hit_pks(&response), vec!["hasvec"], "response: {response}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// A scoped index searches one partition. This is the isolation property the whole +/// partition column exists for, so it is asserted from both sides: each tenant sees +/// its own item and not the other's. +#[tokio::test] +async fn a_scoped_search_sees_only_its_own_partition() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_scope"); + create_vector_table(&name, 2, "COSINE", true).await; + + put_vector(&name, "a_item", Some("tenant_a"), &[1.0, 0.0]).await; + put_vector(&name, "b_item", Some("tenant_b"), &[1.0, 0.0]).await; + + let a = search_until_pks(&name, &[1.0, 0.0], 10, Some("tenant_a"), &["a_item"]).await; + assert_eq!(hit_pks(&a), vec!["a_item"], "tenant_a response: {a}"); + + let b = search_until_pks(&name, &[1.0, 0.0], 10, Some("tenant_b"), &["b_item"]).await; + assert_eq!(hit_pks(&b), vec!["b_item"], "tenant_b response: {b}"); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Moving an item between partitions must move its row, not duplicate it. A row +/// keyed by base item makes this work; keying by partition would leave the old row +/// and the item would be findable under both tenants. +#[tokio::test] +async fn changing_the_partition_attribute_moves_the_row() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_move"); + create_vector_table(&name, 2, "COSINE", true).await; + + put_vector(&name, "mover", Some("tenant_a"), &[1.0, 0.0]).await; + put_vector(&name, "mover", Some("tenant_b"), &[1.0, 0.0]).await; + + // Wait for the destination partition first, then assert the source is empty. + // That ordering is what makes the absence assertion sound: one apply performs + // the delete and the insert in a single transaction, so the row appearing under + // tenant_b proves the removal from tenant_a has already committed. Asserting the + // absence first would pass while the move had simply not propagated. + let b = search_until_pks(&name, &[1.0, 0.0], 10, Some("tenant_b"), &["mover"]).await; + assert_eq!(hit_pks(&b), vec!["mover"], "tenant_b response: {b}"); + + let a = search(&name, &[1.0, 0.0], 10, Some("tenant_a")).await; + assert!( + hit_pks(&a).is_empty(), + "the old partition must no longer hold the row: {a}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Dot product ranks the other way round from cosine and euclidean. A single +/// ordering would silently return the worst matches here, so the direction is +/// asserted over the wire rather than only in a unit test. +#[tokio::test] +async fn dot_product_ranks_larger_scores_first() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_dot"); + create_vector_table(&name, 2, "DOT_PRODUCT", false).await; + + put_vector(&name, "small", None, &[0.5, 0.0]).await; + put_vector(&name, "large", None, &[4.0, 0.0]).await; + put_vector(&name, "negative", None, &[-3.0, 0.0]).await; + + let response = + search_until_pks(&name, &[1.0, 0.0], 3, None, &["large", "small", "negative"]).await; + assert_eq!( + hit_pks(&response), + vec!["large", "small", "negative"], + "response: {response}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// The vector attribute is withheld unless the caller names it. +/// +/// Confirmed as the service's behaviour: it is returned only when explicitly asked +/// for in `ProjectionExpression`, even under a Projection of `ALL`. Asserted in both +/// directions, because either half alone would pass against a broken implementation: +/// "absent by default" passes if the attribute is never returned at all, and +/// "present when named" passes if it is always returned. +#[tokio::test] +async fn the_vector_attribute_is_returned_only_when_named() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_vecproj"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "a", None, &[1.0, 0.0]).await; + + // Default: withheld, but the rest of the item is still there, so this is not + // passing merely because nothing was returned. + let response = search_until_count(&name, &[1.0, 0.0], 5, None, 1).await; + let item = response + .pointer("/SearchResults/0/Item") + .unwrap_or_else(|| panic!("no item in: {response}")); + assert!( + item.get("emb").is_none(), + "the vector must be withheld by default: {item}" + ); + assert!( + item.get("pk").is_some(), + "the rest of the item must still be returned: {item}" + ); + + // Named explicitly: returned. + let body = format!( + r#"{{ + "TableName": "{name}", + "IndexName": "vidx", + "SearchVector": [{{"N": "1"}}, {{"N": "0"}}], + "TopK": 5, + "ProjectionExpression": "pk, emb" + }}"# + ); + let (status, text) = call("SearchVectors", &body).await; + assert_eq!( + status, 200, + "SearchVectors with a projection failed: {text}" + ); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let item = json + .pointer("/SearchResults/0/Item") + .unwrap_or_else(|| panic!("no item in: {text}")); + assert!( + item.get("emb").is_some(), + "the vector must be returned when named in ProjectionExpression: {item}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// A vector index holds 32-bit floats, so a client writing more precision than an +/// `f32` carries reads the narrowed value back from a search, while the base item +/// keeps exactly what was written. +/// +/// The service's own validation is the evidence for the width: it rejects a +/// component outside `[-3.4028235E38, 3.4028235E38]`, which is exactly `f32::MAX`, +/// and names the expected type "32-bit floating point number". Reading the narrowed +/// value back could not be measured directly, because `SearchVectors` is not served +/// by the standard DynamoDB endpoint. +/// +/// Both halves are asserted deliberately. Without the base-table half this would +/// also pass against an implementation that had simply corrupted the item on write. +#[tokio::test] +async fn the_index_narrows_a_vector_to_f32_but_the_item_keeps_its_own_precision() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_vecf32"); + create_vector_table(&name, 2, "COSINE", false).await; + + // More decimal places than an f32 carries. Nearest f32 is 0.12345679. No + // trailing zero, because `N` normalisation trims one and that would confound + // the base-item assertion below with a second, unrelated effect. + let written = "0.1234567890123456789"; + let body = format!( + r#"{{ + "TableName": "{name}", + "Item": {{"pk": {{"S": "a"}}, "emb": {{"L": [{{"N": "{written}"}}, {{"N": "0"}}]}}}} + }}"# + ); + let (status, text) = call("PutItem", &body).await; + assert_eq!(status, 200, "PutItem failed: {text}"); + + // The base item is untouched: narrowing belongs to the index, not the write. + let (status, text) = call( + "GetItem", + &format!( + r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "a"}}}}, "ConsistentRead": true}}"# + ), + ) + .await; + assert_eq!(status, 200, "GetItem failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + assert_eq!( + json.pointer("/Item/emb/L/0/N").and_then(|v| v.as_str()), + Some(written), + "the base item must keep the client's own precision: {text}" + ); + + // The search returns what the index stored, which is the f32. + let body = format!( + r#"{{ + "TableName": "{name}", + "IndexName": "vidx", + "SearchVector": [{{"N": "1"}}, {{"N": "0"}}], + "TopK": 5, + "ProjectionExpression": "pk, emb" + }}"# + ); + let json = search_body_until(&body, "the written item to be indexed", |r| { + r.pointer("/SearchResults/0/Item/emb/L/0/N").is_some() + }) + .await; + assert_eq!( + json.pointer("/SearchResults/0/Item/emb/L/0/N") + .and_then(|v| v.as_str()), + Some("0.12345679"), + "the index must return the narrowed f32: {json}" + ); + // The exactly-representable component must not acquire a decimal point. + assert_eq!( + json.pointer("/SearchResults/0/Item/emb/L/1/N") + .and_then(|v| v.as_str()), + Some("0"), + "an exact component must round-trip unchanged: {json}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// A two-HASH search schema is rejected with the service's measured message. +/// +/// This mattered because the contract accepted it and then could not honour it: the +/// query side requires a condition for every declared HASH, while the backend +/// resolves the scope from the first and demotes the rest to filters. Measured +/// against the live service on 2026-08-06. +#[tokio::test] +async fn a_two_hash_search_schema_is_rejected() { + if skip_unless_supported().await { + return; + } + let name = table_name("twohash"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [ + {{"AttributeName": "pk", "AttributeType": "S"}}, + {{"AttributeName": "a", "AttributeType": "S"}}, + {{"AttributeName": "b", "AttributeType": "S"}} + ], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": 4, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "SearchSchema": [ + {{"AttributeName": "a", "SearchSchemaElementType": "HASH"}}, + {{"AttributeName": "b", "SearchSchemaElementType": "HASH"}} + ], + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 400, "two HASH elements must be rejected: {text}"); + assert!( + text.contains("Member must have HASH count less than or equal to 1"), + "expected the measured HASH-count message, got: {text}" + ); +} + +/// An unknown distance function is rejected, and the enum value set is listed in +/// the service's order. +/// +/// The order is measured and is neither alphabetical nor the enum's declaration +/// order, so it is asserted explicitly: an earlier version guessed alphabetical. +#[tokio::test] +async fn an_unknown_distance_function_lists_the_measured_enum_order() { + if skip_unless_supported().await { + return; + } + let name = table_name("baddf"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": 4, + "DistanceFunction": "MANHATTAN", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!( + status, 400, + "an unknown distance function must be rejected: {text}" + ); + assert!( + text.contains("[DOT_PRODUCT, COSINE, EUCLIDEAN]"), + "the enum value set must be listed in the service's measured order, got: {text}" + ); +} + +/// Adding a vector index to a table that already holds items backfills them, so a +/// search finds data written before the index existed. +/// +/// This is the whole point of the `UpdateTable` create path, and it is what makes +/// the difference between an index and a filter over new writes. Items are written +/// first, deliberately, so a backfill that did nothing would fail here rather than +/// pass because the write path happened to cover it. +#[tokio::test] +async fn adding_a_vector_index_backfills_the_items_already_there() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_backfill"); + // A plain table: no vector index at creation. + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST" + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + wait_for_active(&name).await; + + // Three items with vectors, plus one without, which must be skipped rather + // than break the scan. + put_vector(&name, "near", None, &[1.0, 0.0]).await; + put_vector(&name, "far", None, &[-1.0, 0.0]).await; + put_vector(&name, "mid", None, &[0.0, 1.0]).await; + let (status, text) = call( + "PutItem", + &format!(r#"{{"TableName": "{name}", "Item": {{"pk": {{"S": "novec"}}}}}}"#), + ) + .await; + assert_eq!(status, 200, "PutItem without a vector failed: {text}"); + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{ + "TableName": "{name}", + "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Dimensions": 2, + "DistanceFunction": "COSINE", + "Projection": {{"ProjectionType": "ALL"}} + }}}}] + }}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable create failed: {text}"); + + // The backfill is asynchronous: UpdateTable returns with the index CREATING, and + // a search is refused until it is ACTIVE. Waiting is not test scaffolding, it is + // what a real client has to do, and the service takes minutes over it. + wait_for_vector_index_active(&name, "vidx").await; + + let response = search_until_count(&name, &[1.0, 0.0], 10, None, 3).await; + let results = response + .get("SearchResults") + .and_then(|r| r.as_array()) + .unwrap_or_else(|| panic!("no results in: {response}")); + assert_eq!( + results.len(), + 3, + "the three vector-carrying items must be backfilled, and the fourth skipped: {response}" + ); + assert_eq!( + results[0].pointer("/Item/pk/S").and_then(|v| v.as_str()), + Some("near"), + "backfilled rows must be searchable in distance order: {response}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// A backfilled index keeps working for writes that arrive afterwards, which is +/// what proves the write-path hook and the backfill agree on the row shape. +#[tokio::test] +async fn an_index_added_by_update_table_indexes_later_writes_too() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_bf_then_write"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST" + }}"# + ); + call("CreateTable", &body).await; + wait_for_active(&name).await; + put_vector(&name, "before", None, &[1.0, 0.0]).await; + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{ + "TableName": "{name}", + "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Dimensions": 2, + "DistanceFunction": "COSINE", + "Projection": {{"ProjectionType": "ALL"}} + }}}}] + }}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable create failed: {text}"); + + // Written while the index is still CREATING, deliberately. The queue does not + // claim rows for a table whose vector index is building, so this write + // accumulates and is applied once the index goes ACTIVE. Moving it after the + // wait would still pass but would stop covering that path. + put_vector(&name, "after", None, &[0.9, 0.1]).await; + + wait_for_vector_index_active(&name, "vidx").await; + + // "before" arrives via the backfill, "after" via the propagation queue once the + // index is ACTIVE, so this converges on both being present. + let response = search_until( + &name, + &[1.0, 0.0], + 10, + None, + "both the backfilled and the later-written item to be present", + |r| { + let pks = hit_pks(r); + pks.iter().any(|k| k == "before") && pks.iter().any(|k| k == "after") + }, + ) + .await; + let results = response + .get("SearchResults") + .and_then(|r| r.as_array()) + .unwrap_or_else(|| panic!("no results in: {response}")); + let keys: Vec<&str> = results + .iter() + .filter_map(|r| r.pointer("/Item/pk/S").and_then(|v| v.as_str())) + .collect(); + assert!( + keys.contains(&"before") && keys.contains(&"after"), + "both the backfilled and the later-written item must be found: {keys:?}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Deleting a vector index stops it serving, and `DescribeTable` stops reporting it. +/// +/// Measured against this backend before the path existed: the delete returned 200 +/// while the index stayed ACTIVE and kept returning hits, which is why both halves +/// are asserted here rather than just the status code. +#[tokio::test] +async fn deleting_a_vector_index_stops_it_serving_and_reporting() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_idxdelete"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "a", None, &[1.0, 0.0]).await; + + // It serves before the delete, so the assertions below cannot pass vacuously. + // This has to converge: the write is propagated asynchronously, so a single + // search here could find nothing and make the precondition assert falsely. + let response = search_until_count(&name, &[1.0, 0.0], 5, None, 1).await; + assert_eq!( + response + .get("SearchResults") + .and_then(|r| r.as_array()) + .map(Vec::len), + Some(1), + "the index must serve before it is deleted: {response}" + ); + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Delete": {{"IndexName": "vidx"}}}}]}}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable delete failed: {text}"); + + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + assert_eq!(status, 200, "DescribeTable failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let still_there = json + .pointer("/Table/VectorIndexes") + .and_then(|v| v.as_array()) + .is_some_and(|a| { + a.iter() + .any(|i| i.pointer("/IndexName").and_then(|n| n.as_str()) == Some("vidx")) + }); + assert!( + !still_there, + "a deleted index must not still be reported: {text}" + ); + + let body = format!( + r#"{{"TableName": "{name}", "IndexName": "vidx", "SearchVector": {}, "TopK": 5}}"#, + vector_json(&[1.0, 0.0]) + ); + let (status, text) = call("SearchVectors", &body).await; + assert_eq!( + status, 400, + "searching a deleted index must fail, not return stale hits: {text}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// The base items survive the index being deleted: only the index goes. +#[tokio::test] +async fn deleting_a_vector_index_leaves_the_items_alone() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_idxdel_items"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "a", None, &[1.0, 0.0]).await; + + call( + "UpdateTable", + &format!( + r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Delete": {{"IndexName": "vidx"}}}}]}}"# + ), + ) + .await; + + let (status, text) = call( + "GetItem", + &format!( + r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "a"}}}}, "ConsistentRead": true}}"# + ), + ) + .await; + assert_eq!(status, 200, "GetItem failed: {text}"); + assert!( + text.contains(r#""emb""#), + "the item and its vector attribute must be untouched: {text}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// Creating an index that already exists, and deleting one that does not. +#[tokio::test] +async fn update_table_rejects_a_duplicate_create_and_a_missing_delete() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_idx_errs"); + create_vector_table(&name, 2, "COSINE", false).await; + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{ + "TableName": "{name}", + "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Dimensions": 2, + "DistanceFunction": "COSINE", + "Projection": {{"ProjectionType": "ALL"}} + }}}}] + }}"# + ), + ) + .await; + assert_eq!(status, 400, "a duplicate create must be rejected: {text}"); + assert!( + text.contains("already exists"), + "unexpected duplicate-create message: {text}" + ); + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Delete": {{"IndexName": "nosuch"}}}}]}}"# + ), + ) + .await; + assert_eq!( + status, 400, + "deleting a missing index must be rejected: {text}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// DescribeTable reports the vector index, and reports it ACTIVE with no +/// `Backfilling` member, which is what the service does for an index created by +/// CreateTable. +#[tokio::test] +async fn describe_table_reports_the_vector_index() { + if skip_unless_supported().await { + return; + } + let name = table_name("pos_describe"); + create_vector_table(&name, 4, "COSINE", false).await; + + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + assert_eq!(status, 200, "DescribeTable failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let vidx = json + .pointer("/Table/VectorIndexes/0") + .unwrap_or_else(|| panic!("no vector index in description: {text}")); + + assert_eq!( + vidx.pointer("/IndexName").and_then(|v| v.as_str()), + Some("vidx") + ); + assert_eq!( + vidx.pointer("/Dimensions").and_then(|v| v.as_u64()), + Some(4) + ); + assert_eq!( + vidx.pointer("/DistanceFunction").and_then(|v| v.as_str()), + Some("COSINE") + ); + assert_eq!( + vidx.pointer("/IndexStatus").and_then(|v| v.as_str()), + Some("ACTIVE") + ); + assert!( + vidx.get("Backfilling").is_none(), + "an ACTIVE index must not carry Backfilling at all: {vidx}" + ); + assert!( + vidx.get("VectorAttribute").is_some(), + "VectorAttribute must be reported: {vidx}" + ); + // The ARN must name this table and this index in the service's shape. A + // malformed or misattributed ARN would ship unnoticed with only the presence + // check the members above get. + let arn = vidx + .pointer("/IndexArn") + .and_then(|v| v.as_str()) + .unwrap_or_else(|| panic!("IndexArn must be reported: {vidx}")); + assert!( + arn.starts_with("arn:aws:dynamodb:") && arn.ends_with(&format!("table/{name}/index/vidx")), + "IndexArn must have the service's table//index/ form: {arn}" + ); + // Emitted as numbers, matching the service's members. Their VALUES are the + // GSI convention in this backend (not live-maintained), so only shape is + // asserted here; a missing member would break a generated client's model. + assert!( + vidx.pointer("/ItemCount").is_some_and(serde_json::Value::is_number), + "ItemCount must be a number: {vidx}" + ); + assert!( + vidx.pointer("/IndexSizeBytes") + .is_some_and(serde_json::Value::is_number), + "IndexSizeBytes must be a number: {vidx}" + ); + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// `SearchVectorsOutput` carries no `Count` field. +/// +/// Measured against the live service in sandbox 964157134968 on 2026-08-10: the +/// response contains only `SearchResults` and, when asked for, +/// `ConsumedCapacity`. Five parameter variations were probed and none produced a +/// `Count`: no projection, `ReturnConsumedCapacity=INDEXES`, a +/// `ProjectionExpression`, a `TopK` larger than the item count, and a projection +/// naming a single non-key attribute. The botocore model agrees: `SearchVectorsOutput` +/// declares exactly those two members. +/// +/// This is asserted rather than left to review because an extra top-level field is +/// invisible to a generated client (it ignores unknown members), so nothing else in +/// the suite would ever catch its return. The `TopK`-exceeds-matches case is +/// included deliberately: a `Count` field is most tempting to add exactly when the +/// result set is shorter than `TopK`. +#[tokio::test] +async fn the_search_response_carries_no_count_field() { + if skip_unless_supported().await { + return; + } + let name = table_name("no_count"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "a", None, &[1.0, 0.0]).await; + put_vector(&name, "b", None, &[0.0, 1.0]).await; + + // Case 1: plain search. Converge first, so "no Count field" is asserted against + // a populated response rather than an empty one. + let response = search_until_count(&name, &[1.0, 0.0], 2, None, 2).await; + assert!( + response.get("Count").is_none(), + "SearchVectors must not return a Count field: {response}" + ); + assert!( + response.get("SearchResults").is_some(), + "SearchResults must be present: {response}" + ); + + // Case 2: TopK larger than the number of matches, where a Count field is the + // most tempting addition. + let response = search_until_count(&name, &[1.0, 0.0], 50, None, 2).await; + assert!( + response.get("Count").is_none(), + "SearchVectors must not return a Count field when TopK exceeds the match \ + count: {response}" + ); + + // Case 3: with ConsumedCapacity requested, so the only two legal top-level + // members are both present and nothing else is. + let body = format!( + r#"{{ + "TableName": "{name}", + "IndexName": "vidx", + "SearchVector": [{{"N": "1"}}, {{"N": "0"}}], + "TopK": 2, + "ReturnConsumedCapacity": "INDEXES" + }}"# + ); + let (status, text) = call("SearchVectors", &body).await; + assert_eq!(status, 200, "SearchVectors failed: {text}"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + let members: Vec<&str> = json + .as_object() + .unwrap_or_else(|| panic!("response is not an object: {text}")) + .keys() + .map(String::as_str) + .collect(); + for m in &members { + assert!( + matches!(*m, "SearchResults" | "ConsumedCapacity"), + "unexpected top-level member '{m}' in SearchVectorsOutput; the service \ + returns only SearchResults and ConsumedCapacity: {text}" + ); + } + assert!( + members.contains(&"ConsumedCapacity"), + "ConsumedCapacity must be present when requested: {text}" + ); +} + +/// A `KEYS_ONLY` vector index still projects its search-schema attributes, so a +/// filtered search works and returns them. +/// +/// The documented rule is specific and is NOT GSI `KEYS_ONLY` semantics: on a +/// vector index, `KEYS_ONLY` projects the base table primary key, the vector +/// attribute, AND any inline filter attributes declared in the SearchSchema +/// (developer guide, "Projections"). GSI `KEYS_ONLY` projects base keys plus the +/// index's own key attributes and nothing else. +/// +/// This went unasserted because `create_vector_table` hardcodes +/// `ProjectionType: ALL`, so all 18 pre-existing search tests exercise the one +/// projection under which the distinction cannot appear. Under `KEYS_ONLY` the +/// inline filter attribute was dropped from the stored payload, and because the +/// filter is evaluated against that payload, `item.get(name)` returned `None` for +/// every row and a filtered search matched nothing at all. +/// +/// Asserted three ways so a partial fix cannot pass: the filtered search must +/// find the item, the filter attribute must come back in the result, and a +/// non-projected attribute must NOT come back (otherwise this would also pass +/// against an implementation that quietly ignored the projection and behaved like +/// `ALL`). +#[tokio::test] +async fn keys_only_vector_index_still_filters_on_its_search_schema() { + if skip_unless_supported().await { + return; + } + let name = table_name("keysonly_filter"); + let body = format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [ + {{"AttributeName": "pk", "AttributeType": "S"}}, + {{"AttributeName": "category", "AttributeType": "S"}} + ], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": 2, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "SearchSchema": [ + {{"AttributeName": "category", "SearchSchemaElementType": "INLINE_FILTER"}} + ], + "Projection": {{"ProjectionType": "KEYS_ONLY"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 200, "CreateTable with KEYS_ONLY failed: {text}"); + wait_for_active(&name).await; + + // `note` is deliberately non-projected under KEYS_ONLY. + let put = format!( + r#"{{ + "TableName": "{name}", + "Item": {{ + "pk": {{"S": "a"}}, + "category": {{"S": "books"}}, + "note": {{"S": "not projected"}}, + "emb": {{"L": [{{"N": "1"}}, {{"N": "0"}}]}} + }} + }}"# + ); + let (status, text) = call("PutItem", &put).await; + assert_eq!(status, 200, "PutItem failed: {text}"); + + let search = format!( + r#"{{ + "TableName": "{name}", + "IndexName": "vidx", + "SearchVector": [{{"N": "1"}}, {{"N": "0"}}], + "TopK": 5, + "SearchConditionExpression": "category = :c", + "ExpressionAttributeValues": {{":c": {{"S": "books"}}}} + }}"# + ); + // Converge on the filtered query itself. Before the projection fix this returned + // an empty result set permanently, so a timeout here is the defect reappearing + // rather than slow propagation. + let json = search_body_until( + &search, + "the KEYS_ONLY index to match its inline filter", + |r| { + r.get("SearchResults") + .and_then(|v| v.as_array()) + .is_some_and(|a| !a.is_empty()) + }, + ) + .await; + let text = json.to_string(); + let results = json + .get("SearchResults") + .and_then(|v| v.as_array()) + .unwrap_or_else(|| panic!("no results array in: {text}")); + assert_eq!( + results.len(), + 1, + "a KEYS_ONLY index must still match its inline filter; the filter \ + attribute has to be projected for the filter to be evaluable: {text}" + ); + + let item = results[0] + .get("Item") + .unwrap_or_else(|| panic!("no item in: {text}")); + assert!( + item.get("category").is_some(), + "KEYS_ONLY must project inline filter attributes, so 'category' must be \ + returned: {item}" + ); + assert!( + item.get("pk").is_some(), + "KEYS_ONLY must project the base table primary key: {item}" + ); + assert!( + item.get("note").is_none(), + "KEYS_ONLY must NOT project an unrelated non-key attribute; returning it \ + would mean the projection was ignored entirely: {item}" + ); +} + +/// A search against a HASH-scoped index must be REFUSED without a condition, not +/// answered with an empty result set. +/// +/// This is the divergence that mattered most of everything found by differential +/// testing against the live service, because it was a silent wrong answer rather +/// than an error. The index scopes to one partition via its HASH element, and with +/// no condition there was no partition to scope to, so the search ran unscoped and +/// returned HTTP 200 with zero results. A caller who forgot the condition was told +/// "no matches" for a request the service rejects outright, which is +/// indistinguishable from an empty table. +/// +/// The code carried a comment asserting that "validation upstream guarantees it is +/// present here". No such validation existed. +/// +/// Data is inserted first, and the positive case asserts a non-empty result, so +/// "zero results" cannot be mistaken for correct behaviour on an empty index: the +/// test would pass against the broken build if it only checked the refusal. +/// +/// Message measured against DynamoDB in us-east-1 on 2026-08-11. +#[tokio::test] +async fn a_hash_scoped_index_requires_a_search_condition() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_hash_required"); + create_vector_table(&name, 2, "COSINE", true).await; + + for (pk, tenant, emb) in [("a", "t1", "1.0"), ("b", "t1", "0.9"), ("c", "t2", "0.1")] { + let (status, text) = call( + "PutItem", + &format!( + r#"{{"TableName": "{name}", "Item": {{ + "pk": {{"S": "{pk}"}}, + "tenant": {{"S": "{tenant}"}}, + "emb": {{"L": [{{"N": "{emb}"}}, {{"N": "0.1"}}]}} + }}}}"# + ), + ) + .await; + assert_eq!(status, 200, "PutItem failed: {text}"); + } + + // The scoped search works and finds data, which is what makes the refusal + // below meaningful rather than vacuous. + let scoped = search_until_count(&name, &[1.0, 0.1], 10, Some("t1"), 2).await; + assert_eq!( + hit_pks(&scoped).len(), + 2, + "the scoped search must find both t1 items before the refusal is meaningful: {scoped}" + ); + + // No SearchConditionExpression at all. + let (status, text) = call( + "SearchVectors", + &format!( + r#"{{"TableName": "{name}", "IndexName": "vidx", + "SearchVector": [{{"N": "1.0"}}, {{"N": "0.1"}}], "TopK": 10}}"# + ), + ) + .await; + assert_eq!( + status, 400, + "an omitted SearchConditionExpression must be refused, not answered with an \ + empty result set: {text}" + ); + assert!( + text.contains("SearchConditionExpression must be provided when SearchSchema has a HASH key"), + "expected the measured service message: {text}" + ); + + // An expression that IS supplied, references only IN-SCHEMA attributes, and + // still omits the HASH leaves the search equally unscoped. The attribute must + // be in the schema for this to test the intended branch: an out-of-schema + // attribute (an earlier version used the base `pk`) is refused by the + // "not in SearchSchema" validation first, so the omitted-HASH rule was never + // exercised. That needs a schema with more than the HASH element, so this + // sub-case builds its own. + let name2 = table_name("vi_hash_omitted"); + let body = format!( + r#"{{ + "TableName": "{name2}", + "AttributeDefinitions": [ + {{"AttributeName": "pk", "AttributeType": "S"}}, + {{"AttributeName": "tenant", "AttributeType": "S"}}, + {{"AttributeName": "category", "AttributeType": "S"}} + ], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST", + "VectorIndexes": [{{ + "IndexName": "vidx", + "Dimensions": 2, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "SearchSchema": [ + {{"AttributeName": "tenant", "SearchSchemaElementType": "HASH"}}, + {{"AttributeName": "category", "SearchSchemaElementType": "INLINE_FILTER"}} + ], + "Projection": {{"ProjectionType": "ALL"}} + }}] + }}"# + ); + let (status, text) = call("CreateTable", &body).await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + wait_for_active(&name2).await; + + // The wording of this one is not measured against the service, so only the + // refusal is asserted, not the text. + let (status, text) = call( + "SearchVectors", + &format!( + r#"{{"TableName": "{name2}", "IndexName": "vidx", + "SearchVector": [{{"N": "1.0"}}, {{"N": "0.1"}}], "TopK": 10, + "SearchConditionExpression": "category = :c", + "ExpressionAttributeValues": {{":c": {{"S": "x"}}}}}}"# + ), + ) + .await; + assert_eq!( + status, 400, + "an in-schema expression that omits the HASH attribute must be refused: {text}" + ); + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name2}"}}"#)).await; +} + +/// An index that is still building must be reported by DescribeTable and refused by +/// SearchVectors, and must serve once it is ACTIVE. +/// +/// The backfill is detached and commits per batch, so a search can now arrive while +/// the index holds only part of the data. The service refuses that, and reports it by +/// saying the table does not have the index at all rather than naming the status: +/// measured four times against DynamoDB on 2026-08-11 against an index in CREATING. +/// +/// Both halves matter. Asserting only the refusal would also pass if the index never +/// became usable, and asserting only the success would also pass if it were searchable +/// while incomplete. +#[tokio::test] +async fn a_building_vector_index_is_visible_but_not_searchable() { + if skip_unless_supported().await { + return; + } + set_backfill_delay(3000).await; + let name = table_name("vi_building"); + let (status, text) = call( + "CreateTable", + &format!( + r#"{{"TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST"}}"# + ), + ) + .await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + wait_for_active(&name).await; + + // More than one batch, so the delay is actually reached. + for i in 0..600 { + put_vector(&name, &format!("k{i:06}"), None, &[1.0, 0.0]).await; + } + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", "Dimensions": 2, "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}}}}}}]}}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable failed: {text}"); + let described: serde_json::Value = serde_json::from_str(&text).expect("json"); + let vi = &described["TableDescription"]["VectorIndexes"][0]; + assert_eq!( + vi["IndexStatus"], "CREATING", + "UpdateTable must return while the index is still building: {text}" + ); + + // DescribeTable must still report it, with its status. + let (_, text) = call("DescribeTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; + let d: serde_json::Value = serde_json::from_str(&text).expect("json"); + assert_eq!( + d["Table"]["VectorIndexes"][0]["IndexStatus"], "CREATING", + "a building index must remain visible in DescribeTable: {text}" + ); + + // But it must not serve. + let (status, text) = call( + "SearchVectors", + &format!( + r#"{{"TableName": "{name}", "IndexName": "vidx", + "SearchVector": [{{"N": "1.0"}}, {{"N": "0.0"}}], "TopK": 5}}"# + ), + ) + .await; + assert_eq!( + status, 400, + "a search against a building index must be refused, not answered from partial \ + data: {text}" + ); + assert!( + text.contains("The table does not have the specified index"), + "expected the measured service message: {text}" + ); + + // And it must serve once ACTIVE, or the refusal above is worthless. + wait_for_vector_index_active(&name, "vidx").await; + let hits = search_until_count(&name, &[1.0, 0.0], 5, None, 5).await; + assert_eq!( + hit_pks(&hits).len(), + 5, + "the index must serve once ACTIVE: {hits}" + ); + set_backfill_delay(0).await; +} + +/// Removing a row during a backfill must not cause another row to be skipped. +/// +/// The backfill pages through the base table. Anchoring those pages on an OFFSET meant +/// that removing any already-scanned row shifted every later position by one, so the +/// next batch skipped a row, which was then missing from the index permanently: no +/// queue entry can repair it, because the skipped row was never written to. Only the +/// removed one was. Pagination is anchored on the key instead. +/// +/// The probe row sits at the batch boundary (index 500, with BATCH = 500) and is the +/// only row pointing in its direction, so its absence is unambiguous rather than being +/// masked by its neighbours. +#[tokio::test] +async fn removing_a_row_during_a_backfill_does_not_skip_another() { + if skip_unless_supported().await { + return; + } + set_backfill_delay(3000).await; + let name = table_name("vi_noskip"); + let (status, text) = call( + "CreateTable", + &format!( + r#"{{"TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + "BillingMode": "PAY_PER_REQUEST"}}"# + ), + ) + .await; + assert_eq!(status, 200, "CreateTable failed: {text}"); + wait_for_active(&name).await; + + for i in 0..600 { + // Only the boundary row points along y. + let v = if i == 500 { [0.0, 1.0] } else { [1.0, 0.0] }; + put_vector(&name, &format!("k{i:06}"), None, &v).await; + } + + let (status, text) = call( + "UpdateTable", + &format!( + r#"{{"TableName": "{name}", "VectorIndexUpdates": [{{"Create": {{ + "IndexName": "vidx", "Dimensions": 2, "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}}}}}}]}}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateTable failed: {text}"); + + // Remove a row from the range the first batch has already scanned. + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + let (status, text) = call( + "DeleteItem", + &format!(r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "k000010"}}}}}}"#), + ) + .await; + assert_eq!(status, 200, "DeleteItem failed: {text}"); + + wait_for_vector_index_active(&name, "vidx").await; + let hits = search_until_pks(&name, &[0.0, 1.0], 1, None, &["k000500"]).await; + assert_eq!( + hit_pks(&hits), + vec!["k000500"], + "the row at the batch boundary must still be indexed after a removal shifted \ + the scan: {hits}" + ); + set_backfill_delay(0).await; +} + +/// Set the vector backfill batch delay via the management API. +/// +/// A backfill over a test-sized table otherwise finishes faster than a client can +/// issue its next request, so a test that wanted a write to land mid-backfill would be +/// racing it and would pass whether or not the behaviour under test is correct. +/// +/// A missing admin password is a hard failure when `EXTENDDB_EXPECT_VECTORS=1`, rather +/// than a skip. The vector suites exist to be run, and the repo already had a suite +/// that skipped itself and reported green when this variable was absent, which is the +/// exact failure mode `EXTENDDB_EXPECT_VECTORS` was introduced to close. +async fn set_backfill_delay(ms: u64) { + let user = std::env::var("EXTENDDB_ADMIN_USER").unwrap_or_else(|_| "admin".into()); + let Ok(pass) = std::env::var("EXTENDDB_ADMIN_PASSWORD") else { + assert!( + std::env::var("EXTENDDB_EXPECT_VECTORS").as_deref() != Ok("1"), + "EXTENDDB_EXPECT_VECTORS=1 but EXTENDDB_ADMIN_PASSWORD is unset, so the \ + backfill tests cannot set the batch delay and would silently pass without \ + testing anything. Run via devtools/run-tests --extenddb \ + --rust-integration, which provisions it." + ); + eprintln!("SKIP: EXTENDDB_ADMIN_PASSWORD unset; backfill delay left alone"); + return; + }; + let ep = std::env::var("EXTENDDB_TEST_ENDPOINT") + .unwrap_or_else(|_| "https://127.0.0.1:18443".to_owned()); + let base = format!("{}/management", ep.trim_end_matches('/')); + let http = reqwest::Client::builder() + .danger_accept_invalid_certs(true) // localhost self-signed; test only + .build() + .expect("reqwest build"); + let r = http + .put(format!("{base}/settings/vector_backfill_batch_delay_ms")) + .basic_auth(&user, Some(&pass)) + .json(&serde_json::json!({ "value": ms.to_string() })) + .send() + .await + .expect("settings send"); + let status = r.status(); + let body = r.text().await.unwrap_or_default(); + assert!( + status.is_success(), + "setting the backfill delay failed ({status}): {body}" + ); +} + +/// Poll until a named vector index reports `ACTIVE`. +/// +/// [`wait_for_active`] only waits on `TableStatus`, which is ACTIVE throughout an +/// index build, so it returns immediately and would let a test search a building +/// index and misread the refusal as a failure. +async fn wait_for_vector_index_active(table: &str, index: &str) { + for _ in 0..600 { + let (status, text) = call("DescribeTable", &format!(r#"{{"TableName": "{table}"}}"#)).await; + if status == 200 { + let d: serde_json::Value = serde_json::from_str(&text).expect("json"); + let found = d["Table"]["VectorIndexes"] + .as_array() + .into_iter() + .flatten() + .any(|vi| vi["IndexName"] == index && vi["IndexStatus"] == "ACTIVE"); + if found { + return; + } + } + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + panic!("vector index {index} on {table} never became ACTIVE"); +} + +/// A vector index requires on-demand billing, and there is a documented cap of +/// five vector indexes per table. +/// +/// Both are documented service constraints that the emulator previously accepted: +/// "Vector indexes are supported only on tables that use on-demand capacity mode" +/// under Requirements and limitations, and "Vector indexes per table: 5" in the +/// quota table. Accepting a combination the service refuses is the more dangerous +/// direction of divergence, because code written against the emulator then fails +/// on first contact with DynamoDB. +/// +/// The PROVISIONED case is asserted twice, once explicitly and once with +/// `BillingMode` omitted, because the field defaults to PROVISIONED when absent +/// and an implementation that only checked the explicit value would pass the first +/// and fail the second. The five-index case asserts the boundary in both +/// directions so an off-by-one cannot hide: five is accepted, six is refused. +#[tokio::test] +async fn vector_indexes_require_on_demand_and_cap_at_five() { + if skip_unless_supported().await { + return; + } + + fn one_index(n: usize) -> String { + format!( + r#"{{ + "IndexName": "vidx{n}", + "Dimensions": 2, + "DistanceFunction": "COSINE", + "VectorAttribute": {{"AttributeName": "emb"}}, + "Projection": {{"ProjectionType": "ALL"}} + }}"# + ) + } + fn body(name: &str, billing: Option<&str>, count: usize) -> String { + let billing_line = match billing { + Some(b) => format!(r#""BillingMode": "{b}","#), + None => String::new(), + }; + let indexes = (0..count).map(one_index).collect::>().join(", "); + format!( + r#"{{ + "TableName": "{name}", + "AttributeDefinitions": [{{"AttributeName": "pk", "AttributeType": "S"}}], + "KeySchema": [{{"AttributeName": "pk", "KeyType": "HASH"}}], + {billing_line} + "VectorIndexes": [{indexes}] + }}"# + ) + } + + // PROVISIONED, stated explicitly. ProvisionedThroughput is supplied so the + // rejection cannot be attributed to a missing-throughput error instead. + let name = table_name("vi_prov"); + let mut prov = body(&name, Some("PROVISIONED"), 1); + prov = prov.replace( + r#""VectorIndexes""#, + r#""ProvisionedThroughput": {"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, "VectorIndexes""#, + ); + let (status, text) = call("CreateTable", &prov).await; + assert_eq!( + status, 400, + "a PROVISIONED table must not accept a vector index: {text}" + ); + assert!( + text.contains("PAY_PER_REQUEST"), + "the error should name the required billing mode: {text}" + ); + + // BillingMode omitted, which defaults to PROVISIONED. + let name = table_name("vi_default"); + let (status, text) = call("CreateTable", &body(&name, None, 1)).await; + assert_eq!( + status, 400, + "an omitted BillingMode defaults to PROVISIONED and must be refused: {text}" + ); + + // Six indexes: over the documented cap of five. + let name = table_name("vi_six"); + let (status, text) = call("CreateTable", &body(&name, Some("PAY_PER_REQUEST"), 6)).await; + assert_eq!( + status, 400, + "six vector indexes exceeds the documented cap of five: {text}" + ); + + // Five indexes: exactly at the cap, must be accepted. This is what makes the + // check above an off-by-one test rather than a blanket refusal. + let name = table_name("vi_five"); + let (status, text) = call("CreateTable", &body(&name, Some("PAY_PER_REQUEST"), 5)).await; + assert_eq!( + status, 200, + "five vector indexes is at the documented cap and must be accepted: {text}" + ); + wait_for_active(&name).await; +} + +/// BatchWriteItem maintains the vector index exactly as PutItem and DeleteItem do. +/// +/// The engine routes batch entries through the same `.put_item` / `.delete_item` +/// paths, so this held in code from the start, but it had no wire coverage: a +/// refactor that gave BatchWriteItem its own storage path could silently stop +/// maintaining the index and every existing test would stay green. Both halves are +/// exercised: puts must appear in the index, and a batched delete must remove one +/// while leaving the other, so a converged-but-empty index cannot pass as "deleted". +#[tokio::test] +async fn batch_write_item_maintains_the_index() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_batch_write"); + create_vector_table(&name, 2, "COSINE", false).await; + + let (status, text) = call( + "BatchWriteItem", + &format!( + r#"{{"RequestItems": {{"{name}": [ + {{"PutRequest": {{"Item": {{"pk": {{"S": "b1"}}, + "emb": {{"L": [{{"N": "1.0"}}, {{"N": "0.0"}}]}}}}}}}}, + {{"PutRequest": {{"Item": {{"pk": {{"S": "b2"}}, + "emb": {{"L": [{{"N": "0.9"}}, {{"N": "0.1"}}]}}}}}}}} + ]}}}}"# + ), + ) + .await; + assert_eq!(status, 200, "BatchWriteItem failed: {text}"); + search_until_pks(&name, &[1.0, 0.0], 10, None, &["b1", "b2"]).await; + + let (status, text) = call( + "BatchWriteItem", + &format!( + r#"{{"RequestItems": {{"{name}": [ + {{"DeleteRequest": {{"Key": {{"pk": {{"S": "b1"}}}}}}}} + ]}}}}"# + ), + ) + .await; + assert_eq!(status, 200, "BatchWriteItem delete failed: {text}"); + // Converging on the survivor rather than on absence: "b2 alone" is ordered + // behind the delete in the same queue, so it cannot be satisfied by a window + // where the delete simply has not applied yet. + search_until_pks(&name, &[1.0, 0.0], 10, None, &["b2"]).await; + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// TransactWriteItems maintains the vector index for both a transactional Put and +/// a transactional Delete. +/// +/// The transactional path enqueues index maintenance in its own storage code +/// (`transactions.rs`), a sibling of the ordinary write paths rather than a caller +/// of them, so wire coverage here guards a genuinely separate implementation. The +/// unit tests cover the enqueue; this proves the queue rows drain into search +/// results over the wire. +#[tokio::test] +async fn transact_write_items_maintains_the_index() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_transact_write"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "keep", None, &[0.5, 0.5]).await; + search_until_pks(&name, &[0.5, 0.5], 10, None, &["keep"]).await; + + let (status, text) = call( + "TransactWriteItems", + &format!( + r#"{{"TransactItems": [ + {{"Put": {{"TableName": "{name}", "Item": {{"pk": {{"S": "txput"}}, + "emb": {{"L": [{{"N": "1.0"}}, {{"N": "0.0"}}]}}}}}}}}, + {{"Delete": {{"TableName": "{name}", "Key": {{"pk": {{"S": "keep"}}}}}}}} + ]}}"# + ), + ) + .await; + assert_eq!(status, 200, "TransactWriteItems failed: {text}"); + // One converged assertion covers both halves: the put must appear AND the + // delete's target must be gone, and since both rode one transaction their + // queue rows drain together. + search_until_pks(&name, &[1.0, 0.0], 10, None, &["txput"]).await; + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// An UpdateItem whose UpdateExpression REMOVEs the vector attribute takes the item +/// out of the index, and one that SETs it back re-admits it. +/// +/// The storage unit tests prove the queue applies a vectorless image as a removal; +/// this proves the whole wire path agrees: expression parsing, the update write, +/// the enqueue, and the drain. The re-admission half exists so the removal cannot +/// pass by accident of a broken index that returns nothing for any query. +#[tokio::test] +async fn update_item_removing_the_vector_attribute_leaves_the_index() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_update_remove"); + create_vector_table(&name, 2, "COSINE", false).await; + put_vector(&name, "a", None, &[1.0, 0.0]).await; + put_vector(&name, "b", None, &[0.9, 0.1]).await; + search_until_pks(&name, &[1.0, 0.0], 10, None, &["a", "b"]).await; + + let (status, text) = call( + "UpdateItem", + &format!( + r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "a"}}}}, + "UpdateExpression": "REMOVE emb"}}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateItem REMOVE failed: {text}"); + // "b alone" is ordered behind the REMOVE in the queue, so it cannot pass in + // the window before the REMOVE applies. + search_until_pks(&name, &[1.0, 0.0], 10, None, &["b"]).await; + + let (status, text) = call( + "UpdateItem", + &format!( + r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "a"}}}}, + "UpdateExpression": "SET emb = :v", + "ExpressionAttributeValues": {{":v": {{"L": [{{"N": "1.0"}}, {{"N": "0.0"}}]}}}}}}"# + ), + ) + .await; + assert_eq!(status, 200, "UpdateItem SET failed: {text}"); + search_until_pks(&name, &[1.0, 0.0], 10, None, &["a", "b"]).await; + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} + +/// A malformed vector cannot slip past write validation inside `if_not_exists`. +/// +/// `SET emb = if_not_exists(emb, :v)` persists `:v` verbatim whenever the +/// attribute is absent, which is exactly the first write of a vector, yet an +/// earlier extraction only surfaced bare `SET emb = :v` for validation. The +/// bypass was concrete: a wrong-dimension vector through the fallback form +/// returned 200 and the item was silently omitted from the index, where the +/// service refuses the write. Both forms must now be refused identically, and +/// the valid fallback form must still work end to end. +#[tokio::test] +async fn if_not_exists_cannot_smuggle_a_malformed_vector() { + if skip_unless_supported().await { + return; + } + let name = table_name("vi_ine_validate"); + create_vector_table(&name, 2, "COSINE", false).await; + + // Wrong dimensions through the fallback form must be refused. + let (status, text) = call( + "UpdateItem", + &format!( + r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "a"}}}}, + "UpdateExpression": "SET emb = if_not_exists(emb, :v)", + "ExpressionAttributeValues": {{":v": {{"L": [{{"N": "1.0"}}, {{"N": "2.0"}}, {{"N": "3.0"}}]}}}}}}"# + ), + ) + .await; + assert_eq!( + status, 400, + "a wrong-dimension vector through if_not_exists must be refused, \ + not accepted and silently omitted from the index: {text}" + ); + + // The valid fallback form must still work and reach the index. + let (status, text) = call( + "UpdateItem", + &format!( + r#"{{"TableName": "{name}", "Key": {{"pk": {{"S": "a"}}}}, + "UpdateExpression": "SET emb = if_not_exists(emb, :v)", + "ExpressionAttributeValues": {{":v": {{"L": [{{"N": "1.0"}}, {{"N": "0.0"}}]}}}}}}"# + ), + ) + .await; + assert_eq!(status, 200, "a valid vector through if_not_exists must work: {text}"); + search_until_pks(&name, &[1.0, 0.0], 10, None, &["a"]).await; + + let _ = call("DeleteTable", &format!(r#"{{"TableName": "{name}"}}"#)).await; +} diff --git a/tests/rust/src/vector_index_unsupported.rs b/tests/rust/src/vector_index_unsupported.rs index b0a2a073..af7ddd36 100644 --- a/tests/rust/src/vector_index_unsupported.rs +++ b/tests/rust/src/vector_index_unsupported.rs @@ -52,7 +52,7 @@ fn http_client() -> reqwest::Client { /// dispatch, so an unsigned request never reaches the capability gate and the /// test would pass for the wrong reason, asserting an auth failure while /// believing it asserted a vector refusal. -async fn call(target: &str, body: &str) -> (u16, String) { +pub(crate) async fn call(target: &str, body: &str) -> (u16, String) { let access_key = std::env::var("AWS_ACCESS_KEY_ID").expect("AWS_ACCESS_KEY_ID must be set"); let secret_key = std::env::var("AWS_SECRET_ACCESS_KEY").expect("AWS_SECRET_ACCESS_KEY must be set"); @@ -142,7 +142,7 @@ fn assert_validation_exception(status: u16, body: &str, expected_message: &str) assert_eq!(message, expected_message); } -fn table_name(suffix: &str) -> String { +pub(crate) fn table_name(suffix: &str) -> String { format!("vec_wire_{}_{}", suffix, uuid::Uuid::new_v4().simple()) } @@ -152,9 +152,7 @@ fn table_name(suffix: &str) -> String { /// answer, because the capability is not otherwise observable over the wire. /// Every test here asserts a refusal, so on a backend that *does* implement /// vector search they must skip rather than fail: the refusals are the contract -/// for non-participating backends only. No in-tree backend implements it today, -/// so this returns false for both, and the mechanism exists for the first one that -/// does. +/// for non-participating backends only. /// /// Deliberately distinguishes the refusal from any other failure. Treating "not a /// 200" as unsupported would make the whole suite skip silently the first time an @@ -188,18 +186,23 @@ async fn backend_supports_vectors() -> bool { panic!("vector support probe failed for an unrelated reason: {status} {text}"); } +/// Whether the running backend implements vector indexes, for the suite that +/// asserts the participating behaviour. Named positively so the caller reads as +/// an opt-in rather than as a double negative. +pub(crate) async fn vectors_supported() -> bool { + !is_real_dynamodb() && backend_supports_vectors().await +} + /// What this run asserts about the backend's vector capability. /// -/// Three states on purpose. A suite that adapts to whatever the backend reports -/// never asserts *which* backend is under test, so a backend that silently gained -/// or lost vector support would change what is exercised and still report green. -/// `EXTENDDB_EXPECT_VECTORS` lets a CI job state its expectation: +/// Three states on purpose. Both vector suites otherwise adapt to whatever the +/// backend reports, so no run asserts *which* backend is under test, and a backend +/// that silently lost vector support would skip the entire positive suite and still +/// report green. `EXTENDDB_EXPECT_VECTORS` lets a CI job state its expectation: /// -/// - `1`: the backend must support vectors, so a positive suite failing to run is -/// an error rather than a skip. No such suite exists yet; it arrives with the -/// first participating backend, and the variable is defined here so that suite -/// inherits the guard rather than inventing one. -/// - `0`: the backend must not, so these refusal tests failing to run is an error. +/// - `1`: the backend must support vectors. The positive suite failing to run is an +/// error rather than a skip. +/// - `0`: the backend must not. The refusal suite failing to run is an error. /// - unset: adapt quietly, so a plain local `cargo test` works against either /// backend without ceremony. /// @@ -221,8 +224,8 @@ pub(crate) fn expect_vectors() -> Option { /// /// The expectation is read *before* probing, not inside the assertion. Reading it /// inside a short-circuiting `&&` meant an invalid value was never validated on a -/// backend without vector support, which is every backend today: the typo guard -/// was dead exactly where it was needed. +/// backend without vector support, so the typo guard was dead in the Postgres job, +/// which is the one job these refusal tests exist for. async fn skip_if_supported() -> bool { if is_real_dynamodb() { return true; @@ -235,10 +238,9 @@ async fn skip_if_supported() -> bool { these refusal tests would skip: either a backend gained vector support \ unnoticed, or this job sets the wrong expectation" ); - // The converse matters here too, and not only in a positive suite. Until a - // backend implements vector search there is no positive suite to notice a run - // that claims support the backend does not have, so the claim would pass - // unchecked. + // The converse is asserted here too rather than left to the positive suite + // alone: a run claiming support against a backend that refuses is wrong + // whichever suite happens to notice first. assert!( !(!supported && expected == Some(true)), "EXTENDDB_EXPECT_VECTORS=1 but the backend refuses vector indexes: either \ diff --git a/tests/test_gsi_async.py b/tests/test_gsi_async.py index 35abbc1c..c71f7b2e 100755 --- a/tests/test_gsi_async.py +++ b/tests/test_gsi_async.py @@ -230,7 +230,7 @@ def test_base_table_read_is_immediately_consistent( def test_gsi_sync_path_with_zero_delay( self, dynamodb_client, gsi_table ): - """When gsi_propagation_delay_ms=0, GSI updates are synchronous. + """When index_propagation_delay_ms=0, GSI updates are synchronous. Sets the system-wide delay to 0, writes an item, and asserts the GSI query returns the item immediately (single query, no @@ -239,8 +239,8 @@ def test_gsi_sync_path_with_zero_delay( table_name = gsi_table # Save original delay and set to 0 (sync mode). - original_delay = extenddb_settings_get("gsi_propagation_delay_ms") - extenddb_settings_set("gsi_propagation_delay_ms", "0") + original_delay = extenddb_settings_get("index_propagation_delay_ms") + extenddb_settings_set("index_propagation_delay_ms", "0") try: pk = f"sync-{uuid.uuid4().hex[:8]}" @@ -265,28 +265,28 @@ def test_gsi_sync_path_with_zero_delay( ExpressionAttributeValues={":pk": {"S": gsi_pk}}, ) assert resp["Count"] == 1, ( - "With gsi_propagation_delay_ms=0, GSI should be " + "With index_propagation_delay_ms=0, GSI should be " "immediately consistent" ) assert resp["Items"][0]["data"]["S"] == "sync-value" finally: # Restore original delay. - extenddb_settings_set("gsi_propagation_delay_ms", original_delay) + extenddb_settings_set("index_propagation_delay_ms", original_delay) def test_gsi_configured_delay_range( self, dynamodb_client, gsi_table ): """GSI propagation delay falls within the configured range. - Sets gsi_propagation_delay_ms to 50, writes items, and measures + Sets index_propagation_delay_ms to 50, writes items, and measures the observed propagation delay. The delay should be within [1, 50]ms (the random range used by the worker). """ table_name = gsi_table # Save original delay and set to 50ms. - original_delay = extenddb_settings_get("gsi_propagation_delay_ms") - extenddb_settings_set("gsi_propagation_delay_ms", "50") + original_delay = extenddb_settings_get("index_propagation_delay_ms") + extenddb_settings_set("index_propagation_delay_ms", "50") try: delays = [] @@ -336,4 +336,4 @@ def test_gsi_configured_delay_range( f"Delays exceeded 500ms: {[f'{d:.1f}' for d in delays]}" ) finally: - extenddb_settings_set("gsi_propagation_delay_ms", original_delay) + extenddb_settings_set("index_propagation_delay_ms", original_delay) diff --git a/tests/test_gsi_async_queue.py b/tests/test_gsi_async_queue.py index 91701dfd..6161a663 100644 --- a/tests/test_gsi_async_queue.py +++ b/tests/test_gsi_async_queue.py @@ -77,7 +77,7 @@ def _set_gsi_delay(config_path, ms): ``--config`` must precede the ``set`` subcommand for the settings CLI. """ - _run_extenddb("settings", "--config", config_path, "set", "gsi_propagation_delay_ms", str(ms)) + _run_extenddb("settings", "--config", config_path, "set", "index_propagation_delay_ms", str(ms)) def _server_pid_on_port(port): diff --git a/tests/test_gsi_async_queue_sqlite.py b/tests/test_gsi_async_queue_sqlite.py index 1c222d63..a725999a 100644 --- a/tests/test_gsi_async_queue_sqlite.py +++ b/tests/test_gsi_async_queue_sqlite.py @@ -155,21 +155,21 @@ def _init(self, gsi_delay_ms: int) -> None: self.admin_password = line.split("Password:", 1)[1].strip() _patch_port(self.config, self.port) # Set the system GSI delay before serving (read at startup). - self._run("settings", "set", "gsi_propagation_delay_ms", str(gsi_delay_ms)) + self._run("settings", "set", "index_propagation_delay_ms", str(gsi_delay_ms)) # Assert the delay is persisted as expected BEFORE the server starts. # These tests run isolated, file-backed servers with their own database, - # so the main devtools/run-tests setting of gsi_propagation_delay_ms=0 on + # so the main devtools/run-tests setting of index_propagation_delay_ms=0 on # the shared server cannot race or clobber this value. The read-back pins # that invariant rather than relying on it implicitly. conn = self._connect() try: row = conn.execute( - "SELECT value FROM settings WHERE key = 'gsi_propagation_delay_ms'" + "SELECT value FROM settings WHERE key = 'index_propagation_delay_ms'" ).fetchone() finally: conn.close() assert row is not None and row[0] == str(gsi_delay_ms), ( - f"expected gsi_propagation_delay_ms={gsi_delay_ms} persisted at startup, " + f"expected index_propagation_delay_ms={gsi_delay_ms} persisted at startup, " f"got {row!r}" )