From 0f75fb8943b8a8b35c4c16b6bcb5ec1df5e41236 Mon Sep 17 00:00:00 2001 From: ila Date: Wed, 22 Jul 2026 15:45:36 +0200 Subject: [PATCH 01/36] Make delta capture transactional --- CMakeLists.txt | 1 + docs/codebase-audit-2026-07-22.md | 134 ++++ src/core/refresh_locks.cpp | 18 + src/include/core/refresh_locks.hpp | 26 + .../rules/transactional_delta_capture.hpp | 40 ++ src/rules/refresh_insert_rule.cpp | 586 +++--------------- src/rules/transactional_delta_capture.cpp | 355 +++++++++++ src/upsert/refresh.cpp | 2 + test/sql/concurrency.test | 61 ++ test/sql/insert_rule.test | 203 +++++- 10 files changed, 935 insertions(+), 491 deletions(-) create mode 100644 docs/codebase-audit-2026-07-22.md create mode 100644 src/include/rules/transactional_delta_capture.hpp create mode 100644 src/rules/transactional_delta_capture.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 326cbe74..9889f8bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,6 +57,7 @@ set(EXTENSION_SOURCES src/upsert/refresh_index_regen.cpp src/rules/incremental_rewrite_rule.cpp src/rules/refresh_insert_rule.cpp + src/rules/transactional_delta_capture.cpp src/rules/schema_evolution.cpp src/rules/column_hider.cpp src/core/refresh_locks.cpp diff --git a/docs/codebase-audit-2026-07-22.md b/docs/codebase-audit-2026-07-22.md new file mode 100644 index 00000000..f19efebb --- /dev/null +++ b/docs/codebase-audit-2026-07-22.md @@ -0,0 +1,134 @@ +# OpenIVM codebase audit and remediation plan + +Date: 2026-07-22 + +This audit covers the current working tree, including the in-progress nonlocal refresh changes. Findings were checked +from correctness, reliability, performance, simplicity, and code-hygiene perspectives. Confirmed bugs were reproduced +with real materialized views and refreshes rather than inferred only from static analysis. + +The remediation rule is correctness first: an incrementalizable query must retain a correct incremental or affected-domain +maintenance path. A bug must not be hidden by weakening a test or silently changing the view to `FULL_REFRESH`. + +## P0: correctness and transaction safety + +- [x] **Capture delta rows in the base DML transaction.** Delta capture is now a streaming logical/physical operator that + appends through the caller's `ClientContext`, so base DML and delta rows share one commit or rollback. A resolved + catalog-level lock prevents refresh from advancing its watermark past an uncommitted delta. Regression coverage includes + rollback, constraint failure, defaults, generated columns, `RETURNING`, and an uncommitted DML/refresh race. +- [ ] **Capture actual `ON CONFLICT` outcomes.** `INSERT ... ON CONFLICT DO UPDATE` and `INSERT OR REPLACE` do not emit the + required delete-old/insert-new delta pair. Derive deltas from rows actually inserted or updated, not merely the input to + `LogicalInsert`. +- [ ] **Make MV lifecycle and native refresh transactional.** CREATE, REPLACE, DROP, ALTER, and `PRAGMA refresh` currently + execute material work through helper autocommit connections. Caller rollback cannot restore a consistent collection of + user view, backing table, metadata, and deltas; failed replacement can destroy the old MV. Use the caller transaction for + native-catalog work and stage cross-catalog replacement before publication. +- [ ] **Continue cascade traversal when the current node has no source deltas.** The root empty-delta fast path returns + before visiting downstream nodes. A child can have independent source deltas or pending parent-delta rows after an earlier + failed cascade. Model `node skipped` separately from `graph traversal complete` and checkpoint each node independently. +- [ ] **Use NULL-safe affected-window partition matching.** `IN` does not select a NULL partition even though SQL window + partitioning groups NULL values. Join the affected-key relation with `IS NOT DISTINCT FROM` for every partition key. +- [ ] **Preserve `SUM` NULL semantics.** Weighted SUM alone cannot distinguish numeric zero from no non-NULL inputs. Persist + a hidden `COUNT(sum_argument)` and render NULL when that count reaches zero. +- [ ] **Use an unconditional row count for group existence.** `COUNT(nullable_expression)=0` does not mean a group is empty. + Persist a hidden `COUNT(*)` solely for deciding whether the group row must be deleted. +- [ ] **Delete the heuristic group-measure update fast path.** `refresh_group_measure.cpp` infers aggregate lineage from SQL + text, output-alias substrings, LPTS formatting, and value strings. A reproduced update to input `v` changed unrelated + aliases `revenue` and `savings`. Use the existing affected-group recompute path until bound aggregate lineage can prove a + safe fast path. +- [ ] **Validate the persisted `RefreshType` ABI.** Unknown values are raw-cast and can pass through refresh without applying + data changes while still advancing the watermark. Give every persisted value an explicit ordinal and decode it through a + single validating migration function; every dispatch must reject unknown values. +- [ ] **Give one refresh-node API ownership of hooks.** Scheduled hooks currently run twice, empty-delta skipping can suppress + replace hooks, cascaded nodes bypass hooks, and hook errors are swallowed. Hooks, skip decisions, refresh, and error + propagation must have exactly-once semantics. +- [ ] **Make crash recovery a durable state transition.** Some full-recompute paths bypass the in-progress journal, while + recovery clears the journal before recovery succeeds. Persist an attempt/checkpoint before every data mutation and clear it + only with successful post-refresh metadata. +- [ ] **Do not consume DuckLake structural-change identity before refresh succeeds.** Source-identity probing currently + persists a new table ID immediately. Carry it as pending activity and commit it with the successful snapshot/watermark. + +## P1: fail-closed metadata and authoritative planning + +- [ ] **Distinguish metadata error, absence, and empty state.** Several getters convert query errors into `{}`, `false`, or a + fallback strategy. Refresh must fail closed on schema/query errors; `optional` should mean genuinely absent metadata only. +- [ ] **Reject missing strategy contracts instead of falling through.** Missing DISTINCT or semi/anti metadata currently + falls into unrelated compiler cases. Validate the selected strategy and all required metadata once before compilation. +- [ ] **Remove runtime type guessing from window lineage.** Bare-name `information_schema` lookup with `LIMIT 1` is ambiguous + across schemas/catalogs, and coercing both comparison sides to VARCHAR changes bound comparison semantics. Persist exact + bound cast chains and source identity at CREATE time. +- [ ] **Make source identity fully qualified.** DML classification, metadata lookup, locking, cleanup, and dependency handling + rely too heavily on bare table or view names. Use catalog, schema, and stable object identity. +- [ ] **Centralize complete per-view DROP cleanup.** Hooks, refresh history/profile rows, dependency rows, and matcher state can + survive DROP and be inherited by a newly created MV of the same name. +- [ ] **Preserve quoted output identifiers.** Planner output names are sanitized for internal use and then reused where the + physical schema requires the original SQL identifier. Quoted names containing punctuation can fail during MV creation. +- [ ] **Classify only after required lineage is valid.** Window lineage can demote a finalized model to `FULL_REFRESH` after + node maintenance and update semantics have already been derived, leaving contradictory model state. +- [ ] **Finish or reverse the `CURRENT_DIFF_RECOMPUTE` removal coherently.** Deterministic SAMPLE, POSITIONAL, and ASOF shapes + are demoted to unconditional full refresh while stale nonlocal strategies, documentation, and benchmark paths remain. + Preserve a typed affected/current-diff recompute path where correctness permits it; delete truly dead strategies. + +## P2: remove SQL-text hacks and duplicate abstractions + +- [ ] **Render Spark SQL by dialect instead of global regex replacement.** Whole-program replacement mutates literals, + comments, and quoted identifiers, while early full-refresh paths bypass conversion entirely. Render casts, timestamps, and + NULL-safe equality through dialect-aware builders/LPTS. +- [ ] **Replace affected-group source substitution with plan-node substitution.** Plain occurrence replacement can modify + literals, comments, CTE references, and longer identifiers. Replace the exact `LogicalGet` occurrence before serialization. +- [ ] **Delete the workload-specific TPC-DI left-join shortcut.** General refresh code hardcodes `fact_market_history`, three + source names, and `sk_company_id`. Retain the generic correct path unless typed lineage proves a generic optimization. +- [ ] **Replace `parser_sql_extractors.cpp` with bound-plan extraction.** Its approximately 1,250 lines duplicate tokenization, + quote handling, clause parsing, alias rewriting, and parenthesis tracking for DISTINCT, filtered aggregates, and semi/anti + metadata already represented by bound logical operators and expressions. +- [ ] **Delete the independent refresh SQL mini-parsers.** Aggregate-filter stripping, SCD2 predicate injection, running-window + parsing, and LPTS CTE parsing belong in logical-plan/AST construction, not four inconsistent scanners of emitted SQL. +- [ ] **Make the delta IR authoritative or remove its diagnostic-only graph.** Per-node maintenance, update semantics, generic + affected domains, lineage facts, and auxiliary-state annotations are populated and logged but do not select compilation. + Keeping them beside parallel `RefreshType`/metadata dispatch creates drift. +- [ ] **Remove the nonfunctional view-matching product surface until it has behavior.** Candidate lookup, canonicalization, + and predicate implication are stubs, but settings, metadata columns, logs, CMake targets, and estimator APIs remain public. + Keep the FK constraint functionality that is actually used. +- [ ] **Replace handwritten JSON and pipe-delimited lineage encoding.** Important semantic metadata is manually scanned and + malformed fields can be skipped while keeping an incomplete lineage arm. Use typed normalized rows or a versioned structured + serializer and reject malformed records as a whole. +- [ ] **Split CREATE and refresh into typed phase programs.** `PlanFunction` and `GenerateRefreshSQL` each span roughly a + thousand lines. Use immutable inputs and results such as `ViewDefinition`, `RefreshMetadataSnapshot`, `PendingDeltaBatch`, + `RefreshStrategy`, and `RefreshProgram`; do not introduce a class hierarchy. +- [ ] **Replace boolean/argument soup with strategy input/result structs.** Compiler functions with 17 parameters, output + booleans, and implicit reclassification hide invariants and make call-order mistakes likely. +- [ ] **Use tagged CREATE operations instead of magic strings.** Cleanup, profiling, schema derivation, and executable SQL are + currently mixed in one string vector and reparsed by prefixes and tab-delimited fields. +- [ ] **Consolidate duplicated profiling and NULL-safe predicate helpers.** CREATE and refresh profiling independently implement + IDs, retention, step collection, and persistence; SQL utilities contain parallel NULL-safe builders. + +## P3: refresh performance and repeated work + +- [ ] **Remove exact base-table counting from normal join compilation.** Standard join refresh performs `COUNT(*)` for every + base table on every refresh after delta activity has already been computed. Use activity/statistics or remove the heuristic. +- [ ] **Avoid repeated parse/bind/plan/model reconstruction.** Refresh planning currently reparses and optimizes the stored + query multiple times. Persist a versioned create-time typed model and rebuild only for migration or invalidation. +- [ ] **Load metadata once.** `GenerateRefreshSQL` issues many independent queries for one view and sometimes reloads the same + field. Load one immutable snapshot; keep writes and recovery checkpoints separate. +- [ ] **Consolidate projection deletes once.** The projection path builds the same grouped/ranked net delta independently for + DELETE and INSERT. Materialize or share the consolidated batch. +- [ ] **Carry delta activity through the full refresh.** Activity checks, join-term selection, max timestamp lookup, and cleanup + repeatedly scan the same delta tables. Capture count/delete/watermark information once per transaction. +- [ ] **Price actual join work in the adaptive model.** The model counts exponential terms but underprices copied plan nodes, + base-leaf appearances, compilation work, and generated SQL size. +- [ ] **Remove the fake `ConstraintCache` or make it a cache.** It repopulates a map that no read path consumes, while FK cost + estimation separately parses textual constraint descriptions. + +## Target refresh flow + +At CREATE time, bind once and persist a versioned typed view definition, delta plan, output schema, exact source identities, +and concrete strategy metadata. + +At refresh time: + +1. Acquire the graph/node transaction and locks. +2. Load one immutable metadata snapshot. +3. Capture one pending delta batch and its watermark. +4. Choose one explicit correctness-preserving strategy; use cost only between valid strategies. +5. Compile a structured refresh program containing data, cascade, cleanup, and checkpoint phases. +6. Execute native work atomically; render SQL once only at a genuine external-dialect boundary. +7. Commit data, cursor/snapshot, source identity, and recovery state together. diff --git a/src/core/refresh_locks.cpp b/src/core/refresh_locks.cpp index b0dbff70..c3d54714 100644 --- a/src/core/refresh_locks.cpp +++ b/src/core/refresh_locks.cpp @@ -5,6 +5,7 @@ namespace duckdb { std::mutex RefreshLocks::map_mutex_; std::unordered_map> RefreshLocks::view_mutexes_; std::unordered_map> RefreshLocks::delta_mutexes_; +std::unordered_map> RefreshLocks::delta_catalog_mutexes_; std::mutex &RefreshLocks::GetViewMutex(const string &view_name) { std::lock_guard guard(map_mutex_); @@ -24,6 +25,15 @@ std::mutex &RefreshLocks::GetDeltaMutex(const string &delta_table_name) { return *entry; } +std::mutex &RefreshLocks::GetDeltaCatalogMutex(const string &catalog_name) { + std::lock_guard guard(map_mutex_); + auto &entry = delta_catalog_mutexes_[catalog_name]; + if (!entry) { + entry = duckdb::unique_ptr(new std::mutex()); + } + return *entry; +} + void RefreshLocks::LockView(const string &view_name) { GetViewMutex(view_name).lock(); } @@ -44,4 +54,12 @@ void RefreshLocks::UnlockDelta(const string &delta_table_name) { GetDeltaMutex(delta_table_name).unlock(); } +void RefreshLocks::LockDeltaCatalog(const string &catalog_name) { + GetDeltaCatalogMutex(catalog_name).lock(); +} + +void RefreshLocks::UnlockDeltaCatalog(const string &catalog_name) { + GetDeltaCatalogMutex(catalog_name).unlock(); +} + } // namespace duckdb diff --git a/src/include/core/refresh_locks.hpp b/src/include/core/refresh_locks.hpp index 904045ca..d0022f86 100644 --- a/src/include/core/refresh_locks.hpp +++ b/src/include/core/refresh_locks.hpp @@ -37,13 +37,39 @@ class RefreshLocks { static void UnlockDelta(const string &delta_table_name); + // Serializes refresh with transactions that write native delta tables in the same catalog. + // The writer holds this lock until transaction commit/rollback so a refresh cannot advance + // its timestamp cursor past an uncommitted delta row. + static void LockDeltaCatalog(const string &catalog_name); + + static void UnlockDeltaCatalog(const string &catalog_name); + private: static std::mutex &GetViewMutex(const string &view_name); static std::mutex &GetDeltaMutex(const string &delta_table_name); + static std::mutex &GetDeltaCatalogMutex(const string &catalog_name); static std::mutex map_mutex_; static std::unordered_map> view_mutexes_; static std::unordered_map> delta_mutexes_; + static std::unordered_map> delta_catalog_mutexes_; +}; + +// RAII guard for catalog-level delta transaction locks. +class DeltaCatalogLockGuard { + string name_; + +public: + explicit DeltaCatalogLockGuard(const string &catalog_name) : name_(catalog_name) { + RefreshLocks::LockDeltaCatalog(name_); + } + ~DeltaCatalogLockGuard() { + RefreshLocks::UnlockDeltaCatalog(name_); + } + DeltaCatalogLockGuard(const DeltaCatalogLockGuard &) = delete; + DeltaCatalogLockGuard &operator=(const DeltaCatalogLockGuard &) = delete; + DeltaCatalogLockGuard(DeltaCatalogLockGuard &&) = delete; + DeltaCatalogLockGuard &operator=(DeltaCatalogLockGuard &&) = delete; }; // RAII guard for delta-table locks. Automatically unlocks on scope exit (including exceptions). diff --git a/src/include/rules/transactional_delta_capture.hpp b/src/include/rules/transactional_delta_capture.hpp new file mode 100644 index 00000000..6cd18d1d --- /dev/null +++ b/src/include/rules/transactional_delta_capture.hpp @@ -0,0 +1,40 @@ +#ifndef TRANSACTIONAL_DELTA_CAPTURE_HPP +#define TRANSACTIONAL_DELTA_CAPTURE_HPP + +#include "duckdb/planner/operator/logical_extension_operator.hpp" + +namespace duckdb { + +class TableCatalogEntry; + +enum class DeltaCaptureMode : uint8_t { INSERT, DELETE, UPDATE }; + +// Streaming plan operator that writes the rows affected by a base-table DML statement +// to its delta table through the caller's transaction before passing the input onward. +class LogicalTransactionalDeltaCapture : public LogicalExtensionOperator { +public: + LogicalTransactionalDeltaCapture(TableCatalogEntry &base_table, TableCatalogEntry &delta_table, + DeltaCaptureMode mode, vector> update_expressions = {}, + vector update_columns = {}, optional_idx row_id_index = {}); + + TableCatalogEntry &base_table; + TableCatalogEntry &delta_table; + DeltaCaptureMode mode; + vector update_columns; + optional_idx row_id_index; + + PhysicalOperator &CreatePlan(ClientContext &context, PhysicalPlanGenerator &planner) override; + vector GetColumnBindings() override; + string GetName() const override; + string GetExtensionName() const override; + bool SupportSerialization() const override { + return false; + } + +protected: + void ResolveTypes() override; +}; + +} // namespace duckdb + +#endif // TRANSACTIONAL_DELTA_CAPTURE_HPP diff --git a/src/rules/refresh_insert_rule.cpp b/src/rules/refresh_insert_rule.cpp index 4f88a18f..3ad38ab9 100644 --- a/src/rules/refresh_insert_rule.cpp +++ b/src/rules/refresh_insert_rule.cpp @@ -7,170 +7,64 @@ #include "core/refresh_locks.hpp" #include "core/sql_utils.hpp" #include "rules/column_hider.hpp" +#include "rules/transactional_delta_capture.hpp" -#include "lpts_pipeline.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" -#include "duckdb/common/column_index.hpp" #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" -#include "duckdb/function/table/read_csv.hpp" #include "duckdb/main/connection.hpp" #include "duckdb/optimizer/optimizer.hpp" #include "duckdb/parser/parsed_data/alter_table_info.hpp" #include "duckdb/parser/parsed_data/drop_info.hpp" -#include "duckdb/parser/parser.hpp" -#include "duckdb/parser/statement/logical_plan_statement.hpp" -#include "duckdb/planner/operator/logical_comparison_join.hpp" -#include "duckdb/parser/tableref/basetableref.hpp" -#include "duckdb/planner/expression.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" -#include "duckdb/planner/expression_iterator.hpp" -#include "duckdb/planner/operator/logical_aggregate.hpp" #include "duckdb/planner/operator/logical_delete.hpp" -#include "duckdb/planner/operator/logical_dummy_scan.hpp" -#include "duckdb/planner/operator/logical_expression_get.hpp" -#include "duckdb/planner/operator/logical_filter.hpp" -#include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/planner/operator/logical_insert.hpp" #include "duckdb/planner/operator/logical_projection.hpp" #include "duckdb/planner/operator/logical_simple.hpp" #include "duckdb/planner/operator/logical_update.hpp" -#include "duckdb/planner/planner.hpp" - -#include -#include namespace duckdb { -// PAC compatibility boundary: delta writes run through a fresh connection, so -// disable PAC checks when that extension is loaded in the caller session. -static void DisablePACIfLoaded(ClientContext &context, Connection &con) { - Value pac_val; - if (context.TryGetCurrentSetting("pac_check", pac_val)) { - con.Query("SET pac_check = false"); - } +static bool IsTrackedBaseTable(ClientContext &context, const string &table_name) { + Connection con(*context.db); + return RefreshMetadata(con).IsBaseTable(table_name); } -// Build the data column list from a delta table catalog entry, excluding metadata columns. -// Returns e.g. "id, name, val" (quoted) — the base table columns only. -static string BuildDeltaDataColumns(TableCatalogEntry &delta_entry) { - string cols; - for (auto &col : delta_entry.GetColumns().Logical()) { - if (col.GetName() == openivm::MULTIPLICITY_COL || col.GetName() == openivm::TIMESTAMP_COL) { - continue; - } - if (!cols.empty()) { - cols += ", "; - } - cols += KeywordHelper::WriteOptionallyQuoted(col.GetName()); +static void ResolveInsertDefaults(OptimizerExtensionInput &input, LogicalInsert &insert) { + if (insert.column_index_map.empty()) { + return; } - return cols; -} - -// Build "INSERT INTO delta_t (col1, col2, ..., mul, ts)" prefix for delta writes. -static string BuildDeltaInsertPrefix(const string &full_delta_table_name, TableCatalogEntry &delta_entry) { - string col_list = BuildDeltaDataColumns(delta_entry); - return "INSERT INTO " + full_delta_table_name + " (" + col_list + ", " + string(openivm::MULTIPLICITY_COL) + ", " + - string(openivm::TIMESTAMP_COL) + ")"; -} - -// Build "SELECT col1, col2, ..., , now()::timestamp FROM " for delta writes. -static string BuildDeltaSelectFrom(TableCatalogEntry &delta_entry, const string &mul_val, const string &source) { - string cols = BuildDeltaDataColumns(delta_entry); - return "SELECT " + cols + ", " + mul_val + ", now()::timestamp FROM " + source; -} - -using BoundColumnNameMap = std::map, string>; - -static void CollectBoundColumnNames(LogicalOperator &op, BoundColumnNameMap &column_names) { - if (op.type == LogicalOperatorType::LOGICAL_GET) { - auto &get = op.Cast(); - auto bindings = get.GetColumnBindings(); - auto column_ids = get.GetColumnIds(); - for (auto &binding : bindings) { - if (binding.column_index >= column_ids.size()) { - continue; + auto child_bindings = insert.children[0]->GetColumnBindings(); + vector> expressions; + for (auto &column : insert.table.GetColumns().Physical()) { + auto mapped_index = insert.column_index_map[column.Physical()]; + if (mapped_index == DConstants::INVALID_INDEX) { + expressions.push_back(insert.bound_defaults[column.StorageOid()]->Copy()); + } else { + if (mapped_index >= child_bindings.size()) { + throw InternalException("OpenIVM insert column mapping is out of range"); } - const auto &column_name = get.GetColumnName(column_ids[binding.column_index]); - column_names[{binding.table_index, binding.column_index}] = - KeywordHelper::WriteOptionallyQuoted(column_name); + expressions.push_back(make_uniq(column.Type(), child_bindings[mapped_index])); } } - for (auto &child : op.children) { - CollectBoundColumnNames(*child, column_names); - } + auto projection = make_uniq(input.optimizer.binder.GenerateTableIndex(), std::move(expressions)); + projection->children.push_back(std::move(insert.children[0])); + insert.children[0] = std::move(projection); + insert.column_index_map = physical_index_vector_t(); + insert.expected_types = insert.table.GetTypes(); } -static void QuoteBoundColumnRefs(Expression &expr, const BoundColumnNameMap &column_names) { - if (expr.GetExpressionClass() == ExpressionClass::BOUND_COLUMN_REF) { - auto &bcr = expr.Cast(); - auto entry = column_names.find({bcr.binding.table_index, bcr.binding.column_index}); - if (entry != column_names.end()) { - bcr.alias = entry->second; - } +static idx_t FindExpressionBindingIndex(LogicalOperator &child, const Expression &expression) { + if (expression.type != ExpressionType::BOUND_COLUMN_REF) { + throw InternalException("OpenIVM expected a bound row-id column reference"); } - ExpressionIterator::EnumerateChildren(expr, [&](unique_ptr &child) { - if (child) { - QuoteBoundColumnRefs(*child, column_names); + auto &column_ref = expression.Cast(); + auto bindings = child.GetColumnBindings(); + for (idx_t index = 0; index < bindings.size(); index++) { + if (bindings[index] == column_ref.binding) { + return index; } - }); -} - -static string QuotedExpressionString(const unique_ptr &expr, const BoundColumnNameMap &column_names) { - auto copy = expr->Copy(); - QuoteBoundColumnRefs(*copy, column_names); - return copy->ToString(); -} - -static string BuildDeltaInsertFromPlan(ClientContext &context, TableCatalogEntry &delta_entry, - const string &full_delta_table_name, unique_ptr &source_plan) { - string prefix = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry); - SqlDialect dialect = openivm::CompileFactsContextSlot::Get(context).target_dialect; - auto ast = LogicalPlanToAst(context, source_plan, dialect); - auto cte_list = AstToCteList(*ast, dialect); - string subquery_string = cte_list->ToQuery(false); - if (!subquery_string.empty() && subquery_string.back() == ';') { - subquery_string.pop_back(); - } - return prefix + " SELECT *, 1, now()::timestamp FROM (" + subquery_string + ")"; -} - -static string BuildDeleteDeltaInsertFromPlan(ClientContext &context, TableCatalogEntry &delta_entry, - const string &full_delta_table_name, const string &full_table_name, - unique_ptr &source_plan) { - string prefix = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry); - string data_cols = BuildDeltaDataColumns(delta_entry); - SqlDialect dialect = openivm::CompileFactsContextSlot::Get(context).target_dialect; - auto ast = LogicalPlanToAst(context, source_plan, dialect); - auto cte_list = AstToCteList(*ast, dialect); - string subquery_string = cte_list->ToQuery(false); - if (!subquery_string.empty() && subquery_string.back() == ';') { - subquery_string.pop_back(); } - // DuckDB DELETE children identify physical rows by rowid; read the base table - // columns back through that rowid set to materialize the negative delta tuple. - return prefix + " SELECT " + data_cols + ", -1, now()::timestamp FROM " + full_table_name + - " WHERE rowid IN (SELECT rowid FROM (" + subquery_string + ") openivm_deleted_rows)"; -} - -static bool IsRowIdColumn(const unique_ptr &expr) { - if (!expr || expr->type != ExpressionType::BOUND_COLUMN_REF) { - return false; - } - auto &col_ref = expr->Cast(); - return StringUtil::CIEquals(col_ref.GetName(), "rowid") || StringUtil::CIEquals(col_ref.alias, "rowid"); -} - -static bool IsSemiJoinOnRowId(LogicalComparisonJoin &join) { - if (join.join_type != JoinType::SEMI) { - return false; - } - for (auto &condition : join.conditions) { - if (IsRowIdColumn(condition.left) || IsRowIdColumn(condition.right)) { - return true; - } - } - return false; + throw InternalException("OpenIVM could not resolve the DML row-id binding"); } RefreshInsertRule::RefreshInsertRule() { @@ -340,379 +234,93 @@ void RefreshInsertRule::RefreshInsertRuleFunction(OptimizerExtensionInput &input return; } - auto root_name = root->GetName(); - if (root_name.rfind("INSERT", 0) != 0 && root_name.rfind("DELETE", 0) != 0 && root_name.rfind("UPDATE", 0) != 0) { - return; + auto dml = root; + while (dml->type != LogicalOperatorType::LOGICAL_INSERT && dml->type != LogicalOperatorType::LOGICAL_DELETE && + dml->type != LogicalOperatorType::LOGICAL_UPDATE) { + if (dml->children.size() != 1) { + return; + } + dml = dml->children[0].get(); } - switch (root->type) { + switch (dml->type) { case LogicalOperatorType::LOGICAL_INSERT: { - auto insert_node = dynamic_cast(root); - auto insert_table_name = insert_node->table.name; - OPENIVM_DEBUG_PRINT("[INSERT RULE] INSERT into '%s'\n", insert_table_name.c_str()); - - if (SqlUtils::IsDelta(insert_table_name) || insert_table_name.empty() || - IncrementalTableNames::IsDataTable(insert_table_name)) { + auto &insert = dml->Cast(); + const auto &table_name = insert.table.name; + if (SqlUtils::IsDelta(table_name) || table_name.empty() || IncrementalTableNames::IsDataTable(table_name) || + insert.table.catalog.GetCatalogType() == "ducklake") { return; } - // DuckLake tables have native change tracking — no delta writes needed - if (insert_node->table.catalog.GetCatalogType() == "ducklake") { - OPENIVM_DEBUG_PRINT("[INSERT RULE] Skipping delta for DuckLake table '%s'\n", insert_table_name.c_str()); + auto delta_table = Catalog::GetEntry( + input.context, insert.table.catalog.GetName(), insert.table.schema.name, SqlUtils::DeltaName(table_name), + OnEntryNotFound::RETURN_NULL); + if (!delta_table || !IsTrackedBaseTable(input.context, table_name)) { return; } - auto delta_table_catalog_entry = Catalog::GetEntry( - input.context, insert_node->table.catalog.GetName(), insert_node->table.schema.name, - SqlUtils::DeltaName(insert_table_name), OnEntryNotFound::RETURN_NULL); - - if (delta_table_catalog_entry) { - Connection con(*input.context.db); - DisablePACIfLoaded(input.context, con); - RefreshMetadata metadata(con); - if (metadata.IsBaseTable(insert_table_name)) { - string full_delta_table_name = SqlUtils::FullDeltaName( - insert_node->table.catalog.GetName(), insert_node->table.schema.name, insert_node->table.name); - if (insert_node->children[0]->type == LogicalOperatorType::LOGICAL_PROJECTION) { - auto &delta_entry_ins = delta_table_catalog_entry->Cast(); - string insert_query = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry_ins); - - auto projection = dynamic_cast(insert_node->children[0].get()); - if (projection->children[0]->type == LogicalOperatorType::LOGICAL_EXPRESSION_GET) { - insert_query += " VALUES "; - auto expression_get = dynamic_cast(projection->children[0].get()); - bool all_values_are_constants = true; - for (auto &expression : expression_get->expressions) { - for (auto &value : expression) { - if (value->type != ExpressionType::VALUE_CONSTANT) { - all_values_are_constants = false; - break; - } - } - if (!all_values_are_constants) { - break; - } - } - if (!all_values_are_constants) { - // DuckDB may bind VALUES literals through casts or other scalar expressions. Serialize the - // planned insert source instead of rejecting an otherwise valid base-table write. - insert_query = BuildDeltaInsertFromPlan(*con.context, delta_entry_ins, - full_delta_table_name, insert_node->children[0]); - } else { - for (auto &expression : expression_get->expressions) { - string values = "("; - for (auto &value : expression) { - auto constant = dynamic_cast(value.get()); - values += constant->value.ToSQLString() + ","; - } - values += "1, now()::timestamp),"; - insert_query += values; - } - insert_query.pop_back(); - } - } else { - auto &delta_entry = delta_table_catalog_entry->Cast(); - insert_query = BuildDeltaInsertFromPlan(*con.context, delta_entry, full_delta_table_name, - insert_node->children[0]); - } - OPENIVM_DEBUG_PRINT("[INSERT RULE] insert_query: %s\n", insert_query.c_str()); - { - DeltaLockGuard guard(SqlUtils::DeltaName(insert_table_name)); - auto r = con.Query(insert_query); - if (r->HasError()) { - throw Exception(ExceptionType::EXECUTOR, - "Cannot insert in delta table after insertion! " + r->GetError()); - } - } - - } else if (insert_node->children[0]->type == LogicalOperatorType::LOGICAL_GET) { - auto get = dynamic_cast(insert_node->children[0].get()); - auto *bind_data = dynamic_cast(get->bind_data.get()); - if (!bind_data) { - throw NotImplementedException( - "Only CSV file imports (read_csv) are supported for IVM delta tracking " - "via LOGICAL_GET. Other table functions are not yet supported."); - } - auto &delta_entry_csv = delta_table_catalog_entry->Cast(); - string prefix_csv = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry_csv); - auto files = bind_data->file_list->GetAllFiles(); - for (auto &file : files) { - auto query = prefix_csv + " SELECT *, 1, now()::timestamp FROM read_csv('" + file.path + "');"; - DeltaLockGuard guard(SqlUtils::DeltaName(insert_table_name)); - auto r = con.Query(query); - if (r->HasError()) { - throw Exception(ExceptionType::EXECUTOR, "Cannot insert in delta table! " + r->GetError()); - } - } - } else { - // Any other insert-source plan shape. DuckDB places a STREAMING_LIMIT/LOGICAL_LIMIT - // directly above the projection for larger INSERT ... SELECT (above a row threshold), - // so children[0] is neither LOGICAL_PROJECTION nor LOGICAL_GET. Serialize the whole - // source plan generically so the delta is still captured. Without this catch-all, such - // inserts updated the base table but silently skipped the delta, leaving the MV stale - // after the next refresh. - auto &delta_entry_other = delta_table_catalog_entry->Cast(); - string insert_query = BuildDeltaInsertFromPlan(*con.context, delta_entry_other, - full_delta_table_name, insert_node->children[0]); - OPENIVM_DEBUG_PRINT("[INSERT RULE] generic-plan delta insert_query: %s\n", insert_query.c_str()); - DeltaLockGuard guard(SqlUtils::DeltaName(insert_table_name)); - auto r = con.Query(insert_query); - if (r->HasError()) { - throw Exception(ExceptionType::EXECUTOR, - "Cannot insert in delta table after insertion! " + r->GetError()); - } - } - } - } - } break; - + ResolveInsertDefaults(input, insert); + auto capture = make_uniq(insert.table, delta_table->Cast(), + DeltaCaptureMode::INSERT); + capture->children.push_back(std::move(insert.children[0])); + insert.children[0] = std::move(capture); + OPENIVM_DEBUG_PRINT("[INSERT RULE] transactional INSERT delta capture for '%s'\n", table_name.c_str()); + break; + } case LogicalOperatorType::LOGICAL_DELETE: { - auto delete_node = dynamic_cast(root); - auto delete_table_name = delete_node->table.name; - OPENIVM_DEBUG_PRINT("[INSERT RULE] DELETE from '%s'\n", delete_table_name.c_str()); - if (SqlUtils::IsDelta(delete_table_name) || IncrementalTableNames::IsDataTable(delete_table_name)) { + auto &delete_op = dml->Cast(); + const auto &table_name = delete_op.table.name; + if (SqlUtils::IsDelta(table_name) || IncrementalTableNames::IsDataTable(table_name) || + delete_op.table.catalog.GetCatalogType() == "ducklake") { return; } - if (delete_node->table.catalog.GetCatalogType() == "ducklake") { - OPENIVM_DEBUG_PRINT("[INSERT RULE] Skipping delta for DuckLake table '%s'\n", delete_table_name.c_str()); + auto delta_table = Catalog::GetEntry( + input.context, delete_op.table.catalog.GetName(), delete_op.table.schema.name, + SqlUtils::DeltaName(table_name), OnEntryNotFound::RETURN_NULL); + if (!delta_table || !IsTrackedBaseTable(input.context, table_name)) { return; } - auto delta_table_catalog_entry = Catalog::GetEntry( - input.context, delete_node->table.catalog.GetName(), delete_node->table.schema.name, - SqlUtils::DeltaName(delete_table_name), OnEntryNotFound::RETURN_NULL); - - if (delta_table_catalog_entry) { - auto full_table_name = SqlUtils::FullName(delete_node->table.catalog.GetName(), - delete_node->table.schema.name, delete_node->table.name); - auto full_delta_table_name = SqlUtils::FullDeltaName( - delete_node->table.catalog.GetName(), delete_node->table.schema.name, delete_node->table.name); - Connection con(*input.context.db); - DisablePACIfLoaded(input.context, con); - RefreshMetadata metadata(con); - if (metadata.IsBaseTable(delete_table_name)) { - auto &delta_entry_del = delta_table_catalog_entry->Cast(); - string insert_string = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry_del) + " " + - BuildDeltaSelectFrom(delta_entry_del, "-1", full_table_name); - if (plan->children[0]->type == LogicalOperatorType::LOGICAL_FILTER) { - auto filter = dynamic_cast(plan->children[0].get()); - bool has_subquery = false; - for (auto &expr : filter->expressions) { - has_subquery = has_subquery || expr->HasSubquery(); - } - if (has_subquery) { - insert_string = BuildDeleteDeltaInsertFromPlan( - *con.context, delta_entry_del, full_delta_table_name, full_table_name, plan->children[0]); - } else { - BoundColumnNameMap column_names; - CollectBoundColumnNames(*filter->children[0], column_names); - insert_string += " where "; - for (idx_t i = 0; i < filter->expressions.size(); i++) { - if (i > 0) { - insert_string += " AND "; - } - insert_string += QuotedExpressionString(filter->expressions[i], column_names); - } - } - } else if (plan->children[0]->type == LogicalOperatorType::LOGICAL_GET) { - auto get = dynamic_cast(plan->children[0].get()); - if (!get->table_filters.filters.empty()) { - insert_string += " where "; - bool first_filter = true; - for (auto &entry : get->table_filters.filters) { - if (!first_filter) { - insert_string += " AND "; - } - first_filter = false; - auto col_name = get->GetColumnName(ColumnIndex(entry.first)); - col_name = KeywordHelper::WriteOptionallyQuoted(col_name); - insert_string += entry.second->ToString(col_name); - } - } - } else if (plan->children[0]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT) { - return; - } else if (plan->children[0]->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { - // DELETE FROM t WHERE rowid IN (subquery) compiles to a SEMI JOIN on - // rowid. LPTS cannot serialise the full SEMI JOIN because `rowid` is a - // virtual column whose binding is lost after the join. Serialise only - // the right child (the subquery returning rowids) and wrap it as - // WHERE rowid IN (...) against the base table. - auto *join = dynamic_cast(plan->children[0].get()); - if (join && IsSemiJoinOnRowId(*join) && !join->children.empty()) { - try { - string prefix_del = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry_del); - string data_cols = BuildDeltaDataColumns(delta_entry_del); - SqlDialect dialect = openivm::CompileFactsContextSlot::Get(*con.context).target_dialect; - auto ast = LogicalPlanToAst(*con.context, join->children[1], dialect); - auto cte_list = AstToCteList(*ast, dialect); - string rowid_sql = cte_list->ToQuery(false); - if (!rowid_sql.empty() && rowid_sql.back() == ';') { - rowid_sql.pop_back(); - } - insert_string = prefix_del + " SELECT " + data_cols + ", -1, now()::timestamp FROM " + - full_table_name + " WHERE rowid IN (" + rowid_sql + ")"; - } catch (...) { - throw NotImplementedException( - "DELETE with rowid IN (subquery) is not yet fully supported for IVM delta tracking"); - } - } else { - try { - insert_string = - BuildDeleteDeltaInsertFromPlan(*con.context, delta_entry_del, full_delta_table_name, - full_table_name, plan->children[0]); - } catch (...) { - throw NotImplementedException( - "DELETE with complex subqueries is not yet fully supported for IVM delta tracking"); - } - } - } else { - try { - string prefix_del = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry_del); - SqlDialect dialect = openivm::CompileFactsContextSlot::Get(*con.context).target_dialect; - auto ast = LogicalPlanToAst(*con.context, plan->children[0], dialect); - auto cte_list = AstToCteList(*ast, dialect); - string subquery_string = cte_list->ToQuery(false); - if (!subquery_string.empty() && subquery_string.back() == ';') { - subquery_string.pop_back(); - } - insert_string = prefix_del + " SELECT *, -1, now()::timestamp FROM (" + subquery_string + ")"; - } catch (...) { - throw NotImplementedException( - "DELETE with complex subqueries is not yet fully supported for IVM delta tracking"); - } - } - - { - DeltaLockGuard guard(SqlUtils::DeltaName(delete_table_name)); - auto r = con.Query(insert_string); - if (r->HasError()) { - throw Exception(ExceptionType::EXECUTOR, - "Cannot insert in delta table after deletion! " + r->GetError()); - } - } - } - } - } break; - + D_ASSERT(delete_op.expressions.size() == 1); + auto row_id_index = FindExpressionBindingIndex(*delete_op.children[0], *delete_op.expressions[0]); + auto capture = make_uniq( + delete_op.table, delta_table->Cast(), DeltaCaptureMode::DELETE, + vector> {}, vector {}, optional_idx(row_id_index)); + capture->children.push_back(std::move(delete_op.children[0])); + delete_op.children[0] = std::move(capture); + OPENIVM_DEBUG_PRINT("[INSERT RULE] transactional DELETE delta capture for '%s'\n", table_name.c_str()); + break; + } case LogicalOperatorType::LOGICAL_UPDATE: { - auto update_node = dynamic_cast(root); - auto update_table_name = update_node->table.name; - if (SqlUtils::IsDelta(update_table_name) || IncrementalTableNames::IsDataTable(update_table_name)) { + auto &update = dml->Cast(); + const auto &table_name = update.table.name; + if (SqlUtils::IsDelta(table_name) || IncrementalTableNames::IsDataTable(table_name) || + update.table.catalog.GetCatalogType() == "ducklake") { return; } - if (update_node->table.catalog.GetCatalogType() == "ducklake") { - OPENIVM_DEBUG_PRINT("[INSERT RULE] Skipping delta for DuckLake table '%s'\n", update_table_name.c_str()); + auto delta_table = Catalog::GetEntry( + input.context, update.table.catalog.GetName(), update.table.schema.name, SqlUtils::DeltaName(table_name), + OnEntryNotFound::RETURN_NULL); + if (!delta_table || !IsTrackedBaseTable(input.context, table_name)) { return; } - auto delta_table_catalog_entry = Catalog::GetEntry( - input.context, update_node->table.catalog.GetName(), update_node->table.schema.name, - SqlUtils::DeltaName(update_table_name), OnEntryNotFound::RETURN_NULL); - - if (delta_table_catalog_entry) { - Connection con(*input.context.db); - DisablePACIfLoaded(input.context, con); - RefreshMetadata metadata(con); - if (!metadata.IsBaseTable(update_table_name)) { - break; - } - { - auto full_table_name = SqlUtils::FullName(update_node->table.catalog.GetName(), - update_node->table.schema.name, update_node->table.name); - auto full_delta_table_name = SqlUtils::FullDeltaName( - update_node->table.catalog.GetName(), update_node->table.schema.name, update_node->table.name); - auto *projection = dynamic_cast(update_node->children[0].get()); - if (!projection) { - OPENIVM_DEBUG_PRINT("[INSERT RULE] UPDATE skipped: no projection child (child type: %s)\n", - LogicalOperatorToString(update_node->children[0]->type).c_str()); - break; - } - - std::map update_values; - string where_string; - BoundColumnNameMap column_names; - CollectBoundColumnNames(*projection->children[0], column_names); - for (size_t i = 0; i < update_node->columns.size(); i++) { - auto column = update_node->columns[i].index; - update_values[to_string(column)] = QuotedExpressionString(projection->expressions[i], column_names); - } - - if (projection->children[0]->type == LogicalOperatorType::LOGICAL_FILTER) { - auto filter = dynamic_cast(projection->children[0].get()); - where_string += " where "; - for (idx_t i = 0; i < filter->expressions.size(); i++) { - if (i > 0) { - where_string += " AND "; - } - where_string += QuotedExpressionString(filter->expressions[i], column_names); - } - } else if (projection->children[0]->type == LogicalOperatorType::LOGICAL_GET) { - auto get = dynamic_cast(projection->children[0].get()); - if (!get->table_filters.filters.empty()) { - where_string += " where "; - bool first_filter = true; - for (auto &entry : get->table_filters.filters) { - if (!first_filter) { - where_string += " AND "; - } - first_filter = false; - auto col_name = get->GetColumnName(ColumnIndex(entry.first)); - col_name = KeywordHelper::WriteOptionallyQuoted(col_name); - where_string += entry.second->ToString(col_name); - } - } - } else if (projection->children[0]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT) { - return; - } else { - throw NotImplementedException("Only simple UPDATE statements are supported in IVM!"); - } - - auto &delta_entry_upd = delta_table_catalog_entry->Cast(); - string prefix_upd = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry_upd); - string select_old = BuildDeltaSelectFrom(delta_entry_upd, "-1", full_table_name) + where_string; - // For select_new: use the update_values map to replace modified columns - string select_new = "SELECT "; - for (auto &col : delta_entry_upd.GetColumns().Logical()) { - if (col.GetName() == openivm::MULTIPLICITY_COL || col.GetName() == openivm::TIMESTAMP_COL) { - continue; - } - // Find the column's positional index in the base table - auto base_columns = update_node->table.GetColumns().GetColumnNames(); - for (size_t i = 0; i < base_columns.size(); i++) { - if (base_columns[i] == col.GetName()) { - if (update_values.find(to_string(i)) != update_values.end()) { - select_new += update_values[to_string(i)] + ", "; - } else { - select_new += KeywordHelper::WriteOptionallyQuoted(col.GetName()) + ", "; - } - break; - } - } - } - select_new += "1, now()::timestamp FROM " + full_table_name + where_string; - - { - DeltaLockGuard guard(SqlUtils::DeltaName(update_table_name)); - // ATOMIC UPDATE DELTA WRITES: - // The old-delete and new-insert rows MUST commit together. If they land - // in separate auto-commit transactions, a concurrent refresh can take a - // snapshot between them — seeing one but not the other. Since both rows - // target the same group, processing only one breaks consolidation (net - // count change should be 0, but ends up +1 or -1 → MV drift). - // - // Combining into a single multi-row INSERT via UNION ALL ensures both - // rows share one commit point and one `now()` value — either both visible - // in a given snapshot or neither. - string combined = prefix_upd + " " + "SELECT * FROM (" + select_old + - ") UNION ALL SELECT * FROM (" + select_new + ")"; - OPENIVM_DEBUG_PRINT("[INSERT RULE] combined UPDATE delta: %s\n", combined.c_str()); - auto r = con.Query(combined); - if (r->HasError()) { - throw Exception(ExceptionType::EXECUTOR, "Cannot insert UPDATE delta rows! " + r->GetError()); - } - } + vector> update_expressions; + update_expressions.reserve(update.expressions.size()); + for (idx_t index = 0; index < update.expressions.size(); index++) { + if (update.expressions[index]->type == ExpressionType::VALUE_DEFAULT) { + update_expressions.push_back(update.bound_defaults[update.columns[index].index]->Copy()); + } else { + update_expressions.push_back(update.expressions[index]->Copy()); } } - } break; + auto row_id_index = update.children[0]->GetColumnBindings().size() - 1; + auto capture = make_uniq( + update.table, delta_table->Cast(), DeltaCaptureMode::UPDATE, + std::move(update_expressions), update.columns, optional_idx(row_id_index)); + capture->children.push_back(std::move(update.children[0])); + update.children[0] = std::move(capture); + OPENIVM_DEBUG_PRINT("[INSERT RULE] transactional UPDATE delta capture for '%s'\n", table_name.c_str()); + break; + } default: return; } } - } // namespace duckdb diff --git a/src/rules/transactional_delta_capture.cpp b/src/rules/transactional_delta_capture.cpp new file mode 100644 index 00000000..f65a2c00 --- /dev/null +++ b/src/rules/transactional_delta_capture.cpp @@ -0,0 +1,355 @@ +#include "rules/transactional_delta_capture.hpp" + +#include "core/openivm_constants.hpp" +#include "core/openivm_debug.hpp" +#include "core/refresh_locks.hpp" + +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/execution/physical_operator.hpp" +#include "duckdb/main/client_context_state.hpp" +#include "duckdb/planner/binder.hpp" +#include "duckdb/planner/expression_binder/check_binder.hpp" +#include "duckdb/storage/data_table.hpp" +#include "duckdb/storage/table/scan_state.hpp" +#include "duckdb/transaction/duck_transaction.hpp" + +namespace duckdb { + +namespace { + +static constexpr const char *DELTA_WRITE_STATE_KEY = "openivm_transactional_delta_write"; + +class TransactionalDeltaWriteState : public ClientContextState { +public: + void Acquire(const string &catalog_name) { + lock_guard guard(lock); + if (catalog_guard) { + if (catalog_name != locked_catalog) { + throw InternalException("OpenIVM delta transaction attempted to modify multiple catalogs"); + } + return; + } + catalog_guard = make_uniq(catalog_name); + locked_catalog = catalog_name; + } + + void TransactionCommit(MetaTransaction &transaction, ClientContext &context) override { + Release(); + } + + void TransactionRollback(MetaTransaction &transaction, ClientContext &context) override { + Release(); + } + + void TransactionRollback(MetaTransaction &transaction, ClientContext &context, + optional_ptr error) override { + Release(); + } + +private: + void Release() { + lock_guard guard(lock); + catalog_guard.reset(); + locked_catalog.clear(); + } + + mutex lock; + string locked_catalog; + unique_ptr catalog_guard; +}; + +static void AcquireDeltaWriteLock(ClientContext &context, const string &catalog_name) { + auto state = context.registered_state->GetOrCreate(DELTA_WRITE_STATE_KEY); + state->Acquire(catalog_name); +} + +class TransactionalDeltaCaptureGlobalState : public GlobalOperatorState { +public: + mutex lock; + unordered_set captured_row_ids; +}; + +class TransactionalDeltaCaptureLocalState : public OperatorState { +public: + TransactionalDeltaCaptureLocalState(ExecutionContext &context, + const vector> &update_expressions, + const vector> &generated_expressions) + : update_executor(context.client, update_expressions), + generated_executor(context.client, generated_expressions) { + vector update_types; + update_types.reserve(update_expressions.size()); + for (auto &expression : update_expressions) { + update_types.push_back(expression->return_type); + } + update_values.Initialize(Allocator::Get(context.client), update_types); + + vector generated_types; + generated_types.reserve(generated_expressions.size()); + for (auto &expression : generated_expressions) { + generated_types.push_back(expression->return_type); + } + generated_values.Initialize(Allocator::Get(context.client), generated_types); + } + + ExpressionExecutor update_executor; + DataChunk update_values; + ExpressionExecutor generated_executor; + DataChunk generated_values; +}; + +class PhysicalTransactionalDeltaCapture : public PhysicalOperator { +public: + PhysicalTransactionalDeltaCapture(PhysicalPlan &physical_plan, PhysicalOperator &child, + TableCatalogEntry &base_table_p, TableCatalogEntry &delta_table_p, + DeltaCaptureMode mode_p, vector> update_expressions_p, + vector> generated_expressions_p, + vector update_columns_p, optional_idx row_id_index_p, + idx_t estimated_cardinality) + : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, child.GetTypes(), estimated_cardinality), + base_table(base_table_p), delta_table(delta_table_p), mode(mode_p), + update_expressions(std::move(update_expressions_p)), + generated_expressions(std::move(generated_expressions_p)), update_columns(std::move(update_columns_p)), + row_id_index(row_id_index_p) { + children.push_back(child); + BuildDeltaColumnMap(); + } + + unique_ptr GetGlobalOperatorState(ClientContext &context) const override { + return make_uniq(); + } + + unique_ptr GetOperatorState(ExecutionContext &context) const override { + return make_uniq(context, update_expressions, generated_expressions); + } + + OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk, + GlobalOperatorState &gstate_p, OperatorState &state_p) const override { + AcquireDeltaWriteLock(context.client, base_table.catalog.GetName()); + auto &gstate = gstate_p.Cast(); + auto &state = state_p.Cast(); + lock_guard guard(gstate.lock); + + if (mode == DeltaCaptureMode::INSERT) { + AppendDelta(context.client, input, 1, state); + } else { + CaptureDeleteOrUpdate(context, input, gstate, state); + } + chunk.Reference(input); + return OperatorResultType::NEED_MORE_INPUT; + } + + bool ParallelOperator() const override { + return true; + } + + string GetName() const override { + return "OPENIVM_TRANSACTIONAL_DELTA_CAPTURE"; + } + +private: + void BuildDeltaColumnMap() { + case_insensitive_map_t base_column_index; + idx_t base_index = 0; + for (auto &column : base_table.GetColumns().Physical()) { + base_column_index[column.Name()] = base_index++; + } + + idx_t generated_index = 0; + for (auto &column : delta_table.GetColumns().Physical()) { + generated_column_map.push_back(DConstants::INVALID_INDEX); + if (column.Name() == openivm::MULTIPLICITY_COL) { + delta_column_map.push_back(DConstants::INVALID_INDEX); + multiplicity_index = column.Physical().index; + continue; + } + if (column.Name() == openivm::TIMESTAMP_COL) { + delta_column_map.push_back(DConstants::INVALID_INDEX); + timestamp_index = column.Physical().index; + continue; + } + auto entry = base_column_index.find(column.Name()); + if (entry == base_column_index.end()) { + if (generated_index >= generated_expressions.size()) { + throw InternalException("OpenIVM could not bind generated delta column '%s'", column.Name()); + } + delta_column_map.push_back(DConstants::INVALID_INDEX); + generated_column_map.back() = generated_index++; + continue; + } + delta_column_map.push_back(entry->second); + } + if (generated_index != generated_expressions.size()) { + throw InternalException("OpenIVM generated delta column count does not match bound expressions"); + } + if (!multiplicity_index.IsValid() || !timestamp_index.IsValid()) { + throw InternalException("OpenIVM delta table is missing multiplicity or timestamp metadata"); + } + } + + void AppendDelta(ClientContext &context, DataChunk &base_rows, int32_t multiplicity, + TransactionalDeltaCaptureLocalState &state) const { + if (base_rows.size() == 0) { + return; + } + auto delta_types = delta_table.GetTypes(); + DataChunk delta_rows; + delta_rows.Initialize(Allocator::Get(context), delta_types); + if (!generated_expressions.empty()) { + state.generated_values.Reset(); + state.generated_executor.Execute(base_rows, state.generated_values); + } + for (idx_t delta_index = 0; delta_index < delta_column_map.size(); delta_index++) { + if (delta_index == multiplicity_index.GetIndex()) { + delta_rows.data[delta_index].Reference(Value::INTEGER(multiplicity)); + } else if (delta_index == timestamp_index.GetIndex()) { + delta_rows.data[delta_index].Reference(Value::TIMESTAMP(Timestamp::GetCurrentTimestamp())); + } else { + auto generated_index = generated_column_map[delta_index]; + if (generated_index != DConstants::INVALID_INDEX) { + delta_rows.data[delta_index].Reference(state.generated_values.data[generated_index]); + } else { + auto base_index = delta_column_map[delta_index]; + D_ASSERT(base_index != DConstants::INVALID_INDEX); + delta_rows.data[delta_index].Reference(base_rows.data[base_index]); + } + } + } + delta_rows.SetCardinality(base_rows); + vector> no_constraints; + delta_table.GetStorage().LocalAppend(delta_table, context, delta_rows, no_constraints); + } + + void CaptureDeleteOrUpdate(ExecutionContext &context, DataChunk &input, + TransactionalDeltaCaptureGlobalState &gstate, + TransactionalDeltaCaptureLocalState &state) const { + D_ASSERT(row_id_index.IsValid()); + input.Flatten(); + auto &row_ids = input.data[row_id_index.GetIndex()]; + auto row_id_data = FlatVector::GetData(row_ids); + SelectionVector selection(input.size()); + idx_t selected_count = 0; + for (idx_t row = 0; row < input.size(); row++) { + if (gstate.captured_row_ids.insert(row_id_data[row]).second) { + selection.set_index(selected_count++, row); + } + } + if (selected_count == 0) { + return; + } + + DataChunk selected_input; + selected_input.InitializeEmpty(input.GetTypes()); + selected_input.Reference(input); + if (selected_count != input.size()) { + selected_input.Slice(selection, selected_count); + } + selected_input.Flatten(); + + vector column_ids; + auto base_types = base_table.GetTypes(); + column_ids.reserve(base_types.size()); + for (idx_t column = 0; column < base_types.size(); column++) { + column_ids.emplace_back(column); + } + DataChunk old_rows; + old_rows.Initialize(Allocator::Get(context.client), base_types); + auto fetch_state = ColumnFetchState(); + auto &transaction = DuckTransaction::Get(context.client, base_table.catalog); + base_table.GetStorage().Fetch(transaction, old_rows, column_ids, selected_input.data[row_id_index.GetIndex()], + selected_count, fetch_state); + AppendDelta(context.client, old_rows, -1, state); + + if (mode != DeltaCaptureMode::UPDATE) { + return; + } + state.update_values.Reset(); + state.update_executor.Execute(selected_input, state.update_values); + DataChunk new_rows; + new_rows.Initialize(Allocator::Get(context.client), base_types); + for (idx_t column = 0; column < base_types.size(); column++) { + new_rows.data[column].Reference(old_rows.data[column]); + } + for (idx_t update_index = 0; update_index < update_columns.size(); update_index++) { + new_rows.data[update_columns[update_index].index].Reference(state.update_values.data[update_index]); + } + new_rows.SetCardinality(old_rows); + AppendDelta(context.client, new_rows, 1, state); + } + + TableCatalogEntry &base_table; + TableCatalogEntry &delta_table; + DeltaCaptureMode mode; + vector> update_expressions; + vector> generated_expressions; + vector update_columns; + optional_idx row_id_index; + vector delta_column_map; + vector generated_column_map; + optional_idx multiplicity_index; + optional_idx timestamp_index; +}; + +} // namespace + +LogicalTransactionalDeltaCapture::LogicalTransactionalDeltaCapture(TableCatalogEntry &base_table_p, + TableCatalogEntry &delta_table_p, + DeltaCaptureMode mode_p, + vector> update_expressions, + vector update_columns_p, + optional_idx row_id_index_p) + : LogicalExtensionOperator(std::move(update_expressions)), base_table(base_table_p), delta_table(delta_table_p), + mode(mode_p), update_columns(std::move(update_columns_p)), row_id_index(row_id_index_p) { +} + +PhysicalOperator &LogicalTransactionalDeltaCapture::CreatePlan(ClientContext &context, PhysicalPlanGenerator &planner) { + D_ASSERT(children.size() == 1); + auto &child = planner.CreatePlan(*children[0]); + planner.dependencies.AddDependency(base_table); + planner.dependencies.AddDependency(delta_table); + + case_insensitive_set_t physical_columns; + for (auto &column : base_table.GetColumns().Physical()) { + physical_columns.insert(column.Name()); + } + vector> generated_expressions; + auto binder = Binder::CreateBinder(context); + physical_index_set_t bound_columns; + CheckBinder generated_binder(*binder, context, base_table.name, base_table.GetColumns(), bound_columns); + for (auto &column : delta_table.GetColumns().Physical()) { + if (column.Name() == openivm::MULTIPLICITY_COL || column.Name() == openivm::TIMESTAMP_COL || + physical_columns.find(column.Name()) != physical_columns.end()) { + continue; + } + auto &base_column = base_table.GetColumns().GetColumn(column.Name()); + if (!base_column.Generated()) { + throw InternalException("OpenIVM delta column '%s' is absent from the base table storage", column.Name()); + } + generated_binder.target_type = base_column.Type(); + auto generated_expression = base_column.GeneratedExpression().Copy(); + generated_expressions.push_back(generated_binder.Bind(generated_expression)); + } + return planner.Make(child, base_table, delta_table, mode, std::move(expressions), + std::move(generated_expressions), std::move(update_columns), + row_id_index, estimated_cardinality); +} + +vector LogicalTransactionalDeltaCapture::GetColumnBindings() { + D_ASSERT(children.size() == 1); + return children[0]->GetColumnBindings(); +} + +string LogicalTransactionalDeltaCapture::GetName() const { + return "OPENIVM_TRANSACTIONAL_DELTA_CAPTURE"; +} + +string LogicalTransactionalDeltaCapture::GetExtensionName() const { + return "openivm_transactional_delta_capture"; +} + +void LogicalTransactionalDeltaCapture::ResolveTypes() { + D_ASSERT(children.size() == 1); + types = children[0]->types; +} + +} // namespace duckdb diff --git a/src/upsert/refresh.cpp b/src/upsert/refresh.cpp index e85c0b92..4b950c91 100644 --- a/src/upsert/refresh.cpp +++ b/src/upsert/refresh.cpp @@ -118,6 +118,8 @@ static bool RefreshViewLocked(ClientContext &context, const string &view_catalog RefreshProfiler profiler(context, vn); auto lock_start = std::chrono::steady_clock::now(); ViewLockGuard view_guard(vn); + auto &view_catalog = Catalog::GetCatalog(context, view_catalog_name); + DeltaCatalogLockGuard delta_catalog_guard(view_catalog.GetName()); // Acquire delta-table locks in sorted order to serialize parallel refreshes that // share base tables (e.g. mv_A and mv_B both reading STOCK → both write to // `delta_STOCK` inside their transactions → "Conflict on tuple deletion!" when diff --git a/test/sql/concurrency.test b/test/sql/concurrency.test index 19e07a77..cb7c4400 100644 --- a/test/sql/concurrency.test +++ b/test/sql/concurrency.test @@ -302,3 +302,64 @@ SELECT count(*) FROM ( ); ---- 0 + +# ========================================== +# Test 6: Refresh waits for an uncommitted delta-producing transaction. +# Without the catalog-level lock, refresh can advance its timestamp cursor while +# the delta row is invisible, then permanently skip that row after it commits. +# ========================================== + +statement ok +CREATE TABLE tx_refresh_race (id INTEGER, v INTEGER); + +statement ok +INSERT INTO tx_refresh_race VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW mv_tx_refresh_race AS SELECT id, v FROM tx_refresh_race; + +concurrentloop threadid 0 2 + +onlyif threadid=0 +statement ok +BEGIN TRANSACTION; + +onlyif threadid=0 +statement ok +INSERT INTO tx_refresh_race VALUES (2, 20); + +onlyif threadid=0 +statement ok +SELECT sleep_ms(2000); + +onlyif threadid=0 +statement ok +COMMIT; + +onlyif threadid=1 +statement ok +SELECT sleep_ms(500); + +onlyif threadid=1 +statement ok +PRAGMA refresh('mv_tx_refresh_race'); + +endloop + +query I +SELECT count(*) FROM ( + SELECT * FROM mv_tx_refresh_race + EXCEPT ALL + SELECT * FROM tx_refresh_race +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_refresh_race + EXCEPT ALL + SELECT * FROM mv_tx_refresh_race +); +---- +0 diff --git a/test/sql/insert_rule.test b/test/sql/insert_rule.test index 6e992aae..8559be65 100644 --- a/test/sql/insert_rule.test +++ b/test/sql/insert_rule.test @@ -1112,8 +1112,8 @@ SELECT count(*) FROM ( # DuckDB places a STREAMING_LIMIT directly above the projection for large LIMIT values, # so the INSERT's child is neither LOGICAL_PROJECTION nor LOGICAL_GET. The delta-capture # rule previously handled only those two shapes and silently skipped delta population for -# any other shape, leaving the MV stale after refresh. The catch-all branch serializes the -# source plan so the delta is captured for these shapes too. +# any other shape, leaving the MV stale after refresh. Transactional capture now wraps the +# DML input as a streaming physical operator, independent of the source-plan shape. # ========================================== statement ok @@ -1157,3 +1157,202 @@ query I SELECT count(*) FROM (SELECT * FROM bulk_base EXCEPT ALL SELECT * FROM mv_bulk); ---- 0 + +# ========================================== +# Regression: delta capture shares the base DML transaction. +# A rollback or failed constraint must roll back the corresponding delta rows too. +# ========================================== + +statement ok +CREATE TABLE tx_atomic_src ( + id INTEGER PRIMARY KEY, + grp INTEGER, + v INTEGER NOT NULL DEFAULT 7 +); + +statement ok +INSERT INTO tx_atomic_src VALUES (1, 1, 10), (2, 1, 20), (3, 2, 30); + +statement ok +CREATE MATERIALIZED VIEW tx_atomic_mv AS +SELECT grp, SUM(v) AS total, COUNT(*) AS n +FROM tx_atomic_src +GROUP BY grp; + +# Batch conflicting DML before one rollback. None of its four logical delta rows may survive. +statement ok +BEGIN TRANSACTION; + +statement ok +INSERT INTO tx_atomic_src (id, grp) VALUES (4, 2); + +statement ok +UPDATE tx_atomic_src SET v = 15 WHERE id = 1; + +statement ok +DELETE FROM tx_atomic_src WHERE id = 2; + +statement ok +ROLLBACK; + +query I +SELECT count(*) FROM openivm_delta_tx_atomic_src; +---- +0 + +statement ok +PRAGMA refresh('tx_atomic_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_atomic_mv + EXCEPT ALL + SELECT grp, SUM(v), COUNT(*) FROM tx_atomic_src GROUP BY grp +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT grp, SUM(v), COUNT(*) FROM tx_atomic_src GROUP BY grp + EXCEPT ALL + SELECT * FROM tx_atomic_mv +); +---- +0 + +# The same batch must remain incremental when committed. +statement ok +BEGIN TRANSACTION; + +statement ok +INSERT INTO tx_atomic_src (id, grp) VALUES (4, 2); + +statement ok +UPDATE tx_atomic_src SET v = 15 WHERE id = 1; + +statement ok +DELETE FROM tx_atomic_src WHERE id = 2; + +statement ok +COMMIT; + +query I +SELECT count(*) FROM openivm_delta_tx_atomic_src; +---- +4 + +statement ok +PRAGMA refresh('tx_atomic_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_atomic_mv + EXCEPT ALL + SELECT grp, SUM(v), COUNT(*) FROM tx_atomic_src GROUP BY grp +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT grp, SUM(v), COUNT(*) FROM tx_atomic_src GROUP BY grp + EXCEPT ALL + SELECT * FROM tx_atomic_mv +); +---- +0 + +# Constraint failures happen after the capture operator has seen input, so these prove +# that both writes are part of the same transaction and roll back together. +statement error +INSERT INTO tx_atomic_src VALUES (1, 9, 999); +---- +Duplicate key + +statement error +UPDATE tx_atomic_src SET v = NULL WHERE id = 1; +---- +NOT NULL + +query I +SELECT count(*) FROM openivm_delta_tx_atomic_src; +---- +4 + +statement ok +PRAGMA refresh('tx_atomic_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_atomic_mv + EXCEPT ALL + SELECT grp, SUM(v), COUNT(*) FROM tx_atomic_src GROUP BY grp +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT grp, SUM(v), COUNT(*) FROM tx_atomic_src GROUP BY grp + EXCEPT ALL + SELECT * FROM tx_atomic_mv +); +---- +0 + +# Generated columns are not physically stored in the base table, so capture must evaluate +# them for both old and new rows. RETURNING adds a projection above the DML operator; it must +# not hide the affected DML plan from the capture rule. +statement ok +CREATE TABLE tx_generated_src ( + a INTEGER, + doubled INTEGER GENERATED ALWAYS AS (a * 2) +); + +statement ok +INSERT INTO tx_generated_src (a) VALUES (1), (2); + +statement ok +CREATE MATERIALIZED VIEW tx_generated_mv AS SELECT a, doubled FROM tx_generated_src; + +query II +INSERT INTO tx_generated_src (a) VALUES (3) RETURNING a, doubled; +---- +3 6 + +query II +UPDATE tx_generated_src SET a = a + 10 WHERE a = 1 RETURNING a, doubled; +---- +11 22 + +query II +DELETE FROM tx_generated_src WHERE a = 2 RETURNING a, doubled; +---- +2 4 + +query I +SELECT count(*) FROM openivm_delta_tx_generated_src; +---- +4 + +statement ok +PRAGMA refresh('tx_generated_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_generated_mv + EXCEPT ALL + SELECT * FROM tx_generated_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_generated_src + EXCEPT ALL + SELECT * FROM tx_generated_mv +); +---- +0 From d88437a8dd409709ac63b8701fbb0812c180a859 Mon Sep 17 00:00:00 2001 From: ila Date: Wed, 22 Jul 2026 17:01:38 +0200 Subject: [PATCH 02/36] Harden transactional delta capture --- src/core/refresh_locks.cpp | 58 +++++++++-- src/core/refresh_metadata.cpp | 28 +++++ src/include/core/refresh_locks.hpp | 82 ++++++++++----- src/include/core/refresh_metadata.hpp | 9 ++ src/rules/refresh_insert_rule.cpp | 120 +++++++++++++++------- src/rules/transactional_delta_capture.cpp | 39 +++++-- src/upsert/refresh.cpp | 45 ++++++-- test/sql/concurrency.test | 97 +++++++++++++++++ test/sql/insert_rule.test | 76 +++++++++++++- 9 files changed, 459 insertions(+), 95 deletions(-) diff --git a/src/core/refresh_locks.cpp b/src/core/refresh_locks.cpp index c3d54714..57e9fc71 100644 --- a/src/core/refresh_locks.cpp +++ b/src/core/refresh_locks.cpp @@ -5,7 +5,41 @@ namespace duckdb { std::mutex RefreshLocks::map_mutex_; std::unordered_map> RefreshLocks::view_mutexes_; std::unordered_map> RefreshLocks::delta_mutexes_; -std::unordered_map> RefreshLocks::delta_catalog_mutexes_; +std::unordered_map> RefreshLocks::delta_catalog_gates_; + +void DeltaCatalogPhaseGate::EnterWrite() { + std::unique_lock guard(lock); + // Once a refresh is waiting, stop admitting new writers so the finite set of + // active transactions can drain. Writers otherwise run concurrently. + condition.wait(guard, [&]() { return active_refreshes == 0 && waiting_refreshes == 0; }); + active_writers++; +} + +void DeltaCatalogPhaseGate::ExitWrite() { + lock_guard guard(lock); + D_ASSERT(active_writers > 0); + active_writers--; + if (active_writers == 0) { + condition.notify_all(); + } +} + +void DeltaCatalogPhaseGate::EnterRefresh() { + std::unique_lock guard(lock); + waiting_refreshes++; + condition.wait(guard, [&]() { return active_writers == 0; }); + waiting_refreshes--; + active_refreshes++; +} + +void DeltaCatalogPhaseGate::ExitRefresh() { + lock_guard guard(lock); + D_ASSERT(active_refreshes > 0); + active_refreshes--; + if (active_refreshes == 0) { + condition.notify_all(); + } +} std::mutex &RefreshLocks::GetViewMutex(const string &view_name) { std::lock_guard guard(map_mutex_); @@ -25,11 +59,11 @@ std::mutex &RefreshLocks::GetDeltaMutex(const string &delta_table_name) { return *entry; } -std::mutex &RefreshLocks::GetDeltaCatalogMutex(const string &catalog_name) { +DeltaCatalogPhaseGate &RefreshLocks::GetDeltaCatalogGate(Catalog &catalog) { std::lock_guard guard(map_mutex_); - auto &entry = delta_catalog_mutexes_[catalog_name]; + auto &entry = delta_catalog_gates_[&catalog]; if (!entry) { - entry = duckdb::unique_ptr(new std::mutex()); + entry = make_uniq(); } return *entry; } @@ -54,12 +88,20 @@ void RefreshLocks::UnlockDelta(const string &delta_table_name) { GetDeltaMutex(delta_table_name).unlock(); } -void RefreshLocks::LockDeltaCatalog(const string &catalog_name) { - GetDeltaCatalogMutex(catalog_name).lock(); +void RefreshLocks::EnterDeltaWrite(Catalog &catalog) { + GetDeltaCatalogGate(catalog).EnterWrite(); +} + +void RefreshLocks::ExitDeltaWrite(Catalog &catalog) { + GetDeltaCatalogGate(catalog).ExitWrite(); +} + +void RefreshLocks::EnterDeltaRefresh(Catalog &catalog) { + GetDeltaCatalogGate(catalog).EnterRefresh(); } -void RefreshLocks::UnlockDeltaCatalog(const string &catalog_name) { - GetDeltaCatalogMutex(catalog_name).unlock(); +void RefreshLocks::ExitDeltaRefresh(Catalog &catalog) { + GetDeltaCatalogGate(catalog).ExitRefresh(); } } // namespace duckdb diff --git a/src/core/refresh_metadata.cpp b/src/core/refresh_metadata.cpp index a64ab84d..6b8ece33 100644 --- a/src/core/refresh_metadata.cpp +++ b/src/core/refresh_metadata.cpp @@ -122,6 +122,28 @@ RefreshMetadata::SourceLocation RefreshMetadata::GetSourceLocation(const string return loc; } +vector RefreshMetadata::GetDeltaSources(const string &view_name, + const string &fallback_catalog, + const string &fallback_schema) { + auto result = con.Query("SELECT table_name, catalog_type, source_catalog, source_schema FROM " + + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name = '" + + SqlUtils::EscapeValue(view_name) + "'"); + vector sources; + if (result->HasError()) { + return sources; + } + for (idx_t row = 0; row < result->RowCount(); row++) { + DeltaSource source; + source.table_name = result->GetValue(0, row).ToString(); + source.catalog_type = result->GetValue(1, row).IsNull() ? "duckdb" : result->GetValue(1, row).ToString(); + source.catalog_name = + result->GetValue(2, row).IsNull() ? fallback_catalog : result->GetValue(2, row).ToString(); + source.schema_name = result->GetValue(3, row).IsNull() ? fallback_schema : result->GetValue(3, row).ToString(); + sources.push_back(std::move(source)); + } + return sources; +} + string RefreshMetadata::ResolveDeltaQualifiedName(const string &view_name, const string &delta_table_name, const string &fallback_catalog, const string &fallback_schema) { auto loc = GetSourceLocation(view_name, delta_table_name, fallback_catalog, fallback_schema); @@ -259,6 +281,12 @@ vector RefreshMetadata::GetDownstreamViews(const string &view_name) { return result; // topological order: downstream parents before fan-in children } +bool RefreshMetadata::HasDownstreamViews(const string &view_name) { + auto result = con.Query("SELECT 1 FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE table_name = '" + + SqlUtils::EscapeValue(SqlUtils::DeltaName(view_name)) + "' LIMIT 1"); + return !result->HasError() && result->RowCount() > 0; +} + vector RefreshMetadata::GetGroupColumns(const string &view_name) { auto result = con.Query("SELECT group_columns FROM " + string(openivm::VIEWS_TABLE) + " WHERE view_name = '" + SqlUtils::EscapeValue(view_name) + "'"); diff --git a/src/include/core/refresh_locks.hpp b/src/include/core/refresh_locks.hpp index d0022f86..cc57236f 100644 --- a/src/include/core/refresh_locks.hpp +++ b/src/include/core/refresh_locks.hpp @@ -3,6 +3,7 @@ #include "duckdb.hpp" +#include #include #include @@ -13,9 +14,26 @@ namespace duckdb { // Per-view mutex: prevents two concurrent refreshes of the same MV (which would // cause write-write conflicts on the MV table). // -// Per-delta-table mutex: serializes the refresh's "read deltas + set last_update" -// critical section with the insert rule's delta row writes. This closes the window -// where concurrent DML deltas could be permanently skipped. +// Per-delta-table mutex: serializes refreshes that consume the same delta table. +// +// Per-catalog phase gate: allows concurrent DML writers and concurrent refreshes, but +// prevents those two phases from overlapping. This closes the window where refresh +// could advance a watermark past a delta row that commits afterward. +class DeltaCatalogPhaseGate { +public: + void EnterWrite(); + void ExitWrite(); + void EnterRefresh(); + void ExitRefresh(); + +private: + mutex lock; + std::condition_variable condition; + idx_t active_writers = 0; + idx_t active_refreshes = 0; + idx_t waiting_refreshes = 0; +}; + class RefreshLocks { public: // --- View-level locks (prevent concurrent refresh of same MV) --- @@ -29,47 +47,61 @@ class RefreshLocks { static void UnlockView(const string &view_name); - // --- Delta-table-level locks (serialize delta reads/writes) --- + // --- Delta-table-level locks (serialize overlapping refreshes) --- - // Blocking lock — held briefly by both refresh (read + timestamp update) - // and insert rule (delta row write). + // Blocking lock held by refresh while consuming and checkpointing one delta table. static void LockDelta(const string &delta_table_name); static void UnlockDelta(const string &delta_table_name); - // Serializes refresh with transactions that write native delta tables in the same catalog. - // The writer holds this lock until transaction commit/rollback so a refresh cannot advance - // its timestamp cursor past an uncommitted delta row. - static void LockDeltaCatalog(const string &catalog_name); - - static void UnlockDeltaCatalog(const string &catalog_name); + static void EnterDeltaWrite(Catalog &catalog); + static void ExitDeltaWrite(Catalog &catalog); + static void EnterDeltaRefresh(Catalog &catalog); + static void ExitDeltaRefresh(Catalog &catalog); private: static std::mutex &GetViewMutex(const string &view_name); static std::mutex &GetDeltaMutex(const string &delta_table_name); - static std::mutex &GetDeltaCatalogMutex(const string &catalog_name); + static DeltaCatalogPhaseGate &GetDeltaCatalogGate(Catalog &catalog); static std::mutex map_mutex_; static std::unordered_map> view_mutexes_; static std::unordered_map> delta_mutexes_; - static std::unordered_map> delta_catalog_mutexes_; + static std::unordered_map> delta_catalog_gates_; }; -// RAII guard for catalog-level delta transaction locks. -class DeltaCatalogLockGuard { - string name_; +// Write guards can be retained by transaction state and released from the commit thread; +// unlike std::mutex ownership, phase-gate membership is not tied to one OS thread. +class DeltaCatalogWriteGuard { + Catalog *catalog; + +public: + explicit DeltaCatalogWriteGuard(Catalog &catalog_p) : catalog(&catalog_p) { + RefreshLocks::EnterDeltaWrite(*catalog); + } + ~DeltaCatalogWriteGuard() { + RefreshLocks::ExitDeltaWrite(*catalog); + } + DeltaCatalogWriteGuard(const DeltaCatalogWriteGuard &) = delete; + DeltaCatalogWriteGuard &operator=(const DeltaCatalogWriteGuard &) = delete; + DeltaCatalogWriteGuard(DeltaCatalogWriteGuard &&) = delete; + DeltaCatalogWriteGuard &operator=(DeltaCatalogWriteGuard &&) = delete; +}; + +class DeltaCatalogRefreshGuard { + Catalog *catalog; public: - explicit DeltaCatalogLockGuard(const string &catalog_name) : name_(catalog_name) { - RefreshLocks::LockDeltaCatalog(name_); + explicit DeltaCatalogRefreshGuard(Catalog &catalog_p) : catalog(&catalog_p) { + RefreshLocks::EnterDeltaRefresh(*catalog); } - ~DeltaCatalogLockGuard() { - RefreshLocks::UnlockDeltaCatalog(name_); + ~DeltaCatalogRefreshGuard() { + RefreshLocks::ExitDeltaRefresh(*catalog); } - DeltaCatalogLockGuard(const DeltaCatalogLockGuard &) = delete; - DeltaCatalogLockGuard &operator=(const DeltaCatalogLockGuard &) = delete; - DeltaCatalogLockGuard(DeltaCatalogLockGuard &&) = delete; - DeltaCatalogLockGuard &operator=(DeltaCatalogLockGuard &&) = delete; + DeltaCatalogRefreshGuard(const DeltaCatalogRefreshGuard &) = delete; + DeltaCatalogRefreshGuard &operator=(const DeltaCatalogRefreshGuard &) = delete; + DeltaCatalogRefreshGuard(DeltaCatalogRefreshGuard &&) = delete; + DeltaCatalogRefreshGuard &operator=(DeltaCatalogRefreshGuard &&) = delete; }; // RAII guard for delta-table locks. Automatically unlocks on scope exit (including exceptions). diff --git a/src/include/core/refresh_metadata.hpp b/src/include/core/refresh_metadata.hpp index 828b2173..95846ba7 100644 --- a/src/include/core/refresh_metadata.hpp +++ b/src/include/core/refresh_metadata.hpp @@ -55,9 +55,17 @@ class RefreshMetadata { string schema_name; string table_name; }; + struct DeltaSource { + string table_name; + string catalog_type; + string catalog_name; + string schema_name; + }; SourceLocation GetSourceLocation(const string &view_name, const string &table_name, const string &fallback_catalog = "", const string &fallback_schema = ""); + vector GetDeltaSources(const string &view_name, const string &fallback_catalog = "", + const string &fallback_schema = ""); string ResolveDeltaQualifiedName(const string &view_name, const string &delta_table_name, const string &fallback_catalog = "", const string &fallback_schema = ""); @@ -80,6 +88,7 @@ class RefreshMetadata { // Get all downstream MV dependents in topological order (closest first). // For table→mv1→mv2→mv3, GetDownstreamViews("mv1") returns ["mv2", "mv3"]. vector GetDownstreamViews(const string &view_name); + bool HasDownstreamViews(const string &view_name); // Get refresh_interval in seconds for a view. Returns -1 if not set (manual only). int64_t GetRefreshInterval(const string &view_name); diff --git a/src/rules/refresh_insert_rule.cpp b/src/rules/refresh_insert_rule.cpp index 3ad38ab9..d5687c4e 100644 --- a/src/rules/refresh_insert_rule.cpp +++ b/src/rules/refresh_insert_rule.cpp @@ -24,9 +24,19 @@ namespace duckdb { -static bool IsTrackedBaseTable(ClientContext &context, const string &table_name) { - Connection con(*context.db); - return RefreshMetadata(con).IsBaseTable(table_name); +static optional_ptr TryGetTrackedDeltaTable(ClientContext &context, TableCatalogEntry &table) { + const auto &table_name = table.name; + if (table_name.empty() || SqlUtils::IsDelta(table_name) || IncrementalTableNames::IsDataTable(table_name) || + table.catalog.GetCatalogType() == "ducklake") { + return nullptr; + } + auto delta_table = + Catalog::GetEntry(context, table.catalog.GetName(), table.schema.name, + SqlUtils::DeltaName(table_name), OnEntryNotFound::RETURN_NULL); + if (!delta_table) { + return nullptr; + } + return &delta_table->Cast(); } static void ResolveInsertDefaults(OptimizerExtensionInput &input, LogicalInsert &insert) { @@ -67,6 +77,59 @@ static idx_t FindExpressionBindingIndex(LogicalOperator &child, const Expression throw InternalException("OpenIVM could not resolve the DML row-id binding"); } +static void ResolveUpdateDefaults(OptimizerExtensionInput &input, LogicalUpdate &update) { + bool has_default = false; + for (auto &expression : update.expressions) { + has_default = has_default || expression->type == ExpressionType::VALUE_DEFAULT; + } + if (!has_default) { + return; + } + + auto child_bindings = update.children[0]->GetColumnBindings(); + auto child_types = update.children[0]->types; + if (child_bindings.empty() || child_bindings.size() != child_types.size()) { + throw InternalException("OpenIVM cannot normalize UPDATE defaults without a row-id input"); + } + + auto projection_index = input.optimizer.binder.GenerateTableIndex(); + vector> projection_expressions; + projection_expressions.reserve(child_bindings.size() + update.expressions.size()); + for (idx_t index = 0; index + 1 < child_bindings.size(); index++) { + projection_expressions.push_back( + make_uniq(child_types[index], child_bindings[index])); + } + + vector default_indexes(update.expressions.size(), DConstants::INVALID_INDEX); + for (idx_t index = 0; index < update.expressions.size(); index++) { + if (update.expressions[index]->type != ExpressionType::VALUE_DEFAULT) { + continue; + } + default_indexes[index] = projection_expressions.size(); + projection_expressions.push_back(update.bound_defaults[update.columns[index].index]->Copy()); + } + + const auto row_id_output_index = projection_expressions.size(); + projection_expressions.push_back(make_uniq(child_types.back(), child_bindings.back())); + for (idx_t index = 0; index < update.expressions.size(); index++) { + auto return_type = update.expressions[index]->return_type; + idx_t output_index; + if (default_indexes[index] != DConstants::INVALID_INDEX) { + output_index = default_indexes[index]; + } else { + output_index = FindExpressionBindingIndex(*update.children[0], *update.expressions[index]); + D_ASSERT(output_index + 1 < child_bindings.size()); + } + update.expressions[index] = + make_uniq(return_type, ColumnBinding(projection_index, output_index)); + } + + auto projection = make_uniq(projection_index, std::move(projection_expressions)); + projection->children.push_back(std::move(update.children[0])); + update.children[0] = std::move(projection); + D_ASSERT(update.children[0]->GetColumnBindings().size() == row_id_output_index + 1); +} + RefreshInsertRule::RefreshInsertRule() { optimize_function = RefreshInsertRuleFunction; optimizer_info = make_shared_ptr(); @@ -247,19 +310,13 @@ void RefreshInsertRule::RefreshInsertRuleFunction(OptimizerExtensionInput &input case LogicalOperatorType::LOGICAL_INSERT: { auto &insert = dml->Cast(); const auto &table_name = insert.table.name; - if (SqlUtils::IsDelta(table_name) || table_name.empty() || IncrementalTableNames::IsDataTable(table_name) || - insert.table.catalog.GetCatalogType() == "ducklake") { - return; - } - auto delta_table = Catalog::GetEntry( - input.context, insert.table.catalog.GetName(), insert.table.schema.name, SqlUtils::DeltaName(table_name), - OnEntryNotFound::RETURN_NULL); - if (!delta_table || !IsTrackedBaseTable(input.context, table_name)) { + auto delta_table = TryGetTrackedDeltaTable(input.context, insert.table); + if (!delta_table) { return; } ResolveInsertDefaults(input, insert); - auto capture = make_uniq(insert.table, delta_table->Cast(), - DeltaCaptureMode::INSERT); + auto capture = + make_uniq(insert.table, *delta_table, DeltaCaptureMode::INSERT); capture->children.push_back(std::move(insert.children[0])); insert.children[0] = std::move(capture); OPENIVM_DEBUG_PRINT("[INSERT RULE] transactional INSERT delta capture for '%s'\n", table_name.c_str()); @@ -268,21 +325,15 @@ void RefreshInsertRule::RefreshInsertRuleFunction(OptimizerExtensionInput &input case LogicalOperatorType::LOGICAL_DELETE: { auto &delete_op = dml->Cast(); const auto &table_name = delete_op.table.name; - if (SqlUtils::IsDelta(table_name) || IncrementalTableNames::IsDataTable(table_name) || - delete_op.table.catalog.GetCatalogType() == "ducklake") { - return; - } - auto delta_table = Catalog::GetEntry( - input.context, delete_op.table.catalog.GetName(), delete_op.table.schema.name, - SqlUtils::DeltaName(table_name), OnEntryNotFound::RETURN_NULL); - if (!delta_table || !IsTrackedBaseTable(input.context, table_name)) { + auto delta_table = TryGetTrackedDeltaTable(input.context, delete_op.table); + if (!delta_table) { return; } D_ASSERT(delete_op.expressions.size() == 1); auto row_id_index = FindExpressionBindingIndex(*delete_op.children[0], *delete_op.expressions[0]); auto capture = make_uniq( - delete_op.table, delta_table->Cast(), DeltaCaptureMode::DELETE, - vector> {}, vector {}, optional_idx(row_id_index)); + delete_op.table, *delta_table, DeltaCaptureMode::DELETE, vector> {}, + vector {}, optional_idx(row_id_index)); capture->children.push_back(std::move(delete_op.children[0])); delete_op.children[0] = std::move(capture); OPENIVM_DEBUG_PRINT("[INSERT RULE] transactional DELETE delta capture for '%s'\n", table_name.c_str()); @@ -291,29 +342,20 @@ void RefreshInsertRule::RefreshInsertRuleFunction(OptimizerExtensionInput &input case LogicalOperatorType::LOGICAL_UPDATE: { auto &update = dml->Cast(); const auto &table_name = update.table.name; - if (SqlUtils::IsDelta(table_name) || IncrementalTableNames::IsDataTable(table_name) || - update.table.catalog.GetCatalogType() == "ducklake") { - return; - } - auto delta_table = Catalog::GetEntry( - input.context, update.table.catalog.GetName(), update.table.schema.name, SqlUtils::DeltaName(table_name), - OnEntryNotFound::RETURN_NULL); - if (!delta_table || !IsTrackedBaseTable(input.context, table_name)) { + auto delta_table = TryGetTrackedDeltaTable(input.context, update.table); + if (!delta_table) { return; } + ResolveUpdateDefaults(input, update); vector> update_expressions; update_expressions.reserve(update.expressions.size()); for (idx_t index = 0; index < update.expressions.size(); index++) { - if (update.expressions[index]->type == ExpressionType::VALUE_DEFAULT) { - update_expressions.push_back(update.bound_defaults[update.columns[index].index]->Copy()); - } else { - update_expressions.push_back(update.expressions[index]->Copy()); - } + update_expressions.push_back(update.expressions[index]->Copy()); } auto row_id_index = update.children[0]->GetColumnBindings().size() - 1; - auto capture = make_uniq( - update.table, delta_table->Cast(), DeltaCaptureMode::UPDATE, - std::move(update_expressions), update.columns, optional_idx(row_id_index)); + auto capture = make_uniq(update.table, *delta_table, DeltaCaptureMode::UPDATE, + std::move(update_expressions), update.columns, + optional_idx(row_id_index)); capture->children.push_back(std::move(update.children[0])); update.children[0] = std::move(capture); OPENIVM_DEBUG_PRINT("[INSERT RULE] transactional UPDATE delta capture for '%s'\n", table_name.c_str()); diff --git a/src/rules/transactional_delta_capture.cpp b/src/rules/transactional_delta_capture.cpp index f65a2c00..101c02fb 100644 --- a/src/rules/transactional_delta_capture.cpp +++ b/src/rules/transactional_delta_capture.cpp @@ -22,16 +22,18 @@ static constexpr const char *DELTA_WRITE_STATE_KEY = "openivm_transactional_delt class TransactionalDeltaWriteState : public ClientContextState { public: - void Acquire(const string &catalog_name) { + void Acquire(Catalog &catalog) { lock_guard guard(lock); if (catalog_guard) { - if (catalog_name != locked_catalog) { - throw InternalException("OpenIVM delta transaction attempted to modify multiple catalogs"); + if (&catalog != locked_catalog) { + throw TransactionException( + "OpenIVM cannot capture delta rows for catalogs '%s' and '%s' in one transaction", + locked_catalog->GetName(), catalog.GetName()); } return; } - catalog_guard = make_uniq(catalog_name); - locked_catalog = catalog_name; + catalog_guard = make_uniq(catalog); + locked_catalog = &catalog; } void TransactionCommit(MetaTransaction &transaction, ClientContext &context) override { @@ -51,17 +53,17 @@ class TransactionalDeltaWriteState : public ClientContextState { void Release() { lock_guard guard(lock); catalog_guard.reset(); - locked_catalog.clear(); + locked_catalog = nullptr; } mutex lock; - string locked_catalog; - unique_ptr catalog_guard; + Catalog *locked_catalog = nullptr; + unique_ptr catalog_guard; }; -static void AcquireDeltaWriteLock(ClientContext &context, const string &catalog_name) { +static void EnterDeltaWritePhase(ClientContext &context, Catalog &catalog) { auto state = context.registered_state->GetOrCreate(DELTA_WRITE_STATE_KEY); - state->Acquire(catalog_name); + state->Acquire(catalog); } class TransactionalDeltaCaptureGlobalState : public GlobalOperatorState { @@ -96,6 +98,7 @@ class TransactionalDeltaCaptureLocalState : public OperatorState { DataChunk update_values; ExpressionExecutor generated_executor; DataChunk generated_values; + bool entered_write_phase = false; }; class PhysicalTransactionalDeltaCapture : public PhysicalOperator { @@ -125,9 +128,12 @@ class PhysicalTransactionalDeltaCapture : public PhysicalOperator { OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk, GlobalOperatorState &gstate_p, OperatorState &state_p) const override { - AcquireDeltaWriteLock(context.client, base_table.catalog.GetName()); auto &gstate = gstate_p.Cast(); auto &state = state_p.Cast(); + if (!state.entered_write_phase) { + EnterDeltaWritePhase(context.client, delta_table.catalog); + state.entered_write_phase = true; + } lock_guard guard(gstate.lock); if (mode == DeltaCaptureMode::INSERT) { @@ -300,6 +306,17 @@ LogicalTransactionalDeltaCapture::LogicalTransactionalDeltaCapture(TableCatalogE optional_idx row_id_index_p) : LogicalExtensionOperator(std::move(update_expressions)), base_table(base_table_p), delta_table(delta_table_p), mode(mode_p), update_columns(std::move(update_columns_p)), row_id_index(row_id_index_p) { + if (mode == DeltaCaptureMode::INSERT) { + if (!expressions.empty() || !update_columns.empty() || row_id_index.IsValid()) { + throw InternalException("OpenIVM INSERT delta capture received UPDATE/DELETE state"); + } + } else if (!row_id_index.IsValid()) { + throw InternalException("OpenIVM DELETE/UPDATE delta capture is missing its row-id column"); + } else if (mode == DeltaCaptureMode::DELETE && (!expressions.empty() || !update_columns.empty())) { + throw InternalException("OpenIVM DELETE delta capture received UPDATE state"); + } else if (mode == DeltaCaptureMode::UPDATE && expressions.size() != update_columns.size()) { + throw InternalException("OpenIVM UPDATE delta capture expression/column counts do not match"); + } } PhysicalOperator &LogicalTransactionalDeltaCapture::CreatePlan(ClientContext &context, PhysicalPlanGenerator &planner) { diff --git a/src/upsert/refresh.cpp b/src/upsert/refresh.cpp index 4b950c91..2441ecfe 100644 --- a/src/upsert/refresh.cpp +++ b/src/upsert/refresh.cpp @@ -118,24 +118,51 @@ static bool RefreshViewLocked(ClientContext &context, const string &view_catalog RefreshProfiler profiler(context, vn); auto lock_start = std::chrono::steady_clock::now(); ViewLockGuard view_guard(vn); - auto &view_catalog = Catalog::GetCatalog(context, view_catalog_name); - DeltaCatalogLockGuard delta_catalog_guard(view_catalog.GetName()); + Connection probe_con(*context.db.get()); + RefreshMetadata probe_meta(probe_con); + auto delta_sources = probe_meta.GetDeltaSources(vn, view_catalog_name, view_schema_name); + vector delta_table_names; + delta_table_names.reserve(delta_sources.size()); + + // Enter the refresh phase for every native source catalog before inspecting delta activity. + // Writers enter the complementary phase through the actual delta table catalog, so this also + // covers views whose target and source catalogs differ. + vector source_catalog_names; + for (auto &source : delta_sources) { + delta_table_names.push_back(source.table_name); + if (source.catalog_type != "ducklake") { + source_catalog_names.push_back(Catalog::GetCatalog(context, source.catalog_name).GetName()); + } + } + std::sort(source_catalog_names.begin(), source_catalog_names.end()); + source_catalog_names.erase(std::unique(source_catalog_names.begin(), source_catalog_names.end()), + source_catalog_names.end()); + vector> delta_catalog_guards; + for (auto &catalog_name : source_catalog_names) { + delta_catalog_guards.push_back(make_uniq(Catalog::GetCatalog(context, catalog_name))); + } // Acquire delta-table locks in sorted order to serialize parallel refreshes that // share base tables (e.g. mv_A and mv_B both reading STOCK → both write to // `delta_STOCK` inside their transactions → "Conflict on tuple deletion!" when // the second tx tries to delete rows the first already processed). Sorting // guarantees the same acquisition order across all views, so no deadlock is // possible between concurrent refreshes. + auto delta_lock_names = delta_table_names; + if (probe_meta.HasDownstreamViews(vn)) { + // A parent refresh produces this delta while a child refresh consumes it. + // Put both operations under the same table lock so a child cannot advance + // its watermark past an uncommitted parent delta. + delta_lock_names.push_back(SqlUtils::DeltaName(vn)); + } vector> delta_guards; - Connection probe_con(*context.db.get()); - RefreshMetadata probe_meta(probe_con); - auto delta_table_names = probe_meta.GetDeltaTables(vn); - std::sort(delta_table_names.begin(), delta_table_names.end()); - delta_table_names.erase(std::unique(delta_table_names.begin(), delta_table_names.end()), delta_table_names.end()); - for (auto &dt : delta_table_names) { + std::sort(delta_lock_names.begin(), delta_lock_names.end()); + delta_lock_names.erase(std::unique(delta_lock_names.begin(), delta_lock_names.end()), delta_lock_names.end()); + for (auto &dt : delta_lock_names) { delta_guards.push_back(make_uniq(dt)); } - profiler.AddStep("acquire_locks", lock_start, to_string(delta_table_names.size()) + " delta locks"); + profiler.AddStep("acquire_locks", lock_start, + to_string(source_catalog_names.size()) + " catalog gates, " + to_string(delta_lock_names.size()) + + " delta locks"); DeltaActivityResult delta_activity; DeltaActivityResult *precomputed_delta_activity = nullptr; if (skip_empty_refresh) { diff --git a/test/sql/concurrency.test b/test/sql/concurrency.test index cb7c4400..ac86cebc 100644 --- a/test/sql/concurrency.test +++ b/test/sql/concurrency.test @@ -318,6 +318,9 @@ INSERT INTO tx_refresh_race VALUES (1, 10); statement ok CREATE MATERIALIZED VIEW mv_tx_refresh_race AS SELECT id, v FROM tx_refresh_race; +statement ok +CREATE TABLE tx_refresh_timing (phase VARCHAR, observed_at TIMESTAMP); + concurrentloop threadid 0 2 onlyif threadid=0 @@ -340,12 +343,27 @@ onlyif threadid=1 statement ok SELECT sleep_ms(500); +onlyif threadid=1 +statement ok +INSERT INTO tx_refresh_timing VALUES ('before', now()::TIMESTAMP); + onlyif threadid=1 statement ok PRAGMA refresh('mv_tx_refresh_race'); +onlyif threadid=1 +statement ok +INSERT INTO tx_refresh_timing VALUES ('after', now()::TIMESTAMP); + endloop +# Prove refresh began during the writer transaction and waited for its commit. +query I +SELECT epoch_ms(max(observed_at)) - epoch_ms(min(observed_at)) >= 1000 +FROM tx_refresh_timing; +---- +true + query I SELECT count(*) FROM ( SELECT * FROM mv_tx_refresh_race @@ -363,3 +381,82 @@ SELECT count(*) FROM ( ); ---- 0 + +# ========================================== +# Test 7: A child refresh must not advance past an uncommitted parent delta. +# The sleep keeps the parent refresh transaction open while the child attempts +# to consume openivm_delta_chain_lock_parent. Cascading is disabled so a missed +# delta cannot be repaired by a later automatic child refresh. +# ========================================== + +statement ok +SET openivm_cascade_refresh = 'off'; + +statement ok +CREATE TABLE chain_lock_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO chain_lock_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW chain_lock_parent AS + SELECT id, v FROM chain_lock_src WHERE sleep_ms(400) IS NULL; + +statement ok +CREATE MATERIALIZED VIEW chain_lock_child AS + SELECT SUM(v) AS total FROM chain_lock_parent; + +statement ok +INSERT INTO chain_lock_src VALUES (2, 20); + +concurrentloop threadid 0 2 + +onlyif threadid=0 +statement ok +PRAGMA refresh('chain_lock_parent'); + +onlyif threadid=1 +statement ok +SELECT sleep_ms(150); + +onlyif threadid=1 +statement ok +PRAGMA refresh('chain_lock_child'); + +endloop + +query I +SELECT count(*) FROM ( + SELECT id, v FROM chain_lock_parent + EXCEPT ALL + SELECT id, v FROM chain_lock_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, v FROM chain_lock_src + EXCEPT ALL + SELECT id, v FROM chain_lock_parent +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT total FROM chain_lock_child + EXCEPT ALL + SELECT SUM(v) AS total FROM chain_lock_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT SUM(v) AS total FROM chain_lock_src + EXCEPT ALL + SELECT total FROM chain_lock_child +); +---- +0 diff --git a/test/sql/insert_rule.test b/test/sql/insert_rule.test index 8559be65..b5f89a96 100644 --- a/test/sql/insert_rule.test +++ b/test/sql/insert_rule.test @@ -1179,7 +1179,8 @@ SELECT grp, SUM(v) AS total, COUNT(*) AS n FROM tx_atomic_src GROUP BY grp; -# Batch conflicting DML before one rollback. None of its four logical delta rows may survive. +# Batch conflicting DML before one rollback. The id=5 row is inserted, updated, and +# deleted in the same transaction; none of the eight logical delta rows may survive. statement ok BEGIN TRANSACTION; @@ -1192,6 +1193,15 @@ UPDATE tx_atomic_src SET v = 15 WHERE id = 1; statement ok DELETE FROM tx_atomic_src WHERE id = 2; +statement ok +INSERT INTO tx_atomic_src VALUES (5, 3, 50); + +statement ok +UPDATE tx_atomic_src SET v = 55 WHERE id = 5; + +statement ok +DELETE FROM tx_atomic_src WHERE id = 5; + statement ok ROLLBACK; @@ -1234,13 +1244,22 @@ UPDATE tx_atomic_src SET v = 15 WHERE id = 1; statement ok DELETE FROM tx_atomic_src WHERE id = 2; +statement ok +INSERT INTO tx_atomic_src VALUES (5, 3, 50); + +statement ok +UPDATE tx_atomic_src SET v = 55 WHERE id = 5; + +statement ok +DELETE FROM tx_atomic_src WHERE id = 5; + statement ok COMMIT; query I SELECT count(*) FROM openivm_delta_tx_atomic_src; ---- -4 +8 statement ok PRAGMA refresh('tx_atomic_mv'); @@ -1278,7 +1297,7 @@ NOT NULL query I SELECT count(*) FROM openivm_delta_tx_atomic_src; ---- -4 +8 statement ok PRAGMA refresh('tx_atomic_mv'); @@ -1356,3 +1375,54 @@ SELECT count(*) FROM ( ); ---- 0 + +# UPDATE DEFAULT must be evaluated once and shared by base-table update and delta capture. +# A sequence default makes duplicate evaluation observable. +statement ok +CREATE SEQUENCE tx_default_seq START 100; + +statement ok +CREATE TABLE tx_default_src ( + id INTEGER, + v BIGINT DEFAULT nextval('tx_default_seq') +); + +statement ok +INSERT INTO tx_default_src VALUES (1, 0); + +statement ok +CREATE MATERIALIZED VIEW tx_default_mv AS SELECT id, v FROM tx_default_src; + +statement ok +UPDATE tx_default_src SET v = DEFAULT WHERE id = 1; + +query I +SELECT v FROM tx_default_src; +---- +100 + +query I +SELECT v FROM openivm_delta_tx_default_src WHERE openivm_multiplicity = 1; +---- +100 + +statement ok +PRAGMA refresh('tx_default_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_default_mv + EXCEPT ALL + SELECT * FROM tx_default_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_default_src + EXCEPT ALL + SELECT * FROM tx_default_mv +); +---- +0 From 696a2658211d983c2899d84684559e3334f27e06 Mon Sep 17 00:00:00 2001 From: ila Date: Wed, 22 Jul 2026 22:14:47 +0200 Subject: [PATCH 03/36] Capture resolved conflict outcomes --- docs/codebase-audit-2026-07-22.md | 7 +- .../rules/transactional_delta_capture.hpp | 21 + src/rules/refresh_insert_rule.cpp | 22 +- src/rules/transactional_delta_capture.cpp | 599 ++++++++++++++---- test/sql/insert_rule.test | 352 ++++++++++ 5 files changed, 878 insertions(+), 123 deletions(-) diff --git a/docs/codebase-audit-2026-07-22.md b/docs/codebase-audit-2026-07-22.md index f19efebb..df6a979e 100644 --- a/docs/codebase-audit-2026-07-22.md +++ b/docs/codebase-audit-2026-07-22.md @@ -15,9 +15,10 @@ maintenance path. A bug must not be hidden by weakening a test or silently chang appends through the caller's `ClientContext`, so base DML and delta rows share one commit or rollback. A resolved catalog-level lock prevents refresh from advancing its watermark past an uncommitted delta. Regression coverage includes rollback, constraint failure, defaults, generated columns, `RETURNING`, and an uncommitted DML/refresh race. -- [ ] **Capture actual `ON CONFLICT` outcomes.** `INSERT ... ON CONFLICT DO UPDATE` and `INSERT OR REPLACE` do not emit the - required delete-old/insert-new delta pair. Derive deltas from rows actually inserted or updated, not merely the input to - `LogicalInsert`. +- [x] **Capture actual `ON CONFLICT` outcomes.** Conflict clauses lower to `LogicalMergeInto`; a transparent physical action + decorator now captures only the INSERT, UPDATE, and DELETE rows selected by DuckDB after match and action predicates are + resolved. Updates emit the required delete-old/insert-new pair, volatile defaults are evaluated once, `RETURNING` is + preserved, and all writes remain in the base DML transaction. The same boundary also covers native `MERGE` actions. - [ ] **Make MV lifecycle and native refresh transactional.** CREATE, REPLACE, DROP, ALTER, and `PRAGMA refresh` currently execute material work through helper autocommit connections. Caller rollback cannot restore a consistent collection of user view, backing table, metadata, and deltas; failed replacement can destroy the old MV. Use the caller transaction for diff --git a/src/include/rules/transactional_delta_capture.hpp b/src/include/rules/transactional_delta_capture.hpp index 6cd18d1d..70cdec82 100644 --- a/src/include/rules/transactional_delta_capture.hpp +++ b/src/include/rules/transactional_delta_capture.hpp @@ -35,6 +35,27 @@ class LogicalTransactionalDeltaCapture : public LogicalExtensionOperator { void ResolveTypes() override; }; +// Transparent wrapper around a LogicalMergeInto. At physical-plan creation it decorates +// DuckDB's resolved action sinks, so only rows that actually INSERT/UPDATE/DELETE are captured. +class LogicalTransactionalMergeDeltaCapture : public LogicalExtensionOperator { +public: + LogicalTransactionalMergeDeltaCapture(TableCatalogEntry &base_table, TableCatalogEntry &delta_table); + + TableCatalogEntry &base_table; + TableCatalogEntry &delta_table; + + PhysicalOperator &CreatePlan(ClientContext &context, PhysicalPlanGenerator &planner) override; + vector GetColumnBindings() override; + string GetName() const override; + string GetExtensionName() const override; + bool SupportSerialization() const override { + return false; + } + +protected: + void ResolveTypes() override; +}; + } // namespace duckdb #endif // TRANSACTIONAL_DELTA_CAPTURE_HPP diff --git a/src/rules/refresh_insert_rule.cpp b/src/rules/refresh_insert_rule.cpp index d5687c4e..61cd59fe 100644 --- a/src/rules/refresh_insert_rule.cpp +++ b/src/rules/refresh_insert_rule.cpp @@ -18,6 +18,7 @@ #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/operator/logical_delete.hpp" #include "duckdb/planner/operator/logical_insert.hpp" +#include "duckdb/planner/operator/logical_merge_into.hpp" #include "duckdb/planner/operator/logical_projection.hpp" #include "duckdb/planner/operator/logical_simple.hpp" #include "duckdb/planner/operator/logical_update.hpp" @@ -297,13 +298,15 @@ void RefreshInsertRule::RefreshInsertRuleFunction(OptimizerExtensionInput &input return; } - auto dml = root; + auto dml_owner = &plan; + auto dml = dml_owner->get(); while (dml->type != LogicalOperatorType::LOGICAL_INSERT && dml->type != LogicalOperatorType::LOGICAL_DELETE && - dml->type != LogicalOperatorType::LOGICAL_UPDATE) { + dml->type != LogicalOperatorType::LOGICAL_UPDATE && dml->type != LogicalOperatorType::LOGICAL_MERGE_INTO) { if (dml->children.size() != 1) { return; } - dml = dml->children[0].get(); + dml_owner = &dml->children[0]; + dml = dml_owner->get(); } switch (dml->type) { @@ -361,6 +364,19 @@ void RefreshInsertRule::RefreshInsertRuleFunction(OptimizerExtensionInput &input OPENIVM_DEBUG_PRINT("[INSERT RULE] transactional UPDATE delta capture for '%s'\n", table_name.c_str()); break; } + case LogicalOperatorType::LOGICAL_MERGE_INTO: { + auto &merge = dml->Cast(); + const auto &table_name = merge.table.name; + auto delta_table = TryGetTrackedDeltaTable(input.context, merge.table); + if (!delta_table) { + return; + } + auto capture = make_uniq(merge.table, *delta_table); + capture->children.push_back(std::move(*dml_owner)); + *dml_owner = std::move(capture); + OPENIVM_DEBUG_PRINT("[INSERT RULE] transactional MERGE action delta capture for '%s'\n", table_name.c_str()); + break; + } default: return; } diff --git a/src/rules/transactional_delta_capture.cpp b/src/rules/transactional_delta_capture.cpp index 101c02fb..77f78965 100644 --- a/src/rules/transactional_delta_capture.cpp +++ b/src/rules/transactional_delta_capture.cpp @@ -6,10 +6,15 @@ #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "duckdb/execution/expression_executor.hpp" +#include "duckdb/execution/operator/persistent/physical_delete.hpp" +#include "duckdb/execution/operator/persistent/physical_insert.hpp" +#include "duckdb/execution/operator/persistent/physical_merge_into.hpp" +#include "duckdb/execution/operator/persistent/physical_update.hpp" #include "duckdb/execution/physical_operator.hpp" #include "duckdb/main/client_context_state.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/expression_binder/check_binder.hpp" +#include "duckdb/planner/expression/bound_reference_expression.hpp" #include "duckdb/storage/data_table.hpp" #include "duckdb/storage/table/scan_state.hpp" #include "duckdb/transaction/duck_transaction.hpp" @@ -66,26 +71,11 @@ static void EnterDeltaWritePhase(ClientContext &context, Catalog &catalog) { state->Acquire(catalog); } -class TransactionalDeltaCaptureGlobalState : public GlobalOperatorState { -public: - mutex lock; - unordered_set captured_row_ids; -}; - -class TransactionalDeltaCaptureLocalState : public OperatorState { +class TransactionalDeltaAppendState { public: - TransactionalDeltaCaptureLocalState(ExecutionContext &context, - const vector> &update_expressions, - const vector> &generated_expressions) - : update_executor(context.client, update_expressions), - generated_executor(context.client, generated_expressions) { - vector update_types; - update_types.reserve(update_expressions.size()); - for (auto &expression : update_expressions) { - update_types.push_back(expression->return_type); - } - update_values.Initialize(Allocator::Get(context.client), update_types); - + TransactionalDeltaAppendState(ExecutionContext &context, + const vector> &generated_expressions) + : generated_executor(context.client, generated_expressions) { vector generated_types; generated_types.reserve(generated_expressions.size()); for (auto &expression : generated_expressions) { @@ -94,64 +84,66 @@ class TransactionalDeltaCaptureLocalState : public OperatorState { generated_values.Initialize(Allocator::Get(context.client), generated_types); } - ExpressionExecutor update_executor; - DataChunk update_values; ExpressionExecutor generated_executor; DataChunk generated_values; bool entered_write_phase = false; }; -class PhysicalTransactionalDeltaCapture : public PhysicalOperator { +class TransactionalDeltaAppender { public: - PhysicalTransactionalDeltaCapture(PhysicalPlan &physical_plan, PhysicalOperator &child, - TableCatalogEntry &base_table_p, TableCatalogEntry &delta_table_p, - DeltaCaptureMode mode_p, vector> update_expressions_p, - vector> generated_expressions_p, - vector update_columns_p, optional_idx row_id_index_p, - idx_t estimated_cardinality) - : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, child.GetTypes(), estimated_cardinality), - base_table(base_table_p), delta_table(delta_table_p), mode(mode_p), - update_expressions(std::move(update_expressions_p)), - generated_expressions(std::move(generated_expressions_p)), update_columns(std::move(update_columns_p)), - row_id_index(row_id_index_p) { - children.push_back(child); + TransactionalDeltaAppender(TableCatalogEntry &base_table_p, TableCatalogEntry &delta_table_p, + vector> generated_expressions_p) + : base_table(base_table_p), delta_table(delta_table_p), + generated_expressions(std::move(generated_expressions_p)) { BuildDeltaColumnMap(); } - unique_ptr GetGlobalOperatorState(ClientContext &context) const override { - return make_uniq(); - } - - unique_ptr GetOperatorState(ExecutionContext &context) const override { - return make_uniq(context, update_expressions, generated_expressions); + void EnterWritePhase(ClientContext &context, TransactionalDeltaAppendState &state) const { + if (state.entered_write_phase) { + return; + } + EnterDeltaWritePhase(context, delta_table.catalog); + state.entered_write_phase = true; } - OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk, - GlobalOperatorState &gstate_p, OperatorState &state_p) const override { - auto &gstate = gstate_p.Cast(); - auto &state = state_p.Cast(); - if (!state.entered_write_phase) { - EnterDeltaWritePhase(context.client, delta_table.catalog); - state.entered_write_phase = true; + void Append(ClientContext &context, DataChunk &base_rows, int32_t multiplicity, + TransactionalDeltaAppendState &state) const { + if (base_rows.size() == 0) { + return; } - lock_guard guard(gstate.lock); - - if (mode == DeltaCaptureMode::INSERT) { - AppendDelta(context.client, input, 1, state); - } else { - CaptureDeleteOrUpdate(context, input, gstate, state); + auto delta_types = delta_table.GetTypes(); + DataChunk delta_rows; + delta_rows.Initialize(Allocator::Get(context), delta_types); + if (!generated_expressions.empty()) { + state.generated_values.Reset(); + state.generated_executor.Execute(base_rows, state.generated_values); } - chunk.Reference(input); - return OperatorResultType::NEED_MORE_INPUT; + for (idx_t delta_index = 0; delta_index < delta_column_map.size(); delta_index++) { + if (delta_index == multiplicity_index.GetIndex()) { + delta_rows.data[delta_index].Reference(Value::INTEGER(multiplicity)); + } else if (delta_index == timestamp_index.GetIndex()) { + delta_rows.data[delta_index].Reference(Value::TIMESTAMP(Timestamp::GetCurrentTimestamp())); + } else { + auto generated_index = generated_column_map[delta_index]; + if (generated_index != DConstants::INVALID_INDEX) { + delta_rows.data[delta_index].Reference(state.generated_values.data[generated_index]); + } else { + auto base_index = delta_column_map[delta_index]; + D_ASSERT(base_index != DConstants::INVALID_INDEX); + delta_rows.data[delta_index].Reference(base_rows.data[base_index]); + } + } + } + delta_rows.SetCardinality(base_rows); + vector> no_constraints; + delta_table.GetStorage().LocalAppend(delta_table, context, delta_rows, no_constraints); } - bool ParallelOperator() const override { - return true; + const vector> &GeneratedExpressions() const { + return generated_expressions; } - string GetName() const override { - return "OPENIVM_TRANSACTIONAL_DELTA_CAPTURE"; - } + mutex capture_lock; private: void BuildDeltaColumnMap() { @@ -193,39 +185,89 @@ class PhysicalTransactionalDeltaCapture : public PhysicalOperator { } } - void AppendDelta(ClientContext &context, DataChunk &base_rows, int32_t multiplicity, - TransactionalDeltaCaptureLocalState &state) const { - if (base_rows.size() == 0) { - return; - } - auto delta_types = delta_table.GetTypes(); - DataChunk delta_rows; - delta_rows.Initialize(Allocator::Get(context), delta_types); - if (!generated_expressions.empty()) { - state.generated_values.Reset(); - state.generated_executor.Execute(base_rows, state.generated_values); + TableCatalogEntry &base_table; + TableCatalogEntry &delta_table; + vector> generated_expressions; + vector delta_column_map; + vector generated_column_map; + optional_idx multiplicity_index; + optional_idx timestamp_index; +}; + +class TransactionalDeltaCaptureGlobalState : public GlobalOperatorState { +public: + unordered_set captured_row_ids; +}; + +class TransactionalDeltaCaptureLocalState : public OperatorState { +public: + TransactionalDeltaCaptureLocalState(ExecutionContext &context, + const vector> &update_expressions, + const vector> &generated_expressions) + : append_state(context, generated_expressions), update_executor(context.client, update_expressions) { + vector update_types; + update_types.reserve(update_expressions.size()); + for (auto &expression : update_expressions) { + update_types.push_back(expression->return_type); } - for (idx_t delta_index = 0; delta_index < delta_column_map.size(); delta_index++) { - if (delta_index == multiplicity_index.GetIndex()) { - delta_rows.data[delta_index].Reference(Value::INTEGER(multiplicity)); - } else if (delta_index == timestamp_index.GetIndex()) { - delta_rows.data[delta_index].Reference(Value::TIMESTAMP(Timestamp::GetCurrentTimestamp())); - } else { - auto generated_index = generated_column_map[delta_index]; - if (generated_index != DConstants::INVALID_INDEX) { - delta_rows.data[delta_index].Reference(state.generated_values.data[generated_index]); - } else { - auto base_index = delta_column_map[delta_index]; - D_ASSERT(base_index != DConstants::INVALID_INDEX); - delta_rows.data[delta_index].Reference(base_rows.data[base_index]); - } - } + update_values.Initialize(Allocator::Get(context.client), update_types); + } + + TransactionalDeltaAppendState append_state; + ExpressionExecutor update_executor; + DataChunk update_values; +}; + +class PhysicalTransactionalDeltaCapture : public PhysicalOperator { +public: + PhysicalTransactionalDeltaCapture(PhysicalPlan &physical_plan, PhysicalOperator &child, + TableCatalogEntry &base_table_p, TableCatalogEntry &delta_table_p, + DeltaCaptureMode mode_p, vector> update_expressions_p, + vector> generated_expressions_p, + vector update_columns_p, optional_idx row_id_index_p, + idx_t estimated_cardinality) + : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, child.GetTypes(), estimated_cardinality), + base_table(base_table_p), delta_table(delta_table_p), mode(mode_p), + update_expressions(std::move(update_expressions_p)), update_columns(std::move(update_columns_p)), + row_id_index(row_id_index_p), appender(make_shared_ptr( + base_table, delta_table, std::move(generated_expressions_p))) { + children.push_back(child); + } + + unique_ptr GetGlobalOperatorState(ClientContext &context) const override { + return make_uniq(); + } + + unique_ptr GetOperatorState(ExecutionContext &context) const override { + return make_uniq(context, update_expressions, + appender->GeneratedExpressions()); + } + + OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk, + GlobalOperatorState &gstate_p, OperatorState &state_p) const override { + auto &gstate = gstate_p.Cast(); + auto &state = state_p.Cast(); + appender->EnterWritePhase(context.client, state.append_state); + lock_guard guard(appender->capture_lock); + + if (mode == DeltaCaptureMode::INSERT) { + appender->Append(context.client, input, 1, state.append_state); + } else { + CaptureDeleteOrUpdate(context, input, gstate, state); } - delta_rows.SetCardinality(base_rows); - vector> no_constraints; - delta_table.GetStorage().LocalAppend(delta_table, context, delta_rows, no_constraints); + chunk.Reference(input); + return OperatorResultType::NEED_MORE_INPUT; + } + + bool ParallelOperator() const override { + return true; + } + + string GetName() const override { + return "OPENIVM_TRANSACTIONAL_DELTA_CAPTURE"; } +private: void CaptureDeleteOrUpdate(ExecutionContext &context, DataChunk &input, TransactionalDeltaCaptureGlobalState &gstate, TransactionalDeltaCaptureLocalState &state) const { @@ -264,7 +306,7 @@ class PhysicalTransactionalDeltaCapture : public PhysicalOperator { auto &transaction = DuckTransaction::Get(context.client, base_table.catalog); base_table.GetStorage().Fetch(transaction, old_rows, column_ids, selected_input.data[row_id_index.GetIndex()], selected_count, fetch_state); - AppendDelta(context.client, old_rows, -1, state); + appender->Append(context.client, old_rows, -1, state.append_state); if (mode != DeltaCaptureMode::UPDATE) { return; @@ -280,22 +322,319 @@ class PhysicalTransactionalDeltaCapture : public PhysicalOperator { new_rows.data[update_columns[update_index].index].Reference(state.update_values.data[update_index]); } new_rows.SetCardinality(old_rows); - AppendDelta(context.client, new_rows, 1, state); + appender->Append(context.client, new_rows, 1, state.append_state); } TableCatalogEntry &base_table; TableCatalogEntry &delta_table; DeltaCaptureMode mode; vector> update_expressions; - vector> generated_expressions; vector update_columns; optional_idx row_id_index; - vector delta_column_map; - vector generated_column_map; - optional_idx multiplicity_index; - optional_idx timestamp_index; + shared_ptr appender; }; +class MergeActionDeltaCaptureGlobalState : public GlobalSinkState { +public: + explicit MergeActionDeltaCaptureGlobalState(unique_ptr child_state_p) + : child_state(std::move(child_state_p)) { + } + + unique_ptr child_state; + unordered_set captured_row_ids; +}; + +class MergeActionDeltaCaptureLocalState : public LocalSinkState { +public: + MergeActionDeltaCaptureLocalState(ExecutionContext &context, PhysicalOperator &action_op, + const vector> &generated_expressions, + const vector &delegated_types, bool has_update_defaults) + : child_state(action_op.GetLocalSinkState(context)), append_state(context, generated_expressions) { + if (action_op.type != PhysicalOperatorType::UPDATE) { + return; + } + auto &update = action_op.Cast(); + vector update_types; + update_types.reserve(update.expressions.size()); + for (auto &expression : update.expressions) { + update_types.push_back(expression->return_type); + } + update_values.Initialize(Allocator::Get(context.client), update_types); + if (has_update_defaults) { + default_executor = make_uniq(context.client, update.bound_defaults); + delegated_input.Initialize(Allocator::Get(context.client), delegated_types); + } + } + + unique_ptr child_state; + TransactionalDeltaAppendState append_state; + unique_ptr default_executor; + DataChunk delegated_input; + DataChunk update_values; + bool capture_complete_for_input = false; + bool delegated_input_prepared = false; +}; + +class PhysicalMergeActionDeltaCapture : public PhysicalOperator { +public: + PhysicalMergeActionDeltaCapture(PhysicalPlan &physical_plan, PhysicalOperator &action_op_p, + TableCatalogEntry &base_table_p, shared_ptr appender_p, + const vector &action_input_types, idx_t estimated_cardinality) + : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, action_op_p.GetTypes(), + estimated_cardinality), + action_op(action_op_p), base_table(base_table_p), appender(std::move(appender_p)) { + if (action_op.type != PhysicalOperatorType::INSERT && action_op.type != PhysicalOperatorType::UPDATE && + action_op.type != PhysicalOperatorType::DELETE_OPERATOR) { + throw InternalException("OpenIVM cannot capture unsupported MERGE action operator %s", action_op.GetName()); + } + NormalizeUpdateDefaults(action_input_types); + } + + unique_ptr GetGlobalSinkState(ClientContext &context) const override { + return make_uniq(action_op.GetGlobalSinkState(context)); + } + + unique_ptr GetLocalSinkState(ExecutionContext &context) const override { + return make_uniq(context, action_op, appender->GeneratedExpressions(), + delegated_types, !default_update_indexes.empty()); + } + + SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override { + auto &gstate = input.global_state.Cast(); + auto &lstate = input.local_state.Cast(); + auto &action_input = PrepareActionInput(chunk, lstate); + if (!lstate.capture_complete_for_input) { + appender->EnterWritePhase(context.client, lstate.append_state); + lock_guard guard(appender->capture_lock); + Capture(context, action_input, gstate, lstate); + lstate.capture_complete_for_input = true; + } + + OperatorSinkInput child_input {*gstate.child_state, *lstate.child_state, input.interrupt_state}; + auto result = action_op.Sink(context, action_input, child_input); + if (result != SinkResultType::BLOCKED) { + lstate.capture_complete_for_input = false; + lstate.delegated_input_prepared = false; + } + return result; + } + + SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override { + auto &gstate = input.global_state.Cast(); + auto &lstate = input.local_state.Cast(); + OperatorSinkCombineInput child_input {*gstate.child_state, *lstate.child_state, input.interrupt_state}; + return action_op.Combine(context, child_input); + } + + void PrepareFinalize(ClientContext &context, GlobalSinkState &gstate_p) const override { + auto &gstate = gstate_p.Cast(); + action_op.PrepareFinalize(context, *gstate.child_state); + } + + SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const override { + auto &gstate = input.global_state.Cast(); + OperatorSinkFinalizeInput child_input {*gstate.child_state, input.interrupt_state}; + auto result = action_op.Finalize(pipeline, event, context, child_input); + if (result != SinkFinalizeType::BLOCKED) { + action_op.sink_state = std::move(gstate.child_state); + } + return result; + } + + unique_ptr GetGlobalSourceState(ClientContext &context) const override { + D_ASSERT(action_op.sink_state); + return action_op.GetGlobalSourceState(context); + } + + unique_ptr GetLocalSourceState(ExecutionContext &context, + GlobalSourceState &gstate) const override { + return action_op.GetLocalSourceState(context, gstate); + } + + SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const override { + return action_op.GetData(context, chunk, input); + } + + bool IsSink() const override { + return true; + } + + bool IsSource() const override { + return action_op.IsSource(); + } + + bool ParallelSink() const override { + return action_op.ParallelSink(); + } + + bool SinkOrderDependent() const override { + return action_op.SinkOrderDependent(); + } + + string GetName() const override { + return "OPENIVM_MERGE_ACTION_DELTA_CAPTURE"; + } + +private: + void NormalizeUpdateDefaults(const vector &action_input_types) { + if (action_op.type != PhysicalOperatorType::UPDATE) { + return; + } + auto &update = action_op.Cast(); + for (idx_t index = 0; index < update.expressions.size(); index++) { + if (update.expressions[index]->GetExpressionType() == ExpressionType::VALUE_DEFAULT) { + default_update_indexes.push_back(index); + } + } + if (default_update_indexes.empty()) { + return; + } + if (action_input_types.empty()) { + throw InternalException("OpenIVM MERGE UPDATE DEFAULT capture is missing its row-id input"); + } + delegated_types.reserve(action_input_types.size() + default_update_indexes.size()); + for (idx_t index = 0; index + 1 < action_input_types.size(); index++) { + delegated_types.push_back(action_input_types[index]); + } + for (auto update_index : default_update_indexes) { + auto &expression = update.expressions[update_index]; + const auto reference_index = delegated_types.size(); + delegated_types.push_back(expression->return_type); + expression = make_uniq(expression->return_type, reference_index); + } + delegated_types.push_back(action_input_types.back()); + } + + DataChunk &PrepareActionInput(DataChunk &input, MergeActionDeltaCaptureLocalState &state) const { + if (default_update_indexes.empty()) { + return input; + } + if (state.delegated_input_prepared) { + return state.delegated_input; + } + D_ASSERT(state.default_executor); + state.default_executor->SetChunk(input); + state.delegated_input.Reset(); + for (idx_t index = 0; index + 1 < input.ColumnCount(); index++) { + state.delegated_input.data[index].Reference(input.data[index]); + } + auto &update = action_op.Cast(); + idx_t delegated_index = input.ColumnCount() - 1; + for (auto update_index : default_update_indexes) { + state.default_executor->ExecuteExpression(update.columns[update_index].index, + state.delegated_input.data[delegated_index++]); + } + state.delegated_input.data[delegated_index].Reference(input.data.back()); + state.delegated_input.SetCardinality(input); + state.delegated_input_prepared = true; + return state.delegated_input; + } + + void Capture(ExecutionContext &context, DataChunk &input, MergeActionDeltaCaptureGlobalState &gstate, + MergeActionDeltaCaptureLocalState &lstate) const { + if (action_op.type == PhysicalOperatorType::INSERT) { + appender->Append(context.client, input, 1, lstate.append_state); + return; + } + + const auto row_id_index = action_op.type == PhysicalOperatorType::DELETE_OPERATOR + ? action_op.Cast().row_id_index + : input.ColumnCount() - 1; + input.Flatten(); + auto row_id_data = FlatVector::GetData(input.data[row_id_index]); + SelectionVector selection(input.size()); + idx_t selected_count = 0; + for (idx_t row = 0; row < input.size(); row++) { + if (gstate.captured_row_ids.insert(row_id_data[row]).second) { + selection.set_index(selected_count++, row); + } + } + if (selected_count == 0) { + return; + } + + DataChunk selected_input; + selected_input.InitializeEmpty(input.GetTypes()); + selected_input.Reference(input); + if (selected_count != input.size()) { + selected_input.Slice(selection, selected_count); + } + selected_input.Flatten(); + + auto base_types = base_table.GetTypes(); + vector column_ids; + column_ids.reserve(base_types.size()); + for (idx_t column = 0; column < base_types.size(); column++) { + column_ids.emplace_back(column); + } + DataChunk old_rows; + old_rows.Initialize(Allocator::Get(context.client), base_types); + auto fetch_state = ColumnFetchState(); + auto &transaction = DuckTransaction::Get(context.client, base_table.catalog); + base_table.GetStorage().Fetch(transaction, old_rows, column_ids, selected_input.data[row_id_index], + selected_count, fetch_state); + appender->Append(context.client, old_rows, -1, lstate.append_state); + + if (action_op.type != PhysicalOperatorType::UPDATE) { + return; + } + auto &update = action_op.Cast(); + lstate.update_values.Reset(); + lstate.update_values.SetCardinality(selected_input); + for (idx_t index = 0; index < update.expressions.size(); index++) { + auto &expression = *update.expressions[index]; + D_ASSERT(expression.GetExpressionType() == ExpressionType::BOUND_REF); + auto &reference = expression.Cast(); + lstate.update_values.data[index].Reference(selected_input.data[reference.index]); + } + + DataChunk new_rows; + new_rows.Initialize(Allocator::Get(context.client), base_types); + for (idx_t column = 0; column < base_types.size(); column++) { + new_rows.data[column].Reference(old_rows.data[column]); + } + for (idx_t index = 0; index < update.columns.size(); index++) { + new_rows.data[update.columns[index].index].Reference(lstate.update_values.data[index]); + } + new_rows.SetCardinality(old_rows); + appender->Append(context.client, new_rows, 1, lstate.append_state); + } + + PhysicalOperator &action_op; + TableCatalogEntry &base_table; + shared_ptr appender; + vector default_update_indexes; + vector delegated_types; +}; + +static vector> BindGeneratedExpressions(ClientContext &context, TableCatalogEntry &base_table, + TableCatalogEntry &delta_table) { + case_insensitive_set_t physical_columns; + for (auto &column : base_table.GetColumns().Physical()) { + physical_columns.insert(column.Name()); + } + vector> generated_expressions; + auto binder = Binder::CreateBinder(context); + physical_index_set_t bound_columns; + CheckBinder generated_binder(*binder, context, base_table.name, base_table.GetColumns(), bound_columns); + for (auto &column : delta_table.GetColumns().Physical()) { + if (column.Name() == openivm::MULTIPLICITY_COL || column.Name() == openivm::TIMESTAMP_COL || + physical_columns.find(column.Name()) != physical_columns.end()) { + continue; + } + auto &base_column = base_table.GetColumns().GetColumn(column.Name()); + if (!base_column.Generated()) { + throw InternalException("OpenIVM delta column '%s' is absent from the base table storage", column.Name()); + } + generated_binder.target_type = base_column.Type(); + auto generated_expression = base_column.GeneratedExpression().Copy(); + generated_expressions.push_back(generated_binder.Bind(generated_expression)); + } + return generated_expressions; +} + } // namespace LogicalTransactionalDeltaCapture::LogicalTransactionalDeltaCapture(TableCatalogEntry &base_table_p, @@ -325,27 +664,7 @@ PhysicalOperator &LogicalTransactionalDeltaCapture::CreatePlan(ClientContext &co planner.dependencies.AddDependency(base_table); planner.dependencies.AddDependency(delta_table); - case_insensitive_set_t physical_columns; - for (auto &column : base_table.GetColumns().Physical()) { - physical_columns.insert(column.Name()); - } - vector> generated_expressions; - auto binder = Binder::CreateBinder(context); - physical_index_set_t bound_columns; - CheckBinder generated_binder(*binder, context, base_table.name, base_table.GetColumns(), bound_columns); - for (auto &column : delta_table.GetColumns().Physical()) { - if (column.Name() == openivm::MULTIPLICITY_COL || column.Name() == openivm::TIMESTAMP_COL || - physical_columns.find(column.Name()) != physical_columns.end()) { - continue; - } - auto &base_column = base_table.GetColumns().GetColumn(column.Name()); - if (!base_column.Generated()) { - throw InternalException("OpenIVM delta column '%s' is absent from the base table storage", column.Name()); - } - generated_binder.target_type = base_column.Type(); - auto generated_expression = base_column.GeneratedExpression().Copy(); - generated_expressions.push_back(generated_binder.Bind(generated_expression)); - } + auto generated_expressions = BindGeneratedExpressions(context, base_table, delta_table); return planner.Make(child, base_table, delta_table, mode, std::move(expressions), std::move(generated_expressions), std::move(update_columns), row_id_index, estimated_cardinality); @@ -369,4 +688,50 @@ void LogicalTransactionalDeltaCapture::ResolveTypes() { types = children[0]->types; } +LogicalTransactionalMergeDeltaCapture::LogicalTransactionalMergeDeltaCapture(TableCatalogEntry &base_table_p, + TableCatalogEntry &delta_table_p) + : base_table(base_table_p), delta_table(delta_table_p) { +} + +PhysicalOperator &LogicalTransactionalMergeDeltaCapture::CreatePlan(ClientContext &context, + PhysicalPlanGenerator &planner) { + D_ASSERT(children.size() == 1); + auto &child = planner.CreatePlan(*children[0]); + if (child.type != PhysicalOperatorType::MERGE_INTO) { + throw InternalException("OpenIVM MERGE delta capture expected a physical MERGE operator"); + } + planner.dependencies.AddDependency(base_table); + planner.dependencies.AddDependency(delta_table); + auto generated_expressions = BindGeneratedExpressions(context, base_table, delta_table); + auto appender = + make_shared_ptr(base_table, delta_table, std::move(generated_expressions)); + auto &merge = child.Cast(); + for (auto &action : merge.actions) { + if (!action->op) { + continue; + } + action->op = planner.Make( + *action->op, base_table, appender, merge.children[0].get().types, estimated_cardinality); + } + return child; +} + +vector LogicalTransactionalMergeDeltaCapture::GetColumnBindings() { + D_ASSERT(children.size() == 1); + return children[0]->GetColumnBindings(); +} + +string LogicalTransactionalMergeDeltaCapture::GetName() const { + return "OPENIVM_TRANSACTIONAL_MERGE_DELTA_CAPTURE"; +} + +string LogicalTransactionalMergeDeltaCapture::GetExtensionName() const { + return "openivm_transactional_merge_delta_capture"; +} + +void LogicalTransactionalMergeDeltaCapture::ResolveTypes() { + D_ASSERT(children.size() == 1); + types = children[0]->types; +} + } // namespace duckdb diff --git a/test/sql/insert_rule.test b/test/sql/insert_rule.test index b5f89a96..43de9508 100644 --- a/test/sql/insert_rule.test +++ b/test/sql/insert_rule.test @@ -1426,3 +1426,355 @@ SELECT count(*) FROM ( ); ---- 0 + +# ========================================== +# Actual outcomes for ON CONFLICT / OR REPLACE +# ========================================== + +statement ok +CREATE TABLE conflict_src ( + id INTEGER PRIMARY KEY, + v INTEGER NOT NULL DEFAULT 7, + label VARCHAR +); + +statement ok +INSERT INTO conflict_src VALUES (1, 10, 'one'), (2, 20, 'two'); + +statement ok +CREATE MATERIALIZED VIEW conflict_mv AS SELECT id, v, label FROM conflict_src; + +# The rejected id=1 input is not an affected row and must not become a delta. +query III +INSERT INTO conflict_src VALUES (1, 999, 'ignored'), (3, 30, 'three') +ON CONFLICT DO NOTHING +RETURNING id, v, label; +---- +3 30 three + +query III +SELECT id, v, openivm_multiplicity +FROM openivm_delta_conflict_src +ORDER BY id, openivm_multiplicity; +---- +3 30 1 + +statement ok +PRAGMA refresh('conflict_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_mv + EXCEPT ALL + SELECT * FROM conflict_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_src + EXCEPT ALL + SELECT * FROM conflict_mv +); +---- +0 + +statement ok +DELETE FROM openivm_delta_conflict_src; + +# Action predicates are resolved by DuckDB before capture: id=2 performs DO NOTHING, +# id=1 updates, and id=4 inserts. +query III +INSERT INTO conflict_src VALUES (1, 5, 'one-updated'), (2, -1, 'ignored'), (4, 40, 'four') +ON CONFLICT(id) DO UPDATE +SET v = conflict_src.v + excluded.v, label = excluded.label +WHERE excluded.v > 0 +RETURNING id, v, label; +---- +1 15 one-updated +4 40 four + +query III +SELECT id, v, openivm_multiplicity +FROM openivm_delta_conflict_src +ORDER BY id, openivm_multiplicity, v; +---- +1 10 -1 +1 15 1 +4 40 1 + +statement ok +PRAGMA refresh('conflict_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_mv + EXCEPT ALL + SELECT * FROM conflict_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_src + EXCEPT ALL + SELECT * FROM conflict_mv +); +---- +0 + +statement ok +DELETE FROM openivm_delta_conflict_src; + +# OR REPLACE is an update for the conflicting row and an insert for the fresh row. +query III +INSERT OR REPLACE INTO conflict_src VALUES (1, 50, 'replaced'), (5, 50, 'five') +RETURNING id, v, label; +---- +1 50 replaced +5 50 five + +query III +SELECT id, v, openivm_multiplicity +FROM openivm_delta_conflict_src +ORDER BY id, openivm_multiplicity, v; +---- +1 15 -1 +1 50 1 +5 50 1 + +statement ok +PRAGMA refresh('conflict_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_mv + EXCEPT ALL + SELECT * FROM conflict_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_src + EXCEPT ALL + SELECT * FROM conflict_mv +); +---- +0 + +# Capture runs before DuckDB's action sink. A failed conflict update must roll back +# both its provisional deltas and the base-table statement. +statement ok +DELETE FROM openivm_delta_conflict_src; + +statement error +INSERT INTO conflict_src VALUES (1, 1, 'bad') +ON CONFLICT(id) DO UPDATE SET v = NULL; +---- +NOT NULL + +query I +SELECT count(*) FROM openivm_delta_conflict_src; +---- +0 + +# Batch conflict update/insert, replacement, UPDATE, and DELETE before one refresh. +# id=6 is inserted, updated, and deleted in the same batch. +statement ok +INSERT INTO conflict_src VALUES (1, 1, 'again'), (6, 60, 'six') +ON CONFLICT(id) DO UPDATE SET v = conflict_src.v + excluded.v, label = excluded.label; + +statement ok +INSERT OR REPLACE INTO conflict_src VALUES (4, 44, 'four-replaced'); + +statement ok +UPDATE conflict_src SET v = 66 WHERE id = 6; + +statement ok +DELETE FROM conflict_src WHERE id = 6; + +query I +SELECT count(*) FROM openivm_delta_conflict_src; +---- +8 + +statement ok +PRAGMA refresh('conflict_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_mv + EXCEPT ALL + SELECT * FROM conflict_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_src + EXCEPT ALL + SELECT * FROM conflict_mv +); +---- +0 + +# A volatile UPDATE DEFAULT is evaluated once and shared by capture and the action sink. +statement ok +CREATE SEQUENCE conflict_default_seq START 100; + +statement ok +CREATE TABLE conflict_default_src ( + id INTEGER PRIMARY KEY, + v BIGINT DEFAULT nextval('conflict_default_seq') +); + +statement ok +INSERT INTO conflict_default_src VALUES (1, 0); + +statement ok +CREATE MATERIALIZED VIEW conflict_default_mv AS SELECT id, v FROM conflict_default_src; + +query II +INSERT INTO conflict_default_src(id) VALUES (1) +ON CONFLICT(id) DO UPDATE SET v = DEFAULT +RETURNING id, v; +---- +1 101 + +query III +SELECT id, v, openivm_multiplicity +FROM openivm_delta_conflict_default_src +ORDER BY openivm_multiplicity; +---- +1 0 -1 +1 101 1 + +statement ok +PRAGMA refresh('conflict_default_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_default_mv + EXCEPT ALL + SELECT * FROM conflict_default_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_default_src + EXCEPT ALL + SELECT * FROM conflict_default_mv +); +---- +0 + +# Generated delta columns are evaluated from both the old and resolved new row. +statement ok +CREATE TABLE conflict_generated_src ( + id INTEGER PRIMARY KEY, + a INTEGER, + doubled INTEGER GENERATED ALWAYS AS (a * 2) +); + +statement ok +INSERT INTO conflict_generated_src(id, a) VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW conflict_generated_mv AS +SELECT id, a, doubled FROM conflict_generated_src; + +statement ok +INSERT INTO conflict_generated_src(id, a) VALUES (1, 5), (2, 20) +ON CONFLICT(id) DO UPDATE SET a = conflict_generated_src.a + excluded.a; + +query IIII +SELECT id, a, doubled, openivm_multiplicity +FROM openivm_delta_conflict_generated_src +ORDER BY id, openivm_multiplicity; +---- +1 10 20 -1 +1 15 30 1 +2 20 40 1 + +statement ok +PRAGMA refresh('conflict_generated_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_generated_mv + EXCEPT ALL + SELECT * FROM conflict_generated_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_generated_src + EXCEPT ALL + SELECT * FROM conflict_generated_mv +); +---- +0 + +# Native MERGE uses the same resolved action boundary. Exercise DELETE, UPDATE, +# and INSERT actions together, including action-specific RETURNING rows. +statement ok +CREATE TABLE merge_src (id INTEGER PRIMARY KEY, v INTEGER); + +statement ok +INSERT INTO merge_src VALUES (1, 10), (2, 20); + +statement ok +CREATE MATERIALIZED VIEW merge_mv AS SELECT id, v FROM merge_src; + +query TII +MERGE INTO merge_src +USING (VALUES (1, 5), (2, -1), (3, 30)) changes(id, v) +ON merge_src.id = changes.id +WHEN MATCHED AND changes.v < 0 THEN DELETE +WHEN MATCHED THEN UPDATE SET v = merge_src.v + changes.v +WHEN NOT MATCHED THEN INSERT VALUES (changes.id, changes.v) +RETURNING merge_action, id, v; +---- +DELETE 2 20 +UPDATE 1 15 +INSERT 3 30 + +query III +SELECT id, v, openivm_multiplicity +FROM openivm_delta_merge_src +ORDER BY id, openivm_multiplicity; +---- +1 10 -1 +1 15 1 +2 20 -1 +3 30 1 + +statement ok +PRAGMA refresh('merge_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM merge_mv + EXCEPT ALL + SELECT * FROM merge_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM merge_src + EXCEPT ALL + SELECT * FROM merge_mv +); +---- +0 From 3006e1a98be41efb8e1334e63512241ca0a0db05 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 23 Jul 2026 09:11:08 +0200 Subject: [PATCH 04/36] Capture actual MERGE update outcomes --- src/rules/transactional_delta_capture.cpp | 565 +++++++++++++++------- test/sql/insert_rule.test | 104 ++++ 2 files changed, 490 insertions(+), 179 deletions(-) diff --git a/src/rules/transactional_delta_capture.cpp b/src/rules/transactional_delta_capture.cpp index 77f78965..1257c684 100644 --- a/src/rules/transactional_delta_capture.cpp +++ b/src/rules/transactional_delta_capture.cpp @@ -18,6 +18,10 @@ #include "duckdb/storage/data_table.hpp" #include "duckdb/storage/table/scan_state.hpp" #include "duckdb/transaction/duck_transaction.hpp" +#include "duckdb/transaction/local_storage.hpp" +#include "duckdb/transaction/transaction_context.hpp" + +#include namespace duckdb { @@ -73,19 +77,21 @@ static void EnterDeltaWritePhase(ClientContext &context, Catalog &catalog) { class TransactionalDeltaAppendState { public: - TransactionalDeltaAppendState(ExecutionContext &context, - const vector> &generated_expressions) - : generated_executor(context.client, generated_expressions) { + TransactionalDeltaAppendState(ClientContext &context, const vector> &generated_expressions, + const vector &delta_types) + : generated_executor(context, generated_expressions) { vector generated_types; generated_types.reserve(generated_expressions.size()); for (auto &expression : generated_expressions) { generated_types.push_back(expression->return_type); } - generated_values.Initialize(Allocator::Get(context.client), generated_types); + generated_values.Initialize(Allocator::Get(context), generated_types); + delta_rows.Initialize(Allocator::Get(context), delta_types); } ExpressionExecutor generated_executor; DataChunk generated_values; + DataChunk delta_rows; bool entered_write_phase = false; }; @@ -93,9 +99,9 @@ class TransactionalDeltaAppender { public: TransactionalDeltaAppender(TableCatalogEntry &base_table_p, TableCatalogEntry &delta_table_p, vector> generated_expressions_p) - : base_table(base_table_p), delta_table(delta_table_p), + : delta_table(delta_table_p), delta_types(delta_table.GetTypes()), generated_expressions(std::move(generated_expressions_p)) { - BuildDeltaColumnMap(); + BuildDeltaColumnMap(base_table_p); } void EnterWritePhase(ClientContext &context, TransactionalDeltaAppendState &state) const { @@ -111,9 +117,8 @@ class TransactionalDeltaAppender { if (base_rows.size() == 0) { return; } - auto delta_types = delta_table.GetTypes(); - DataChunk delta_rows; - delta_rows.Initialize(Allocator::Get(context), delta_types); + auto &delta_rows = state.delta_rows; + delta_rows.Reset(); if (!generated_expressions.empty()) { state.generated_values.Reset(); state.generated_executor.Execute(base_rows, state.generated_values); @@ -136,6 +141,7 @@ class TransactionalDeltaAppender { } delta_rows.SetCardinality(base_rows); vector> no_constraints; + lock_guard guard(append_lock); delta_table.GetStorage().LocalAppend(delta_table, context, delta_rows, no_constraints); } @@ -143,10 +149,12 @@ class TransactionalDeltaAppender { return generated_expressions; } - mutex capture_lock; + const vector &DeltaTypes() const { + return delta_types; + } private: - void BuildDeltaColumnMap() { + void BuildDeltaColumnMap(TableCatalogEntry &base_table) { case_insensitive_map_t base_column_index; idx_t base_index = 0; for (auto &column : base_table.GetColumns().Physical()) { @@ -185,17 +193,77 @@ class TransactionalDeltaAppender { } } - TableCatalogEntry &base_table; TableCatalogEntry &delta_table; + vector delta_types; vector> generated_expressions; vector delta_column_map; vector generated_column_map; optional_idx multiplicity_index; optional_idx timestamp_index; + mutable mutex append_lock; }; +class TransactionalBaseRowFetcher { +public: + explicit TransactionalBaseRowFetcher(TableCatalogEntry &base_table_p) + : base_table(base_table_p), base_types(base_table.GetTypes()) { + column_ids.reserve(base_types.size()); + for (idx_t column = 0; column < base_types.size(); column++) { + column_ids.emplace_back(column); + } + } + + void Fetch(ClientContext &context, Vector &row_ids, idx_t count, DataChunk &rows, + ColumnFetchState &fetch_state) const { + rows.Reset(); + auto &transaction = DuckTransaction::Get(context, base_table.catalog); + base_table.GetStorage().Fetch(transaction, rows, column_ids, row_ids, count, fetch_state); + } + + bool CanFetch(ClientContext &context, row_t row_id) const { + auto &transaction = DuckTransaction::Get(context, base_table.catalog); + auto &storage = base_table.GetStorage(); + if (row_id < MAX_ROW_ID) { + return storage.CanFetch(transaction, row_id); + } + return transaction.GetLocalStorage().CanFetch(storage, row_id); + } + + const vector &Types() const { + return base_types; + } + +private: + TableCatalogEntry &base_table; + vector base_types; + vector column_ids; +}; + +static idx_t SelectUnseenRows(DataChunk &input, idx_t row_id_index, unordered_set &captured_row_ids, + SelectionVector &selection, DataChunk &selected_input) { + auto &row_ids = input.data[row_id_index]; + row_ids.Flatten(input.size()); + auto row_id_data = FlatVector::GetData(row_ids); + idx_t selected_count = 0; + for (idx_t row = 0; row < input.size(); row++) { + if (captured_row_ids.insert(row_id_data[row]).second) { + selection.set_index(selected_count++, row); + } + } + if (selected_count == 0) { + return 0; + } + selected_input.Reference(input); + if (selected_count != input.size()) { + selected_input.Slice(selection, selected_count); + } + selected_input.data[row_id_index].Flatten(selected_count); + return selected_count; +} + class TransactionalDeltaCaptureGlobalState : public GlobalOperatorState { public: + mutex lock; unordered_set captured_row_ids; }; @@ -203,19 +271,33 @@ class TransactionalDeltaCaptureLocalState : public OperatorState { public: TransactionalDeltaCaptureLocalState(ExecutionContext &context, const vector> &update_expressions, - const vector> &generated_expressions) - : append_state(context, generated_expressions), update_executor(context.client, update_expressions) { + const vector> &generated_expressions, + const vector &delta_types, const vector &base_types, + const vector &input_types, bool capture_existing_rows) + : append_state(context.client, generated_expressions, delta_types), + update_executor(context.client, update_expressions), selection(STANDARD_VECTOR_SIZE) { vector update_types; update_types.reserve(update_expressions.size()); for (auto &expression : update_expressions) { update_types.push_back(expression->return_type); } update_values.Initialize(Allocator::Get(context.client), update_types); + if (!capture_existing_rows) { + return; + } + old_rows.Initialize(Allocator::Get(context.client), base_types); + new_rows.Initialize(Allocator::Get(context.client), base_types); + selected_input.InitializeEmpty(input_types); } TransactionalDeltaAppendState append_state; ExpressionExecutor update_executor; DataChunk update_values; + DataChunk selected_input; + DataChunk old_rows; + DataChunk new_rows; + SelectionVector selection; + ColumnFetchState fetch_state; }; class PhysicalTransactionalDeltaCapture : public PhysicalOperator { @@ -227,10 +309,11 @@ class PhysicalTransactionalDeltaCapture : public PhysicalOperator { vector update_columns_p, optional_idx row_id_index_p, idx_t estimated_cardinality) : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, child.GetTypes(), estimated_cardinality), - base_table(base_table_p), delta_table(delta_table_p), mode(mode_p), - update_expressions(std::move(update_expressions_p)), update_columns(std::move(update_columns_p)), - row_id_index(row_id_index_p), appender(make_shared_ptr( - base_table, delta_table, std::move(generated_expressions_p))) { + mode(mode_p), update_expressions(std::move(update_expressions_p)), + update_columns(std::move(update_columns_p)), row_id_index(row_id_index_p), + appender(make_shared_ptr(base_table_p, delta_table_p, + std::move(generated_expressions_p))), + row_fetcher(base_table_p) { children.push_back(child); } @@ -239,8 +322,9 @@ class PhysicalTransactionalDeltaCapture : public PhysicalOperator { } unique_ptr GetOperatorState(ExecutionContext &context) const override { - return make_uniq(context, update_expressions, - appender->GeneratedExpressions()); + return make_uniq( + context, update_expressions, appender->GeneratedExpressions(), appender->DeltaTypes(), row_fetcher.Types(), + children[0].get().types, mode != DeltaCaptureMode::INSERT); } OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk, @@ -248,11 +332,11 @@ class PhysicalTransactionalDeltaCapture : public PhysicalOperator { auto &gstate = gstate_p.Cast(); auto &state = state_p.Cast(); appender->EnterWritePhase(context.client, state.append_state); - lock_guard guard(appender->capture_lock); if (mode == DeltaCaptureMode::INSERT) { appender->Append(context.client, input, 1, state.append_state); } else { + lock_guard guard(gstate.lock); CaptureDeleteOrUpdate(context, input, gstate, state); } chunk.Reference(input); @@ -272,87 +356,231 @@ class PhysicalTransactionalDeltaCapture : public PhysicalOperator { TransactionalDeltaCaptureGlobalState &gstate, TransactionalDeltaCaptureLocalState &state) const { D_ASSERT(row_id_index.IsValid()); - input.Flatten(); - auto &row_ids = input.data[row_id_index.GetIndex()]; - auto row_id_data = FlatVector::GetData(row_ids); - SelectionVector selection(input.size()); - idx_t selected_count = 0; - for (idx_t row = 0; row < input.size(); row++) { - if (gstate.captured_row_ids.insert(row_id_data[row]).second) { - selection.set_index(selected_count++, row); - } - } + state.selected_input.Reset(); + auto selected_count = SelectUnseenRows(input, row_id_index.GetIndex(), gstate.captured_row_ids, state.selection, + state.selected_input); if (selected_count == 0) { return; } - DataChunk selected_input; - selected_input.InitializeEmpty(input.GetTypes()); - selected_input.Reference(input); - if (selected_count != input.size()) { - selected_input.Slice(selection, selected_count); - } - selected_input.Flatten(); - - vector column_ids; - auto base_types = base_table.GetTypes(); - column_ids.reserve(base_types.size()); - for (idx_t column = 0; column < base_types.size(); column++) { - column_ids.emplace_back(column); - } - DataChunk old_rows; - old_rows.Initialize(Allocator::Get(context.client), base_types); - auto fetch_state = ColumnFetchState(); - auto &transaction = DuckTransaction::Get(context.client, base_table.catalog); - base_table.GetStorage().Fetch(transaction, old_rows, column_ids, selected_input.data[row_id_index.GetIndex()], - selected_count, fetch_state); - appender->Append(context.client, old_rows, -1, state.append_state); + row_fetcher.Fetch(context.client, state.selected_input.data[row_id_index.GetIndex()], selected_count, + state.old_rows, state.fetch_state); + appender->Append(context.client, state.old_rows, -1, state.append_state); if (mode != DeltaCaptureMode::UPDATE) { return; } state.update_values.Reset(); - state.update_executor.Execute(selected_input, state.update_values); - DataChunk new_rows; - new_rows.Initialize(Allocator::Get(context.client), base_types); - for (idx_t column = 0; column < base_types.size(); column++) { - new_rows.data[column].Reference(old_rows.data[column]); + state.update_executor.Execute(state.selected_input, state.update_values); + state.new_rows.Reset(); + for (idx_t column = 0; column < row_fetcher.Types().size(); column++) { + state.new_rows.data[column].Reference(state.old_rows.data[column]); } for (idx_t update_index = 0; update_index < update_columns.size(); update_index++) { - new_rows.data[update_columns[update_index].index].Reference(state.update_values.data[update_index]); + state.new_rows.data[update_columns[update_index].index].Reference(state.update_values.data[update_index]); } - new_rows.SetCardinality(old_rows); - appender->Append(context.client, new_rows, 1, state.append_state); + state.new_rows.SetCardinality(state.old_rows); + appender->Append(context.client, state.new_rows, 1, state.append_state); } - TableCatalogEntry &base_table; - TableCatalogEntry &delta_table; DeltaCaptureMode mode; vector> update_expressions; vector update_columns; optional_idx row_id_index; shared_ptr appender; + TransactionalBaseRowFetcher row_fetcher; +}; + +class MergeDeltaCaptureExecutionState { +public: + mutex lock; + std::condition_variable preimage_ready; + unordered_set capturing_row_ids; + unordered_set captured_row_ids; + idx_t finalized_actions = 0; +}; + +class MergeDeltaCaptureCoordinator { +public: + MergeDeltaCaptureCoordinator(TableCatalogEntry &base_table, shared_ptr appender_p, + const vector &action_input_types_p, idx_t action_count_p) + : row_fetcher(base_table), appender(std::move(appender_p)), action_input_types(action_input_types_p), + action_count(action_count_p) { + } + + shared_ptr GetExecutionState(ClientContext &context) { + // Action sinks are separate physical operators, but their preimages form one statement-level delta. + const auto query_id = context.transaction.GetActiveQuery(); + lock_guard guard(lock); + for (auto entry = states.begin(); entry != states.end();) { + if (entry->second.expired()) { + entry = states.erase(entry); + } else { + entry++; + } + } + auto existing = states.find(query_id); + if (existing != states.end()) { + if (auto state = existing->second.lock()) { + return state; + } + } + auto state = make_shared_ptr(); + states[query_id] = state; + return state; + } + + void CapturePreimages(ClientContext &context, MergeDeltaCaptureExecutionState &state, Vector &input_row_ids, + idx_t count, vector &reserved_row_ids, Vector &fetch_row_ids, DataChunk &rows, + ColumnFetchState &fetch_state, TransactionalDeltaAppendState &append_state) const { + UnifiedVectorFormat row_id_data; + input_row_ids.ToUnifiedFormat(count, row_id_data); + auto row_ids = UnifiedVectorFormat::GetData(row_id_data); + reserved_row_ids.clear(); + { + // Reserve before fetching so actions touching the same row wait, while disjoint rows remain parallel. + lock_guard guard(state.lock); + for (idx_t row = 0; row < count; row++) { + auto index = row_id_data.sel->get_index(row); + D_ASSERT(row_id_data.validity.RowIsValid(index)); + auto row_id = row_ids[index]; + if (state.captured_row_ids.find(row_id) != state.captured_row_ids.end() || + !state.capturing_row_ids.insert(row_id).second) { + continue; + } + reserved_row_ids.push_back(row_id); + } + } + + if (!reserved_row_ids.empty()) { + auto fetch_data = FlatVector::GetData(fetch_row_ids); + for (idx_t index = 0; index < reserved_row_ids.size(); index++) { + fetch_data[index] = reserved_row_ids[index]; + } + try { + row_fetcher.Fetch(context, fetch_row_ids, reserved_row_ids.size(), rows, fetch_state); + D_ASSERT(rows.size() == reserved_row_ids.size()); + appender->EnterWritePhase(context, append_state); + appender->Append(context, rows, -1, append_state); + lock_guard guard(state.lock); + for (auto row_id : reserved_row_ids) { + state.capturing_row_ids.erase(row_id); + state.captured_row_ids.insert(row_id); + } + state.preimage_ready.notify_all(); + } catch (std::exception &) { + lock_guard guard(state.lock); + for (auto row_id : reserved_row_ids) { + state.capturing_row_ids.erase(row_id); + } + state.preimage_ready.notify_all(); + throw; + } + } + + unique_lock guard(state.lock); + state.preimage_ready.wait(guard, [&]() { + for (idx_t row = 0; row < count; row++) { + auto index = row_id_data.sel->get_index(row); + if (state.capturing_row_ids.find(row_ids[index]) != state.capturing_row_ids.end()) { + return false; + } + } + return true; + }); + } + + void FinalizeAction(ClientContext &context, MergeDeltaCaptureExecutionState &state) const { + { + lock_guard guard(state.lock); + state.finalized_actions++; + if (state.finalized_actions != action_count) { + return; + } + D_ASSERT(state.capturing_row_ids.empty()); + } + OPENIVM_DEBUG_PRINT("[INSERT RULE] finalizing MERGE deltas for %zu target rows\n", + state.captured_row_ids.size()); + TransactionalDeltaAppendState append_state(context, appender->GeneratedExpressions(), appender->DeltaTypes()); + appender->EnterWritePhase(context, append_state); + Vector row_ids(LogicalType::ROW_TYPE, STANDARD_VECTOR_SIZE); + DataChunk rows; + rows.Initialize(Allocator::Get(context), row_fetcher.Types()); + ColumnFetchState fetch_state; + auto row_id_data = FlatVector::GetData(row_ids); + idx_t count = 0; + idx_t invisible_count = 0; + for (auto row_id : state.captured_row_ids) { + if (!row_fetcher.CanFetch(context, row_id)) { + invisible_count++; + continue; + } + row_id_data[count++] = row_id; + if (count != STANDARD_VECTOR_SIZE) { + continue; + } + row_fetcher.Fetch(context, row_ids, count, rows, fetch_state); + appender->Append(context, rows, 1, append_state); + count = 0; + } + if (count > 0) { + row_fetcher.Fetch(context, row_ids, count, rows, fetch_state); + appender->Append(context, rows, 1, append_state); + } + OPENIVM_DEBUG_PRINT("[INSERT RULE] skipped %zu invisible MERGE target postimages\n", invisible_count); + } + + const TransactionalBaseRowFetcher &RowFetcher() const { + return row_fetcher; + } + + const shared_ptr &Appender() const { + return appender; + } + + const vector &ActionInputTypes() const { + return action_input_types; + } + +private: + TransactionalBaseRowFetcher row_fetcher; + shared_ptr appender; + vector action_input_types; + idx_t action_count; + mutex lock; + unordered_map> states; }; class MergeActionDeltaCaptureGlobalState : public GlobalSinkState { public: - explicit MergeActionDeltaCaptureGlobalState(unique_ptr child_state_p) - : child_state(std::move(child_state_p)) { + MergeActionDeltaCaptureGlobalState(unique_ptr child_state_p, + shared_ptr execution_state_p) + : child_state(std::move(child_state_p)), execution_state(std::move(execution_state_p)) { } unique_ptr child_state; - unordered_set captured_row_ids; + shared_ptr execution_state; + unordered_set captured_delete_insert_row_ids; }; class MergeActionDeltaCaptureLocalState : public LocalSinkState { public: MergeActionDeltaCaptureLocalState(ExecutionContext &context, PhysicalOperator &action_op, - const vector> &generated_expressions, - const vector &delegated_types, bool has_update_defaults) - : child_state(action_op.GetLocalSinkState(context)), append_state(context, generated_expressions) { - if (action_op.type != PhysicalOperatorType::UPDATE) { + const TransactionalDeltaAppender &appender, const vector &base_types, + const vector &action_input_types, + const vector &delegated_types, bool capture_delete_insert_update, + bool has_update_defaults) + : child_state(action_op.GetLocalSinkState(context)), + append_state(context.client, appender.GeneratedExpressions(), appender.DeltaTypes()), + selection(STANDARD_VECTOR_SIZE), fetch_row_ids(LogicalType::ROW_TYPE, STANDARD_VECTOR_SIZE) { + if (action_op.type == PhysicalOperatorType::INSERT) { + return; + } + preimage_rows.Initialize(Allocator::Get(context.client), base_types); + if (!capture_delete_insert_update) { return; } + selected_input.InitializeEmpty(action_input_types); auto &update = action_op.Cast(); vector update_types; update_types.reserve(update.expressions.size()); @@ -360,6 +588,7 @@ class MergeActionDeltaCaptureLocalState : public LocalSinkState { update_types.push_back(expression->return_type); } update_values.Initialize(Allocator::Get(context.client), update_types); + pending_new_rows.Initialize(Allocator::Get(context.client), base_types); if (has_update_defaults) { default_executor = make_uniq(context.client, update.bound_defaults); delegated_input.Initialize(Allocator::Get(context.client), delegated_types); @@ -371,50 +600,74 @@ class MergeActionDeltaCaptureLocalState : public LocalSinkState { unique_ptr default_executor; DataChunk delegated_input; DataChunk update_values; - bool capture_complete_for_input = false; - bool delegated_input_prepared = false; + DataChunk selected_input; + DataChunk preimage_rows; + DataChunk pending_new_rows; + SelectionVector selection; + Vector fetch_row_ids; + ColumnFetchState fetch_state; + vector reserved_row_ids; + bool prepared_for_input = false; }; class PhysicalMergeActionDeltaCapture : public PhysicalOperator { public: PhysicalMergeActionDeltaCapture(PhysicalPlan &physical_plan, PhysicalOperator &action_op_p, - TableCatalogEntry &base_table_p, shared_ptr appender_p, - const vector &action_input_types, idx_t estimated_cardinality) + shared_ptr coordinator_p, idx_t estimated_cardinality) : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, action_op_p.GetTypes(), estimated_cardinality), - action_op(action_op_p), base_table(base_table_p), appender(std::move(appender_p)) { + action_op(action_op_p), coordinator(std::move(coordinator_p)), + capture_delete_insert_update(action_op.type == PhysicalOperatorType::UPDATE && + action_op.Cast().update_is_del_and_insert) { if (action_op.type != PhysicalOperatorType::INSERT && action_op.type != PhysicalOperatorType::UPDATE && action_op.type != PhysicalOperatorType::DELETE_OPERATOR) { throw InternalException("OpenIVM cannot capture unsupported MERGE action operator %s", action_op.GetName()); } - NormalizeUpdateDefaults(action_input_types); + if (capture_delete_insert_update) { + NormalizeUpdateDefaults(coordinator->ActionInputTypes()); + } } unique_ptr GetGlobalSinkState(ClientContext &context) const override { - return make_uniq(action_op.GetGlobalSinkState(context)); + return make_uniq(action_op.GetGlobalSinkState(context), + coordinator->GetExecutionState(context)); } unique_ptr GetLocalSinkState(ExecutionContext &context) const override { - return make_uniq(context, action_op, appender->GeneratedExpressions(), - delegated_types, !default_update_indexes.empty()); + auto &capture_input_types = delegated_types.empty() ? coordinator->ActionInputTypes() : delegated_types; + return make_uniq( + context, action_op, *coordinator->Appender(), coordinator->RowFetcher().Types(), capture_input_types, + delegated_types, capture_delete_insert_update, !default_update_indexes.empty()); } SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override { auto &gstate = input.global_state.Cast(); auto &lstate = input.local_state.Cast(); auto &action_input = PrepareActionInput(chunk, lstate); - if (!lstate.capture_complete_for_input) { - appender->EnterWritePhase(context.client, lstate.append_state); - lock_guard guard(appender->capture_lock); - Capture(context, action_input, gstate, lstate); - lstate.capture_complete_for_input = true; + if (action_op.type != PhysicalOperatorType::INSERT) { + const auto row_id_index = action_op.type == PhysicalOperatorType::DELETE_OPERATOR + ? action_op.Cast().row_id_index + : action_input.ColumnCount() - 1; + coordinator->CapturePreimages(context.client, *gstate.execution_state, action_input.data[row_id_index], + action_input.size(), lstate.reserved_row_ids, lstate.fetch_row_ids, + lstate.preimage_rows, lstate.fetch_state, lstate.append_state); + if (capture_delete_insert_update && !lstate.prepared_for_input) { + PrepareDeleteInsertRows(context, action_input, row_id_index, gstate, lstate); + lstate.prepared_for_input = true; + } } OperatorSinkInput child_input {*gstate.child_state, *lstate.child_state, input.interrupt_state}; auto result = action_op.Sink(context, action_input, child_input); - if (result != SinkResultType::BLOCKED) { - lstate.capture_complete_for_input = false; - lstate.delegated_input_prepared = false; + if (result == SinkResultType::BLOCKED) { + return result; + } + coordinator->Appender()->EnterWritePhase(context.client, lstate.append_state); + if (action_op.type == PhysicalOperatorType::INSERT) { + coordinator->Appender()->Append(context.client, action_input, 1, lstate.append_state); + } else if (capture_delete_insert_update) { + coordinator->Appender()->Append(context.client, lstate.pending_new_rows, 1, lstate.append_state); + lstate.prepared_for_input = false; } return result; } @@ -426,24 +679,20 @@ class PhysicalMergeActionDeltaCapture : public PhysicalOperator { return action_op.Combine(context, child_input); } - void PrepareFinalize(ClientContext &context, GlobalSinkState &gstate_p) const override { - auto &gstate = gstate_p.Cast(); - action_op.PrepareFinalize(context, *gstate.child_state); - } - SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context, OperatorSinkFinalizeInput &input) const override { auto &gstate = input.global_state.Cast(); OperatorSinkFinalizeInput child_input {*gstate.child_state, input.interrupt_state}; auto result = action_op.Finalize(pipeline, event, context, child_input); if (result != SinkFinalizeType::BLOCKED) { - action_op.sink_state = std::move(gstate.child_state); + coordinator->FinalizeAction(context, *gstate.execution_state); } return result; } unique_ptr GetGlobalSourceState(ClientContext &context) const override { - D_ASSERT(action_op.sink_state); + auto &gstate = sink_state->Cast(); + action_op.sink_state = std::move(gstate.child_state); return action_op.GetGlobalSourceState(context); } @@ -457,31 +706,12 @@ class PhysicalMergeActionDeltaCapture : public PhysicalOperator { return action_op.GetData(context, chunk, input); } - bool IsSink() const override { - return true; - } - - bool IsSource() const override { - return action_op.IsSource(); - } - - bool ParallelSink() const override { - return action_op.ParallelSink(); - } - - bool SinkOrderDependent() const override { - return action_op.SinkOrderDependent(); - } - string GetName() const override { return "OPENIVM_MERGE_ACTION_DELTA_CAPTURE"; } private: - void NormalizeUpdateDefaults(const vector &action_input_types) { - if (action_op.type != PhysicalOperatorType::UPDATE) { - return; - } + void NormalizeUpdateDefaults(const vector &input_types) { auto &update = action_op.Cast(); for (idx_t index = 0; index < update.expressions.size(); index++) { if (update.expressions[index]->GetExpressionType() == ExpressionType::VALUE_DEFAULT) { @@ -491,12 +721,12 @@ class PhysicalMergeActionDeltaCapture : public PhysicalOperator { if (default_update_indexes.empty()) { return; } - if (action_input_types.empty()) { + if (input_types.empty()) { throw InternalException("OpenIVM MERGE UPDATE DEFAULT capture is missing its row-id input"); } - delegated_types.reserve(action_input_types.size() + default_update_indexes.size()); - for (idx_t index = 0; index + 1 < action_input_types.size(); index++) { - delegated_types.push_back(action_input_types[index]); + delegated_types.reserve(input_types.size() + default_update_indexes.size()); + for (idx_t index = 0; index + 1 < input_types.size(); index++) { + delegated_types.push_back(input_types[index]); } for (auto update_index : default_update_indexes) { auto &expression = update.expressions[update_index]; @@ -504,14 +734,14 @@ class PhysicalMergeActionDeltaCapture : public PhysicalOperator { delegated_types.push_back(expression->return_type); expression = make_uniq(expression->return_type, reference_index); } - delegated_types.push_back(action_input_types.back()); + delegated_types.push_back(input_types.back()); } DataChunk &PrepareActionInput(DataChunk &input, MergeActionDeltaCaptureLocalState &state) const { if (default_update_indexes.empty()) { return input; } - if (state.delegated_input_prepared) { + if (state.prepared_for_input) { return state.delegated_input; } D_ASSERT(state.default_executor); @@ -528,83 +758,43 @@ class PhysicalMergeActionDeltaCapture : public PhysicalOperator { } state.delegated_input.data[delegated_index].Reference(input.data.back()); state.delegated_input.SetCardinality(input); - state.delegated_input_prepared = true; return state.delegated_input; } - void Capture(ExecutionContext &context, DataChunk &input, MergeActionDeltaCaptureGlobalState &gstate, - MergeActionDeltaCaptureLocalState &lstate) const { - if (action_op.type == PhysicalOperatorType::INSERT) { - appender->Append(context.client, input, 1, lstate.append_state); - return; - } - - const auto row_id_index = action_op.type == PhysicalOperatorType::DELETE_OPERATOR - ? action_op.Cast().row_id_index - : input.ColumnCount() - 1; - input.Flatten(); - auto row_id_data = FlatVector::GetData(input.data[row_id_index]); - SelectionVector selection(input.size()); - idx_t selected_count = 0; - for (idx_t row = 0; row < input.size(); row++) { - if (gstate.captured_row_ids.insert(row_id_data[row]).second) { - selection.set_index(selected_count++, row); - } - } + void PrepareDeleteInsertRows(ExecutionContext &context, DataChunk &input, idx_t row_id_index, + MergeActionDeltaCaptureGlobalState &gstate, + MergeActionDeltaCaptureLocalState &lstate) const { + lstate.selected_input.Reset(); + auto selected_count = SelectUnseenRows(input, row_id_index, gstate.captured_delete_insert_row_ids, + lstate.selection, lstate.selected_input); + lstate.pending_new_rows.Reset(); if (selected_count == 0) { return; } - - DataChunk selected_input; - selected_input.InitializeEmpty(input.GetTypes()); - selected_input.Reference(input); - if (selected_count != input.size()) { - selected_input.Slice(selection, selected_count); - } - selected_input.Flatten(); - - auto base_types = base_table.GetTypes(); - vector column_ids; - column_ids.reserve(base_types.size()); - for (idx_t column = 0; column < base_types.size(); column++) { - column_ids.emplace_back(column); - } - DataChunk old_rows; - old_rows.Initialize(Allocator::Get(context.client), base_types); - auto fetch_state = ColumnFetchState(); - auto &transaction = DuckTransaction::Get(context.client, base_table.catalog); - base_table.GetStorage().Fetch(transaction, old_rows, column_ids, selected_input.data[row_id_index], - selected_count, fetch_state); - appender->Append(context.client, old_rows, -1, lstate.append_state); - - if (action_op.type != PhysicalOperatorType::UPDATE) { - return; - } + coordinator->RowFetcher().Fetch(context.client, lstate.selected_input.data[row_id_index], selected_count, + lstate.preimage_rows, lstate.fetch_state); + D_ASSERT(lstate.preimage_rows.size() == selected_count); auto &update = action_op.Cast(); lstate.update_values.Reset(); - lstate.update_values.SetCardinality(selected_input); + lstate.update_values.SetCardinality(lstate.selected_input); for (idx_t index = 0; index < update.expressions.size(); index++) { auto &expression = *update.expressions[index]; D_ASSERT(expression.GetExpressionType() == ExpressionType::BOUND_REF); auto &reference = expression.Cast(); - lstate.update_values.data[index].Reference(selected_input.data[reference.index]); + lstate.update_values.data[index].Reference(lstate.selected_input.data[reference.index]); } - - DataChunk new_rows; - new_rows.Initialize(Allocator::Get(context.client), base_types); - for (idx_t column = 0; column < base_types.size(); column++) { - new_rows.data[column].Reference(old_rows.data[column]); + for (idx_t column = 0; column < coordinator->RowFetcher().Types().size(); column++) { + lstate.pending_new_rows.data[column].Reference(lstate.preimage_rows.data[column]); } for (idx_t index = 0; index < update.columns.size(); index++) { - new_rows.data[update.columns[index].index].Reference(lstate.update_values.data[index]); + lstate.pending_new_rows.data[update.columns[index].index].Reference(lstate.update_values.data[index]); } - new_rows.SetCardinality(old_rows); - appender->Append(context.client, new_rows, 1, lstate.append_state); + lstate.pending_new_rows.SetCardinality(lstate.preimage_rows); } PhysicalOperator &action_op; - TableCatalogEntry &base_table; - shared_ptr appender; + shared_ptr coordinator; + bool capture_delete_insert_update; vector default_update_indexes; vector delegated_types; }; @@ -706,12 +896,29 @@ PhysicalOperator &LogicalTransactionalMergeDeltaCapture::CreatePlan(ClientContex auto appender = make_shared_ptr(base_table, delta_table, std::move(generated_expressions)); auto &merge = child.Cast(); + idx_t action_count = 0; + bool requires_serial_execution = false; for (auto &action : merge.actions) { if (!action->op) { continue; } - action->op = planner.Make( - *action->op, base_table, appender, merge.children[0].get().types, estimated_cardinality); + action_count++; + if (action->op->type == PhysicalOperatorType::UPDATE && + action->op->Cast().update_is_del_and_insert) { + requires_serial_execution = true; + } + } + auto coordinator = make_shared_ptr(base_table, appender, + merge.children[0].get().types, action_count); + for (auto &action : merge.actions) { + if (!action->op) { + continue; + } + action->op = planner.Make(*action->op, coordinator, estimated_cardinality); + } + if (requires_serial_execution) { + // DuckDB and capture must choose the same first match for delete-and-insert updates. + merge.parallel = false; } return child; } diff --git a/test/sql/insert_rule.test b/test/sql/insert_rule.test index 43de9508..305adfeb 100644 --- a/test/sql/insert_rule.test +++ b/test/sql/insert_rule.test @@ -1778,3 +1778,107 @@ SELECT count(*) FROM ( ); ---- 0 + +# Duplicate source matches can make DuckDB choose a different update row than the +# input's first occurrence when it sorts and deduplicates target row IDs. Capture +# the target's actual final value rather than predicting which source row wins. +statement ok +CREATE TABLE merge_duplicate_src (id INTEGER PRIMARY KEY, v INTEGER); + +statement ok +INSERT INTO merge_duplicate_src SELECT i, 0 FROM range(2500) r(i); + +statement ok +CREATE MATERIALIZED VIEW merge_duplicate_mv AS SELECT id, v FROM merge_duplicate_src; + +statement ok +CREATE TABLE merge_duplicate_changes AS +SELECT i AS id, x AS v +FROM range(2500) r(i), (VALUES (1), (2)) values_table(x); + +statement ok +MERGE INTO merge_duplicate_src +USING merge_duplicate_changes +ON merge_duplicate_src.id = merge_duplicate_changes.id +WHEN MATCHED THEN UPDATE SET v = merge_duplicate_changes.v; + +statement ok +PRAGMA refresh('merge_duplicate_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM merge_duplicate_mv + EXCEPT ALL + SELECT * FROM merge_duplicate_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM merge_duplicate_src + EXCEPT ALL + SELECT * FROM merge_duplicate_mv +); +---- +0 + +# A MERGE can update a committed row while deleting a row inserted earlier in +# the same transaction. The deleted local row ID must not be refetched as a +# positive postimage when the MERGE actions finalize. +statement ok +CREATE TABLE merge_mixed_visibility_src (id INTEGER PRIMARY KEY, v INTEGER); + +statement ok +INSERT INTO merge_mixed_visibility_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW merge_mixed_visibility_mv AS +SELECT id, v FROM merge_mixed_visibility_src; + +statement ok +BEGIN TRANSACTION; + +statement ok +INSERT INTO merge_mixed_visibility_src VALUES (2, 20); + +statement ok +MERGE INTO merge_mixed_visibility_src +USING (VALUES (1, 5), (2, -1)) changes(id, v) +ON merge_mixed_visibility_src.id = changes.id +WHEN MATCHED AND changes.v < 0 THEN DELETE +WHEN MATCHED THEN UPDATE SET v = merge_mixed_visibility_src.v + changes.v; + +statement ok +COMMIT; + +query III +SELECT id, v, openivm_multiplicity +FROM openivm_delta_merge_mixed_visibility_src +ORDER BY id, v, openivm_multiplicity; +---- +1 10 -1 +1 15 1 +2 20 -1 +2 20 1 + +statement ok +PRAGMA refresh('merge_mixed_visibility_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM merge_mixed_visibility_mv + EXCEPT ALL + SELECT * FROM merge_mixed_visibility_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM merge_mixed_visibility_src + EXCEPT ALL + SELECT * FROM merge_mixed_visibility_mv +); +---- +0 From 21e2eb1c93591fb294c6baafcb1a28f9c2c54d71 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 23 Jul 2026 09:55:42 +0200 Subject: [PATCH 05/36] Harden refresh compilation and lineage correctness --- benchmark/queries/tpcc/ducklake_0309.sql | 2 +- benchmark/queries/tpcc/ducklake_0316.sql | 2 + benchmark/queries/tpcc/ducklake_0317.sql | 2 + benchmark/queries/tpcc/query_2094.sql | 2 +- benchmark/queries/tpcc/query_2095.sql | 2 +- benchmark/queries/tpcc/query_2097.sql | 2 +- benchmark/queries/tpcc/query_2099.sql | 2 +- benchmark/queries/tpcc/query_2101.sql | 2 +- benchmark/queries/tpcc/query_2102.sql | 2 +- benchmark/queries/tpcc/query_2103.sql | 2 +- benchmark/queries/tpcc/query_2114.sql | 2 +- benchmark/queries/tpcc/query_2115.sql | 2 +- benchmark/queries/tpcc/query_2116.sql | 2 +- benchmark/queries/tpcc/query_2117.sql | 2 +- benchmark/queries/tpcc/query_2118.sql | 2 +- benchmark/queries/tpcc/query_2119.sql | 2 +- benchmark/queries/tpcc/query_2121.sql | 2 +- benchmark/queries/tpcc/query_2122.sql | 2 +- benchmark/queries/tpcc/query_2123.sql | 2 +- benchmark/queries/tpcc/query_2124.sql | 2 +- benchmark/queries/tpcc/query_2125.sql | 2 +- benchmark/queries/tpcc/query_2126.sql | 2 +- benchmark/src/rewriter_benchmark.cpp | 101 ++- docs/internals/cost_model.md | 1 - docs/refresh/refresh-strategies.md | 2 +- src/core/ivm_delta_model.cpp | 4 +- src/core/ivm_view_classifier.cpp | 20 +- src/core/parser.cpp | 210 +++++- src/core/parser_ddl.cpp | 167 ++++- src/core/parser_parse.cpp | 50 +- src/core/parser_plan_helpers.cpp | 163 ++++- src/core/refresh_locks.cpp | 39 ++ src/core/refresh_metadata.cpp | 18 +- src/core/sql_utils.cpp | 37 + src/delta/operators/nonlocal.cpp | 7 +- src/include/core/ivm_view_classifier.hpp | 6 +- src/include/core/openivm_constants.hpp | 13 +- src/include/core/parser.hpp | 8 + src/include/core/parser_ddl.hpp | 17 +- src/include/core/refresh_locks.hpp | 23 + src/include/core/refresh_metadata.hpp | 4 + src/include/core/sql_utils.hpp | 4 + src/include/upsert/refresh.hpp | 1 + src/include/upsert/refresh_compiler.hpp | 3 +- src/include/upsert/refresh_cost_model.hpp | 2 +- src/openivm_extension.cpp | 22 +- src/upsert/refresh.cpp | 133 ++++ src/upsert/refresh_compiler.cpp | 8 +- src/upsert/refresh_cost_model.cpp | 18 +- src/upsert/refresh_sql.cpp | 176 ++++- src/upsert/refresh_window.cpp | 134 ++-- test/sql/aggregate.test | 65 ++ test/sql/auto_refresh_asof.test | 14 +- test/sql/cascade_window_partition_joined.test | 657 ++++++++++++++++++ test/sql/compile_refresh.test | 31 +- test/sql/compile_refresh_asof.test | 25 +- test/sql/compile_spark_dialect_hardening.test | 38 + test/sql/nonlocal_operator_recompute.test | 271 +++++++- .../sql/nonlocal_operator_recompute_asof.test | 156 ++++- test/sql/transactional_lifecycle.test | 421 +++++++++++ 60 files changed, 2835 insertions(+), 278 deletions(-) create mode 100644 benchmark/queries/tpcc/ducklake_0316.sql create mode 100644 benchmark/queries/tpcc/ducklake_0317.sql create mode 100644 test/sql/transactional_lifecycle.test diff --git a/benchmark/queries/tpcc/ducklake_0309.sql b/benchmark/queries/tpcc/ducklake_0309.sql index da97cb12..6ae8099d 100644 --- a/benchmark/queries/tpcc/ducklake_0309.sql +++ b/benchmark/queries/tpcc/ducklake_0309.sql @@ -1140,5 +1140,5 @@ SELECT min_balance, max_balance FROM topk_projected -ORDER BY total_amount DESC, warehouse_id ASC, district_id ASC, customer_id ASC +ORDER BY ROUND(total_amount, 10) DESC, warehouse_id ASC, district_id ASC, customer_id ASC LIMIT 25 OFFSET 2 diff --git a/benchmark/queries/tpcc/ducklake_0316.sql b/benchmark/queries/tpcc/ducklake_0316.sql new file mode 100644 index 00000000..5b954630 --- /dev/null +++ b/benchmark/queries/tpcc/ducklake_0316.sql @@ -0,0 +1,2 @@ +-- {"operators": "DUCKLAKE,OUTER_JOIN,WINDOW,CAST", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": true, "has_case": false, "tables": "CUSTOMER,OORDER", "ducklake": true, "openivm_verified": true} +SELECT c.C_W_ID AS warehouse_id, c.C_D_ID AS district_id, c.C_ID AS customer_id, o.O_ID AS order_id, ROW_NUMBER() OVER (PARTITION BY c.C_W_ID ORDER BY o.O_ID DESC NULLS LAST, c.C_D_ID, c.C_ID) AS rn FROM dl.CUSTOMER c LEFT JOIN dl.OORDER o ON c.C_W_ID = o.O_W_ID AND c.C_D_ID = o.O_D_ID AND CAST(CAST(c.C_ID AS BIGINT) AS VARCHAR) = CAST(o.O_C_ID AS VARCHAR); diff --git a/benchmark/queries/tpcc/ducklake_0317.sql b/benchmark/queries/tpcc/ducklake_0317.sql new file mode 100644 index 00000000..df0dcd33 --- /dev/null +++ b/benchmark/queries/tpcc/ducklake_0317.sql @@ -0,0 +1,2 @@ +-- {"operators": "DUCKLAKE,OUTER_JOIN,WINDOW,CAST", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": true, "has_case": false, "tables": "CUSTOMER,OORDER", "ducklake": true, "openivm_verified": true} +SELECT c.C_W_ID AS warehouse_id, c.C_D_ID AS district_id, c.C_ID AS customer_id, o.O_ID AS order_id, ROW_NUMBER() OVER (PARTITION BY c.C_W_ID ORDER BY o.O_ID DESC NULLS LAST, c.C_D_ID, c.C_ID) AS rn FROM dl.CUSTOMER c LEFT JOIN dl.OORDER o ON c.C_W_ID = o.O_W_ID AND c.C_D_ID = o.O_D_ID AND CAST(TRY_CAST(CAST(c.C_ID AS VARCHAR) AS BIGINT) AS VARCHAR) = CAST(o.O_C_ID AS VARCHAR); diff --git a/benchmark/queries/tpcc/query_2094.sql b/benchmark/queries/tpcc/query_2094.sql index c2afbacf..bbc2630d 100644 --- a/benchmark/queries/tpcc/query_2094.sql +++ b/benchmark/queries/tpcc/query_2094.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,FILTER", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"} +-- {"operators": "ASOF_JOIN,FILTER", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"} SELECT ol.OL_W_ID, ol.OL_D_ID, ol.OL_O_ID, o.O_ENTRY_D FROM ORDER_LINE ol ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D WHERE ol.OL_AMOUNT > 0; diff --git a/benchmark/queries/tpcc/query_2095.sql b/benchmark/queries/tpcc/query_2095.sql index 1a7f5c28..32d2e803 100644 --- a/benchmark/queries/tpcc/query_2095.sql +++ b/benchmark/queries/tpcc/query_2095.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,LEFT_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"} +-- {"operators": "ASOF_JOIN,LEFT_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"} SELECT ol.OL_W_ID, ol.OL_I_ID, o.O_C_ID FROM ORDER_LINE ol ASOF LEFT JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D; diff --git a/benchmark/queries/tpcc/query_2097.sql b/benchmark/queries/tpcc/query_2097.sql index 1b9c9c07..8092a007 100644 --- a/benchmark/queries/tpcc/query_2097.sql +++ b/benchmark/queries/tpcc/query_2097.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,INNER_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER,CUSTOMER"} +-- {"operators": "ASOF_JOIN,INNER_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER,CUSTOMER", "non_incr_reason": "op:ASOF_JOIN"} SELECT c.C_W_ID, c.C_D_ID, c.C_ID, ol.OL_AMOUNT FROM ORDER_LINE ol ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D JOIN CUSTOMER c ON c.C_W_ID = o.O_W_ID AND c.C_D_ID = o.O_D_ID AND c.C_ID = o.O_C_ID; diff --git a/benchmark/queries/tpcc/query_2099.sql b/benchmark/queries/tpcc/query_2099.sql index 9dc99455..1b1c280c 100644 --- a/benchmark/queries/tpcc/query_2099.sql +++ b/benchmark/queries/tpcc/query_2099.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,CTE", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"} +-- {"operators": "ASOF_JOIN,CTE", "complexity": "high", "is_incremental": false, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"} WITH delivered AS (SELECT * FROM ORDER_LINE WHERE OL_DELIVERY_D IS NOT NULL) SELECT d.OL_W_ID, d.OL_O_ID, o.O_C_ID FROM delivered d ASOF JOIN OORDER o ON d.OL_W_ID = o.O_W_ID AND d.OL_D_ID = o.O_D_ID AND d.OL_DELIVERY_D >= o.O_ENTRY_D; diff --git a/benchmark/queries/tpcc/query_2101.sql b/benchmark/queries/tpcc/query_2101.sql index d4ef5a78..9b2c30d5 100644 --- a/benchmark/queries/tpcc/query_2101.sql +++ b/benchmark/queries/tpcc/query_2101.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,FULL_OUTER_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER,STOCK"} +-- {"operators": "ASOF_JOIN,FULL_OUTER_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER,STOCK", "non_incr_reason": "op:ASOF_JOIN"} SELECT COALESCE(s.S_W_ID, ol.OL_W_ID) AS w_id, COALESCE(s.S_I_ID, ol.OL_I_ID) AS item_id FROM ORDER_LINE ol ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D FULL OUTER JOIN STOCK s ON s.S_W_ID = ol.OL_SUPPLY_W_ID AND s.S_I_ID = ol.OL_I_ID; diff --git a/benchmark/queries/tpcc/query_2102.sql b/benchmark/queries/tpcc/query_2102.sql index 0f7bc45d..755de67e 100644 --- a/benchmark/queries/tpcc/query_2102.sql +++ b/benchmark/queries/tpcc/query_2102.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,SUBQUERY", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"} +-- {"operators": "ASOF_JOIN,SUBQUERY", "complexity": "high", "is_incremental": false, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"} SELECT * FROM (SELECT ol.OL_W_ID, ol.OL_D_ID, o.O_ENTRY_D FROM ORDER_LINE ol ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D) q WHERE q.O_ENTRY_D IS NOT NULL; diff --git a/benchmark/queries/tpcc/query_2103.sql b/benchmark/queries/tpcc/query_2103.sql index 549ed6ff..7dc6c58b 100644 --- a/benchmark/queries/tpcc/query_2103.sql +++ b/benchmark/queries/tpcc/query_2103.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"} +-- {"operators": "ASOF_JOIN,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"} SELECT ol.OL_W_ID, u.label, o.O_ENTRY_D FROM ORDER_LINE ol CROSS JOIN UNNEST(['delivery', 'entry']) AS u(label) ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D; diff --git a/benchmark/queries/tpcc/query_2114.sql b/benchmark/queries/tpcc/query_2114.sql index fa1905ab..73e749c8 100644 --- a/benchmark/queries/tpcc/query_2114.sql +++ b/benchmark/queries/tpcc/query_2114.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE", "complexity": "medium", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK"} +-- {"operators": "SAMPLE", "complexity": "medium", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "non_incr_reason": "op:SAMPLE"} SELECT S_W_ID, S_I_ID, S_QUANTITY FROM STOCK USING SAMPLE reservoir(10 ROWS) REPEATABLE (42); diff --git a/benchmark/queries/tpcc/query_2115.sql b/benchmark/queries/tpcc/query_2115.sql index 3bd115fa..a40c1566 100644 --- a/benchmark/queries/tpcc/query_2115.sql +++ b/benchmark/queries/tpcc/query_2115.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE,AGGREGATE", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE"} +-- {"operators": "SAMPLE,AGGREGATE", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE", "non_incr_reason": "op:SAMPLE"} SELECT OL_W_ID, SUM(OL_AMOUNT) AS sampled_amount FROM (SELECT * FROM ORDER_LINE USING SAMPLE reservoir(25 ROWS) REPEATABLE (7)) sampled_order_line GROUP BY OL_W_ID; diff --git a/benchmark/queries/tpcc/query_2116.sql b/benchmark/queries/tpcc/query_2116.sql index 13b26c43..b9677f7c 100644 --- a/benchmark/queries/tpcc/query_2116.sql +++ b/benchmark/queries/tpcc/query_2116.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE,WINDOW", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "openivm_verified": true} +-- {"operators": "SAMPLE,WINDOW", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "openivm_verified": true, "non_incr_reason": "op:SAMPLE"} SELECT S_W_ID, S_I_ID, S_QUANTITY, ROW_NUMBER() OVER (PARTITION BY S_W_ID ORDER BY S_QUANTITY DESC) AS sampled_rank FROM STOCK USING SAMPLE reservoir(20 ROWS) REPEATABLE (11); diff --git a/benchmark/queries/tpcc/query_2117.sql b/benchmark/queries/tpcc/query_2117.sql index 63293c5e..c000b6f4 100644 --- a/benchmark/queries/tpcc/query_2117.sql +++ b/benchmark/queries/tpcc/query_2117.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE,INNER_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ITEM,STOCK"} +-- {"operators": "SAMPLE,INNER_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ITEM,STOCK", "non_incr_reason": "op:SAMPLE"} SELECT i.I_ID, i.I_NAME, s.S_W_ID, s.S_QUANTITY FROM (SELECT * FROM STOCK USING SAMPLE reservoir(20 ROWS) REPEATABLE (13)) s JOIN ITEM i ON s.S_I_ID = i.I_ID; diff --git a/benchmark/queries/tpcc/query_2118.sql b/benchmark/queries/tpcc/query_2118.sql index 81fcd811..1cd4f687 100644 --- a/benchmark/queries/tpcc/query_2118.sql +++ b/benchmark/queries/tpcc/query_2118.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE,LEFT_JOIN,AGGREGATE", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ITEM,STOCK"} +-- {"operators": "SAMPLE,LEFT_JOIN,AGGREGATE", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ITEM,STOCK", "non_incr_reason": "op:SAMPLE"} SELECT i.I_IM_ID, COUNT(s.S_I_ID) AS sampled_stock_rows FROM ITEM i LEFT JOIN (SELECT * FROM STOCK USING SAMPLE reservoir(30 ROWS) REPEATABLE (17)) s ON i.I_ID = s.S_I_ID GROUP BY i.I_IM_ID; diff --git a/benchmark/queries/tpcc/query_2119.sql b/benchmark/queries/tpcc/query_2119.sql index e40b396d..83f2b310 100644 --- a/benchmark/queries/tpcc/query_2119.sql +++ b/benchmark/queries/tpcc/query_2119.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK"} +-- {"operators": "SAMPLE,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "non_incr_reason": "op:SAMPLE"} SELECT s.S_W_ID, u.threshold FROM (SELECT * FROM STOCK USING SAMPLE reservoir(15 ROWS) REPEATABLE (19)) s CROSS JOIN UNNEST([10, 50, 90]) AS u(threshold) WHERE s.S_QUANTITY >= u.threshold; diff --git a/benchmark/queries/tpcc/query_2121.sql b/benchmark/queries/tpcc/query_2121.sql index 732881e9..152d82a9 100644 --- a/benchmark/queries/tpcc/query_2121.sql +++ b/benchmark/queries/tpcc/query_2121.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE,PIVOT,AGGREGATE", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK"} +-- {"operators": "SAMPLE,PIVOT,AGGREGATE", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "non_incr_reason": "op:SAMPLE"} SELECT * FROM (PIVOT (SELECT * FROM STOCK USING SAMPLE reservoir(20 ROWS) REPEATABLE (29)) ON S_W_ID IN (1, 2, 3) USING SUM(S_QUANTITY) GROUP BY S_I_ID) p; diff --git a/benchmark/queries/tpcc/query_2122.sql b/benchmark/queries/tpcc/query_2122.sql index 7feee2ff..6a6f9539 100644 --- a/benchmark/queries/tpcc/query_2122.sql +++ b/benchmark/queries/tpcc/query_2122.sql @@ -1,2 +1,2 @@ --- {"operators": "POSITIONAL_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM"} +-- {"operators": "POSITIONAL_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM", "non_incr_reason": "join:POSITIONAL"} SELECT s.S_W_ID, s.S_I_ID, i.I_ID, i.I_NAME FROM STOCK s POSITIONAL JOIN ITEM i; diff --git a/benchmark/queries/tpcc/query_2123.sql b/benchmark/queries/tpcc/query_2123.sql index 863124f0..57c5aecb 100644 --- a/benchmark/queries/tpcc/query_2123.sql +++ b/benchmark/queries/tpcc/query_2123.sql @@ -1,2 +1,2 @@ --- {"operators": "POSITIONAL_JOIN,AGGREGATE", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM"} +-- {"operators": "POSITIONAL_JOIN,AGGREGATE", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM", "non_incr_reason": "join:POSITIONAL"} SELECT s.S_W_ID, COUNT(i.I_ID) AS paired_items, SUM(s.S_QUANTITY) AS paired_qty FROM STOCK s POSITIONAL JOIN ITEM i GROUP BY s.S_W_ID; diff --git a/benchmark/queries/tpcc/query_2124.sql b/benchmark/queries/tpcc/query_2124.sql index 803f2958..30324486 100644 --- a/benchmark/queries/tpcc/query_2124.sql +++ b/benchmark/queries/tpcc/query_2124.sql @@ -1,2 +1,2 @@ --- {"operators": "POSITIONAL_JOIN,WINDOW", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "CUSTOMER,OORDER", "openivm_verified": true} +-- {"operators": "POSITIONAL_JOIN,WINDOW", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "CUSTOMER,OORDER", "openivm_verified": true, "non_incr_reason": "join:POSITIONAL"} SELECT c.C_W_ID, c.C_ID, o.O_ID, ROW_NUMBER() OVER (ORDER BY c.C_W_ID, c.C_ID) AS pos_rank FROM CUSTOMER c POSITIONAL JOIN OORDER o; diff --git a/benchmark/queries/tpcc/query_2125.sql b/benchmark/queries/tpcc/query_2125.sql index 4a930fc1..4bb920f4 100644 --- a/benchmark/queries/tpcc/query_2125.sql +++ b/benchmark/queries/tpcc/query_2125.sql @@ -1,2 +1,2 @@ --- {"operators": "POSITIONAL_JOIN,LEFT_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM,ORDER_LINE"} +-- {"operators": "POSITIONAL_JOIN,LEFT_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM,ORDER_LINE", "non_incr_reason": "join:POSITIONAL"} SELECT s.S_W_ID, s.S_I_ID, i.I_NAME, ol.OL_AMOUNT FROM STOCK s POSITIONAL JOIN ITEM i LEFT JOIN ORDER_LINE ol ON ol.OL_I_ID = i.I_ID AND ol.OL_SUPPLY_W_ID = s.S_W_ID; diff --git a/benchmark/queries/tpcc/query_2126.sql b/benchmark/queries/tpcc/query_2126.sql index 884a3789..7a475bcf 100644 --- a/benchmark/queries/tpcc/query_2126.sql +++ b/benchmark/queries/tpcc/query_2126.sql @@ -1,2 +1,2 @@ --- {"operators": "POSITIONAL_JOIN,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM"} +-- {"operators": "POSITIONAL_JOIN,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM", "non_incr_reason": "join:POSITIONAL"} SELECT s.S_W_ID, i.I_ID, u.label FROM STOCK s POSITIONAL JOIN ITEM i CROSS JOIN UNNEST(['left_pos', 'right_pos']) AS u(label); diff --git a/benchmark/src/rewriter_benchmark.cpp b/benchmark/src/rewriter_benchmark.cpp index eae802e0..4290e702 100644 --- a/benchmark/src/rewriter_benchmark.cpp +++ b/benchmark/src/rewriter_benchmark.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,55 @@ using openivm_bench::ReadAllBytes; using openivm_bench::Timestamp; using openivm_bench::WriteAllBytes; +static bool ReadAllBytesUntil(int fd, void *buf, size_t n, std::chrono::steady_clock::time_point deadline, + bool &timed_out) { + char *p = static_cast(buf); + while (n > 0) { + auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + timed_out = true; + return false; + } + + int remaining_ms = + static_cast(std::chrono::duration_cast(deadline - now).count()); + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + pfd.revents = 0; + + int ret = poll(&pfd, 1, std::min(remaining_ms, 500)); + if (ret < 0) { + if (errno == EINTR) { + continue; + } + return false; + } + if (ret == 0) { + continue; + } + if (pfd.revents & POLLIN) { + ssize_t r = read(fd, p, n); + if (r < 0) { + if (errno == EINTR) { + continue; + } + return false; + } + if (r == 0) { + return false; + } + p += r; + n -= static_cast(r); + continue; + } + if (pfd.revents & (POLLHUP | POLLERR | POLLNVAL)) { + return false; + } + } + return true; +} + static string FormatNumber(double v) { std::ostringstream oss; oss << std::setprecision(5) << std::defaultfloat << v; @@ -1019,34 +1069,51 @@ struct ForkWorker { int ret = poll(&pfd, 1, poll_ms); if (ret > 0 && (pfd.revents & POLLIN)) { - if (!ReadAllBytes(from_child_fd, &result_phase, sizeof(result_phase)) || - !ReadAllBytes(from_child_fd, &result_incremental, sizeof(result_incremental)) || - !ReadAllBytes(from_child_fd, &result_correct, sizeof(result_correct)) || - !ReadAllBytes(from_child_fd, &result_time_select_ms, sizeof(result_time_select_ms)) || - !ReadAllBytes(from_child_fd, &result_time_mv_ms, sizeof(result_time_mv_ms)) || - !ReadAllBytes(from_child_fd, &result_time_refresh_ms, sizeof(result_time_refresh_ms)) || - !ReadAllBytes(from_child_fd, &result_time_verify_ms, sizeof(result_time_verify_ms))) { + bool timed_out = false; + auto read_field = [&](void *buf, size_t n) { + return ReadAllBytesUntil(from_child_fd, buf, n, deadline, timed_out); + }; + auto finish_read_failure = [&]() { + if (timed_out) { + kill(child_pid, SIGKILL); + waitpid(child_pid, nullptr, 0); + child_pid = -1; + result_phase = PHASE_TIMEOUT; + result_error = "timeout"; + return; + } int status; waitpid(child_pid, &status, 0); child_pid = -1; result_phase = PHASE_CRASH; + if (WIFEXITED(status)) { + result_error = "child exited with code " + std::to_string(WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + result_error = "child killed by signal " + std::to_string(WTERMSIG(status)); + } else { + result_error = "child died while writing result"; + } + }; + + if (!read_field(&result_phase, sizeof(result_phase)) || + !read_field(&result_incremental, sizeof(result_incremental)) || + !read_field(&result_correct, sizeof(result_correct)) || + !read_field(&result_time_select_ms, sizeof(result_time_select_ms)) || + !read_field(&result_time_mv_ms, sizeof(result_time_mv_ms)) || + !read_field(&result_time_refresh_ms, sizeof(result_time_refresh_ms)) || + !read_field(&result_time_verify_ms, sizeof(result_time_verify_ms))) { + finish_read_failure(); return; } uint32_t err_len = 0; - if (!ReadAllBytes(from_child_fd, &err_len, sizeof(err_len))) { - int status; - waitpid(child_pid, &status, 0); - child_pid = -1; - result_phase = PHASE_CRASH; + if (!read_field(&err_len, sizeof(err_len))) { + finish_read_failure(); return; } if (err_len > 0) { result_error.resize(err_len); - if (!ReadAllBytes(from_child_fd, &result_error[0], err_len)) { - int status; - waitpid(child_pid, &status, 0); - child_pid = -1; - result_phase = PHASE_CRASH; + if (!read_field(&result_error[0], err_len)) { + finish_read_failure(); return; } } diff --git a/docs/internals/cost_model.md b/docs/internals/cost_model.md index 75503d42..378ba71b 100644 --- a/docs/internals/cost_model.md +++ b/docs/internals/cost_model.md @@ -121,7 +121,6 @@ match the refresh path that actually runs: affected groups. - `WINDOW_PARTITION` prices delta scanning plus the affected partition fraction of the base scan. -- `CURRENT_DIFF_RECOMPUTE` prices the same compute and replace work as full recompute. - `DISTINCT_INCREMENTAL` adds aux-state maintenance over affected distinct tuples. - `SEMI_ANTI_RECOMPUTE` prices aux-state/domain recompute for supported semi/anti projection shapes. diff --git a/docs/refresh/refresh-strategies.md b/docs/refresh/refresh-strategies.md index 565fb050..58d2b47c 100644 --- a/docs/refresh/refresh-strategies.md +++ b/docs/refresh/refresh-strategies.md @@ -88,7 +88,7 @@ Returns a single row: |---|---:|---:|---:|---:|---| | incremental | 1200.0 | 50000.0 | 1200.0 | 50000.0 | false | -- `decision`: the selected strategy. Values include `incremental`, `group_recompute`, `window_partition`, `current_diff_recompute`, `distinct_incremental`, `semi_anti_recompute`, and `full`. +- `decision`: the selected strategy. Values include `incremental`, `group_recompute`, `window_partition`, `distinct_incremental`, `semi_anti_recompute`, and `full`. - `incremental_cost`: estimated cost of the selected non-full strategy. For affected-domain strategies, this is not a pure delta-only cost. - `recompute_cost`: estimated cost of a full DELETE + INSERT. - `incremental_predicted_ms`: learned prediction for the selected non-full strategy, or the static estimate before calibration. diff --git a/src/core/ivm_delta_model.cpp b/src/core/ivm_delta_model.cpp index 2387eee5..d161d3c8 100644 --- a/src/core/ivm_delta_model.cpp +++ b/src/core/ivm_delta_model.cpp @@ -639,10 +639,8 @@ void PopulateDeltaViewModelLineage(DeltaViewModel &model, const CreateMVPlanFact if (analysis.found_asof_join && (!has_lineage || AsofWindowPartitionReadsRightSideDirectly(direct_lineage_ops, model) || !WindowLineageCoversAllSources(model.window_lineage_ops, facts))) { - model.type = RefreshType::CURRENT_DIFF_RECOMPUTE; + model.type = RefreshType::FULL_REFRESH; model.window_lineage_ops.clear(); - AddUnique(model.features, DeltaModelFeature::CURRENT_DIFF_RECOMPUTE); - AddUnique(model.strategy_reasons, DeltaStrategyReason::ASOF_CURRENT_DIFF_RECOMPUTE); ValidateDeltaViewModelInvariants(model); return; } diff --git a/src/core/ivm_view_classifier.cpp b/src/core/ivm_view_classifier.cpp index 07b4d8dd..86212442 100644 --- a/src/core/ivm_view_classifier.cpp +++ b/src/core/ivm_view_classifier.cpp @@ -530,27 +530,21 @@ static void SelectRefreshType(DeltaViewModel &model, const PlanAnalysis &analysi auto has_argminmax = std::any_of(analysis.aggregate_types.begin(), analysis.aggregate_types.end(), [](const string &agg_type) { return agg_type == "arg_min" || agg_type == "arg_max"; }); - auto select_current_diff = [&](DeltaStrategyReason reason) { - model.type = RefreshType::CURRENT_DIFF_RECOMPUTE; - AddUnique(model.features, DeltaModelFeature::CURRENT_DIFF_RECOMPUTE); - AddUnique(model.strategy_reasons, reason); - }; if (input.has_unsupported_incremental_construct) { model.type = RefreshType::FULL_REFRESH; } else if (!analysis.incremental_compatible) { model.type = RefreshType::FULL_REFRESH; model.warn_unsupported_incremental = true; } else if (analysis.found_sample) { - select_current_diff(DeltaStrategyReason::SAMPLE_CURRENT_DIFF_RECOMPUTE); + model.type = RefreshType::FULL_REFRESH; } else if (analysis.found_positional_join) { - select_current_diff(DeltaStrategyReason::POSITIONAL_CURRENT_DIFF_RECOMPUTE); + model.type = RefreshType::FULL_REFRESH; } else if (analysis.found_asof_join && analysis.found_window && !model.window_partition_columns.empty()) { model.type = RefreshType::WINDOW_PARTITION; } else if (analysis.found_asof_join && analysis.found_aggregation && !model.group_columns.empty()) { model.type = RefreshType::GROUP_RECOMPUTE; - AddUnique(model.strategy_reasons, DeltaStrategyReason::ASOF_CURRENT_DIFF_RECOMPUTE); } else if (analysis.found_asof_join) { - select_current_diff(DeltaStrategyReason::ASOF_CURRENT_DIFF_RECOMPUTE); + model.type = RefreshType::FULL_REFRESH; } else if (analysis.found_window) { model.type = RefreshType::WINDOW_PARTITION; } else if (analysis.found_grouping_sets) { @@ -659,12 +653,6 @@ const char *DeltaStrategyReasonName(DeltaStrategyReason reason) { return "SEMI_ANTI_AGGREGATE_GROUP_FALLBACK"; case DeltaStrategyReason::OUTER_JOIN_AGGREGATE_RECOMPUTE: return "OUTER_JOIN_AGGREGATE_RECOMPUTE"; - case DeltaStrategyReason::ASOF_CURRENT_DIFF_RECOMPUTE: - return "ASOF_CURRENT_DIFF_RECOMPUTE"; - case DeltaStrategyReason::SAMPLE_CURRENT_DIFF_RECOMPUTE: - return "SAMPLE_CURRENT_DIFF_RECOMPUTE"; - case DeltaStrategyReason::POSITIONAL_CURRENT_DIFF_RECOMPUTE: - return "POSITIONAL_CURRENT_DIFF_RECOMPUTE"; default: return "UNKNOWN"; } @@ -702,8 +690,6 @@ const char *DeltaModelFeatureName(DeltaModelFeature feature) { return "SAMPLE_GLOBAL_RECOMPUTE"; case DeltaModelFeature::POSITIONAL_GLOBAL_RECOMPUTE: return "POSITIONAL_GLOBAL_RECOMPUTE"; - case DeltaModelFeature::CURRENT_DIFF_RECOMPUTE: - return "CURRENT_DIFF_RECOMPUTE"; case DeltaModelFeature::FULL_ONLY: return "FULL_ONLY"; default: diff --git a/src/core/parser.cpp b/src/core/parser.cpp index f5b24ec6..dbe03bdc 100644 --- a/src/core/parser.cpp +++ b/src/core/parser.cpp @@ -6,6 +6,7 @@ #include "core/parser_ddl.hpp" #include "core/parser_plan_helpers.hpp" #include "core/parser_sql_extractors.hpp" +#include "core/refresh_locks.hpp" #include "core/refresh_metadata.hpp" #include "core/ivm_delta_model.hpp" #include "core/ivm_view_classifier.hpp" @@ -19,6 +20,7 @@ #include "duckdb/main/client_data.hpp" #include "duckdb/main/settings.hpp" #include "duckdb/parser/parser.hpp" +#include "duckdb/parser/statement/drop_statement.hpp" #include "duckdb/planner/expression/bound_aggregate_expression.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/operator/logical_top_n.hpp" @@ -243,12 +245,8 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // Handle ALTER MATERIALIZED VIEW — just execute the metadata UPDATE if (!parse_data_ref.alter_sql.empty()) { - auto r = con.Query(parse_data_ref.alter_sql); - if (r->HasError()) { - throw CatalogException("Failed to alter materialized view: " + r->GetError()); - } - // Return via the DDL executor with no DDL to run (the UPDATE already executed) - ConfigureDDLExecutorResult(result); + result.parameters.push_back(Value(parse_data_ref.alter_sql)); + ConfigureDDLExecutorResult(result, DDLExecutionMode::CALLER_TRANSACTION); return result; } @@ -258,7 +256,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC ForwardPacSettingsIfLoaded(context, con); auto name_resolution_start = create_profile_now(); - auto full_view_name = SqlUtils::ExtractTableName(statement->query); + auto full_view_name = parse_data_ref.target_name; // Keep the user's raw AS-query as the source of truth for original-SQL fallback. // Do not recover this from DuckDB's parsed QueryNode::ToString(): that path is a // best-effort pretty-printer and has segfaulted on set-operation query nodes with @@ -306,6 +304,10 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC string data_table = IncrementalTableNames::DataTableName(view_name); string qdt = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(data_table); string qvn = view_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(view_name); + bool staged_cross_catalog_replace = target_is_ducklake && parse_data_ref.is_replace; + string staged_data_table = "openivm_stage_" + view_name; + string staged_qdt = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(staged_data_table); + string initial_load_target = staged_cross_catalog_replace ? staged_qdt : qdt; string view_query = original_view_query; // will be overwritten by LPTS for DDL string top_k_suffix; // ORDER BY … LIMIT k, appended to the CREATE VIEW string top_k_order_suffix; // ORDER BY only, used when fallback stored SQL already applied LIMIT @@ -947,7 +949,12 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC vector cleanup_ddl; vector metadata_ddl; vector aux_metadata_ddl; + vector> staged_aux_tables; + unordered_map aux_state_targets; auto add_cleanup = [&](const string &query) { + if (staged_cross_catalog_replace) { + return; + } cleanup_ddl.push_back(string(OPENIVM_DDL_CLEANUP_PREFIX) + query); }; auto add_profile_marker = [&](const string &step_name, const string &detail = string()) { @@ -957,6 +964,24 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC ddl.push_back(string(OPENIVM_DDL_PROFILE_RECORD_PREFIX) + view_name + "\t" + step_name + "\t" + to_string(duration_ms) + "\t" + detail); }; + auto get_aux_state_target = [&](const string &aux_table) { + auto existing = aux_state_targets.find(aux_table); + if (existing != aux_state_targets.end()) { + return existing->second; + } + auto published_target = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(aux_table); + if (!staged_cross_catalog_replace) { + aux_state_targets.emplace(aux_table, published_target); + return published_target; + } + auto staged_target = + internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted("openivm_stage_" + aux_table); + ddl.push_back("drop table if exists " + staged_target); + cleanup_ddl.push_back(string(OPENIVM_DDL_CLEANUP_PREFIX) + "DROP TABLE IF EXISTS " + staged_target); + staged_aux_tables.emplace_back(staged_target, published_target); + aux_state_targets.emplace(aux_table, staged_target); + return staged_target; + }; for (const auto &step : create_profile_steps) { add_profile_record(step.step_name, step.duration_ms, step.detail); } @@ -965,7 +990,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC "; lpts_fallback=" + string(lpts_fallback ? "true" : "false")); AppendCreateMVSystemTablesDDL(ddl, view_name, parse_data_ref.is_replace); - if (parse_data_ref.is_replace) { + if (parse_data_ref.is_replace && !staged_cross_catalog_replace) { add_profile_marker("create_mv_replace_cleanup"); string qvn_drop = view_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(view_name); string qdt_drop = internal_catalog_prefix + @@ -1066,10 +1091,10 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC if (view_model.HasDistinctAux()) { add_profile_marker("create_mv_distinct_aux"); const auto &meta = view_model.distinct_aux; - string aux_target = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table); + string aux_target = get_aux_state_target(meta.aux_table); string aux_create = BuildDistinctAuxStateCreateSQL(aux_target, meta.cols, meta.source_exprs, "(" + meta.input_sql + ")", "", - /*replace=*/false); + /*replace=*/parse_data_ref.is_replace && !staged_cross_catalog_replace); ddl.push_back(aux_create); add_cleanup("DROP TABLE IF EXISTS " + internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table)); @@ -1081,10 +1106,11 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC add_profile_marker("create_mv_count_distinct_aux"); const auto &meta = view_model.count_distinct_aux; string source_table = QualifyCreateSourceTable(meta.source, current_catalog, current_schema, default_db); - string aux_target = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table); + string aux_target = get_aux_state_target(meta.aux_table); string aux_create = BuildCountDistinctAuxStateCreateSQL(aux_target, source_table, meta.group_cols, meta.group_source_exprs, - meta.distinct_col, meta.distinct_expr, meta.filter, /*replace=*/false); + meta.distinct_col, meta.distinct_expr, meta.filter, + /*replace=*/parse_data_ref.is_replace && !staged_cross_catalog_replace); ddl.push_back(aux_create); add_cleanup("DROP TABLE IF EXISTS " + internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table)); @@ -1097,10 +1123,10 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC const auto &req = view_model.filtered_group_count_aux; const auto &meta = req.meta; string source_table = QualifyCreateSourceTable(req.create_source, current_catalog, current_schema, default_db); - string aux_target = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table); + string aux_target = get_aux_state_target(meta.aux_table); string aux_create = BuildFilteredGroupCountAuxStateCreateSQL( aux_target, source_table, meta.group_col, meta.sum_col, meta.source_group_expr, meta.source_sum_expr, - /*replace=*/false); + /*replace=*/parse_data_ref.is_replace && !staged_cross_catalog_replace); ddl.push_back(aux_create); add_cleanup("DROP TABLE IF EXISTS " + internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table)); @@ -1115,10 +1141,11 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC QualifyCreateSourceTable(meta.left_table, current_catalog, current_schema, default_db); string right_source_table = QualifyCreateSourceTable(meta.right_table, current_catalog, current_schema, default_db); - string aux_target = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table); + string aux_target = get_aux_state_target(meta.aux_table); string aux_create = BuildSemiAntiAuxStateCreateSQL( aux_target, left_source_table, meta.left_alias, right_source_table, meta.right_alias, meta.predicate, - meta.post_filter, meta.right_filter, meta.left_cols, meta.left_exprs, /*replace=*/false, meta.null_aware, + meta.post_filter, meta.right_filter, meta.left_cols, meta.left_exprs, + /*replace=*/parse_data_ref.is_replace && !staged_cross_catalog_replace, meta.null_aware, meta.null_aware_right_expr); ddl.push_back(aux_create); add_cleanup("DROP TABLE IF EXISTS " + internal_catalog_prefix + @@ -1213,7 +1240,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC if (!current_catalog.empty() && current_catalog != default_db) { con.Query("USE " + current_catalog_schema); } - string initial_load_statement = "CREATE TABLE " + qdt + " AS " + view_query; + string initial_load_statement = "CREATE TABLE " + initial_load_target + " AS " + view_query; string diagnostic; diagnostic += "\n[OpenIVM initial-load diagnostic]\n"; diagnostic += "view_name: " + view_name + "\n"; @@ -1234,7 +1261,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC false, diagnostic); } if (SqlUtils::GetBoolSetting(context, "openivm_explain_initial_load_only", false)) { - ConfigureDDLExecutorResult(result); + ConfigureDDLExecutorResult(result, DDLExecutionMode::CALLER_TRANSACTION); return result; } } @@ -1247,13 +1274,28 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC if (!current_catalog.empty() && current_catalog != default_db) { ddl.push_back("use " + current_catalog_schema); } + if (staged_cross_catalog_replace) { + ddl.push_back("drop table if exists " + staged_qdt); + cleanup_ddl.push_back(string(OPENIVM_DDL_CLEANUP_PREFIX) + "DROP TABLE IF EXISTS " + staged_qdt); + } if (view_model.HasSemiAntiAux()) { const auto &meta = view_model.semi_anti_aux; - string aux_target = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table); - ddl.push_back(BuildSemiAntiInitialDataSQL(qdt, aux_target, meta.join_type, meta.left_cols, meta.output_cols, - meta.null_aware, meta.null_aware_left_col)); + string aux_target = get_aux_state_target(meta.aux_table); + ddl.push_back(BuildSemiAntiInitialDataSQL(initial_load_target, aux_target, meta.join_type, meta.left_cols, + meta.output_cols, meta.null_aware, meta.null_aware_left_col)); } else { - ddl.push_back("create table " + qdt + " as " + view_query); + ddl.push_back("create table " + initial_load_target + " as " + view_query); + } + if (staged_cross_catalog_replace) { + // DuckDB cannot make the DuckLake objects and native metadata atomic + // together. Materialize the expensive replacement under an unpublished + // name first; CREATE OR REPLACE publishes it only after the query succeeds. + for (auto &entry : staged_aux_tables) { + ddl.push_back("create or replace table " + entry.second + " as select * from " + entry.first); + ddl.push_back("drop table " + entry.first); + } + ddl.push_back("create or replace table " + qdt + " as select * from " + staged_qdt); + ddl.push_back("drop table " + staged_qdt); } if (!view_catalog_prefix.empty()) { // Keep the same connection after a DuckLake CTAS. Reopening here can force @@ -1299,10 +1341,12 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC } string view_tail = having_where + top_k_view_suffix; if (internal_cols.empty()) { - ddl.push_back("create view " + qvn + " as select * from " + qdt + view_tail); + ddl.push_back(string(staged_cross_catalog_replace ? "create or replace view " : "create view ") + qvn + + " as select * from " + qdt + view_tail); } else { - ddl.push_back("create view " + qvn + " as select * exclude (" + SqlUtils::JoinQuotedColumns(internal_cols) + - ") from " + qdt + view_tail); + ddl.push_back(string(staged_cross_catalog_replace ? "create or replace view " : "create view ") + qvn + + " as select * exclude (" + SqlUtils::JoinQuotedColumns(internal_cols) + ") from " + qdt + + view_tail); } } @@ -1356,7 +1400,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // Delta table for the MV — based on the DATA table (has all columns) add_profile_marker("create_mv_mv_delta_table"); string qdv = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(SqlUtils::DeltaName(view_name)); - ddl.push_back(string(OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX) + qdv + "\t" + qdt); + ddl.push_back(BuildCreateDeltaFromDataOperation(qdv, qdt, staged_cross_catalog_replace)); add_cleanup("DROP TABLE IF EXISTS " + qdv); // --- Index DDL (for aggregate group queries) --- @@ -1382,6 +1426,12 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // Record source-table metadata only after physical MV objects exist. If a later // DuckLake publish fails, the DDL executor removes these rows before the retry. add_profile_marker("create_mv_source_metadata", "rows=" + to_string(source_metadata_ddl.size())); + if (staged_cross_catalog_replace) { + ddl.push_back("DELETE FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name = '" + + SqlUtils::EscapeSingleQuotes(view_name) + "'"); + ddl.push_back("DELETE FROM " + string(openivm::HISTORY_TABLE) + " WHERE view_name = '" + + SqlUtils::EscapeSingleQuotes(view_name) + "'"); + } ddl.insert(ddl.end(), source_metadata_ddl.begin(), source_metadata_ddl.end()); add_cleanup("DELETE FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name = '" + SqlUtils::EscapeSingleQuotes(view_name) + "'"); @@ -1467,7 +1517,113 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC } // Return DDL executor table function - ConfigureDDLExecutorResult(result); + bool caller_transactional_ddl = + !target_is_ducklake && (view_catalog_prefix.empty() || view_target_catalog == default_db); + ConfigureDDLExecutorResult(result, caller_transactional_ddl ? DDLExecutionMode::CALLER_TRANSACTION + : DDLExecutionMode::STAGED_CROSS_CATALOG); return result; } + +string MaterializedViewLifecycleQuery(ClientContext &context, const FunctionParameters ¶meters) { + auto query = StringValue::Get(parameters.values[0]); + auto parse_result = MaterializedViewParserExtension::ParseFunction(nullptr, query); + if (parse_result.type != ParserExtensionResultType::PARSE_SUCCESSFUL) { + throw ParserException("OpenIVM could not parse the materialized-view lifecycle statement"); + } + auto view_name = dynamic_cast(*parse_result.parse_data).target_name; + auto lock_view_name = SqlUtils::LastIdentifierPart(view_name); + auto plan_result = + MaterializedViewParserExtension::PlanFunction(nullptr, context, std::move(parse_result.parse_data)); + if (plan_result.function.name == OPENIVM_TRANSACTIONAL_DDL_FUNCTION) { + if (!lock_view_name.empty()) { + TransactionalMVLockState::Get(context).Acquire({lock_view_name}, {}); + } + return RenderTransactionalDDL(plan_result.parameters); + } + if (!lock_view_name.empty()) { + ViewLockGuard view_guard(lock_view_name); + ExecuteStagedDDL(context, plan_result.parameters); + } else { + ExecuteStagedDDL(context, plan_result.parameters); + } + return "SELECT true AS \"MATERIALIZED VIEW CREATION\""; +} + +string MaterializedViewDropQuery(ClientContext &context, const FunctionParameters ¶meters) { + auto query = StringValue::Get(parameters.values[0]); + ParserOptions options = context.GetParserOptions(); + options.extensions = nullptr; + Parser parser(options); + parser.ParseQuery(query); + if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::DROP_STATEMENT) { + throw InternalException("OpenIVM DROP rewrite expected one DROP VIEW statement"); + } + auto &drop = parser.statements[0]->Cast(); + if (drop.info->type != CatalogType::VIEW_ENTRY) { + throw InternalException("OpenIVM DROP rewrite expected DROP VIEW"); + } + + string catalog_name = drop.info->catalog; + string schema_name = drop.info->schema; + QueryErrorContext error_context; + auto view_entry = Catalog::GetEntry(context, catalog_name, schema_name, + EntryLookupInfo(CatalogType::VIEW_ENTRY, drop.info->name, error_context), + OnEntryNotFound::RETURN_NULL); + if (view_entry) { + catalog_name = view_entry->ParentCatalog().GetName(); + schema_name = view_entry->ParentSchema().name; + } + if (catalog_name == INVALID_CATALOG || schema_name == INVALID_SCHEMA) { + auto &default_entry = ClientData::Get(context).catalog_search_path->GetDefault(); + if (catalog_name == INVALID_CATALOG) { + catalog_name = default_entry.catalog; + } + if (schema_name == INVALID_SCHEMA) { + schema_name = default_entry.schema.empty() ? DEFAULT_SCHEMA : default_entry.schema; + } + } + if (catalog_name.empty()) { + catalog_name = Catalog::GetSystemCatalog(context).GetName(); + } + if (schema_name.empty()) { + schema_name = DEFAULT_SCHEMA; + } + drop.info->catalog = catalog_name; + drop.info->schema = schema_name; + + string program = BuildDropViewStatement(*drop.info) + ";\n"; + Connection con(*context.db); + auto tracked = con.Query("SELECT 1 FROM " + string(openivm::VIEWS_TABLE) + " WHERE view_name = '" + + SqlUtils::EscapeValue(drop.info->name) + "'"); + if (tracked->HasError() || tracked->RowCount() == 0) { + return program; + } + + RefreshMetadata metadata(con); + auto delta_tables = metadata.GetDeltaTables(drop.info->name); + auto prefix = SqlUtils::QualifiedPrefix(catalog_name, schema_name); + program += "DROP TABLE IF EXISTS " + prefix + + KeywordHelper::WriteOptionallyQuoted(IncrementalTableNames::DataTableName(drop.info->name)) + ";\n"; + program += "DROP TABLE IF EXISTS " + prefix + + KeywordHelper::WriteOptionallyQuoted(SqlUtils::DeltaName(drop.info->name)) + ";\n"; + program += "DELETE FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name = '" + + SqlUtils::EscapeValue(drop.info->name) + "';\n"; + program += "DELETE FROM " + string(openivm::VIEWS_TABLE) + " WHERE view_name = '" + + SqlUtils::EscapeValue(drop.info->name) + "';\n"; + + for (auto &delta_table : delta_tables) { + if (metadata.IsDuckLakeTable(drop.info->name, delta_table)) { + continue; + } + auto remaining = con.Query("SELECT count(*) FROM " + string(openivm::DELTA_TABLES_TABLE) + + " WHERE table_name = '" + SqlUtils::EscapeValue(delta_table) + "'"); + if (!remaining->HasError() && remaining->RowCount() > 0 && remaining->GetValue(0, 0).GetValue() == 1) { + program += "DROP TABLE IF EXISTS " + prefix + KeywordHelper::WriteOptionallyQuoted(delta_table) + ";\n"; + } + } + auto view_delta_table = SqlUtils::DeltaName(drop.info->name); + delta_tables.push_back(view_delta_table); + TransactionalMVLockState::Get(context).Acquire({drop.info->name}, delta_tables); + return program; +} } // namespace duckdb diff --git a/src/core/parser_ddl.cpp b/src/core/parser_ddl.cpp index 88889573..499dd75a 100644 --- a/src/core/parser_ddl.cpp +++ b/src/core/parser_ddl.cpp @@ -3,8 +3,13 @@ #include "core/openivm_constants.hpp" #include "core/openivm_debug.hpp" #include "core/sql_utils.hpp" +#include "duckdb/catalog/catalog.hpp" #include "duckdb/common/printer.hpp" #include "duckdb/function/table_function.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" +#include "duckdb/parser/parser.hpp" +#include "duckdb/parser/statement/drop_statement.hpp" +#include "duckdb/planner/binder.hpp" #include #include @@ -160,16 +165,26 @@ struct DeltaSchemaDDL { idx_t column_count = 0; }; -void ParseCreateDeltaFromDataPayload(const string &payload, string &delta_table, string &data_table) { +void ParseCreateDeltaFromDataPayload(const string &payload, bool &replace, string &delta_table, string &data_table) { auto first = payload.find('\t'); if (first == string::npos) { return; } - delta_table = payload.substr(0, first); - data_table = payload.substr(first + 1); + auto second = payload.find('\t', first + 1); + if (second == string::npos) { + return; + } + auto mode = payload.substr(0, first); + if (mode != "create" && mode != "replace") { + return; + } + replace = mode == "replace"; + delta_table = payload.substr(first + 1, second - first - 1); + data_table = payload.substr(second + 1); } -DeltaSchemaDDL BuildCreateDeltaFromDataSQL(Connection &conn, const string &delta_table, const string &data_table) { +DeltaSchemaDDL BuildCreateDeltaFromDataSQL(Connection &conn, const string &delta_table, const string &data_table, + bool replace) { auto described = conn.Query("DESCRIBE SELECT * FROM " + data_table); if (described->HasError()) { throw CatalogException("Could not derive IVM delta schema from data table '" + data_table + @@ -209,7 +224,8 @@ DeltaSchemaDDL BuildCreateDeltaFromDataSQL(Connection &conn, const string &delta columns.push_back(string(openivm::MULTIPLICITY_COL) + " INTEGER DEFAULT 1"); columns.push_back(string(openivm::TIMESTAMP_COL) + " TIMESTAMP DEFAULT now()"); DeltaSchemaDDL result; - result.sql = "create table if not exists " + delta_table + " (" + StringUtil::Join(columns, ", ") + ")"; + result.sql = string(replace ? "create or replace table " : "create table if not exists ") + delta_table + " (" + + StringUtil::Join(columns, ", ") + ")"; result.column_count = described->RowCount(); return result; } @@ -317,17 +333,18 @@ void ExecuteDDL(ClientContext &context, const vector &ddl) { } if (StringUtil::StartsWith(q, OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX)) { flush_pending(); + bool replace = false; string delta_table; string data_table; - ParseCreateDeltaFromDataPayload(q.substr(strlen(OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX)), delta_table, - data_table); + ParseCreateDeltaFromDataPayload(q.substr(strlen(OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX)), replace, + delta_table, data_table); if (delta_table.empty() || data_table.empty()) { fail_ddl("malformed delta-schema payload"); } auto ddl_start = std::chrono::steady_clock::now(); DeltaSchemaDDL derived; try { - derived = BuildCreateDeltaFromDataSQL(*conn, delta_table, data_table); + derived = BuildCreateDeltaFromDataSQL(*conn, delta_table, data_table, replace); } catch (std::exception &ex) { profiler.AddStep(current_profile_step, ddl_start, current_profile_detail + "; delta_schema_derivation_failed=true"); @@ -382,9 +399,141 @@ void DDLExecutorExecuteFunction(ClientContext &context, TableFunctionInput &data } // namespace -void ConfigureDDLExecutorResult(ParserExtensionPlanResult &result) { +string BuildCreateDeltaFromDataOperation(const string &delta_table, const string &data_table, bool replace) { + return string(OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX) + (replace ? "replace\t" : "create\t") + delta_table + + "\t" + data_table; +} + +string BuildDropViewStatement(const DropInfo &drop_info) { + return "SELECT * FROM openivm_execute_drop_view('" + SqlUtils::EscapeValue(drop_info.catalog) + "', '" + + SqlUtils::EscapeValue(drop_info.schema) + "', '" + SqlUtils::EscapeValue(drop_info.name) + "', " + + (drop_info.cascade ? "true" : "false") + ", " + + (drop_info.if_not_found == OnEntryNotFound::RETURN_NULL ? "true" : "false") + ")"; +} + +struct DropViewBindData : public TableFunctionData { + DropInfo info; +}; + +struct DropViewGlobalState : public GlobalTableFunctionState { + bool finished = false; +}; + +unique_ptr BindDropView(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + if (input.inputs.size() != 5) { + throw InternalException("OpenIVM DROP VIEW executor expected five arguments"); + } + auto result = make_uniq(); + result->info.type = CatalogType::VIEW_ENTRY; + result->info.catalog = StringValue::Get(input.inputs[0]); + result->info.schema = StringValue::Get(input.inputs[1]); + result->info.name = StringValue::Get(input.inputs[2]); + result->info.cascade = BooleanValue::Get(input.inputs[3]); + result->info.if_not_found = + BooleanValue::Get(input.inputs[4]) ? OnEntryNotFound::RETURN_NULL : OnEntryNotFound::THROW_EXCEPTION; + if (!input.binder) { + throw InternalException("OpenIVM DROP VIEW executor requires a binder"); + } + auto &catalog = Catalog::GetCatalog(context, result->info.catalog); + input.binder->GetStatementProperties().RegisterDBModify(catalog, context, + DatabaseModificationType::DROP_CATALOG_ENTRY); + return_types.push_back(LogicalType::BOOLEAN); + names.emplace_back("Success"); + return std::move(result); +} + +unique_ptr InitDropView(ClientContext &context, TableFunctionInitInput &input) { + return make_uniq(); +} + +void ExecuteDropView(ClientContext &context, TableFunctionInput &input, DataChunk &output) { + auto &state = input.global_state->Cast(); + if (state.finished) { + return; + } + auto &bind_data = input.bind_data->Cast(); + auto &catalog = Catalog::GetCatalog(context, bind_data.info.catalog); + DropInfo info(bind_data.info); + catalog.DropEntry(context, info); + output.SetValue(0, 0, Value::BOOLEAN(true)); + output.SetCardinality(1); + state.finished = true; +} + +string RenderTransactionalDDL(const vector ¶meters) { + string sql; + auto append_statement = [&](const string &statement) { + if (statement.empty()) { + return; + } + sql += statement; + if (statement.back() != ';') { + sql += ";"; + } + sql += "\n"; + }; + for (auto ¶meter : parameters) { + auto statement = parameter.GetValue(); + if (statement.empty() || StringUtil::StartsWith(statement, OPENIVM_DDL_CLEANUP_PREFIX) || + StringUtil::StartsWith(statement, OPENIVM_DDL_PROFILE_PREFIX) || + StringUtil::StartsWith(statement, OPENIVM_DDL_PROFILE_RECORD_PREFIX)) { + continue; + } + if (StringUtil::StartsWith(statement, OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX)) { + bool replace = false; + string delta_table; + string data_table; + ParseCreateDeltaFromDataPayload(statement.substr(strlen(OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX)), + replace, delta_table, data_table); + if (delta_table.empty() || data_table.empty()) { + throw InternalException("Malformed OpenIVM delta-schema operation"); + } + append_statement(string(replace ? "CREATE OR REPLACE TABLE " : "CREATE TABLE ") + delta_table + + " AS SELECT *, 1::INTEGER AS " + string(openivm::MULTIPLICITY_COL) + + ", now()::TIMESTAMP AS " + string(openivm::TIMESTAMP_COL) + " FROM " + data_table + + " LIMIT 0"); + append_statement("ALTER TABLE " + delta_table + " ALTER " + string(openivm::MULTIPLICITY_COL) + + " SET DEFAULT 1"); + append_statement("ALTER TABLE " + delta_table + " ALTER " + string(openivm::TIMESTAMP_COL) + + " SET DEFAULT now()"); + continue; + } + + try { + Parser parser; + parser.ParseQuery(statement); + if (parser.statements.size() == 1 && parser.statements[0]->type == StatementType::DROP_STATEMENT) { + auto &drop = parser.statements[0]->Cast(); + if (drop.info->type == CatalogType::VIEW_ENTRY) { + append_statement(BuildDropViewStatement(*drop.info)); + continue; + } + } + } catch (std::exception &) { + // The normal DuckDB parser will produce the authoritative error when the + // transactional program is executed. + } + append_statement(statement); + } + append_statement("SELECT true AS \"MATERIALIZED VIEW CREATION\""); + return sql; +} + +void ExecuteStagedDDL(ClientContext &context, const vector ¶meters) { + vector ddl; + ddl.reserve(parameters.size()); + for (auto ¶meter : parameters) { + ddl.push_back(parameter.GetValue()); + } + ExecuteDDL(context, ddl); +} + +void ConfigureDDLExecutorResult(ParserExtensionPlanResult &result, DDLExecutionMode mode) { result.function = TableFunction("openivm_ddl_executor", {}, DDLExecutorExecuteFunction, DDLExecutorBindFunction, DDLExecutorInitFunction); + result.function.name = + mode == DDLExecutionMode::CALLER_TRANSACTION ? OPENIVM_TRANSACTIONAL_DDL_FUNCTION : OPENIVM_STAGED_DDL_FUNCTION; result.requires_valid_transaction = true; result.return_type = StatementReturnType::QUERY_RESULT; } diff --git a/src/core/parser_parse.cpp b/src/core/parser_parse.cpp index ebb84865..04c4a915 100644 --- a/src/core/parser_parse.cpp +++ b/src/core/parser_parse.cpp @@ -3,12 +3,54 @@ #include "core/openivm_constants.hpp" #include "core/openivm_debug.hpp" #include "core/sql_utils.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/parser/parser.hpp" +#include "duckdb/parser/statement/drop_statement.hpp" +#include "duckdb/parser/statement/pragma_statement.hpp" #include namespace duckdb { +static unique_ptr BuildInternalPragma(const string &name, const string &query) { + auto statement = make_uniq(); + statement->info->name = name; + statement->info->parameters.push_back(make_uniq(Value(query))); + return std::move(statement); +} + +ParserOverrideResult MaterializedViewParserExtension::OverrideFunction(ParserExtensionInfo *info, const string &query, + ParserOptions &options) { + try { + auto extension_result = ParseFunction(info, query); + if (extension_result.type == ParserExtensionResultType::PARSE_SUCCESSFUL) { + vector> statements; + statements.push_back(BuildInternalPragma("openivm_materialized_view_lifecycle", query)); + return ParserOverrideResult(std::move(statements)); + } + + // DuckDB parses DROP VIEW natively, so the regular parser-extension fallback + // never sees it. Route only that structured statement through OpenIVM so its + // backing objects and metadata are dropped in the same caller transaction. + ParserOptions native_options = options; + native_options.extensions = nullptr; + Parser parser(native_options); + parser.ParseQuery(query); + if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::DROP_STATEMENT) { + return ParserOverrideResult(); + } + auto &drop = parser.statements[0]->Cast(); + if (drop.info->type != CatalogType::VIEW_ENTRY) { + return ParserOverrideResult(); + } + vector> statements; + statements.push_back(BuildInternalPragma("openivm_materialized_view_drop", query)); + return ParserOverrideResult(std::move(statements)); + } catch (std::exception &ex) { + return ParserOverrideResult(ex); + } +} + ParserExtensionParseResult MaterializedViewParserExtension::ParseFunction(ParserExtensionInfo *info, const string &query) { auto query_lower = SqlUtils::SQLToLowercase(StringUtil::Replace(query, ";", "")); @@ -49,7 +91,9 @@ ParserExtensionParseResult MaterializedViewParserExtension::ParseFunction(Parser alter_parser.ParseQuery("SELECT 1"); auto parse_data = make_uniq_base(std::move(alter_parser.statements[0])); - dynamic_cast(*parse_data).alter_sql = update_sql; + auto &materialized_view_data = dynamic_cast(*parse_data); + materialized_view_data.alter_sql = update_sql; + materialized_view_data.target_name = alter_view_name; return ParserExtensionParseResult(std::move(parse_data)); } @@ -84,7 +128,9 @@ ParserExtensionParseResult MaterializedViewParserExtension::ParseFunction(Parser auto parse_data = make_uniq_base(std::move(p.statements[0]), refresh_interval); - dynamic_cast(*parse_data).is_replace = is_replace; + auto &materialized_view_data = dynamic_cast(*parse_data); + materialized_view_data.is_replace = is_replace; + materialized_view_data.target_name = SqlUtils::ExtractTableName(query_lower); return ParserExtensionParseResult(std::move(parse_data)); } diff --git a/src/core/parser_plan_helpers.cpp b/src/core/parser_plan_helpers.cpp index c7fd3888..19711842 100644 --- a/src/core/parser_plan_helpers.cpp +++ b/src/core/parser_plan_helpers.cpp @@ -530,14 +530,26 @@ static string ProjectionOutputName(const unique_ptr &expr, idx_t exp return bcr.GetName(); } -static BoundColumnRefExpression *GetColumnRefThroughCasts(Expression *expr) { +static BoundColumnRefExpression *GetColumnRefThroughCasts(Expression *expr, string *cast_type = nullptr) { + if (cast_type) { + cast_type->clear(); + } + vector cast_specs; while (expr && expr->expression_class == ExpressionClass::BOUND_CAST) { auto &cast = expr->Cast(); + if (cast_type) { + cast_specs.push_back(SqlUtils::BuildCastSpec(expr->return_type.ToString(), cast.try_cast)); + } expr = cast.child.get(); } if (!expr || expr->type != ExpressionType::BOUND_COLUMN_REF) { return nullptr; } + if (cast_type) { + for (auto it = cast_specs.rbegin(); it != cast_specs.rend(); it++) { + *cast_type = SqlUtils::ComposeCastSpecs(*it, *cast_type); + } + } return &expr->Cast(); } @@ -742,6 +754,25 @@ static bool GetLogicalGetColumnName(LogicalGet &get, idx_t column_index, string return false; } +static bool GetLogicalGetColumnType(LogicalGet &get, idx_t column_index, LogicalType &type) { + if (column_index < get.returned_types.size()) { + type = get.returned_types[column_index]; + return true; + } + if (get.GetTable().get()) { + auto &ids = get.GetColumnIds(); + if (column_index < ids.size()) { + auto base_idx = ids[column_index].GetPrimaryIndex(); + auto &cols = get.GetTable().get()->GetColumns(); + if (base_idx < cols.LogicalColumnCount()) { + type = cols.GetColumn(LogicalIndex(base_idx)).Type(); + return true; + } + } + } + return false; +} + static bool ResolveBindingToGetColumn(ColumnBinding binding, const CreateMVPlanFacts &facts, LogicalGet *&get, string &column) { idx_t table_index = binding.table_index; @@ -794,6 +825,53 @@ static bool ResolveBindingToOccurrenceRef(ColumnBinding binding, const CreateMVP return true; } +static bool ResolveBindingToOccurrenceRefWithCast(ColumnBinding binding, const CreateMVPlanFacts &facts, + OccurrenceColumnRef &out, string &cast_type, + const LogicalType &expression_type) { + idx_t table_index = binding.table_index; + idx_t column_index = binding.column_index; + for (int depth = 0; depth < 16; depth++) { + auto get_it = facts.gets_by_index.find(table_index); + if (get_it != facts.gets_by_index.end()) { + auto *get = get_it->second; + if (!GetLogicalGetColumnName(*get, column_index, out.column)) { + return false; + } + auto occurrence_it = facts.occurrence_by_index.find(get->table_index); + if (occurrence_it == facts.occurrence_by_index.end()) { + return false; + } + out.table = occurrence_it->second.table; + out.occurrence = occurrence_it->second.occurrence; + LogicalType base_type; + if (cast_type.empty() && GetLogicalGetColumnType(*get, column_index, base_type) && + !(base_type == expression_type)) { + cast_type = expression_type.ToString(); + } + return true; + } + auto proj_it = facts.projections_by_index.find(table_index); + if (proj_it == facts.projections_by_index.end()) { + return false; + } + auto *proj = proj_it->second; + if (column_index >= proj->expressions.size()) { + return false; + } + string projection_cast; + auto *next = GetColumnRefThroughCasts(proj->expressions[column_index].get(), &projection_cast); + if (!next) { + return false; + } + if (!projection_cast.empty()) { + cast_type = SqlUtils::ComposeCastSpecs(cast_type, projection_cast); + } + table_index = next->binding.table_index; + column_index = next->binding.column_index; + } + return false; +} + static void AddJoinEdgesFromFacts(CreateMVPlanFacts &facts) { for (auto *join : facts.comparison_joins) { if (join->join_type != JoinType::INNER) { @@ -1063,15 +1141,31 @@ struct WindowLineageOp { string source_table; idx_t source_occurrence = 0; string source_col; + string source_cast; string lookup_table; idx_t lookup_occurrence = 0; string lookup_col; + string lookup_cast; string lookup_output_col; + string lookup_output_cast; }; struct WindowLookupEdge { OccurrenceColumnRef lookup_join; + string lookup_cast; OccurrenceColumnRef changed_join; + string changed_cast; +}; + +struct WindowEquivalenceEdge { + OccurrenceColumnRef first; + string first_cast; + OccurrenceColumnRef second; + string second_cast; + + bool HasCast() const { + return !first_cast.empty() || !second_cast.empty(); + } }; static void CollectLeafColumnBindings(Expression *expr, vector &bindings) { @@ -1133,7 +1227,7 @@ static bool ResolveBindingToOccurrenceRefs(ColumnBinding binding, const CreateMV } static void CollectInnerJoinEdgesOccurrence(LogicalOperator *op, const CreateMVPlanFacts &facts, - vector> &edges) { + vector &edges) { if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) { auto &join = op->Cast(); @@ -1142,15 +1236,18 @@ static void CollectInnerJoinEdgesOccurrence(LogicalOperator *op, const CreateMVP if (cond.comparison != ExpressionType::COMPARE_EQUAL) { continue; } - auto *left = GetColumnRefThroughCasts(cond.left.get()); - auto *right = GetColumnRefThroughCasts(cond.right.get()); + string left_cast; + string right_cast; + auto *left = GetColumnRefThroughCasts(cond.left.get(), &left_cast); + auto *right = GetColumnRefThroughCasts(cond.right.get(), &right_cast); if (!left || !right) { continue; } OccurrenceColumnRef lref, rref; - if (ResolveBindingToOccurrenceRef(left->binding, facts, lref) && - ResolveBindingToOccurrenceRef(right->binding, facts, rref)) { - edges.emplace_back(std::move(lref), std::move(rref)); + if (ResolveBindingToOccurrenceRefWithCast(left->binding, facts, lref, left_cast, left->return_type) && + ResolveBindingToOccurrenceRefWithCast(right->binding, facts, rref, right_cast, + right->return_type)) { + edges.push_back({std::move(lref), std::move(left_cast), std::move(rref), std::move(right_cast)}); } } } @@ -1165,35 +1262,39 @@ static void CollectWindowLookupEdges(LogicalOperator *op, const CreateMVPlanFact if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) { auto &join = op->Cast(); - auto add_lookup_edge = [&](OccurrenceColumnRef lookup_ref, OccurrenceColumnRef changed_ref) { - edges.push_back({std::move(lookup_ref), std::move(changed_ref)}); + auto add_lookup_edge = [&](OccurrenceColumnRef lookup_ref, string lookup_cast, OccurrenceColumnRef changed_ref, + string changed_cast) { + edges.push_back( + {std::move(lookup_ref), std::move(lookup_cast), std::move(changed_ref), std::move(changed_cast)}); }; for (auto &cond : join.conditions) { if (cond.comparison != ExpressionType::COMPARE_EQUAL) { continue; } - auto *left = GetColumnRefThroughCasts(cond.left.get()); - auto *right = GetColumnRefThroughCasts(cond.right.get()); + string left_cast; + string right_cast; + auto *left = GetColumnRefThroughCasts(cond.left.get(), &left_cast); + auto *right = GetColumnRefThroughCasts(cond.right.get(), &right_cast); if (!left || !right) { continue; } OccurrenceColumnRef lref, rref; - if (!ResolveBindingToOccurrenceRef(left->binding, facts, lref) || - !ResolveBindingToOccurrenceRef(right->binding, facts, rref)) { + if (!ResolveBindingToOccurrenceRefWithCast(left->binding, facts, lref, left_cast, left->return_type) || + !ResolveBindingToOccurrenceRefWithCast(right->binding, facts, rref, right_cast, right->return_type)) { continue; } switch (join.join_type) { case JoinType::INNER: - add_lookup_edge(lref, rref); - add_lookup_edge(rref, lref); + add_lookup_edge(lref, left_cast, rref, right_cast); + add_lookup_edge(rref, right_cast, lref, left_cast); break; case JoinType::LEFT: - add_lookup_edge(lref, rref); - add_lookup_edge(rref, lref); + add_lookup_edge(lref, left_cast, rref, right_cast); + add_lookup_edge(rref, right_cast, lref, left_cast); break; case JoinType::RIGHT: - add_lookup_edge(rref, lref); - add_lookup_edge(lref, rref); + add_lookup_edge(rref, right_cast, lref, left_cast); + add_lookup_edge(lref, left_cast, rref, right_cast); break; default: break; @@ -1215,7 +1316,8 @@ static void CollectWindowPartitionRefs(LogicalOperator *op, const CreateMVPlanFa } auto &win_expr = expr->Cast(); for (auto &part : win_expr.partitions) { - auto *bcr = GetColumnRefThroughCasts(part.get()); + string partition_cast; + auto *bcr = GetColumnRefThroughCasts(part.get(), &partition_cast); if (!bcr) { continue; } @@ -1237,6 +1339,7 @@ static void CollectWindowPartitionRefs(LogicalOperator *op, const CreateMVPlanFa op.source_table = ref.table; op.source_occurrence = ref.occurrence; op.source_col = ref.column; + op.source_cast = partition_cast; direct_ops.push_back(std::move(op)); } } @@ -1256,9 +1359,11 @@ static bool SameRef(const OccurrenceColumnRef &a, const OccurrenceColumnRef &b) static bool SameOp(const WindowLineageOp &a, const WindowLineageOp &b) { return a.kind == b.kind && StringUtil::CIEquals(a.output_col, b.output_col) && StringUtil::CIEquals(a.source_table, b.source_table) && a.source_occurrence == b.source_occurrence && - StringUtil::CIEquals(a.source_col, b.source_col) && StringUtil::CIEquals(a.lookup_table, b.lookup_table) && - a.lookup_occurrence == b.lookup_occurrence && StringUtil::CIEquals(a.lookup_col, b.lookup_col) && - StringUtil::CIEquals(a.lookup_output_col, b.lookup_output_col); + StringUtil::CIEquals(a.source_col, b.source_col) && StringUtil::CIEquals(a.source_cast, b.source_cast) && + StringUtil::CIEquals(a.lookup_table, b.lookup_table) && a.lookup_occurrence == b.lookup_occurrence && + StringUtil::CIEquals(a.lookup_col, b.lookup_col) && StringUtil::CIEquals(a.lookup_cast, b.lookup_cast) && + StringUtil::CIEquals(a.lookup_output_col, b.lookup_output_col) && + StringUtil::CIEquals(a.lookup_output_cast, b.lookup_output_cast); } static void AddUniqueLineageOp(vector &ops, WindowLineageOp op) { @@ -1288,9 +1393,12 @@ static RefreshMetadata::WindowPartitionLineageOp ToMetadataWindowLineageOp(const metadata_op.output_col = op.output_col; metadata_op.source = op.source_table; metadata_op.source_col = op.source_col; + metadata_op.source_cast = op.source_cast; metadata_op.lookup = op.lookup_table; metadata_op.lookup_col = op.lookup_col; + metadata_op.lookup_cast = op.lookup_cast; metadata_op.lookup_out = op.lookup_output_col; + metadata_op.lookup_out_cast = op.lookup_output_cast; return metadata_op; } @@ -1320,7 +1428,7 @@ bool BuildWindowPartitionLineageOps(const CreateMVPlanFacts &facts, const vector } } - vector> edges; + vector edges; CollectInnerJoinEdgesOccurrence(plan, facts, edges); vector lookup_edges; CollectWindowLookupEdges(plan, facts, lookup_edges); @@ -1335,6 +1443,9 @@ bool BuildWindowPartitionLineageOps(const CreateMVPlanFacts &facts, const vector changed = false; vector next_ops = partition_ops; for (auto &edge : edges) { + if (edge.HasCast()) { + continue; + } for (auto &direct : partition_ops) { OccurrenceColumnRef direct_ref; direct_ref.table = direct.source_table; @@ -1358,6 +1469,7 @@ bool BuildWindowPartitionLineageOps(const CreateMVPlanFacts &facts, const vector equivalent.source_table = other.table; equivalent.source_occurrence = other.occurrence; equivalent.source_col = other.column; + equivalent.source_cast = direct.source_cast; AddUniqueLineageOp(next_ops, std::move(equivalent)); changed = true; } @@ -1387,10 +1499,13 @@ bool BuildWindowPartitionLineageOps(const CreateMVPlanFacts &facts, const vector lookup.source_table = edge.changed_join.table; lookup.source_occurrence = edge.changed_join.occurrence; lookup.source_col = edge.changed_join.column; + lookup.source_cast = edge.changed_cast; lookup.lookup_table = direct.source_table; lookup.lookup_occurrence = direct.source_occurrence; lookup.lookup_col = edge.lookup_join.column; + lookup.lookup_cast = edge.lookup_cast; lookup.lookup_output_col = direct.source_col; + lookup.lookup_output_cast = direct.source_cast; AddUniqueLineageOp(ops, std::move(lookup)); } } diff --git a/src/core/refresh_locks.cpp b/src/core/refresh_locks.cpp index 57e9fc71..79ee855b 100644 --- a/src/core/refresh_locks.cpp +++ b/src/core/refresh_locks.cpp @@ -104,4 +104,43 @@ void RefreshLocks::ExitDeltaRefresh(Catalog &catalog) { GetDeltaCatalogGate(catalog).ExitRefresh(); } +TransactionalMVLockState &TransactionalMVLockState::Get(ClientContext &context) { + return *context.registered_state->GetOrCreate("openivm_transactional_mv_locks"); +} + +void TransactionalMVLockState::Acquire(const vector &view_names, const vector &delta_table_names) { + auto sorted_views = view_names; + std::sort(sorted_views.begin(), sorted_views.end()); + sorted_views.erase(std::unique(sorted_views.begin(), sorted_views.end()), sorted_views.end()); + for (auto &view_name : sorted_views) { + if (locked_views.insert(view_name).second) { + view_guards.push_back(make_uniq(view_name)); + } + } + + auto sorted_deltas = delta_table_names; + std::sort(sorted_deltas.begin(), sorted_deltas.end()); + sorted_deltas.erase(std::unique(sorted_deltas.begin(), sorted_deltas.end()), sorted_deltas.end()); + for (auto &delta_table : sorted_deltas) { + if (locked_delta_tables.insert(delta_table).second) { + delta_guards.push_back(make_uniq(delta_table)); + } + } +} + +void TransactionalMVLockState::TransactionCommit(MetaTransaction &transaction, ClientContext &context) { + Release(); +} + +void TransactionalMVLockState::TransactionRollback(MetaTransaction &transaction, ClientContext &context) { + Release(); +} + +void TransactionalMVLockState::Release() { + delta_guards.clear(); + view_guards.clear(); + locked_delta_tables.clear(); + locked_views.clear(); +} + } // namespace duckdb diff --git a/src/core/refresh_metadata.cpp b/src/core/refresh_metadata.cpp index 6b8ece33..a965be70 100644 --- a/src/core/refresh_metadata.cpp +++ b/src/core/refresh_metadata.cpp @@ -31,7 +31,11 @@ RefreshType RefreshMetadata::GetViewType(const string &view_name) { if (result->HasError() || result->RowCount() == 0) { throw ParserException("Materialized view '%s' does not exist in IVM metadata.", view_name); } - return static_cast(result->GetValue(0, 0).GetValue()); + auto raw_type = result->GetValue(0, 0).GetValue(); + if (raw_type == static_cast(openivm::LEGACY_CURRENT_DIFF_RECOMPUTE_TYPE)) { + return RefreshType::FULL_REFRESH; + } + return static_cast(raw_type); } bool RefreshMetadata::HasMinMax(const string &view_name) { @@ -946,12 +950,15 @@ bool RefreshMetadata::GetWindowPartitionLineage(const string &view_name, vector< !ExtractJsonString(object, "source_col", op.source_col)) { continue; } + ExtractJsonString(object, "source_cast", op.source_cast); if (op.kind == "lookup") { if (!ExtractJsonString(object, "lookup", op.lookup) || !ExtractJsonString(object, "lookup_col", op.lookup_col) || !ExtractJsonString(object, "lookup_out", op.lookup_out)) { continue; } + ExtractJsonString(object, "lookup_cast", op.lookup_cast); + ExtractJsonString(object, "lookup_out_cast", op.lookup_out_cast); } else if (op.kind != "direct") { continue; } @@ -970,10 +977,19 @@ string RefreshMetadata::WindowPartitionLineageToJson(const vector(c)) || c == '_'; } @@ -43,6 +46,40 @@ static string TrimSQLFragment(const string &input) { return input.substr(start, end - start); } +string SqlUtils::BuildCastSpec(const string &target_type, bool try_cast) { + if (!try_cast) { + return target_type; + } + return string(CAST_EXPRESSION_PREFIX) + "TRY_CAST(" + CAST_COLUMN_PLACEHOLDER + " AS " + target_type + ")"; +} + +string SqlUtils::ApplyCastSpec(const string &column_expression, const string &cast_spec) { + if (cast_spec.empty()) { + return column_expression; + } + if (!StringUtil::StartsWith(cast_spec, CAST_EXPRESSION_PREFIX)) { + return "CAST(" + column_expression + " AS " + cast_spec + ")"; + } + string expression = cast_spec.substr(strlen(CAST_EXPRESSION_PREFIX)); + auto placeholder = expression.find(CAST_COLUMN_PLACEHOLDER); + if (placeholder == string::npos) { + throw InternalException("OpenIVM cast expression is missing its column placeholder"); + } + expression.replace(placeholder, strlen(CAST_COLUMN_PLACEHOLDER), column_expression); + return expression; +} + +string SqlUtils::ComposeCastSpecs(const string &outer_cast_spec, const string &inner_cast_spec) { + if (outer_cast_spec.empty()) { + return inner_cast_spec; + } + if (inner_cast_spec.empty()) { + return outer_cast_spec; + } + string inner_expression = ApplyCastSpec(CAST_COLUMN_PLACEHOLDER, inner_cast_spec); + return string(CAST_EXPRESSION_PREFIX) + ApplyCastSpec(inner_expression, outer_cast_spec); +} + static bool ReadCreateTargetName(const string &sql, const string &object_keyword, string &out) { string lower = StringUtil::Lower(sql); size_t pos = lower.find("create"); diff --git a/src/delta/operators/nonlocal.cpp b/src/delta/operators/nonlocal.cpp index 1c823e2c..7b2aa0e7 100644 --- a/src/delta/operators/nonlocal.cpp +++ b/src/delta/operators/nonlocal.cpp @@ -17,17 +17,16 @@ static DeltaPlanFragment CompileNonLocalDeltaGuard(const DeltaOperatorInput &inp DeltaPlanFragment CompileAsofJoinDelta(const DeltaOperatorInput &input) { return CompileNonLocalDeltaGuard(input, DeltaOperatorStrategy::ASOF_AFFECTED_RECOMPUTE, "ASOF_JOIN", - "WINDOW_PARTITION, GROUP_RECOMPUTE, or CURRENT_DIFF_RECOMPUTE"); + "WINDOW_PARTITION, GROUP_RECOMPUTE, or FULL_REFRESH"); } DeltaPlanFragment CompilePositionalJoinDelta(const DeltaOperatorInput &input) { return CompileNonLocalDeltaGuard(input, DeltaOperatorStrategy::POSITIONAL_GLOBAL_RECOMPUTE, "POSITIONAL_JOIN", - "CURRENT_DIFF_RECOMPUTE"); + "FULL_REFRESH"); } DeltaPlanFragment CompileSampleDelta(const DeltaOperatorInput &input) { - return CompileNonLocalDeltaGuard(input, DeltaOperatorStrategy::SAMPLE_GLOBAL_RECOMPUTE, "SAMPLE", - "CURRENT_DIFF_RECOMPUTE"); + return CompileNonLocalDeltaGuard(input, DeltaOperatorStrategy::SAMPLE_GLOBAL_RECOMPUTE, "SAMPLE", "FULL_REFRESH"); } } // namespace duckdb diff --git a/src/include/core/ivm_view_classifier.hpp b/src/include/core/ivm_view_classifier.hpp index a4b7fac4..331cfb2b 100644 --- a/src/include/core/ivm_view_classifier.hpp +++ b/src/include/core/ivm_view_classifier.hpp @@ -17,10 +17,7 @@ enum class DeltaStrategyReason { NESTED_AGGREGATE_GROUP_FALLBACK, REPEATED_CTE_AGGREGATE_GROUP_FALLBACK, SEMI_ANTI_AGGREGATE_GROUP_FALLBACK, - OUTER_JOIN_AGGREGATE_RECOMPUTE, - ASOF_CURRENT_DIFF_RECOMPUTE, - SAMPLE_CURRENT_DIFF_RECOMPUTE, - POSITIONAL_CURRENT_DIFF_RECOMPUTE + OUTER_JOIN_AGGREGATE_RECOMPUTE }; enum class DeltaModelFeature { @@ -39,7 +36,6 @@ enum class DeltaModelFeature { ASOF_STATEFUL, SAMPLE_GLOBAL_RECOMPUTE, POSITIONAL_GLOBAL_RECOMPUTE, - CURRENT_DIFF_RECOMPUTE, FULL_ONLY }; diff --git a/src/include/core/openivm_constants.hpp b/src/include/core/openivm_constants.hpp index 05bb3e78..8fdef1a1 100644 --- a/src/include/core/openivm_constants.hpp +++ b/src/include/core/openivm_constants.hpp @@ -67,6 +67,8 @@ constexpr const char *DISABLED_OPTIMIZERS = TEMPLATE_DATA_DEPENDENT_OPTIMIZERS; // DELIM joins to eliminate. Pure robustness guard — not flag-gated. constexpr const char *REFRESH_DISABLED_OPTIMIZERS = "deliminator"; +constexpr uint8_t LEGACY_CURRENT_DIFF_RECOMPUTE_TYPE = 10; + } // namespace openivm enum class RefreshType : uint8_t { @@ -78,11 +80,10 @@ enum class RefreshType : uint8_t { WINDOW_PARTITION, // window functions — partition-level recompute GROUP_RECOMPUTE, // inner-DISTINCT-under-AGG fallback: DELETE+INSERT only the GROUP BY keys touched by source deltas TOP_K, // Legacy enum value; current top-k support strips ORDER BY/LIMIT into the user-facing view - DISTINCT_INCREMENTAL, // inner-DISTINCT-under-AGG with aux state (openivm_distinct_aux_state=true): DBSP-correct - // distinct(R)=sgn(R[t]); per-tuple count table emits ±1 only on count transitions - SEMI_ANTI_RECOMPUTE, // SEMI/ANTI join aux state: per-left-tuple match counts, transition-scoped MV updates - CURRENT_DIFF_RECOMPUTE, // exact recompute inside incremental refresh; emits MV deltas from old/current diff - COUNT_DISTINCT_INCREMENTAL // COUNT(DISTINCT x) with per-(group,x) multiplicity aux state + DISTINCT_INCREMENTAL, // inner-DISTINCT-under-AGG with aux state (openivm_distinct_aux_state=true): DBSP-correct + // distinct(R)=sgn(R[t]); per-tuple count table emits ±1 only on count transitions + SEMI_ANTI_RECOMPUTE, // SEMI/ANTI join aux state: per-left-tuple match counts, transition-scoped MV updates + COUNT_DISTINCT_INCREMENTAL = 11 // COUNT(DISTINCT x) with per-(group,x) multiplicity aux state }; enum class GroupRecomputeAffectedMode : uint8_t { SOURCE_DELTA, SOURCE_DELTA_RELAX_AGGREGATE_FILTER, CURRENT_DIFF }; @@ -105,8 +106,6 @@ inline const char *RefreshTypeName(RefreshType type) { return "DISTINCT_INCREMENTAL"; case RefreshType::SEMI_ANTI_RECOMPUTE: return "SEMI_ANTI_RECOMPUTE"; - case RefreshType::CURRENT_DIFF_RECOMPUTE: - return "CURRENT_DIFF_RECOMPUTE"; case RefreshType::COUNT_DISTINCT_INCREMENTAL: return "COUNT_DISTINCT_INCREMENTAL"; case RefreshType::TOP_K: diff --git a/src/include/core/parser.hpp b/src/include/core/parser.hpp index 06e93537..bebef42a 100644 --- a/src/include/core/parser.hpp +++ b/src/include/core/parser.hpp @@ -13,18 +13,25 @@ class MaterializedViewParserExtension : public ParserExtension { explicit MaterializedViewParserExtension() { parse_function = ParseFunction; plan_function = PlanFunction; + parser_override = OverrideFunction; } static ParserExtensionParseResult ParseFunction(ParserExtensionInfo *info, const string &query); + static ParserOverrideResult OverrideFunction(ParserExtensionInfo *info, const string &query, + ParserOptions &options); static ParserExtensionPlanResult PlanFunction(ParserExtensionInfo *info, ClientContext &context, unique_ptr parse_data); }; +string MaterializedViewLifecycleQuery(ClientContext &context, const FunctionParameters ¶meters); +string MaterializedViewDropQuery(ClientContext &context, const FunctionParameters ¶meters); + struct MaterializedViewParseData : ParserExtensionParseData { unique_ptr statement; int64_t refresh_interval = -1; // seconds, -1 = not specified (manual only) bool is_replace = false; // CREATE OR REPLACE: drop old MV before creating string alter_sql; // non-empty for ALTER MATERIALIZED VIEW (executed directly in plan function) + string target_name; // parsed catalog-qualified CREATE/ALTER target unique_ptr Copy() const override { auto copy = make_uniq_base(statement->Copy()); @@ -32,6 +39,7 @@ struct MaterializedViewParseData : ParserExtensionParseData { data.refresh_interval = refresh_interval; data.is_replace = is_replace; data.alter_sql = alter_sql; + data.target_name = target_name; return copy; } diff --git a/src/include/core/parser_ddl.hpp b/src/include/core/parser_ddl.hpp index 37c8b856..af11bf18 100644 --- a/src/include/core/parser_ddl.hpp +++ b/src/include/core/parser_ddl.hpp @@ -6,12 +6,27 @@ namespace duckdb { +struct DropInfo; + static constexpr const char *OPENIVM_DDL_CLEANUP_PREFIX = "openivm_cleanup:"; static constexpr const char *OPENIVM_DDL_PROFILE_PREFIX = "openivm_profile:"; static constexpr const char *OPENIVM_DDL_PROFILE_RECORD_PREFIX = "openivm_profile_record:"; static constexpr const char *OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX = "openivm_create_delta_from_data:"; +static constexpr const char *OPENIVM_TRANSACTIONAL_DDL_FUNCTION = "openivm_transactional_ddl"; +static constexpr const char *OPENIVM_STAGED_DDL_FUNCTION = "openivm_staged_ddl"; + +enum class DDLExecutionMode : uint8_t { CALLER_TRANSACTION, STAGED_CROSS_CATALOG }; -void ConfigureDDLExecutorResult(ParserExtensionPlanResult &result); +void ConfigureDDLExecutorResult(ParserExtensionPlanResult &result, + DDLExecutionMode mode = DDLExecutionMode::STAGED_CROSS_CATALOG); +string RenderTransactionalDDL(const vector ¶meters); +void ExecuteStagedDDL(ClientContext &context, const vector ¶meters); +string BuildCreateDeltaFromDataOperation(const string &delta_table, const string &data_table, bool replace); +string BuildDropViewStatement(const DropInfo &drop_info); +unique_ptr BindDropView(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names); +unique_ptr InitDropView(ClientContext &context, TableFunctionInitInput &input); +void ExecuteDropView(ClientContext &context, TableFunctionInput &input, DataChunk &output); } // namespace duckdb diff --git a/src/include/core/refresh_locks.hpp b/src/include/core/refresh_locks.hpp index cc57236f..9f1f5493 100644 --- a/src/include/core/refresh_locks.hpp +++ b/src/include/core/refresh_locks.hpp @@ -2,10 +2,12 @@ #define REFRESH_LOCKS_HPP #include "duckdb.hpp" +#include "duckdb/main/client_context_state.hpp" #include #include #include +#include namespace duckdb { @@ -161,6 +163,27 @@ class TryViewLockGuard { TryViewLockGuard &operator=(TryViewLockGuard &&) = delete; }; +// Native lifecycle and refresh programs are returned as multiple DuckDB +// statements. Keep their logical locks until the surrounding caller transaction +// commits or rolls back, rather than releasing them between statements. +class TransactionalMVLockState : public ClientContextState { +public: + static TransactionalMVLockState &Get(ClientContext &context); + + void Acquire(const vector &view_names, const vector &delta_table_names); + + void TransactionCommit(MetaTransaction &transaction, ClientContext &context) override; + void TransactionRollback(MetaTransaction &transaction, ClientContext &context) override; + +private: + void Release(); + + unordered_set locked_views; + unordered_set locked_delta_tables; + vector> view_guards; + vector> delta_guards; +}; + } // namespace duckdb #endif // REFRESH_LOCKS_HPP diff --git a/src/include/core/refresh_metadata.hpp b/src/include/core/refresh_metadata.hpp index 95846ba7..2c9f9e23 100644 --- a/src/include/core/refresh_metadata.hpp +++ b/src/include/core/refresh_metadata.hpp @@ -244,9 +244,13 @@ class RefreshMetadata { string output_col; string source; string source_col; + // A target type for one CAST, or an encoded expression template for nested/TRY_CAST chains. + string source_cast; string lookup; string lookup_col; + string lookup_cast; string lookup_out; + string lookup_out_cast; }; bool GetWindowPartitionLineage(const string &view_name, vector &out); diff --git a/src/include/core/sql_utils.hpp b/src/include/core/sql_utils.hpp index 00609c06..2835a28d 100644 --- a/src/include/core/sql_utils.hpp +++ b/src/include/core/sql_utils.hpp @@ -50,6 +50,10 @@ class SqlUtils { static string BuildNullSafeMatch(const vector &columns, const string &lhs_alias, const string &rhs_alias); static string BuildNullSafeKeyPredicate(const vector &columns, const string &left_prefix, const string &right_prefix); + /// Cast specs retain the legacy single-target-type representation while encoding nested/TRY_CAST expressions. + static string BuildCastSpec(const string &target_type, bool try_cast); + static string ComposeCastSpecs(const string &outer_cast_spec, const string &inner_cast_spec); + static string ApplyCastSpec(const string &column_expression, const string &cast_spec); static string BuildFullRecomputeSQL(const string &data_table, const string &view_query_sql); static string ReplaceAllOccurrences(string haystack, const string &needle, const string &replacement); static vector ReplaceEachPlainOccurrence(const string &haystack, const string &needle, diff --git a/src/include/upsert/refresh.hpp b/src/include/upsert/refresh.hpp index a619e0e9..919480f6 100644 --- a/src/include/upsert/refresh.hpp +++ b/src/include/upsert/refresh.hpp @@ -11,6 +11,7 @@ namespace duckdb { // Generates refresh SQL for each view (including cascaded views) and executes it // under a per-view lock. This ensures concurrent refresh of the same view is serialized. void UpsertDeltaQueriesLocked(ClientContext &context, const FunctionParameters ¶meters); +string TransactionalRefreshQuery(ClientContext &context, const FunctionParameters ¶meters); } // namespace duckdb diff --git a/src/include/upsert/refresh_compiler.hpp b/src/include/upsert/refresh_compiler.hpp index 9511508f..322c7a03 100644 --- a/src/include/upsert/refresh_compiler.hpp +++ b/src/include/upsert/refresh_compiler.hpp @@ -44,7 +44,8 @@ string CompileAggregateGroups(const string &view_name, optional_ptr &column_names, const string &view_query_sql = "", bool has_minmax = false, bool list_mode = false, const string &delta_ts_filter = "", const string &catalog_prefix = "", - bool insert_only = false, const vector &column_types = {}); + bool insert_only = false, const vector &column_types = {}, + bool *out_full_recompute = nullptr); string CompileProjectionsFilters(const string &view_name, const vector &column_names, const string &delta_ts_filter = "", const string &catalog_prefix = "", bool insert_only = false); diff --git a/src/include/upsert/refresh_cost_model.hpp b/src/include/upsert/refresh_cost_model.hpp index faba6fa1..2ed558bf 100644 --- a/src/include/upsert/refresh_cost_model.hpp +++ b/src/include/upsert/refresh_cost_model.hpp @@ -30,7 +30,7 @@ struct RefreshCostEstimate { // meaning of `incremental_compute` / `incremental_upsert`. For "incremental" // views the fields hold delta-driven IVM cost. For fixed strategy views they // hold that strategy's affected-domain cost. Known labels: "incremental", - // "group_recompute", "window_partition", "current_diff_recompute", + // "group_recompute", "window_partition", // "distinct_incremental", "semi_anti_recompute", and "full". string strategy_label; diff --git a/src/openivm_extension.cpp b/src/openivm_extension.cpp index 18ab9ea4..4ffb8bc1 100644 --- a/src/openivm_extension.cpp +++ b/src/openivm_extension.cpp @@ -29,6 +29,7 @@ #include "duckdb/parser/tableref/subqueryref.hpp" #include "duckdb/planner/planner.hpp" #include "core/parser.hpp" +#include "core/parser_ddl.hpp" #include "rules/incremental_rewrite_rule.hpp" #include "rules/refresh_insert_rule.hpp" #include "core/openivm_debug.hpp" @@ -157,6 +158,10 @@ static void LoadInternal(ExtensionLoader &loader) { // after the extension is loaded. Entry points also set the current ClientContext // explicitly because pre-existing local settings override global defaults. db_config.SetOption(PreserveInsertionOrderSetting::SettingIndex, Value::BOOLEAN(false)); + // CREATE/ALTER MATERIALIZED VIEW and transactional DROP VIEW are expanded into + // ordinary DuckDB statements by OpenIVM's parser override. FALLBACK leaves every + // statement OpenIVM does not recognize with DuckDB's native parser. + db_config.SetOption(AllowParserOverrideExtensionSetting::SettingIndex, Value("fallback")); db_config.AddExtensionOption("openivm_files_path", "path for compiled SQL reference files", LogicalType::VARCHAR); db_config.AddExtensionOption("openivm_refresh_mode", "refresh strategy: incremental, full, or auto", @@ -356,6 +361,15 @@ static void LoadInternal(ExtensionLoader &loader) { OptimizerExtension::Register(db_config, std::move(incremental_rewrite_rule)); OptimizerExtension::Register(db_config, std::move(refresh_insert_rule)); + loader.RegisterFunction(PragmaFunction::PragmaCall("openivm_materialized_view_lifecycle", + MaterializedViewLifecycleQuery, {LogicalType::VARCHAR})); + loader.RegisterFunction(PragmaFunction::PragmaCall("openivm_materialized_view_drop", MaterializedViewDropQuery, + {LogicalType::VARCHAR})); + loader.RegisterFunction(TableFunction( + "openivm_execute_drop_view", + {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::BOOLEAN}, + ExecuteDropView, BindDropView, InitDropView)); + TableFunction compute_delta_function("ComputeDelta", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, ComputeDeltaFunction, ComputeDeltaBind, ComputeDeltaInit); @@ -387,13 +401,11 @@ static void LoadInternal(ExtensionLoader &loader) { con.Commit(); - // Use the locked pragma_function_t variant: generates SQL and executes it under a - // per-view mutex, preventing concurrent refresh from double-applying deltas. auto refresh_options = - PragmaFunction::PragmaCall("refresh_options", UpsertDeltaQueriesLocked, + PragmaFunction::PragmaCall("refresh_options", TransactionalRefreshQuery, {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}); loader.RegisterFunction(refresh_options); - auto refresh = PragmaFunction::PragmaCall("refresh", UpsertDeltaQueriesLocked, {LogicalType::VARCHAR}); + auto refresh = PragmaFunction::PragmaCall("refresh", TransactionalRefreshQuery, {LogicalType::VARCHAR}); loader.RegisterFunction(refresh); auto declare_rely_fk = PragmaFunction::PragmaCall( "openivm_declare_rely_fk", @@ -421,7 +433,7 @@ static void LoadInternal(ExtensionLoader &loader) { PragmaFunction::PragmaCall("refresh_history", RefreshCostHistoryQuery, {LogicalType::VARCHAR}); loader.RegisterFunction(refresh_history); auto refresh_cross_system = PragmaFunction::PragmaCall( - "refresh_cross_system", UpsertDeltaQueriesLocked, + "refresh_cross_system", TransactionalRefreshQuery, {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}); loader.RegisterFunction(refresh_cross_system); diff --git a/src/upsert/refresh.cpp b/src/upsert/refresh.cpp index 2441ecfe..68a70837 100644 --- a/src/upsert/refresh.cpp +++ b/src/upsert/refresh.cpp @@ -581,4 +581,137 @@ void UpsertDeltaQueriesLocked(ClientContext &context, const FunctionParameters & } } +static string BuildTransactionalRefreshViewSQL(ClientContext &context, Connection &metadata_con, + const string &view_catalog_name, const string &view_schema_name, + const string &view_name, const string &attached_db_catalog_name, + const string &attached_db_schema_name) { + RefreshMetadata metadata(metadata_con); + auto delta_tables = metadata.GetDeltaTables(view_name); + DeltaActivityResult conservative_activity; + conservative_activity.has_join = metadata.HasJoin(view_name) || delta_tables.size() > 1; + conservative_activity.tables_with_changes = delta_tables.size(); + conservative_activity.any_has_deletes = true; + conservative_activity.all_ducklake = false; + conservative_activity.active_delta_table_names = delta_tables; + + // The planning connection cannot see transaction-local delta rows. Compile all + // registered native sources conservatively; the returned SQL executes through + // the caller context and therefore sees exactly the caller's transaction. + return GenerateRefreshSQL(context, view_catalog_name, view_schema_name, view_name, false, attached_db_catalog_name, + attached_db_schema_name, nullptr, nullptr, nullptr, &conservative_activity, nullptr); +} + +string TransactionalRefreshQuery(ClientContext &context, const FunctionParameters ¶meters) { + string view_catalog_name; + string view_schema_name; + string attached_db_catalog_name; + string attached_db_schema_name; + string view_name; + bool cross_system = false; + Connection metadata_con(*context.db); + + if (parameters.values.size() == 3) { + view_catalog_name = StringValue::Get(parameters.values[0]); + view_schema_name = StringValue::Get(parameters.values[1]); + view_name = StringValue::Get(parameters.values[2]); + } else if (parameters.values.size() == 5) { + view_catalog_name = StringValue::Get(parameters.values[0]); + view_schema_name = StringValue::Get(parameters.values[1]); + attached_db_catalog_name = StringValue::Get(parameters.values[2]); + attached_db_schema_name = StringValue::Get(parameters.values[3]); + view_name = StringValue::Get(parameters.values[4]); + cross_system = true; + } else if (parameters.values.size() == 1) { + view_name = StringValue::Get(parameters.values[0]); + auto resolved = ResolveViewCatalogFromContext(context, metadata_con, view_name); + view_catalog_name = resolved.view_catalog_name; + view_schema_name = resolved.view_schema_name; + cross_system = resolved.cross_system; + } else { + throw InvalidInputException("OpenIVM refresh received an unsupported argument list"); + } + if (RefreshMetadata(metadata_con).GetViewQuery(view_name).empty()) { + throw CatalogException("Materialized view '%s' does not exist", view_name); + } + + if (!view_catalog_name.empty()) { + auto default_result = metadata_con.Query("SELECT current_database()"); + if (!default_result->HasError() && default_result->RowCount() > 0 && !default_result->GetValue(0, 0).IsNull() && + default_result->GetValue(0, 0).ToString() != view_catalog_name) { + cross_system = true; + } + } + if (cross_system) { + // DuckDB cannot commit writes to the native metadata catalog and an + // attached external catalog in one transaction. Keep the staged path for + // that boundary; native catalogs use the caller-transaction program below. + UpsertDeltaQueriesLocked(context, parameters); + return "SELECT true AS Success"; + } + + RefreshMetadata metadata(metadata_con); + string cascade_mode = "downstream"; + Value cascade_value; + if (context.TryGetCurrentSetting("openivm_cascade_refresh", cascade_value) && !cascade_value.IsNull()) { + cascade_mode = StringUtil::Lower(cascade_value.ToString()); + } + + vector refresh_order; + if (cascade_mode == "upstream" || cascade_mode == "both") { + auto upstream = metadata.GetUpstreamViews(view_name); + refresh_order.insert(refresh_order.end(), upstream.begin(), upstream.end()); + } + refresh_order.push_back(view_name); + if (cascade_mode == "downstream" || cascade_mode == "both") { + auto downstream = metadata.GetDownstreamViews(view_name); + refresh_order.insert(refresh_order.end(), downstream.begin(), downstream.end()); + } + + string program; + unordered_set seen; + vector lock_views; + vector lock_deltas; + for (auto &node : refresh_order) { + if (!seen.insert(node).second) { + continue; + } + auto location = ResolveViewLocation(metadata_con, node, view_catalog_name, view_schema_name); + if (location.cross_system) { + throw NotImplementedException( + "Transactional native refresh cannot include cross-catalog dependent view '%s'", node); + } + lock_views.push_back(node); + auto node_deltas = metadata.GetDeltaTables(node); + lock_deltas.insert(lock_deltas.end(), node_deltas.begin(), node_deltas.end()); + if (metadata.HasDownstreamViews(node)) { + lock_deltas.push_back(SqlUtils::DeltaName(node)); + } + + string hook_sql; + string hook_mode; + auto hooks = metadata_con.Query("SELECT hook_sql, mode FROM openivm_refresh_hooks" + " WHERE view_name = '" + + SqlUtils::EscapeValue(node) + "'"); + if (!hooks->HasError() && hooks->RowCount() > 0) { + hook_sql = hooks->GetValue(0, 0).ToString(); + hook_mode = StringUtil::Lower(hooks->GetValue(1, 0).ToString()); + } + if (!hook_sql.empty() && hook_mode == "before") { + program += hook_sql + ";\n"; + } + if (hook_mode != "replace") { + program += + BuildTransactionalRefreshViewSQL(context, metadata_con, location.catalog_name, location.schema_name, + node, attached_db_catalog_name, attached_db_schema_name); + program += "\n"; + } + if (!hook_sql.empty() && (hook_mode == "after" || hook_mode == "replace")) { + program += hook_sql + ";\n"; + } + } + TransactionalMVLockState::Get(context).Acquire(lock_views, lock_deltas); + program += "SELECT true AS Success"; + return program; +} + } // namespace duckdb diff --git a/src/upsert/refresh_compiler.cpp b/src/upsert/refresh_compiler.cpp index 9263da60..2e6dff40 100644 --- a/src/upsert/refresh_compiler.cpp +++ b/src/upsert/refresh_compiler.cpp @@ -1146,7 +1146,7 @@ string CompileAggregateGroups(const string &view_name, optional_ptr &column_names, const string &view_query_sql, bool has_minmax, bool list_mode, const string &delta_ts_filter, const string &catalog_prefix, bool /*insert_only*/, - const vector &column_types) { + const vector &column_types, bool *out_full_recompute) { string data_table = catalog_prefix + SqlUtils::QuoteIdentifier(IncrementalTableNames::DataTableName(view_name)); // Any non-summable column (VARCHAR literal, CASE result, LIST) forces a full @@ -1164,7 +1164,11 @@ string CompileSimpleAggregates(const string &view_name, const vector &co } } - if (has_minmax || has_non_summable_col) { + bool full_recompute = has_minmax || has_non_summable_col; + if (out_full_recompute) { + *out_full_recompute = full_recompute; + } + if (full_recompute) { string delete_query = "DELETE FROM " + data_table + ";\n"; string insert_query = "INSERT INTO " + data_table + " " + view_query_sql + ";\n"; return delete_query + insert_query; diff --git a/src/upsert/refresh_cost_model.cpp b/src/upsert/refresh_cost_model.cpp index f266ccca..d7e98acc 100644 --- a/src/upsert/refresh_cost_model.cpp +++ b/src/upsert/refresh_cost_model.cpp @@ -106,8 +106,6 @@ static const char *StrategyLabelForRefreshType(RefreshType view_type) { return "group_recompute"; case RefreshType::WINDOW_PARTITION: return "window_partition"; - case RefreshType::CURRENT_DIFF_RECOMPUTE: - return "current_diff_recompute"; case RefreshType::DISTINCT_INCREMENTAL: return "distinct_incremental"; case RefreshType::SEMI_ANTI_RECOMPUTE: @@ -701,17 +699,7 @@ RefreshCostEstimate EstimateRefreshCost(ClientContext &context, LogicalOperator string strategy_label = StrategyLabelForRefreshType(view_type); double strategy_compute = incremental_compute; double strategy_upsert = incremental_upsert; - if (view_type == RefreshType::CURRENT_DIFF_RECOMPUTE) { - // Current-diff recomputes the result, diffs it against the stored MV (a scan of ~mv_card), and - // applies only the *changed* rows — not a full replace. So it beats full recompute when few rows - // change (saves the full rewrite) and loses when most of the MV changes (the diff scan is then - // pure overhead). Pricing the apply as recompute_replace (= full rewrite) made it a permanent tie - // with full, so the model never preferred full even when the delta touched most of the MV (the - // recurring S05 near-tie). Model the diff scan + changed-row apply instead. - double changed = std::min(estimated_delta_result, mv_card); - strategy_compute = recompute_compute + mv_card; - strategy_upsert = changed * 2.0; - } else if (view_type == RefreshType::GROUP_RECOMPUTE) { + if (view_type == RefreshType::GROUP_RECOMPUTE) { // Estimated affected MV keys = Σᵢ delta_Tᵢ × (mv_card / actual_card_Tᵢ). // Each source contributes a per-table view-query variant (substitute T_i // with delta_T_i in view_query_sql), so compute scales with N_active sources. @@ -772,8 +760,8 @@ RefreshCostEstimate EstimateRefreshCost(ClientContext &context, LogicalOperator // overrides each side once it has enough history. // - will_skip: an empty delta with skip-empty on is short-circuited entirely → ~free. // - genuine incremental (strategy strictly cheaper than recompute and not skipped) pays the - // incremental setup floor; recompute-equivalent strategies (CURRENT_DIFF_RECOMPUTE, - // FULL_REFRESH, TOP_K) and empty-delta-with-skip-off pay only the recompute setup, so the + // incremental setup floor; recompute-equivalent strategies (FULL_REFRESH, TOP_K) and + // empty-delta-with-skip-off pay only the recompute setup, so the // decision is driven purely by the throughput term and the label is not spuriously flipped. bool will_skip = !has_active_delta && skip_empty_enabled; bool genuine_incremental = !will_skip && strategy_total < recompute_total; diff --git a/src/upsert/refresh_sql.cpp b/src/upsert/refresh_sql.cpp index 3f4b58e8..5a634e24 100644 --- a/src/upsert/refresh_sql.cpp +++ b/src/upsert/refresh_sql.cpp @@ -9,6 +9,7 @@ #include "upsert/refresh_compiler.hpp" #include "upsert/refresh_cost_model.hpp" #include "lpts_pipeline.hpp" +#include "sql_dialect.hpp" #include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" #include "duckdb/catalog/entry_lookup_info.hpp" @@ -44,6 +45,27 @@ static string SparkPortableRefreshSQL(string sql) { return sql; } +static string RenderStoredViewQueryForDialect(ClientContext &context, const string &view_query_sql, + const vector &output_names, SqlDialect dialect) { + Parser parser(context.GetParserOptions()); + parser.ParseQuery(view_query_sql); + if (parser.statements.size() != 1) { + throw ParserException("Expected one stored view query, found %llu", + static_cast(parser.statements.size())); + } + Planner planner(context); + planner.CreatePlan(parser.statements[0]->Copy()); + auto plan = std::move(planner.plan); + auto ast = LogicalPlanToAst(context, plan, dialect); + auto cte_list = AstToCteList(*ast, dialect); + auto rendered = cte_list->ToQuery(true, output_names); + if (!rendered.empty() && rendered.back() == ';') { + rendered.pop_back(); + } + StringUtil::Trim(rendered); + return rendered; +} + struct SemiAntiSourceInput { string table_sql; string delta_sql; @@ -70,7 +92,7 @@ struct RefreshPlan { return skip_projection_key_delta || refresh_type == RefreshType::WINDOW_PARTITION || refresh_type == RefreshType::GROUP_RECOMPUTE || refresh_type == RefreshType::DISTINCT_INCREMENTAL || refresh_type == RefreshType::COUNT_DISTINCT_INCREMENTAL || - refresh_type == RefreshType::SEMI_ANTI_RECOMPUTE || refresh_type == RefreshType::CURRENT_DIFF_RECOMPUTE; + refresh_type == RefreshType::SEMI_ANTI_RECOMPUTE || refresh_type == RefreshType::FULL_REFRESH; } const char *DeltaProductionSkipReason() const { @@ -88,8 +110,8 @@ struct RefreshPlan { return "GROUP_RECOMPUTE"; case RefreshType::WINDOW_PARTITION: return "WINDOW_PARTITION"; - case RefreshType::CURRENT_DIFF_RECOMPUTE: - return "CURRENT_DIFF_RECOMPUTE"; + case RefreshType::FULL_REFRESH: + return "FULL_REFRESH"; default: return "UNKNOWN"; } @@ -741,10 +763,6 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam throw ParserException("View not found! Please call IVM with a materialized view."); } RefreshType view_query_type = metadata.GetViewType(view_name); - bool emit_cascade_delta_for_recompute = - active_facts.force_view_delta_cascade && - (view_query_type == RefreshType::WINDOW_PARTITION || view_query_type == RefreshType::GROUP_RECOMPUTE || - view_query_type == RefreshType::CURRENT_DIFF_RECOMPUTE); OPENIVM_DEBUG_PRINT("[UPSERT] View: %s, Type: %d, Query: %s\n", view_name.c_str(), (int)view_query_type, view_query_sql.c_str()); auto delta_table_names = metadata.GetDeltaTables(view_name); @@ -765,9 +783,17 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam if (!flag_result->HasError() && flag_result->RowCount() > 0 && !flag_result->GetValue(0, 0).IsNull() && flag_result->GetValue(0, 0).GetValue()) { Printer::Print("Warning: recovering '" + view_name + "' from interrupted refresh via full recompute."); - metadata.SetRefreshInProgress(view_name, false); - return BuildRecomputeQuery(metadata, view_name, view_query_sql, cross_system, attached_db_catalog_name, - attached_db_schema_name, internal_catalog_prefix, out_post_meta); + auto recovery_query = + BuildRecomputeQuery(metadata, view_name, view_query_sql, cross_system, attached_db_catalog_name, + attached_db_schema_name, internal_catalog_prefix, out_post_meta); + if (cross_system) { + metadata.SetRefreshInProgress(view_name, false); + } else { + recovery_query += "\nUPDATE " + string(openivm::VIEWS_TABLE) + + " SET refresh_in_progress = false WHERE view_name = '" + + SqlUtils::EscapeValue(view_name) + "';\n"; + } + return recovery_query; } } add_profile_step("generate_refresh_sql.recovery_check", recovery_start); @@ -841,7 +867,16 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam refresh_plan.metadata_requires_full_refresh = metadata_requires_full_refresh; refresh_plan.adaptive_recompute = adaptive_recompute; - if (refresh_plan.RequiresFullRecompute()) { + string delta_view_name_bare = SqlUtils::DeltaName(view_name); + string delta_view_name = internal_catalog_prefix + delta_view_name_bare; + auto downstream_check = con.Query("SELECT COUNT(*) FROM " + string(openivm::DELTA_TABLES_TABLE) + + " WHERE table_name = '" + SqlUtils::EscapeValue(delta_view_name_bare) + "'"); + bool has_downstream = !downstream_check->HasError() && downstream_check->RowCount() > 0 && + downstream_check->GetValue(0, 0).GetValue() > 0; + bool full_recompute_needs_cascade_delta = has_downstream || active_facts.force_view_delta_cascade; + bool use_full_recompute = refresh_plan.RequiresFullRecompute(); + + if (use_full_recompute && !full_recompute_needs_cascade_delta) { auto full_refresh_start = profile_now(); auto recompute_query = BuildRecomputeQuery(metadata, view_name, view_query_sql, cross_system, attached_db_catalog_name, @@ -853,6 +888,11 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam "; sql_bytes=" + to_string(recompute_query.size())); return recompute_query; } + RefreshType dispatch_refresh_type = use_full_recompute ? RefreshType::FULL_REFRESH : view_query_type; + refresh_plan.refresh_type = dispatch_refresh_type; + bool emit_cascade_delta_for_recompute = + active_facts.force_view_delta_cascade && (dispatch_refresh_type == RefreshType::WINDOW_PARTITION || + dispatch_refresh_type == RefreshType::GROUP_RECOMPUTE); auto column_metadata_start = profile_now(); vector column_names; vector column_types; @@ -968,7 +1008,7 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam const vector *aggregate_cascade_specs_ptr = nullptr; { bool force_view_delta_cascade = active_facts.force_view_delta_cascade; - bool eligible_refresh_type = (view_query_type == RefreshType::AGGREGATE_GROUP); + bool eligible_refresh_type = (dispatch_refresh_type == RefreshType::AGGREGATE_GROUP); if (force_view_delta_cascade && eligible_refresh_type && !source_has_full_outer && !group_cols.empty()) { auto active_delta_table_names = fast_paths.active_delta_table_names; if (active_delta_table_names.empty() && active_facts.compile_only) { @@ -999,9 +1039,9 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam } } - OPENIVM_DEBUG_PRINT("[UPSERT] Compiling upsert for type: %s\n", RefreshTypeName(view_query_type)); + OPENIVM_DEBUG_PRINT("[UPSERT] Compiling upsert for type: %s\n", RefreshTypeName(dispatch_refresh_type)); auto dispatch_start = profile_now(); - switch (view_query_type) { + switch (dispatch_refresh_type) { case RefreshType::AGGREGATE_HAVING: { bool having_merge = SqlUtils::GetBoolSetting(context, "openivm_having_merge", true); if (having_merge) { @@ -1071,6 +1111,7 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam case RefreshType::SIMPLE_AGGREGATE: { bool sa_insert_only = has_argminmax ? false : insert_only; RefreshMetadata::FilteredGroupCountAuxMeta aux_meta; + bool simple_aggregate_full_recompute = false; if (metadata.GetFilteredGroupCountAuxMeta(view_name, aux_meta)) { if (!active_facts.compile_only) { EnsureFilteredGroupCountAuxState(metadata, con, view_name, aux_meta, delta_table_names, @@ -1091,11 +1132,11 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam aux_meta.group_col.c_str(), aux_meta.sum_col.c_str(), aux_meta.comparison_op.c_str(), aux_meta.threshold_sql.c_str()); } else { - upsert_query = - CompileSimpleAggregates(view_name, column_names, view_query_sql, has_minmax, list_mode, delta_ts_filter, - internal_catalog_prefix, sa_insert_only, column_types); + upsert_query = CompileSimpleAggregates(view_name, column_names, view_query_sql, has_minmax, list_mode, + delta_ts_filter, internal_catalog_prefix, sa_insert_only, + column_types, &simple_aggregate_full_recompute); } - if (!has_minmax && aux_meta.aux_table.empty()) { + if (!has_minmax && aux_meta.aux_table.empty() && !simple_aggregate_full_recompute) { AppendSimpleAggregateEmptySourceNulling(metadata, upsert_query, view_name, column_names, data_table, view_catalog_name, view_schema_name, attached_db_catalog_name, attached_db_schema_name); @@ -1267,11 +1308,31 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam case RefreshType::TOP_K: [[fallthrough]]; case RefreshType::FULL_REFRESH: { - throw InternalException("FULL_REFRESH views should not reach incremental upsert compilation"); + string full_recompute_query = view_query_sql; + if (active_facts.target_dialect == SqlDialect::SPARK) { + vector output_names; + for (auto &column_name : column_names) { + if (column_name != string(openivm::MULTIPLICITY_COL)) { + output_names.push_back(column_name); + } + } + con.BeginTransaction(); + try { + full_recompute_query = RenderStoredViewQueryForDialect(*con.context, view_query_sql, output_names, + active_facts.target_dialect); + con.Rollback(); + } catch (...) { + con.Rollback(); + throw; + } + } + upsert_query = CompileFullRecompute(view_name, full_recompute_query, internal_catalog_prefix); + OPENIVM_DEBUG_PRINT("[UPSERT] Compiling upsert for type: %s\n", RefreshTypeName(dispatch_refresh_type)); + break; } } add_profile_step("generate_refresh_sql.dispatch", dispatch_start, - "refresh_type=" + string(RefreshTypeName(view_query_type)) + + "refresh_type=" + string(RefreshTypeName(dispatch_refresh_type)) + "; upsert_bytes=" + to_string(upsert_query.size())); OPENIVM_DEBUG_PRINT("[UPSERT] Upsert query:\n%s\n", upsert_query.c_str()); string delta_query; @@ -1280,24 +1341,20 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam string post_companion; string compact_delta_view_query; string delete_from_view_query; - string delta_view_name_bare = SqlUtils::DeltaName(view_name); - string delta_view_name = internal_catalog_prefix + delta_view_name_bare; - auto downstream_check = con.Query("SELECT COUNT(*) FROM " + string(openivm::DELTA_TABLES_TABLE) + - " WHERE table_name = '" + SqlUtils::EscapeValue(delta_view_name_bare) + "'"); - bool has_downstream = !downstream_check->HasError() && downstream_check->RowCount() > 0 && - downstream_check->GetValue(0, 0).GetValue() > 0; - // force_view_delta_cascade biases has_downstream=true for AGGREGATE_GROUP - // and AGGREGATE_HAVING so the per-key retract companion is emitted even - // when no downstream MV is currently registered in openivm_delta_tables. + // force_view_delta_cascade biases has_downstream=true for recompute paths + // that can emit their own view-delta rows even when no downstream MV is + // currently registered in openivm_delta_tables. // - // Scope: AGGREGATE_GROUP and AGGREGATE_HAVING only. The SIMPLE_AGGREGATE + // Scope: AGGREGATE_GROUP, AGGREGATE_HAVING, and FULL_REFRESH. The SIMPLE_AGGREGATE // snapshot companion relies on CREATE TEMP TABLE / DROP TABLE pre/post // pairs that not all dialects can carry across statement boundaries. + // Spark FULL_REFRESH uses a split-safe signed old/new companion below. // WINDOW_PARTITION and GROUP_RECOMPUTE go through the // emit_cascade_delta_for_recompute path below. { - if (active_facts.force_view_delta_cascade && - (view_query_type == RefreshType::AGGREGATE_GROUP || view_query_type == RefreshType::AGGREGATE_HAVING)) { + if (active_facts.force_view_delta_cascade && (dispatch_refresh_type == RefreshType::AGGREGATE_GROUP || + dispatch_refresh_type == RefreshType::AGGREGATE_HAVING || + dispatch_refresh_type == RefreshType::FULL_REFRESH)) { has_downstream = true; } } @@ -1310,8 +1367,10 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam } bool recompute_handles_own_cascade_delta = aggregate_recompute_emits_cascade_delta || - (emit_cascade_delta_for_recompute && - (view_query_type == RefreshType::WINDOW_PARTITION || view_query_type == RefreshType::GROUP_RECOMPUTE)); + (emit_cascade_delta_for_recompute && (dispatch_refresh_type == RefreshType::WINDOW_PARTITION || + dispatch_refresh_type == RefreshType::GROUP_RECOMPUTE)); + bool split_safe_full_refresh_cascade = + dispatch_refresh_type == RefreshType::FULL_REFRESH && active_facts.target_dialect == SqlDialect::SPARK; auto build_snapshot_companion = [&]() { string col_list; for (auto &col : column_names) { @@ -1370,6 +1429,47 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam OPENIVM_DEBUG_PRINT("[UPSERT] Pre-companion: %s\n", pre_companion.c_str()); OPENIVM_DEBUG_PRINT("[UPSERT] Post-companion: %s\n", post_companion.c_str()); }; + auto build_split_safe_full_refresh_companion = [&]() { + string col_list; + for (auto &col : column_names) { + if (!col_list.empty()) { + col_list += ", "; + } + col_list += DialectQuoteIdent(col, active_facts.target_dialect); + } + + string select_old; + string select_new; + bool first = true; + for (auto &col : column_names) { + if (!first) { + select_old += ", "; + select_new += ", "; + } + first = false; + if (col == string(openivm::MULTIPLICITY_COL)) { + select_old += "-1"; + select_new += "1"; + } else { + select_old += DialectQuoteIdent(col, active_facts.target_dialect); + select_new += DialectQuoteIdent(col, active_facts.target_dialect); + } + } + + string qdvn = delta_view_name.find('.') == string::npos ? KeywordHelper::WriteOptionallyQuoted(delta_view_name) + : delta_view_name; + pre_companion = "DELETE FROM " + qdvn + " WHERE 1=1"; + if (!delta_ts_filter.empty()) { + pre_companion += " AND " + delta_ts_filter; + } + pre_companion += ";\n"; + pre_companion += + "INSERT INTO " + qdvn + " (" + col_list + ") SELECT " + select_old + " FROM " + data_table + ";\n"; + post_companion = + "INSERT INTO " + qdvn + " (" + col_list + ") SELECT " + select_new + " FROM " + data_table + ";\n"; + OPENIVM_DEBUG_PRINT("[UPSERT] Split-safe full-refresh pre-companion: %s\n", pre_companion.c_str()); + OPENIVM_DEBUG_PRINT("[UPSERT] Split-safe full-refresh post-companion: %s\n", post_companion.c_str()); + }; auto build_affected_snapshot_companion = [&](const vector &keys) { string col_list = SqlUtils::JoinQuotedColumns(column_names); string select_old, select_new; @@ -1427,7 +1527,11 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam : refresh_plan.DeltaProductionSkipReason()); delta_query = ""; if (has_downstream && !recompute_handles_own_cascade_delta) { - build_snapshot_companion(); + if (split_safe_full_refresh_cascade) { + build_split_safe_full_refresh_companion(); + } else { + build_snapshot_companion(); + } } } else { string compute_delta = "select * from ComputeDelta('" + SqlUtils::EscapeValue(internal_catalog_name) + "','" + @@ -1577,7 +1681,7 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam auto assembly_start = profile_now(); if (has_downstream) { - if (skip_empty_enabled && !recompute_handles_own_cascade_delta) { + if (skip_empty_enabled && !recompute_handles_own_cascade_delta && !split_safe_full_refresh_cascade) { compact_delta_view_query = BuildCompactDeltaViewSQL(view_name, delta_view_name, column_names, delta_ts_filter); OPENIVM_DEBUG_PRINT("[UPSERT] Compact delta-view query:\n%s\n", compact_delta_view_query.c_str()); diff --git a/src/upsert/refresh_window.cpp b/src/upsert/refresh_window.cpp index 511f1481..eea7717f 100644 --- a/src/upsert/refresh_window.cpp +++ b/src/upsert/refresh_window.cpp @@ -124,11 +124,37 @@ static string ResolveStandardTrackedTableSQL(RefreshMetadata &metadata, const st return SqlUtils::FullName(loc.catalog_name, loc.schema_name, resolved_name); } +static string BuildLineageColumnExpr(const string &column_name, const string &cast_type, const string &alias = "") { + string column = + alias.empty() ? SqlUtils::QuoteIdentifier(column_name) : alias + "." + SqlUtils::QuoteIdentifier(column_name); + return SqlUtils::ApplyCastSpec(column, cast_type); +} + +static string DescribeColumnType(Connection &con, const string &table_sql, const string &column_name) { + auto result = con.Query("DESCRIBE SELECT " + SqlUtils::QuoteIdentifier(column_name) + " FROM " + table_sql); + if (result->HasError() || result->RowCount() == 0) { + return ""; + } + return StringUtil::Upper(result->GetValue(1, 0).ToString()); +} + +static bool LookupCastMetadataIsSafe(Connection &con, const string &delta_table, const string &source_col, + const string &lookup_table, const string &lookup_col, const string &source_cast, + const string &lookup_cast) { + if (!source_cast.empty() || !lookup_cast.empty()) { + return true; + } + string source_type = DescribeColumnType(con, delta_table, source_col); + string lookup_type = DescribeColumnType(con, lookup_table, lookup_col); + return !source_type.empty() && !lookup_type.empty() && StringUtil::CIEquals(source_type, lookup_type); +} + static string BuildStandardChangedValuesSQL(RefreshMetadata &metadata, const string &view_name, const vector &delta_table_names, const string &table_name, - const string &source_col, const string &output_col, - const string &delta_ts_filter, const string &view_catalog_name, - const string &view_schema_name, const string &attached_db_catalog_name, + const string &source_col, const string &source_cast, + const string &output_col, const string &delta_ts_filter, + const string &view_catalog_name, const string &view_schema_name, + const string &attached_db_catalog_name, const string &attached_db_schema_name) { string delta_table = ResolveStandardTrackedTableSQL(metadata, view_name, delta_table_names, table_name, /*delta_table=*/true, view_catalog_name, view_schema_name, @@ -137,11 +163,11 @@ static string BuildStandardChangedValuesSQL(RefreshMetadata &metadata, const str return ""; } string filter = delta_ts_filter.empty() ? "" : " WHERE " + delta_ts_filter; - return "SELECT DISTINCT " + SqlUtils::QuoteIdentifier(source_col) + " AS " + SqlUtils::QuoteIdentifier(output_col) + - " FROM " + delta_table + filter; + return "SELECT DISTINCT " + BuildLineageColumnExpr(source_col, source_cast) + " AS " + + SqlUtils::QuoteIdentifier(output_col) + " FROM " + delta_table + filter; } -static string BuildStandardLookupChangedKeysSQL(RefreshMetadata &metadata, const string &view_name, +static string BuildStandardLookupChangedKeysSQL(RefreshMetadata &metadata, const string &view_name, Connection &con, const vector &delta_table_names, const RefreshMetadata::WindowPartitionLineageOp &op, const string &delta_ts_filter, const string &view_catalog_name, @@ -156,24 +182,31 @@ static string BuildStandardLookupChangedKeysSQL(RefreshMetadata &metadata, const if (delta_table.empty() || lookup_table.empty()) { return ""; } - string qsource_col = SqlUtils::QuoteIdentifier(op.source_col); - string qlookup_col = SqlUtils::QuoteIdentifier(op.lookup_col); - string qlookup_out = SqlUtils::QuoteIdentifier(op.lookup_out); + if (!LookupCastMetadataIsSafe(con, delta_table, op.source_col, lookup_table, op.lookup_col, op.source_cast, + op.lookup_cast)) { + OPENIVM_DEBUG_PRINT("[UPSERT] WINDOW_PARTITION lookup cast metadata is incomplete for '%s' — " + "full recompute fallback\n", + view_name.c_str()); + return ""; + } + string source_expr = BuildLineageColumnExpr(op.source_col, op.source_cast); + string lookup_expr = BuildLineageColumnExpr(op.lookup_col, op.lookup_cast); + string lookup_out = BuildLineageColumnExpr(op.lookup_out, op.lookup_out_cast); string qoutput_col = SqlUtils::QuoteIdentifier(op.output_col); string filter = delta_ts_filter.empty() ? "" : " WHERE " + delta_ts_filter; - return "SELECT DISTINCT " + qlookup_out + " AS " + qoutput_col + " FROM " + lookup_table + " WHERE " + qlookup_col + - " IN (SELECT DISTINCT " + qsource_col + " FROM " + delta_table + filter + ")"; + return "SELECT DISTINCT " + lookup_out + " AS " + qoutput_col + " FROM " + lookup_table + " WHERE " + lookup_expr + + " IN (SELECT DISTINCT " + source_expr + " FROM " + delta_table + filter + ")"; } -static bool BuildLineageStandardAffectedKeysSQL(RefreshMetadata &metadata, const string &view_name, - const vector &delta_table_names, - const vector &partition_cols, const string &delta_ts_filter, - const string &view_catalog_name, const string &view_schema_name, - const string &attached_db_catalog_name, - const string &attached_db_schema_name, string &affected_keys_sql, - string &key_cols, string &key_tuple) { +enum class LineageAffectedKeysResult : uint8_t { UNAVAILABLE, AVAILABLE, UNSAFE }; + +static LineageAffectedKeysResult BuildLineageStandardAffectedKeysSQL( + RefreshMetadata &metadata, Connection &con, const string &view_name, const vector &delta_table_names, + const vector &partition_cols, const string &delta_ts_filter, const string &view_catalog_name, + const string &view_schema_name, const string &attached_db_catalog_name, const string &attached_db_schema_name, + string &affected_keys_sql, string &key_cols, string &key_tuple) { if (partition_cols.size() != 1) { - return false; + return LineageAffectedKeysResult::UNAVAILABLE; } auto parsed = SplitPartitionSpec(partition_cols[0]); key_cols = SqlUtils::QuoteIdentifier(parsed.first); @@ -181,7 +214,7 @@ static bool BuildLineageStandardAffectedKeysSQL(RefreshMetadata &metadata, const vector lineage_ops; if (!metadata.GetWindowPartitionLineage(view_name, lineage_ops)) { - return false; + return LineageAffectedKeysResult::UNAVAILABLE; } vector arms; @@ -192,16 +225,17 @@ static bool BuildLineageStandardAffectedKeysSQL(RefreshMetadata &metadata, const } string arm_sql; if (op.kind == "direct") { - arm_sql = BuildStandardChangedValuesSQL(metadata, view_name, delta_table_names, op.source, op.source_col, - op.output_col, delta_ts_filter, view_catalog_name, view_schema_name, - attached_db_catalog_name, attached_db_schema_name); + arm_sql = + BuildStandardChangedValuesSQL(metadata, view_name, delta_table_names, op.source, op.source_col, + op.source_cast, op.output_col, delta_ts_filter, view_catalog_name, + view_schema_name, attached_db_catalog_name, attached_db_schema_name); } else if (op.kind == "lookup") { - arm_sql = BuildStandardLookupChangedKeysSQL(metadata, view_name, delta_table_names, op, delta_ts_filter, - view_catalog_name, view_schema_name, attached_db_catalog_name, - attached_db_schema_name); + arm_sql = BuildStandardLookupChangedKeysSQL(metadata, view_name, con, delta_table_names, op, + delta_ts_filter, view_catalog_name, view_schema_name, + attached_db_catalog_name, attached_db_schema_name); } if (arm_sql.empty()) { - continue; + return LineageAffectedKeysResult::UNSAFE; } arms.push_back("(" + arm_sql + ")"); covered_sources.insert(StripTrackedPrefix(op.source)); @@ -209,11 +243,11 @@ static bool BuildLineageStandardAffectedKeysSQL(RefreshMetadata &metadata, const for (auto &dt : delta_table_names) { if (!covered_sources.count(StripTrackedPrefix(dt))) { - return false; + return LineageAffectedKeysResult::UNAVAILABLE; } } if (arms.empty()) { - return false; + return LineageAffectedKeysResult::UNAVAILABLE; } string union_sql; @@ -224,7 +258,7 @@ static bool BuildLineageStandardAffectedKeysSQL(RefreshMetadata &metadata, const union_sql += arms[i]; } affected_keys_sql = "SELECT DISTINCT " + key_cols + " FROM (" + union_sql + ") openivm_changed_partitions"; - return true; + return LineageAffectedKeysResult::AVAILABLE; } static bool AllWindowPartitionSourcesCovered(const vector &delta_table_names, @@ -300,15 +334,15 @@ static bool BuildDuckLakeWindowSourceSpecs(RefreshMetadata &metadata, Connection } static string BuildDuckLakeChangedValuesSQL(const DuckLakeWindowSourceSpec &spec, const string &source_col, - const string &output_col) { - string qsource_col = SqlUtils::QuoteIdentifier(source_col); + const string &source_cast, const string &output_col) { + string source_expr = BuildLineageColumnExpr(source_col, source_cast); string qoutput_col = SqlUtils::QuoteIdentifier(output_col); string insertions = - "SELECT " + qsource_col + " AS " + qoutput_col + " FROM " + + "SELECT " + source_expr + " AS " + qoutput_col + " FROM " + SqlUtils::DuckLakeTableFunction("ducklake_table_insertions", spec.loc.catalog_name, spec.loc.schema_name, spec.loc.table_name, spec.old_snap, spec.current_snap); string deletions = - "SELECT " + qsource_col + " AS " + qoutput_col + " FROM " + + "SELECT " + source_expr + " AS " + qoutput_col + " FROM " + SqlUtils::DuckLakeTableFunction("ducklake_table_deletions", spec.loc.catalog_name, spec.loc.schema_name, spec.loc.table_name, spec.old_snap, spec.current_snap); return "(" + insertions + " UNION ALL " + deletions + ")"; @@ -317,18 +351,18 @@ static string BuildDuckLakeChangedValuesSQL(const DuckLakeWindowSourceSpec &spec static string BuildDuckLakeLookupChangedKeysSQL(const DuckLakeWindowSourceSpec &source_spec, const DuckLakeWindowSourceSpec &lookup_spec, const RefreshMetadata::WindowPartitionLineageOp &op) { - string changed = BuildDuckLakeChangedValuesSQL(source_spec, op.source_col, "openivm_join_key"); + string changed = BuildDuckLakeChangedValuesSQL(source_spec, op.source_col, op.source_cast, "openivm_join_key"); string lookup_table = SqlUtils::FullName(lookup_spec.loc.catalog_name, lookup_spec.loc.schema_name, lookup_spec.loc.table_name); - string lookup_col = SqlUtils::QuoteIdentifier(op.lookup_col); - string lookup_out = SqlUtils::QuoteIdentifier(op.lookup_out); + string lookup_col = BuildLineageColumnExpr(op.lookup_col, op.lookup_cast, "l"); + string lookup_out = BuildLineageColumnExpr(op.lookup_out, op.lookup_out_cast, "l"); string output_col = SqlUtils::QuoteIdentifier(op.output_col); - string current_lookup = "SELECT l." + lookup_out + " AS " + output_col + " FROM " + changed + " c JOIN " + - lookup_table + " AS l AT (VERSION => " + to_string(lookup_spec.current_snap) + ") ON l." + + string current_lookup = "SELECT " + lookup_out + " AS " + output_col + " FROM " + changed + " c JOIN " + + lookup_table + " AS l AT (VERSION => " + to_string(lookup_spec.current_snap) + ") ON " + lookup_col + " = c.openivm_join_key"; - string old_lookup = "SELECT l." + lookup_out + " AS " + output_col + " FROM " + changed + " c JOIN " + - lookup_table + " AS l AT (VERSION => " + to_string(lookup_spec.old_snap) + ") ON l." + - lookup_col + " = c.openivm_join_key"; + string old_lookup = "SELECT " + lookup_out + " AS " + output_col + " FROM " + changed + " c JOIN " + lookup_table + + " AS l AT (VERSION => " + to_string(lookup_spec.old_snap) + ") ON " + lookup_col + + " = c.openivm_join_key"; return "(" + current_lookup + " UNION ALL " + old_lookup + ")"; } @@ -386,7 +420,7 @@ static bool BuildLineageDuckLakeAffectedKeysSQL(RefreshMetadata &metadata, Conne continue; } if (op.kind == "direct") { - arms.push_back(BuildDuckLakeChangedValuesSQL(*source_spec, op.source_col, op.output_col)); + arms.push_back(BuildDuckLakeChangedValuesSQL(*source_spec, op.source_col, op.source_cast, op.output_col)); covered_sources.insert(StripDataPrefix(source_spec->metadata_key)); continue; } @@ -558,12 +592,16 @@ string BuildWindowPartitionRefresh(RefreshMetadata &metadata, Connection &con, c "[UPSERT] Compiling upsert for type: WINDOW_PARTITION (DuckLake, full recompute fallback)\n"); return "DELETE FROM " + data_table + ";\n" + "INSERT INTO " + data_table + " " + view_query_sql + ";\n"; } - if (delta_table_names.size() > 1) { - have_lineage_affected_keys = BuildLineageStandardAffectedKeysSQL( - metadata, view_name, delta_table_names, partition_cols, delta_ts_filter, view_catalog_name, - view_schema_name, attached_db_catalog_name, attached_db_schema_name, affected_keys_sql, affected_key_cols, - affected_key_tuple); + auto lineage_result = BuildLineageStandardAffectedKeysSQL( + metadata, con, view_name, delta_table_names, partition_cols, delta_ts_filter, view_catalog_name, + view_schema_name, attached_db_catalog_name, attached_db_schema_name, affected_keys_sql, affected_key_cols, + affected_key_tuple); + if (lineage_result == LineageAffectedKeysResult::UNSAFE) { + OPENIVM_DEBUG_PRINT("[UPSERT] WINDOW_PARTITION lineage is unsafe for '%s' — full recompute fallback\n", + view_name.c_str()); + return CompileFullRecompute(view_name, view_query_sql, internal_catalog_prefix); } + have_lineage_affected_keys = lineage_result == LineageAffectedKeysResult::AVAILABLE; if (!have_lineage_affected_keys && delta_table_names.size() > 1 && !AllWindowPartitionSourcesCovered(delta_table_names, partition_delta_specs)) { OPENIVM_DEBUG_PRINT("[UPSERT] WINDOW_PARTITION lineage incomplete for '%s' (%zu sources) — full recompute " diff --git a/test/sql/aggregate.test b/test/sql/aggregate.test index 9009ebfc..dec4f4a8 100644 --- a/test/sql/aggregate.test +++ b/test/sql/aggregate.test @@ -2216,6 +2216,71 @@ SELECT COUNT(*) FROM ( ---- 0 +# UNION-ALL-of-scalar-aggregates over separate sources, including an empty source. +# The SIMPLE_AGGREGATE full-recompute path must not run empty-source nulling over +# the whole UNION result. +statement ok +CREATE TABLE ua_multi_a (id INT, amount INT); + +statement ok +CREATE TABLE ua_multi_b (id INT, amount INT); + +statement ok +CREATE TABLE ua_multi_empty (id INT, amount INT); + +statement ok +INSERT INTO ua_multi_a VALUES (1, 10), (2, 20); + +statement ok +INSERT INTO ua_multi_b VALUES (1, 30); + +statement ok +CREATE MATERIALIZED VIEW mv_ua_multi_counts AS + SELECT 'a' AS kind, COUNT(*) AS n FROM ua_multi_a + UNION ALL + SELECT 'b', COUNT(*) FROM ua_multi_b + UNION ALL + SELECT 'empty', COUNT(*) FROM ua_multi_empty; + +query I +SELECT type FROM openivm_views WHERE view_name = 'mv_ua_multi_counts'; +---- +1 + +statement ok +INSERT INTO ua_multi_a VALUES (3, 40); + +statement ok +DELETE FROM ua_multi_b WHERE id = 1; + +statement ok +PRAGMA refresh('mv_ua_multi_counts'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM mv_ua_multi_counts + EXCEPT ALL + SELECT * FROM ( + SELECT 'a', COUNT(*) FROM ua_multi_a + UNION ALL SELECT 'b', COUNT(*) FROM ua_multi_b + UNION ALL SELECT 'empty', COUNT(*) FROM ua_multi_empty + ) q +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM ( + SELECT 'a', COUNT(*) FROM ua_multi_a + UNION ALL SELECT 'b', COUNT(*) FROM ua_multi_b + UNION ALL SELECT 'empty', COUNT(*) FROM ua_multi_empty + ) q + EXCEPT ALL SELECT * FROM mv_ua_multi_counts +); +---- +0 + # LIST with large batch of mixed inserts/deletes statement ok CREATE TABLE list_heavy (bucket INT, token VARCHAR); diff --git a/test/sql/auto_refresh_asof.test b/test/sql/auto_refresh_asof.test index eb6e20ae..354232a9 100644 --- a/test/sql/auto_refresh_asof.test +++ b/test/sql/auto_refresh_asof.test @@ -1,5 +1,5 @@ # name: test/sql/auto_refresh_asof.test -# description: ASOF-join current-diff materialized view under the adaptive cost model. +# description: ASOF-join full-refresh materialized view under the adaptive cost model. # group: [sql] require openivm @@ -27,7 +27,7 @@ INSERT INTO cm_asof_prices VALUES ('A', TIMESTAMP '2024-01-01 10:18:00', 180); statement ok -CREATE MATERIALIZED VIEW mv_cm_current_diff AS +CREATE MATERIALIZED VIEW mv_cm_asof_full AS SELECT t.symbol, t.ts AS trade_ts, t.qty, p.ts AS price_ts, p.price FROM cm_asof_trades t ASOF JOIN cm_asof_prices p @@ -38,11 +38,11 @@ statement ok INSERT INTO cm_asof_prices VALUES ('A', TIMESTAMP '2024-01-01 10:19:00', 190); statement ok -PRAGMA refresh('mv_cm_current_diff'); +PRAGMA refresh('mv_cm_asof_full'); query I -SELECT CASE WHEN method IN ('full', 'current_diff_recompute') THEN 1 ELSE 0 END -FROM openivm_refresh_history WHERE view_name = 'mv_cm_current_diff' ORDER BY refresh_timestamp DESC LIMIT 1; +SELECT CASE WHEN method = 'full' THEN 1 ELSE 0 END +FROM openivm_refresh_history WHERE view_name = 'mv_cm_asof_full' ORDER BY refresh_timestamp DESC LIMIT 1; ---- 1 @@ -53,14 +53,14 @@ SELECT count(*) FROM ( ASOF JOIN cm_asof_prices p ON t.symbol = p.symbol AND t.ts >= p.ts - EXCEPT ALL SELECT * FROM mv_cm_current_diff + EXCEPT ALL SELECT * FROM mv_cm_asof_full ); ---- 0 query I SELECT count(*) FROM ( - SELECT * FROM mv_cm_current_diff + SELECT * FROM mv_cm_asof_full EXCEPT ALL SELECT t.symbol, t.ts AS trade_ts, t.qty, p.ts AS price_ts, p.price FROM cm_asof_trades t diff --git a/test/sql/cascade_window_partition_joined.test b/test/sql/cascade_window_partition_joined.test index 4c93100d..91959a43 100644 --- a/test/sql/cascade_window_partition_joined.test +++ b/test/sql/cascade_window_partition_joined.test @@ -158,3 +158,660 @@ SELECT COUNT(*) FROM ( ); ---- 0 + +# ========================================== +# Casted lookup keys: joined WINDOW_PARTITION affected-key lineage must +# preserve casts from the join predicate. TPC-DI securities has this shape: +# partition by a left-table key, and refresh after a right-table update through +# CAST(left_key AS VARCHAR) = CAST(right_key AS VARCHAR). +# ========================================== + +statement ok +CREATE TABLE cwj_cast_security ( + symbol VARCHAR, + cik BIGINT, + pts INT +); + +statement ok +CREATE TABLE cwj_cast_company ( + company_id VARCHAR, + name VARCHAR, + rank_score INT +); + +statement ok +INSERT INTO cwj_cast_security VALUES + ('SYM_A', 100, 10), + ('SYM_A', 101, 20), + ('SYM_B', 200, 5); + +statement ok +INSERT INTO cwj_cast_company VALUES + ('100', 'Acme', 1), + ('101', 'Beta', 5), + ('200', 'Solo', 3); + +statement ok +CREATE MATERIALIZED VIEW cwj_cast_mv AS + SELECT s.symbol, + s.cik, + c.name AS company_name, + c.rank_score, + ROW_NUMBER() OVER ( + PARTITION BY s.symbol + ORDER BY c.rank_score DESC NULLS LAST, s.pts DESC, s.cik + ) AS rn + FROM cwj_cast_security s + LEFT JOIN cwj_cast_company c + ON CAST(s.cik AS VARCHAR) = CAST(c.company_id AS VARCHAR); + +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) LIKE + '%CAST%cik%AS VARCHAR%IN (SELECT DISTINCT %company_id%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'cwj_cast_mv', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":false}' +) +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +UPDATE cwj_cast_company +SET rank_score = 9, name = 'Acme Updated' +WHERE company_id = '100'; + +statement ok +PRAGMA refresh('cwj_cast_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT symbol, cik, company_name, rank_score, rn + FROM openivm_data_cwj_cast_mv + EXCEPT ALL + SELECT s.symbol, + s.cik, + c.name AS company_name, + c.rank_score, + ROW_NUMBER() OVER ( + PARTITION BY s.symbol + ORDER BY c.rank_score DESC NULLS LAST, s.pts DESC, s.cik + ) AS rn + FROM cwj_cast_security s + LEFT JOIN cwj_cast_company c + ON CAST(s.cik AS VARCHAR) = CAST(c.company_id AS VARCHAR) +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT s.symbol, + s.cik, + c.name AS company_name, + c.rank_score, + ROW_NUMBER() OVER ( + PARTITION BY s.symbol + ORDER BY c.rank_score DESC NULLS LAST, s.pts DESC, s.cik + ) AS rn + FROM cwj_cast_security s + LEFT JOIN cwj_cast_company c + ON CAST(s.cik AS VARCHAR) = CAST(c.company_id AS VARCHAR) + EXCEPT ALL + SELECT symbol, cik, company_name, rank_score, rn + FROM openivm_data_cwj_cast_mv +); +---- +0 + +# Explicit non-VARCHAR casts must be preserved in lookup lineage. + +statement ok +CREATE TABLE cwj_cast_int_security ( + symbol VARCHAR, + key_txt VARCHAR, + pts INT +); + +statement ok +CREATE TABLE cwj_cast_int_company ( + key_int INTEGER, + name VARCHAR, + rank_score INT +); + +statement ok +INSERT INTO cwj_cast_int_security VALUES + ('SYM_C', '0300', 10), + ('SYM_C', '301', 20), + ('SYM_D', '400', 5); + +statement ok +INSERT INTO cwj_cast_int_company VALUES + (300, 'Cedar', 2), + (301, 'Delta', 6), + (400, 'Echo', 4); + +statement ok +CREATE MATERIALIZED VIEW cwj_cast_int_mv AS + SELECT s.symbol, + s.key_txt, + c.name AS company_name, + c.rank_score, + ROW_NUMBER() OVER ( + PARTITION BY s.symbol + ORDER BY c.rank_score DESC NULLS LAST, s.pts DESC, s.key_txt + ) AS rn + FROM cwj_cast_int_security s + LEFT JOIN cwj_cast_int_company c + ON CAST(s.key_txt AS INTEGER) = c.key_int; + +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) LIKE + '%CAST%key_txt%AS INTEGER%IN (SELECT DISTINCT %key_int%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE + '%CAST%key_int%AS VARCHAR%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'cwj_cast_int_mv', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":false}' +) +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +UPDATE cwj_cast_int_company +SET rank_score = 12, name = 'Cedar Updated' +WHERE key_int = 300; + +statement ok +PRAGMA refresh('cwj_cast_int_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT symbol, key_txt, company_name, rank_score, rn + FROM openivm_data_cwj_cast_int_mv + EXCEPT ALL + SELECT s.symbol, + s.key_txt, + c.name AS company_name, + c.rank_score, + ROW_NUMBER() OVER ( + PARTITION BY s.symbol + ORDER BY c.rank_score DESC NULLS LAST, s.pts DESC, s.key_txt + ) AS rn + FROM cwj_cast_int_security s + LEFT JOIN cwj_cast_int_company c + ON CAST(s.key_txt AS INTEGER) = c.key_int +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT s.symbol, + s.key_txt, + c.name AS company_name, + c.rank_score, + ROW_NUMBER() OVER ( + PARTITION BY s.symbol + ORDER BY c.rank_score DESC NULLS LAST, s.pts DESC, s.key_txt + ) AS rn + FROM cwj_cast_int_security s + LEFT JOIN cwj_cast_int_company c + ON CAST(s.key_txt AS INTEGER) = c.key_int + EXCEPT ALL + SELECT symbol, key_txt, company_name, rank_score, rn + FROM openivm_data_cwj_cast_int_mv +); +---- +0 + +# Simulate metadata written before lookup_cast was persisted. Because the raw +# key types differ, refresh must fail closed to a full recompute rather than +# inventing a VARCHAR comparison ('0300' must continue to match integer 300). +statement ok +UPDATE openivm_views +SET lineage_json = regexp_replace(lineage_json, ',"lookup_cast":"INTEGER"', '', 'g') +WHERE view_name = 'cwj_cast_int_mv'; + +query I +SELECT CASE WHEN + string_agg(lower(sql), ' ' ORDER BY stmt_order) LIKE + '%delete from %openivm_data_cwj_cast_int_mv%;%' + AND string_agg(lower(sql), ' ' ORDER BY stmt_order) LIKE + '%insert into %openivm_data_cwj_cast_int_mv%' + AND string_agg(lower(sql), ' ' ORDER BY stmt_order) NOT LIKE + '%cast%key_int%as varchar%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'cwj_cast_int_mv', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":false}' +) +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +UPDATE cwj_cast_int_company +SET rank_score = 14, name = 'Cedar Legacy Updated' +WHERE key_int = 300; + +statement ok +PRAGMA refresh('cwj_cast_int_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT symbol, key_txt, company_name, rank_score, rn + FROM openivm_data_cwj_cast_int_mv + EXCEPT ALL + SELECT s.symbol, + s.key_txt, + c.name AS company_name, + c.rank_score, + ROW_NUMBER() OVER ( + PARTITION BY s.symbol + ORDER BY c.rank_score DESC NULLS LAST, s.pts DESC, s.key_txt + ) AS rn + FROM cwj_cast_int_security s + LEFT JOIN cwj_cast_int_company c + ON CAST(s.key_txt AS INTEGER) = c.key_int +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT s.symbol, + s.key_txt, + c.name AS company_name, + c.rank_score, + ROW_NUMBER() OVER ( + PARTITION BY s.symbol + ORDER BY c.rank_score DESC NULLS LAST, s.pts DESC, s.key_txt + ) AS rn + FROM cwj_cast_int_security s + LEFT JOIN cwj_cast_int_company c + ON CAST(s.key_txt AS INTEGER) = c.key_int + EXCEPT ALL + SELECT symbol, key_txt, company_name, rank_score, rn + FROM openivm_data_cwj_cast_int_mv +); +---- +0 + +# Inner-join equivalence through a casted join key must not add a raw direct +# arm for the other side. Right-side changes should look up the partition key +# from the partition-side table so the affected key keeps the MV partition type. + +statement ok +CREATE TABLE cwj_inner_cast_l ( + k BIGINT, + payload VARCHAR, + pts INT +); + +statement ok +CREATE TABLE cwj_inner_cast_r ( + kstr VARCHAR, + score INT +); + +statement ok +INSERT INTO cwj_inner_cast_l VALUES + (10, 'left-a', 1), + (20, 'left-b', 2), + (30, 'left-c', 3); + +statement ok +INSERT INTO cwj_inner_cast_r VALUES + ('10', 100), + ('20', 200), + ('30', 300); + +statement ok +CREATE MATERIALIZED VIEW cwj_inner_cast_mv AS + SELECT l.k, + l.payload, + r.score, + ROW_NUMBER() OVER ( + PARTITION BY l.k + ORDER BY r.score DESC, l.pts + ) AS rn + FROM cwj_inner_cast_l l + JOIN cwj_inner_cast_r r + ON CAST(l.k AS VARCHAR) = r.kstr; + +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) LIKE + '%CAST%k%AS VARCHAR%IN (SELECT DISTINCT %kstr%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE + '%SELECT DISTINCT kstr AS k FROM %openivm_delta_cwj_inner_cast_r%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'cwj_inner_cast_mv', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":false}' +) +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +UPDATE cwj_inner_cast_r +SET score = 125 +WHERE kstr = '10'; + +statement ok +PRAGMA refresh('cwj_inner_cast_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT k, payload, score, rn + FROM openivm_data_cwj_inner_cast_mv + EXCEPT ALL + SELECT l.k, + l.payload, + r.score, + ROW_NUMBER() OVER ( + PARTITION BY l.k + ORDER BY r.score DESC, l.pts + ) AS rn + FROM cwj_inner_cast_l l + JOIN cwj_inner_cast_r r + ON CAST(l.k AS VARCHAR) = r.kstr +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.k, + l.payload, + r.score, + ROW_NUMBER() OVER ( + PARTITION BY l.k + ORDER BY r.score DESC, l.pts + ) AS rn + FROM cwj_inner_cast_l l + JOIN cwj_inner_cast_r r + ON CAST(l.k AS VARCHAR) = r.kstr + EXCEPT ALL + SELECT k, payload, score, rn + FROM openivm_data_cwj_inner_cast_mv +); +---- +0 + +# Nested CAST chains in a join key must be reproduced exactly in affected-key +# lookup SQL. Collapsing this chain to its outer VARCHAR cast makes '0300' stop +# matching '300' and leaves the window partition stale after a lookup update. +statement ok +CREATE TABLE cwj_nested_cast_left ( + grp VARCHAR, + key_txt VARCHAR, + pts INTEGER +); + +statement ok +CREATE TABLE cwj_nested_cast_right ( + key_txt VARCHAR, + score INTEGER +); + +statement ok +INSERT INTO cwj_nested_cast_left VALUES + ('nested', '0300', 1), + ('nested', '301', 2); + +statement ok +INSERT INTO cwj_nested_cast_right VALUES + ('300', 3), + ('301', 8); + +statement ok +CREATE MATERIALIZED VIEW cwj_nested_cast_mv AS + SELECT l.grp, + l.key_txt, + r.score, + ROW_NUMBER() OVER ( + PARTITION BY l.grp + ORDER BY r.score DESC NULLS LAST, l.pts, l.key_txt + ) AS rn + FROM cwj_nested_cast_left l + LEFT JOIN cwj_nested_cast_right r + ON CAST(CAST(l.key_txt AS INTEGER) AS VARCHAR) = r.key_txt; + +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) LIKE + '%CAST(CAST%key_txt%AS INTEGER) AS VARCHAR%IN (SELECT DISTINCT %key_txt%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'cwj_nested_cast_mv', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":false}' +) +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +UPDATE cwj_nested_cast_right +SET score = 12 +WHERE key_txt = '300'; + +statement ok +PRAGMA refresh('cwj_nested_cast_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT grp, key_txt, score, rn + FROM openivm_data_cwj_nested_cast_mv + EXCEPT ALL + SELECT l.grp, + l.key_txt, + r.score, + ROW_NUMBER() OVER ( + PARTITION BY l.grp + ORDER BY r.score DESC NULLS LAST, l.pts, l.key_txt + ) AS rn + FROM cwj_nested_cast_left l + LEFT JOIN cwj_nested_cast_right r + ON CAST(CAST(l.key_txt AS INTEGER) AS VARCHAR) = r.key_txt +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.grp, + l.key_txt, + r.score, + ROW_NUMBER() OVER ( + PARTITION BY l.grp + ORDER BY r.score DESC NULLS LAST, l.pts, l.key_txt + ) AS rn + FROM cwj_nested_cast_left l + LEFT JOIN cwj_nested_cast_right r + ON CAST(CAST(l.key_txt AS INTEGER) AS VARCHAR) = r.key_txt + EXCEPT ALL + SELECT grp, key_txt, score, rn + FROM openivm_data_cwj_nested_cast_mv +); +---- +0 + +# Legacy self-join lineage can have a valid direct arm and an unsafe lookup arm +# for the same physical source. The unsafe arm must invalidate the whole +# affected-key program instead of being hidden by source-level coverage. +statement ok +CREATE TABLE cwj_legacy_self ( + id INTEGER, + grp VARCHAR, + key_txt VARCHAR, + key_int INTEGER, + score INTEGER, + pts INTEGER +); + +statement ok +INSERT INTO cwj_legacy_self VALUES + (1, 'target', '0300', NULL, 0, 1), + (2, 'lookup', NULL, 300, 3, 2), + (3, 'target', '301', NULL, 0, 3), + (4, 'lookup', NULL, 301, 8, 4); + +statement ok +CREATE MATERIALIZED VIEW cwj_legacy_self_mv AS + SELECT l.id, + l.grp, + r.score, + ROW_NUMBER() OVER ( + PARTITION BY l.grp + ORDER BY r.score DESC NULLS LAST, l.pts, l.id + ) AS rn + FROM cwj_legacy_self l + LEFT JOIN cwj_legacy_self r + ON CAST(l.key_txt AS INTEGER) = r.key_int; + +statement ok +UPDATE openivm_views +SET lineage_json = regexp_replace(lineage_json, ',"lookup_cast":"INTEGER"', '', 'g') +WHERE view_name = 'cwj_legacy_self_mv'; + +query I +SELECT COUNT(*) +FROM openivm_compile_with_facts( + 'cwj_legacy_self_mv', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":false}' +) +WHERE stmt_kind = 'data' + AND lower(trim(sql)) = 'delete from openivm_data_cwj_legacy_self_mv;'; +---- +1 + +statement ok +UPDATE cwj_legacy_self SET score = 12 WHERE id = 2; + +statement ok +PRAGMA refresh('cwj_legacy_self_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT id, grp, score, rn + FROM openivm_data_cwj_legacy_self_mv + EXCEPT ALL + SELECT l.id, + l.grp, + r.score, + ROW_NUMBER() OVER ( + PARTITION BY l.grp + ORDER BY r.score DESC NULLS LAST, l.pts, l.id + ) AS rn + FROM cwj_legacy_self l + LEFT JOIN cwj_legacy_self r + ON CAST(l.key_txt AS INTEGER) = r.key_int +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.id, + l.grp, + r.score, + ROW_NUMBER() OVER ( + PARTITION BY l.grp + ORDER BY r.score DESC NULLS LAST, l.pts, l.id + ) AS rn + FROM cwj_legacy_self l + LEFT JOIN cwj_legacy_self r + ON CAST(l.key_txt AS INTEGER) = r.key_int + EXCEPT ALL + SELECT id, grp, score, rn + FROM openivm_data_cwj_legacy_self_mv +); +---- +0 + +# TRY_CAST is semantically distinct from CAST on malformed keys and must remain +# present when the affected-key lookup expression is reconstructed. +statement ok +INSERT INTO cwj_nested_cast_left VALUES ('try', 'not-an-integer', 1); + +statement ok +CREATE MATERIALIZED VIEW cwj_try_cast_mv AS + SELECT l.grp, + l.key_txt, + r.score, + ROW_NUMBER() OVER ( + PARTITION BY l.grp + ORDER BY r.score DESC NULLS LAST, l.pts, l.key_txt + ) AS rn + FROM cwj_nested_cast_left l + LEFT JOIN cwj_nested_cast_right r + ON CAST(TRY_CAST(l.key_txt AS INTEGER) AS VARCHAR) = r.key_txt; + +query I +SELECT CASE WHEN + string_agg(sql, ' ' ORDER BY stmt_order) LIKE + '%CAST(TRY_CAST%key_txt%AS INTEGER) AS VARCHAR%IN (SELECT DISTINCT %key_txt%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'cwj_try_cast_mv', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":false}' +) +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +UPDATE cwj_nested_cast_right +SET score = 14 +WHERE key_txt = '300'; + +statement ok +PRAGMA refresh('cwj_try_cast_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT grp, key_txt, score, rn + FROM openivm_data_cwj_try_cast_mv + EXCEPT ALL + SELECT l.grp, + l.key_txt, + r.score, + ROW_NUMBER() OVER ( + PARTITION BY l.grp + ORDER BY r.score DESC NULLS LAST, l.pts, l.key_txt + ) AS rn + FROM cwj_nested_cast_left l + LEFT JOIN cwj_nested_cast_right r + ON CAST(TRY_CAST(l.key_txt AS INTEGER) AS VARCHAR) = r.key_txt +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.grp, + l.key_txt, + r.score, + ROW_NUMBER() OVER ( + PARTITION BY l.grp + ORDER BY r.score DESC NULLS LAST, l.pts, l.key_txt + ) AS rn + FROM cwj_nested_cast_left l + LEFT JOIN cwj_nested_cast_right r + ON CAST(TRY_CAST(l.key_txt AS INTEGER) AS VARCHAR) = r.key_txt + EXCEPT ALL + SELECT grp, key_txt, score, rn + FROM openivm_data_cwj_try_cast_mv +); +---- +0 diff --git a/test/sql/compile_refresh.test b/test/sql/compile_refresh.test index ea235919..d65cd275 100644 --- a/test/sql/compile_refresh.test +++ b/test/sql/compile_refresh.test @@ -159,6 +159,35 @@ WHERE stmt_kind = 'data'; ---- 1 +# ========================================== +# Test 6: forced full refresh still dispatches through the full-recompute +# compiler when compile-for-cascade asks for view deltas. +# ========================================== + +statement ok +SET openivm_refresh_mode = 'full'; + +query I +SELECT CASE WHEN string_agg(lower(sql), ' ' ORDER BY stmt_order) + LIKE '%delete from openivm_data_mv_compile_only_proj;%insert into openivm_data_mv_compile_only_proj select%' + AND string_agg(lower(sql), ' ' ORDER BY stmt_order) + LIKE '%from %compile_only_proj_src%' + AND string_agg(lower(sql), ' ' ORDER BY stmt_order) + LIKE '%insert into openivm_delta_mv_compile_only_proj%' + AND string_agg(lower(sql), ' ' ORDER BY stmt_order) + NOT LIKE '%openivm_net%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'mv_compile_only_proj', + '{"target_dialect":"duckdb","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +SET openivm_refresh_mode = 'incremental'; + statement ok CREATE TABLE compile_only_left(id INT, k INT); @@ -361,7 +390,7 @@ WHERE stmt_kind = 'data'; 1 # ========================================== -# Test 7: openivm_compile_with_facts on a non-existent view fails cleanly +# Test 9: openivm_compile_with_facts on a non-existent view fails cleanly # ========================================== statement error diff --git a/test/sql/compile_refresh_asof.test b/test/sql/compile_refresh_asof.test index bbcf22db..53edbff3 100644 --- a/test/sql/compile_refresh_asof.test +++ b/test/sql/compile_refresh_asof.test @@ -1,5 +1,5 @@ # name: test/sql/compile_refresh_asof.test -# description: compile-only CURRENT_DIFF_RECOMPUTE for an ASOF view (split from compile_refresh.test). +# description: compile-only full refresh for an ASOF projection view. # group: [sql] require openivm @@ -8,8 +8,7 @@ statement ok SET openivm_files_path='__TEST_DIR__'; # ========================================== -# Test 6: CURRENT_DIFF_RECOMPUTE skips local no-op refreshes, but -# compile-only still emits the external refresh template +# Test 6: ASOF projections fall back to the full-refresh template # ========================================== statement ok @@ -29,7 +28,7 @@ INSERT INTO compile_only_asof_prices VALUES ('A', TIMESTAMP '2024-04-01 10:10:00', 110); statement ok -CREATE MATERIALIZED VIEW mv_compile_only_current_diff AS +CREATE MATERIALIZED VIEW mv_compile_only_asof_full AS SELECT t.symbol AS symbol, t.ts AS trade_ts, @@ -41,30 +40,32 @@ CREATE MATERIALIZED VIEW mv_compile_only_current_diff AS AND t.ts >= p.ts; query I -SELECT type FROM openivm_views WHERE view_name = 'mv_compile_only_current_diff'; +SELECT type FROM openivm_views WHERE view_name = 'mv_compile_only_asof_full'; ---- -10 +3 query I SELECT CASE WHEN coalesce(string_agg(lower(sql), ' ' ORDER BY stmt_order), '') - LIKE '%delete from %openivm_data_mv_compile_only_current_diff%' + LIKE '%delete from %openivm_data_mv_compile_only_asof_full%' + AND coalesce(string_agg(lower(sql), ' ' ORDER BY stmt_order), '') + LIKE '%insert into %openivm_data_mv_compile_only_asof_full%' THEN 1 ELSE 0 END FROM openivm_compile_with_facts( - 'mv_compile_only_current_diff', + 'mv_compile_only_asof_full', '{"target_dialect":"duckdb"}' ) WHERE stmt_kind = 'data'; ---- -0 +1 query I SELECT CASE WHEN string_agg(lower(sql), ' ' ORDER BY stmt_order) - LIKE '%delete from %openivm_data_mv_compile_only_current_diff%' + LIKE '%delete from %openivm_data_mv_compile_only_asof_full%' AND string_agg(lower(sql), ' ' ORDER BY stmt_order) - LIKE '%insert into %openivm_data_mv_compile_only_current_diff%' + LIKE '%insert into %openivm_data_mv_compile_only_asof_full%' THEN 1 ELSE 0 END FROM openivm_compile_with_facts( - 'mv_compile_only_current_diff', + 'mv_compile_only_asof_full', '{"target_dialect":"duckdb","compile_only":true}' ) WHERE stmt_kind = 'data'; diff --git a/test/sql/compile_spark_dialect_hardening.test b/test/sql/compile_spark_dialect_hardening.test index cdcc5c3b..246eefd3 100644 --- a/test/sql/compile_spark_dialect_hardening.test +++ b/test/sql/compile_spark_dialect_hardening.test @@ -157,3 +157,41 @@ FROM openivm_compile_with_facts( WHERE stmt_kind = 'data'; ---- 1 + +# Spark full-refresh cascades must be split-safe: emit signed old/new rows +# directly around the recompute, with no temp-backed snapshot companion. +statement ok +CREATE TABLE sph_full_src("from" INT, val INT); + +statement ok +INSERT INTO sph_full_src VALUES (1, 10), (2, 20); + +statement ok +CREATE MATERIALIZED VIEW sph_full_mv AS + SELECT "from", val FROM sph_full_src; + +statement ok +INSERT INTO sph_full_src VALUES (3, 30); + +statement ok +SET openivm_refresh_mode = 'full'; + +query I +SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%TEMP TABLE%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%INSERT INTO openivm_delta_sph_full_mv%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%SELECT `from`, `val`, -1 FROM openivm_data_sph_full_mv%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%SELECT `from`, `val`, 1 FROM openivm_data_sph_full_mv%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%DELETE FROM openivm_data_sph_full_mv%' + AND string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%INSERT INTO openivm_data_sph_full_mv%`sph_full_src`%AS `from`%' + AND string_agg(sql, ' ' ORDER BY stmt_order) NOT LIKE '%"from"%' + THEN 1 ELSE 0 END +FROM openivm_compile_with_facts( + 'sph_full_mv', + '{"target_dialect":"spark","compile_only":true,"force_view_delta_cascade":true}' +) +WHERE stmt_kind = 'data'; +---- +1 + +statement ok +SET openivm_refresh_mode = 'incremental'; diff --git a/test/sql/nonlocal_operator_recompute.test b/test/sql/nonlocal_operator_recompute.test index 94dcda4b..212c846c 100644 --- a/test/sql/nonlocal_operator_recompute.test +++ b/test/sql/nonlocal_operator_recompute.test @@ -9,7 +9,7 @@ require openivm # ========================================== -# SAMPLE is exact current/new diff recompute, not a local delta +# SAMPLE falls back to full refresh; it is not a local delta # ========================================== statement ok @@ -36,7 +36,7 @@ CREATE MATERIALIZED VIEW mv_oprec_sample AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_sample'; ---- -10 +3 statement ok INSERT INTO oprec_sample VALUES (7, 1, 70), (8, 3, 80); @@ -95,8 +95,61 @@ SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_sample_volatile'; ---- 3 +# Persisted legacy CURRENT_DIFF_RECOMPUTE metadata (type 10) maps to the +# current full-refresh strategy during refresh. + +statement ok +CREATE TABLE oprec_legacy_type10 (id INT, grp INT, v INT); + +statement ok +INSERT INTO oprec_legacy_type10 VALUES + (1, 1, 10), + (2, 1, 20), + (3, 2, 30), + (4, 2, 40); + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_legacy_type10 AS + SELECT id, grp, v + FROM oprec_legacy_type10 USING SAMPLE reservoir(3 ROWS) REPEATABLE (31); + +statement ok +UPDATE openivm_views SET type = 10 WHERE view_name = 'mv_oprec_legacy_type10'; + +statement ok +INSERT INTO oprec_legacy_type10 VALUES (5, 3, 50); + +statement ok +DELETE FROM oprec_legacy_type10 WHERE id = 2; + +statement ok +UPDATE oprec_legacy_type10 SET v = 15 WHERE id = 1; + +statement ok +PRAGMA refresh('mv_oprec_legacy_type10'); + +query I +SELECT count(*) FROM ( + SELECT id, grp, v + FROM oprec_legacy_type10 USING SAMPLE reservoir(3 ROWS) REPEATABLE (31) + EXCEPT ALL + SELECT id, grp, v FROM mv_oprec_legacy_type10 +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, grp, v FROM mv_oprec_legacy_type10 + EXCEPT ALL + SELECT id, grp, v + FROM oprec_legacy_type10 USING SAMPLE reservoir(3 ROWS) REPEATABLE (31) +); +---- +0 + # ========================================== -# POSITIONAL JOIN is exact current/new diff recompute +# POSITIONAL JOIN falls back to full refresh # ========================================== statement ok @@ -129,7 +182,7 @@ CREATE MATERIALIZED VIEW mv_oprec_positional AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_positional'; ---- -10 +3 statement ok INSERT INTO oprec_pos_l VALUES (4, 'd'); @@ -183,6 +236,212 @@ SELECT count(*) FROM ( ---- 0 +# ========================================== +# FULL_REFRESH non-local projections still emit cascade deltas +# ========================================== + +statement ok +SET openivm_cascade_refresh = 'downstream'; + +statement ok +CREATE TABLE oprec_sample_cascade (id INT, grp INT, v INT); + +statement ok +INSERT INTO oprec_sample_cascade VALUES + (1, 1, 10), + (2, 1, 20), + (3, 1, 30), + (4, 2, 40), + (5, 2, 50), + (6, 3, 60); + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_sample_cascade_parent AS + SELECT id, grp, v + FROM oprec_sample_cascade USING SAMPLE reservoir(4 ROWS) REPEATABLE (29); + +query I +SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_sample_cascade_parent'; +---- +3 + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_sample_cascade_child AS + SELECT grp, COUNT(*) AS cnt, SUM(v) AS total_v + FROM mv_oprec_sample_cascade_parent + GROUP BY grp; + +statement ok +INSERT INTO oprec_sample_cascade VALUES (7, 2, 70), (8, 4, 80); + +statement ok +DELETE FROM oprec_sample_cascade WHERE id = 2; + +statement ok +UPDATE oprec_sample_cascade SET v = 15 WHERE id = 1; + +statement ok +PRAGMA refresh('mv_oprec_sample_cascade_parent'); + +query I +SELECT count(*) FROM ( + SELECT id, grp, v + FROM oprec_sample_cascade USING SAMPLE reservoir(4 ROWS) REPEATABLE (29) + EXCEPT ALL + SELECT id, grp, v FROM mv_oprec_sample_cascade_parent +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, grp, v FROM mv_oprec_sample_cascade_parent + EXCEPT ALL + SELECT id, grp, v + FROM oprec_sample_cascade USING SAMPLE reservoir(4 ROWS) REPEATABLE (29) +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT grp, COUNT(*) AS cnt, SUM(v) AS total_v + FROM ( + SELECT id, grp, v + FROM oprec_sample_cascade USING SAMPLE reservoir(4 ROWS) REPEATABLE (29) + ) s + GROUP BY grp + EXCEPT ALL + SELECT grp, cnt, total_v FROM mv_oprec_sample_cascade_child +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT grp, cnt, total_v FROM mv_oprec_sample_cascade_child + EXCEPT ALL + SELECT grp, COUNT(*) AS cnt, SUM(v) AS total_v + FROM ( + SELECT id, grp, v + FROM oprec_sample_cascade USING SAMPLE reservoir(4 ROWS) REPEATABLE (29) + ) s + GROUP BY grp +); +---- +0 + +statement ok +CREATE TABLE oprec_pos_cascade_l (id INT, lval INT); + +statement ok +CREATE TABLE oprec_pos_cascade_r (rid INT, rval INT); + +statement ok +INSERT INTO oprec_pos_cascade_l VALUES (1, 10), (2, 20), (3, 30); + +statement ok +INSERT INTO oprec_pos_cascade_r VALUES (10, 100), (20, 200); + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_pos_cascade_parent AS + SELECT + l.id AS id, + l.lval AS lval, + r.rid AS rid, + r.rval AS rval, + coalesce(l.id, 0) + coalesce(r.rid, 0) AS id_score, + coalesce(l.lval, 0) + coalesce(r.rval, 0) AS val_score + FROM oprec_pos_cascade_l l + POSITIONAL JOIN oprec_pos_cascade_r r; + +query I +SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_pos_cascade_parent'; +---- +3 + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_pos_cascade_child AS + SELECT COUNT(*) AS cnt, SUM(id_score) AS total_ids, SUM(val_score) AS total_vals + FROM mv_oprec_pos_cascade_parent; + +statement ok +INSERT INTO oprec_pos_cascade_l VALUES (4, 40); + +statement ok +DELETE FROM oprec_pos_cascade_r WHERE rid = 20; + +statement ok +UPDATE oprec_pos_cascade_l SET lval = 15 WHERE id = 1; + +statement ok +PRAGMA refresh('mv_oprec_pos_cascade_parent'); + +query I +SELECT count(*) FROM ( + SELECT + l.id AS id, + l.lval AS lval, + r.rid AS rid, + r.rval AS rval, + coalesce(l.id, 0) + coalesce(r.rid, 0) AS id_score, + coalesce(l.lval, 0) + coalesce(r.rval, 0) AS val_score + FROM oprec_pos_cascade_l l + POSITIONAL JOIN oprec_pos_cascade_r r + EXCEPT ALL + SELECT id, lval, rid, rval, id_score, val_score FROM mv_oprec_pos_cascade_parent +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, lval, rid, rval, id_score, val_score FROM mv_oprec_pos_cascade_parent + EXCEPT ALL + SELECT + l.id AS id, + l.lval AS lval, + r.rid AS rid, + r.rval AS rval, + coalesce(l.id, 0) + coalesce(r.rid, 0) AS id_score, + coalesce(l.lval, 0) + coalesce(r.rval, 0) AS val_score + FROM oprec_pos_cascade_l l + POSITIONAL JOIN oprec_pos_cascade_r r +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT + COUNT(*) AS cnt, + SUM(coalesce(l.id, 0) + coalesce(r.rid, 0)) AS total_ids, + SUM(coalesce(l.lval, 0) + coalesce(r.rval, 0)) AS total_vals + FROM oprec_pos_cascade_l l + POSITIONAL JOIN oprec_pos_cascade_r r + EXCEPT ALL + SELECT cnt, total_ids, total_vals FROM mv_oprec_pos_cascade_child +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT cnt, total_ids, total_vals FROM mv_oprec_pos_cascade_child + EXCEPT ALL + SELECT + COUNT(*) AS cnt, + SUM(coalesce(l.id, 0) + coalesce(r.rid, 0)) AS total_ids, + SUM(coalesce(l.lval, 0) + coalesce(r.rval, 0)) AS total_vals + FROM oprec_pos_cascade_l l + POSITIONAL JOIN oprec_pos_cascade_r r +); +---- +0 + +statement ok +SET openivm_cascade_refresh = 'off'; + # ========================================== # Hidden non-local operators inside materialized CTEs are still classified conservatively # ========================================== @@ -214,7 +473,7 @@ CREATE MATERIALIZED VIEW mv_oprec_cte_sample AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_cte_sample'; ---- -10 +3 statement ok INSERT INTO oprec_cte_sample VALUES (7, 2, 70), (8, 4, 80); @@ -286,7 +545,7 @@ CREATE MATERIALIZED VIEW mv_oprec_cte_positional AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_cte_positional'; ---- -10 +3 statement ok INSERT INTO oprec_cte_pos_l VALUES (4, 40); diff --git a/test/sql/nonlocal_operator_recompute_asof.test b/test/sql/nonlocal_operator_recompute_asof.test index 91addd50..957f47a3 100644 --- a/test/sql/nonlocal_operator_recompute_asof.test +++ b/test/sql/nonlocal_operator_recompute_asof.test @@ -5,7 +5,7 @@ require openivm # ========================================== -# ASOF projections use exact current/new diff recompute +# ASOF projections fall back to full refresh # ========================================== statement ok @@ -42,7 +42,7 @@ CREATE MATERIALIZED VIEW mv_oprec_asof_projection AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_asof_projection'; ---- -10 +3 statement ok INSERT INTO oprec_asof_prices VALUES ('A', TIMESTAMP '2024-01-01 10:19:00', 190); @@ -95,6 +95,151 @@ SELECT count(*) FROM ( ---- 0 +# FULL_REFRESH ASOF projections still emit signed deltas for downstream cascade. + +statement ok +SET openivm_cascade_refresh = 'downstream'; + +statement ok +CREATE TABLE oprec_asof_cascade_trades (symbol VARCHAR, ts TIMESTAMP, qty INT); + +statement ok +CREATE TABLE oprec_asof_cascade_prices (symbol VARCHAR, ts TIMESTAMP, price INT); + +statement ok +INSERT INTO oprec_asof_cascade_trades VALUES + ('A', TIMESTAMP '2024-01-02 10:00:00', 1), + ('A', TIMESTAMP '2024-01-02 10:20:00', 2), + ('B', TIMESTAMP '2024-01-02 10:15:00', 3); + +statement ok +INSERT INTO oprec_asof_cascade_prices VALUES + ('A', TIMESTAMP '2024-01-02 09:00:00', 100), + ('A', TIMESTAMP '2024-01-02 10:18:00', 180), + ('B', TIMESTAMP '2024-01-02 10:12:00', 120); + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_asof_cascade_parent AS + SELECT + t.symbol AS symbol, + t.ts AS trade_ts, + t.qty AS qty, + p.ts AS price_ts, + p.price AS price + FROM oprec_asof_cascade_trades t + ASOF JOIN oprec_asof_cascade_prices p + ON t.symbol = p.symbol + AND t.ts >= p.ts; + +query I +SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_asof_cascade_parent'; +---- +3 + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_asof_cascade_child AS + SELECT symbol, COUNT(*) AS cnt, SUM(qty * price) AS total_value + FROM mv_oprec_asof_cascade_parent + GROUP BY symbol; + +statement ok +INSERT INTO oprec_asof_cascade_prices VALUES ('A', TIMESTAMP '2024-01-02 10:19:00', 190); + +statement ok +INSERT INTO oprec_asof_cascade_trades VALUES ('A', TIMESTAMP '2024-01-02 10:21:00', 4); + +statement ok +DELETE FROM oprec_asof_cascade_prices WHERE symbol = 'B' AND ts = TIMESTAMP '2024-01-02 10:12:00'; + +statement ok +UPDATE oprec_asof_cascade_trades SET qty = 6 WHERE symbol = 'A' AND ts = TIMESTAMP '2024-01-02 10:00:00'; + +statement ok +PRAGMA refresh('mv_oprec_asof_cascade_parent'); + +query I +SELECT count(*) FROM ( + SELECT + t.symbol AS symbol, + t.ts AS trade_ts, + t.qty AS qty, + p.ts AS price_ts, + p.price AS price + FROM oprec_asof_cascade_trades t + ASOF JOIN oprec_asof_cascade_prices p + ON t.symbol = p.symbol + AND t.ts >= p.ts + EXCEPT ALL + SELECT symbol, trade_ts, qty, price_ts, price FROM mv_oprec_asof_cascade_parent +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT symbol, trade_ts, qty, price_ts, price FROM mv_oprec_asof_cascade_parent + EXCEPT ALL + SELECT + t.symbol AS symbol, + t.ts AS trade_ts, + t.qty AS qty, + p.ts AS price_ts, + p.price AS price + FROM oprec_asof_cascade_trades t + ASOF JOIN oprec_asof_cascade_prices p + ON t.symbol = p.symbol + AND t.ts >= p.ts +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT symbol, COUNT(*) AS cnt, SUM(qty * price) AS total_value + FROM ( + SELECT + t.symbol AS symbol, + t.ts AS trade_ts, + t.qty AS qty, + p.ts AS price_ts, + p.price AS price + FROM oprec_asof_cascade_trades t + ASOF JOIN oprec_asof_cascade_prices p + ON t.symbol = p.symbol + AND t.ts >= p.ts + ) q + GROUP BY symbol + EXCEPT ALL + SELECT symbol, cnt, total_value FROM mv_oprec_asof_cascade_child +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT symbol, cnt, total_value FROM mv_oprec_asof_cascade_child + EXCEPT ALL + SELECT symbol, COUNT(*) AS cnt, SUM(qty * price) AS total_value + FROM ( + SELECT + t.symbol AS symbol, + t.ts AS trade_ts, + t.qty AS qty, + p.ts AS price_ts, + p.price AS price + FROM oprec_asof_cascade_trades t + ASOF JOIN oprec_asof_cascade_prices p + ON t.symbol = p.symbol + AND t.ts >= p.ts + ) q + GROUP BY symbol +); +---- +0 + +statement ok +SET openivm_cascade_refresh = 'off'; + # Grouped ASOF outputs use group recompute with current/new affected keys. statement ok @@ -285,8 +430,9 @@ SELECT count(*) FROM ( ---- 0 -# ASOF windows whose partition key comes from the RHS match must use current/new diff: -# a newer RHS row can move existing left rows out of the old matched partition. +# ASOF windows whose partition key comes from the RHS match fall back to full +# refresh: a newer RHS row can move existing left rows out of the old matched +# partition, so partition-local recompute is not safe. statement ok CREATE TABLE oprec_asof_rhs_win_trades (symbol VARCHAR, ts TIMESTAMP, qty INT); @@ -329,7 +475,7 @@ CREATE MATERIALIZED VIEW mv_oprec_asof_rhs_window AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_asof_rhs_window'; ---- -10 +3 statement ok INSERT INTO oprec_asof_rhs_win_prices VALUES ('A', TIMESTAMP '2024-02-02 10:15:00', 150); diff --git a/test/sql/transactional_lifecycle.test b/test/sql/transactional_lifecycle.test new file mode 100644 index 00000000..18cc7680 --- /dev/null +++ b/test/sql/transactional_lifecycle.test @@ -0,0 +1,421 @@ +# name: test/sql/transactional_lifecycle.test +# description: Materialized-view lifecycle operations participate in the caller transaction +# group: [sql] + +require openivm + +require parquet + +statement ok +INSTALL ducklake; + +statement ok +LOAD ducklake; + +load __TEST_DIR__/openivm_transactional_lifecycle.db + +# CREATE rollback removes every object created for the MV. +statement ok +CREATE TABLE lifecycle_create_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_create_src VALUES (1, 10); + +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE MATERIALIZED VIEW lifecycle_create_mv AS +SELECT id, v FROM lifecycle_create_src; + +statement ok +ROLLBACK; + +query IIII +SELECT + (SELECT count(*) FROM duckdb_views() WHERE view_name = 'lifecycle_create_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_data_lifecycle_create_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_delta_lifecycle_create_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_views'); +---- +0 0 0 0 + +# REPLACE rollback restores the old definition, data, and refresh metadata. +statement ok +CREATE TABLE lifecycle_replace_src (id INTEGER, old_v INTEGER, new_v INTEGER); + +statement ok +INSERT INTO lifecycle_replace_src VALUES (1, 10, 100); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_replace_mv REFRESH EVERY '5 minutes' AS +SELECT id, old_v AS v FROM lifecycle_replace_src; + +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE OR REPLACE MATERIALIZED VIEW lifecycle_replace_mv REFRESH EVERY '10 minutes' AS +SELECT id, new_v AS v FROM lifecycle_replace_src; + +statement ok +ROLLBACK; + +query II +SELECT id, v FROM lifecycle_replace_mv; +---- +1 10 + +query I +SELECT refresh_interval FROM openivm_views WHERE view_name = 'lifecycle_replace_mv'; +---- +300 + +# A replacement that fails during initial materialization also preserves the old MV. +statement ok +CREATE TABLE lifecycle_failed_replace_src (id INTEGER, txt VARCHAR); + +statement ok +INSERT INTO lifecycle_failed_replace_src VALUES (1, 'not-an-integer'); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_failed_replace_mv AS +SELECT id, txt FROM lifecycle_failed_replace_src; + +statement error +CREATE OR REPLACE MATERIALIZED VIEW lifecycle_failed_replace_mv AS +SELECT id, CAST(txt AS INTEGER) AS txt FROM lifecycle_failed_replace_src; +---- +Could not convert + +query II +SELECT id, txt FROM lifecycle_failed_replace_mv; +---- +1 not-an-integer + +# ALTER rollback leaves the previous schedule unchanged. +statement ok +BEGIN TRANSACTION; + +statement ok +ALTER MATERIALIZED VIEW lifecycle_replace_mv SET REFRESH EVERY '30 minutes'; + +statement ok +ROLLBACK; + +query I +SELECT refresh_interval FROM openivm_views WHERE view_name = 'lifecycle_replace_mv'; +---- +300 + +# DROP rollback restores the user view, backing state, metadata, and refreshability. +statement ok +CREATE TABLE lifecycle_drop_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_drop_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_drop_mv AS +SELECT id, v FROM lifecycle_drop_src; + +statement ok +BEGIN TRANSACTION; + +statement ok +DROP VIEW lifecycle_drop_mv; + +statement ok +ROLLBACK; + +query III +SELECT + (SELECT count(*) FROM duckdb_views() WHERE view_name = 'lifecycle_drop_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_data_lifecycle_drop_mv'), + (SELECT count(*) FROM openivm_views WHERE view_name = 'lifecycle_drop_mv'); +---- +1 1 1 + +statement ok +INSERT INTO lifecycle_drop_src VALUES (2, 20); + +statement ok +PRAGMA refresh('lifecycle_drop_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_drop_mv + EXCEPT ALL + SELECT * FROM lifecycle_drop_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_drop_src + EXCEPT ALL + SELECT * FROM lifecycle_drop_mv +); +---- +0 + +# Refresh rollback restores the MV and leaves its source delta available for retry. +statement ok +CREATE TABLE lifecycle_refresh_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_refresh_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_refresh_mv AS +SELECT id, v FROM lifecycle_refresh_src; + +statement ok +INSERT INTO lifecycle_refresh_src VALUES (2, 20); + +statement ok +BEGIN TRANSACTION; + +statement ok +PRAGMA refresh('lifecycle_refresh_mv'); + +query II +SELECT id, v FROM lifecycle_refresh_mv ORDER BY id; +---- +1 10 +2 20 + +statement ok +ROLLBACK; + +query II +SELECT id, v FROM lifecycle_refresh_mv ORDER BY id; +---- +1 10 + +query I +SELECT count(*) FROM openivm_delta_lifecycle_refresh_src; +---- +1 + +statement ok +PRAGMA refresh('lifecycle_refresh_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_refresh_mv + EXCEPT ALL + SELECT * FROM lifecycle_refresh_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_refresh_src + EXCEPT ALL + SELECT * FROM lifecycle_refresh_mv +); +---- +0 + +# Transaction-local DML is visible to refresh and rolls back together. +statement ok +CREATE TABLE lifecycle_local_refresh_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_local_refresh_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_local_refresh_mv AS +SELECT id, v FROM lifecycle_local_refresh_src; + +statement ok +BEGIN TRANSACTION; + +statement ok +INSERT INTO lifecycle_local_refresh_src VALUES (2, 20); + +statement ok +PRAGMA refresh('lifecycle_local_refresh_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_local_refresh_mv + EXCEPT ALL + SELECT * FROM lifecycle_local_refresh_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_local_refresh_src + EXCEPT ALL + SELECT * FROM lifecycle_local_refresh_mv +); +---- +0 + +statement ok +ROLLBACK; + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_local_refresh_mv + EXCEPT ALL + SELECT * FROM lifecycle_local_refresh_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_local_refresh_src + EXCEPT ALL + SELECT * FROM lifecycle_local_refresh_mv +); +---- +0 + +# A refresh keeps its OpenIVM locks until the surrounding transaction ends. +# The second connection starts while the first transaction is sleeping after +# refresh and must wait for rollback before it can consume the restored delta. +statement ok +CREATE TABLE lifecycle_lock_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_lock_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_lock_mv AS +SELECT id, v FROM lifecycle_lock_src; + +statement ok +INSERT INTO lifecycle_lock_src VALUES (2, 20); + +statement ok +CREATE TABLE lifecycle_lock_timing (phase VARCHAR, observed_at TIMESTAMP); + +concurrentloop threadid 0 2 + +onlyif threadid=0 +statement ok +BEGIN TRANSACTION; + +onlyif threadid=0 +statement ok +PRAGMA refresh('lifecycle_lock_mv'); + +onlyif threadid=0 +statement ok +SELECT sleep_ms(1200); + +onlyif threadid=0 +statement ok +ROLLBACK; + +onlyif threadid=1 +statement ok +SELECT sleep_ms(200); + +onlyif threadid=1 +statement ok +INSERT INTO lifecycle_lock_timing VALUES ('before', now()::TIMESTAMP); + +onlyif threadid=1 +statement ok +PRAGMA refresh('lifecycle_lock_mv'); + +onlyif threadid=1 +statement ok +INSERT INTO lifecycle_lock_timing VALUES ('after', now()::TIMESTAMP); + +endloop + +query I +SELECT epoch_ms(max(observed_at)) - epoch_ms(min(observed_at)) >= 700 +FROM lifecycle_lock_timing; +---- +true + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_lock_mv + EXCEPT ALL + SELECT * FROM lifecycle_lock_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_lock_src + EXCEPT ALL + SELECT * FROM lifecycle_lock_mv +); +---- +0 + +# DuckLake cannot share the native metadata transaction. Its replacement data +# is therefore built under an unpublished staging name and published only after +# materialization succeeds. +statement ok +ATTACH '__TEST_DIR__/openivm_transactional_lifecycle.ducklake' AS lifecycle_dl (TYPE ducklake); + +statement ok +CREATE TABLE lifecycle_dl.replace_src (id INTEGER, old_v INTEGER, txt VARCHAR); + +statement ok +INSERT INTO lifecycle_dl.replace_src VALUES (1, 10, 'bad'); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_dl.replace_mv AS +SELECT id, old_v AS v FROM lifecycle_dl.replace_src; + +statement error +CREATE OR REPLACE MATERIALIZED VIEW lifecycle_dl.replace_mv AS +SELECT id, CAST(txt AS INTEGER) AS v FROM lifecycle_dl.replace_src; +---- +Could not convert + +query II +SELECT id, v FROM lifecycle_dl.replace_mv; +---- +1 10 + +query I +SELECT count(*) FROM duckdb_tables() +WHERE database_name = 'lifecycle_dl' + AND table_name = 'openivm_stage_replace_mv'; +---- +0 + +statement ok +CREATE OR REPLACE MATERIALIZED VIEW lifecycle_dl.replace_mv AS +SELECT id, length(txt) AS v FROM lifecycle_dl.replace_src; + +statement ok +INSERT INTO lifecycle_dl.replace_src VALUES (2, 20, 'four'); + +statement ok +PRAGMA refresh('replace_mv'); + +query I +SELECT count(*) FROM ( + SELECT id, v FROM lifecycle_dl.replace_mv + EXCEPT ALL + SELECT id, length(txt) AS v FROM lifecycle_dl.replace_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, length(txt) AS v FROM lifecycle_dl.replace_src + EXCEPT ALL + SELECT id, v FROM lifecycle_dl.replace_mv +); +---- +0 From a059f61a76aff4c09e42aab796ac44785f28c51c Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 23 Jul 2026 10:48:57 +0200 Subject: [PATCH 06/36] Finish transactional MV lifecycle --- docs/codebase-audit-2026-07-22.md | 10 ++- src/core/parser.cpp | 15 +++- src/core/parser_ddl.cpp | 97 +++++++++++++++++++++-- src/core/refresh_locks.cpp | 92 ++++++++++++++------- src/include/core/parser_ddl.hpp | 2 +- src/include/core/refresh_locks.hpp | 42 ++++++---- src/rules/transactional_delta_capture.cpp | 14 ++-- src/upsert/refresh.cpp | 40 ++++++++-- 8 files changed, 242 insertions(+), 70 deletions(-) diff --git a/docs/codebase-audit-2026-07-22.md b/docs/codebase-audit-2026-07-22.md index df6a979e..2c7d583a 100644 --- a/docs/codebase-audit-2026-07-22.md +++ b/docs/codebase-audit-2026-07-22.md @@ -19,10 +19,12 @@ maintenance path. A bug must not be hidden by weakening a test or silently chang decorator now captures only the INSERT, UPDATE, and DELETE rows selected by DuckDB after match and action predicates are resolved. Updates emit the required delete-old/insert-new pair, volatile defaults are evaluated once, `RETURNING` is preserved, and all writes remain in the base DML transaction. The same boundary also covers native `MERGE` actions. -- [ ] **Make MV lifecycle and native refresh transactional.** CREATE, REPLACE, DROP, ALTER, and `PRAGMA refresh` currently - execute material work through helper autocommit connections. Caller rollback cannot restore a consistent collection of - user view, backing table, metadata, and deltas; failed replacement can destroy the old MV. Use the caller transaction for - native-catalog work and stage cross-catalog replacement before publication. +- [x] **Make MV lifecycle and native refresh transactional.** Native CREATE, REPLACE, DROP, ALTER, and refresh inside an + explicit caller transaction now mutate the user view, backing state, metadata, and deltas atomically and retain their + OpenIVM locks through commit or rollback. Autocommit refresh temporarily retains the established locked executor because + DuckDB query-pragma preprocessing ends a transaction before its returned program completes; `refresh.cpp` records the + native-operator follow-up. DuckLake replacement materializes data and auxiliary state under unpublished staging names + before publishing them. - [ ] **Continue cascade traversal when the current node has no source deltas.** The root empty-delta fast path returns before visiting downstream nodes. A child can have independent source deltas or pending parent-delta rows after an earlier failed cascade. Model `node skipped` separately from `graph traversal complete` and checkpoint each node independently. diff --git a/src/core/parser.cpp b/src/core/parser.cpp index dbe03bdc..7cb3a5b7 100644 --- a/src/core/parser.cpp +++ b/src/core/parser.cpp @@ -379,6 +379,12 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC select_planner.CreatePlan(std::move(select_parser.statements[0])); auto select_plan = std::move(select_planner.plan); visible_output_count = select_planner.names.size(); + for (auto &name : select_planner.names) { + if (StringUtil::CIEquals(name, openivm::MULTIPLICITY_COL) || + StringUtil::CIEquals(name, openivm::TIMESTAMP_COL)) { + throw BinderException("Materialized-view output uses reserved OpenIVM column '%s'", name); + } + } add_create_profile_step("create_compile_select_plan", select_parse_plan_start); // Inline CTEs without running the full optimizer, which can reshape plans @@ -1536,9 +1542,11 @@ string MaterializedViewLifecycleQuery(ClientContext &context, const FunctionPara MaterializedViewParserExtension::PlanFunction(nullptr, context, std::move(parse_result.parse_data)); if (plan_result.function.name == OPENIVM_TRANSACTIONAL_DDL_FUNCTION) { if (!lock_view_name.empty()) { - TransactionalMVLockState::Get(context).Acquire({lock_view_name}, {}); + auto &default_entry = ClientData::Get(context).catalog_search_path->GetDefault(); + auto &catalog = Catalog::GetCatalog(context, default_entry.catalog); + TransactionalMVLockState::Get(context).Acquire({lock_view_name}, {}, {&catalog}); } - return RenderTransactionalDDL(plan_result.parameters); + return RenderTransactionalDDL(context, plan_result.parameters); } if (!lock_view_name.empty()) { ViewLockGuard view_guard(lock_view_name); @@ -1623,7 +1631,8 @@ string MaterializedViewDropQuery(ClientContext &context, const FunctionParameter } auto view_delta_table = SqlUtils::DeltaName(drop.info->name); delta_tables.push_back(view_delta_table); - TransactionalMVLockState::Get(context).Acquire({drop.info->name}, delta_tables); + auto &catalog = Catalog::GetCatalog(context, catalog_name); + TransactionalMVLockState::Get(context).Acquire({drop.info->name}, delta_tables, {&catalog}); return program; } } // namespace duckdb diff --git a/src/core/parser_ddl.cpp b/src/core/parser_ddl.cpp index 499dd75a..8611bccd 100644 --- a/src/core/parser_ddl.cpp +++ b/src/core/parser_ddl.cpp @@ -461,9 +461,23 @@ void ExecuteDropView(ClientContext &context, TableFunctionInput &input, DataChun state.finished = true; } -string RenderTransactionalDDL(const vector ¶meters) { +string RenderTransactionalDDL(ClientContext &context, const vector ¶meters) { + struct ProfileRow { + string view_name; + string step_name; + int64_t duration_ms; + string detail; + }; + struct PendingProfileMarker { + string view_name; + string step_name; + string detail; + idx_t statement_start; + }; + string sql; - auto append_statement = [&](const string &statement) { + idx_t logical_statement_count = 0; + auto append_statement = [&](const string &statement, bool logical_statement = true) { if (statement.empty()) { return; } @@ -472,12 +486,57 @@ string RenderTransactionalDDL(const vector ¶meters) { sql += ";"; } sql += "\n"; + if (logical_statement) { + logical_statement_count++; + } + }; + Value profile_value; + bool profile_enabled = context.TryGetCurrentSetting("openivm_profile_refresh", profile_value) && + !profile_value.IsNull() && BooleanValue::Get(profile_value); + vector profile_rows; + unique_ptr pending_profile; + auto finish_profile_marker = [&]() { + if (!pending_profile) { + return; + } + auto statement_count = logical_statement_count - pending_profile->statement_start; + auto detail = pending_profile->detail; + if (!detail.empty()) { + detail += "; "; + } + detail += + "statements=" + to_string(statement_count) + "; transactional_program=true; duration_not_measured=true"; + profile_rows.push_back({pending_profile->view_name, pending_profile->step_name, 0, std::move(detail)}); + pending_profile.reset(); }; for (auto ¶meter : parameters) { auto statement = parameter.GetValue(); - if (statement.empty() || StringUtil::StartsWith(statement, OPENIVM_DDL_CLEANUP_PREFIX) || - StringUtil::StartsWith(statement, OPENIVM_DDL_PROFILE_PREFIX) || - StringUtil::StartsWith(statement, OPENIVM_DDL_PROFILE_RECORD_PREFIX)) { + if (statement.empty() || StringUtil::StartsWith(statement, OPENIVM_DDL_CLEANUP_PREFIX)) { + continue; + } + if (StringUtil::StartsWith(statement, OPENIVM_DDL_PROFILE_RECORD_PREFIX)) { + if (profile_enabled) { + string view_name; + string step_name; + string detail; + int64_t duration_ms = 0; + ParseCreateMVProfileRecord(statement.substr(strlen(OPENIVM_DDL_PROFILE_RECORD_PREFIX)), view_name, + step_name, duration_ms, detail); + profile_rows.push_back({std::move(view_name), std::move(step_name), duration_ms, std::move(detail)}); + } + continue; + } + if (StringUtil::StartsWith(statement, OPENIVM_DDL_PROFILE_PREFIX)) { + if (profile_enabled) { + finish_profile_marker(); + string view_name; + string step_name; + string detail; + ParseCreateMVProfileMarker(statement.substr(strlen(OPENIVM_DDL_PROFILE_PREFIX)), view_name, step_name, + detail); + pending_profile = make_uniq(PendingProfileMarker { + std::move(view_name), std::move(step_name), std::move(detail), logical_statement_count}); + } continue; } if (StringUtil::StartsWith(statement, OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX)) { @@ -493,10 +552,11 @@ string RenderTransactionalDDL(const vector ¶meters) { " AS SELECT *, 1::INTEGER AS " + string(openivm::MULTIPLICITY_COL) + ", now()::TIMESTAMP AS " + string(openivm::TIMESTAMP_COL) + " FROM " + data_table + " LIMIT 0"); - append_statement("ALTER TABLE " + delta_table + " ALTER " + string(openivm::MULTIPLICITY_COL) + - " SET DEFAULT 1"); + append_statement( + "ALTER TABLE " + delta_table + " ALTER " + string(openivm::MULTIPLICITY_COL) + " SET DEFAULT 1", false); append_statement("ALTER TABLE " + delta_table + " ALTER " + string(openivm::TIMESTAMP_COL) + - " SET DEFAULT now()"); + " SET DEFAULT now()", + false); continue; } @@ -516,6 +576,27 @@ string RenderTransactionalDDL(const vector ¶meters) { } append_statement(statement); } + if (profile_enabled) { + finish_profile_marker(); + if (!profile_rows.empty()) { + auto view_name = profile_rows.front().view_name; + profile_rows.push_back( + {view_name, "create_mv_total", 0, "transactional_program=true; duration_not_measured=true"}); + auto now = std::chrono::steady_clock::now().time_since_epoch(); + auto refresh_id = view_name + "_create_tx_" + + to_string(std::chrono::duration_cast(now).count()); + for (idx_t step_order = 0; step_order < profile_rows.size(); step_order++) { + auto &row = profile_rows[step_order]; + append_statement("INSERT OR REPLACE INTO " + string(openivm::PROFILE_TABLE) + + " (refresh_id, view_name, step_order, step_name, duration_ms, detail) VALUES ('" + + SqlUtils::EscapeValue(refresh_id) + "', '" + SqlUtils::EscapeValue(row.view_name) + + "', " + to_string(step_order) + ", '" + SqlUtils::EscapeValue(row.step_name) + + "', " + to_string(row.duration_ms) + ", '" + SqlUtils::EscapeValue(row.detail) + + "')", + false); + } + } + } append_statement("SELECT true AS \"MATERIALIZED VIEW CREATION\""); return sql; } diff --git a/src/core/refresh_locks.cpp b/src/core/refresh_locks.cpp index 79ee855b..f6a91a2c 100644 --- a/src/core/refresh_locks.cpp +++ b/src/core/refresh_locks.cpp @@ -7,38 +7,49 @@ std::unordered_map> RefreshLocks::view_mutexes_; std::unordered_map> RefreshLocks::delta_mutexes_; std::unordered_map> RefreshLocks::delta_catalog_gates_; -void DeltaCatalogPhaseGate::EnterWrite() { +static bool HasOtherOwner(const unordered_map &owners, ClientContext &owner) { + return !owners.empty() && (owners.size() > 1 || owners.find(&owner) == owners.end()); +} + +void DeltaCatalogPhaseGate::EnterWrite(ClientContext &owner) { std::unique_lock guard(lock); // Once a refresh is waiting, stop admitting new writers so the finite set of - // active transactions can drain. Writers otherwise run concurrently. - condition.wait(guard, [&]() { return active_refreshes == 0 && waiting_refreshes == 0; }); - active_writers++; + // active transactions can drain. The refresh owner's own later DML remains + // reentrant, which lets transaction-local DML and refresh compose safely. + condition.wait(guard, [&]() { + bool owner_is_active = active_writers.find(&owner) != active_writers.end() || + active_refreshes.find(&owner) != active_refreshes.end(); + return !HasOtherOwner(active_refreshes, owner) && (waiting_refreshes == 0 || owner_is_active); + }); + active_writers[&owner]++; } -void DeltaCatalogPhaseGate::ExitWrite() { +void DeltaCatalogPhaseGate::ExitWrite(ClientContext &owner) { lock_guard guard(lock); - D_ASSERT(active_writers > 0); - active_writers--; - if (active_writers == 0) { - condition.notify_all(); + auto entry = active_writers.find(&owner); + D_ASSERT(entry != active_writers.end() && entry->second > 0); + if (--entry->second == 0) { + active_writers.erase(entry); } + condition.notify_all(); } -void DeltaCatalogPhaseGate::EnterRefresh() { +void DeltaCatalogPhaseGate::EnterRefresh(ClientContext &owner) { std::unique_lock guard(lock); waiting_refreshes++; - condition.wait(guard, [&]() { return active_writers == 0; }); + condition.wait(guard, [&]() { return !HasOtherOwner(active_writers, owner); }); waiting_refreshes--; - active_refreshes++; + active_refreshes[&owner]++; } -void DeltaCatalogPhaseGate::ExitRefresh() { +void DeltaCatalogPhaseGate::ExitRefresh(ClientContext &owner) { lock_guard guard(lock); - D_ASSERT(active_refreshes > 0); - active_refreshes--; - if (active_refreshes == 0) { - condition.notify_all(); + auto entry = active_refreshes.find(&owner); + D_ASSERT(entry != active_refreshes.end() && entry->second > 0); + if (--entry->second == 0) { + active_refreshes.erase(entry); } + condition.notify_all(); } std::mutex &RefreshLocks::GetViewMutex(const string &view_name) { @@ -88,27 +99,52 @@ void RefreshLocks::UnlockDelta(const string &delta_table_name) { GetDeltaMutex(delta_table_name).unlock(); } -void RefreshLocks::EnterDeltaWrite(Catalog &catalog) { - GetDeltaCatalogGate(catalog).EnterWrite(); +void RefreshLocks::EnterDeltaWrite(ClientContext &owner, Catalog &catalog) { + GetDeltaCatalogGate(catalog).EnterWrite(owner); } -void RefreshLocks::ExitDeltaWrite(Catalog &catalog) { - GetDeltaCatalogGate(catalog).ExitWrite(); +void RefreshLocks::ExitDeltaWrite(ClientContext &owner, Catalog &catalog) { + GetDeltaCatalogGate(catalog).ExitWrite(owner); } -void RefreshLocks::EnterDeltaRefresh(Catalog &catalog) { - GetDeltaCatalogGate(catalog).EnterRefresh(); +void RefreshLocks::EnterDeltaRefresh(ClientContext &owner, Catalog &catalog) { + GetDeltaCatalogGate(catalog).EnterRefresh(owner); } -void RefreshLocks::ExitDeltaRefresh(Catalog &catalog) { - GetDeltaCatalogGate(catalog).ExitRefresh(); +void RefreshLocks::ExitDeltaRefresh(ClientContext &owner, Catalog &catalog) { + GetDeltaCatalogGate(catalog).ExitRefresh(owner); } TransactionalMVLockState &TransactionalMVLockState::Get(ClientContext &context) { - return *context.registered_state->GetOrCreate("openivm_transactional_mv_locks"); + auto state = context.registered_state->GetOrCreate("openivm_transactional_mv_locks"); + state->owner = &context; + return *state; +} + +optional_ptr TransactionalMVLockState::TryGet(ClientContext &context) { + return context.registered_state->Get("openivm_transactional_mv_locks"); +} + +bool TransactionalMVLockState::OwnsRefreshDelta(Catalog &catalog, const string &delta_table_name) const { + return locked_catalogs.find(&catalog) != locked_catalogs.end() && + locked_delta_tables.find(delta_table_name) != locked_delta_tables.end(); } -void TransactionalMVLockState::Acquire(const vector &view_names, const vector &delta_table_names) { +void TransactionalMVLockState::Acquire(const vector &view_names, const vector &delta_table_names, + const vector &source_catalogs) { + if (!owner) { + throw InternalException("OpenIVM transactional lock state has no owning client context"); + } + auto sorted_catalogs = source_catalogs; + std::sort(sorted_catalogs.begin(), sorted_catalogs.end(), + [](Catalog *left, Catalog *right) { return left->GetName() < right->GetName(); }); + sorted_catalogs.erase(std::unique(sorted_catalogs.begin(), sorted_catalogs.end()), sorted_catalogs.end()); + for (auto catalog : sorted_catalogs) { + if (locked_catalogs.insert(catalog).second) { + catalog_guards.push_back(make_uniq(*owner, *catalog)); + } + } + auto sorted_views = view_names; std::sort(sorted_views.begin(), sorted_views.end()); sorted_views.erase(std::unique(sorted_views.begin(), sorted_views.end()), sorted_views.end()); @@ -139,8 +175,10 @@ void TransactionalMVLockState::TransactionRollback(MetaTransaction &transaction, void TransactionalMVLockState::Release() { delta_guards.clear(); view_guards.clear(); + catalog_guards.clear(); locked_delta_tables.clear(); locked_views.clear(); + locked_catalogs.clear(); } } // namespace duckdb diff --git a/src/include/core/parser_ddl.hpp b/src/include/core/parser_ddl.hpp index af11bf18..19dc8a3b 100644 --- a/src/include/core/parser_ddl.hpp +++ b/src/include/core/parser_ddl.hpp @@ -19,7 +19,7 @@ enum class DDLExecutionMode : uint8_t { CALLER_TRANSACTION, STAGED_CROSS_CATALOG void ConfigureDDLExecutorResult(ParserExtensionPlanResult &result, DDLExecutionMode mode = DDLExecutionMode::STAGED_CROSS_CATALOG); -string RenderTransactionalDDL(const vector ¶meters); +string RenderTransactionalDDL(ClientContext &context, const vector ¶meters); void ExecuteStagedDDL(ClientContext &context, const vector ¶meters); string BuildCreateDeltaFromDataOperation(const string &delta_table, const string &data_table, bool replace); string BuildDropViewStatement(const DropInfo &drop_info); diff --git a/src/include/core/refresh_locks.hpp b/src/include/core/refresh_locks.hpp index 9f1f5493..0002e2e0 100644 --- a/src/include/core/refresh_locks.hpp +++ b/src/include/core/refresh_locks.hpp @@ -23,16 +23,16 @@ namespace duckdb { // could advance a watermark past a delta row that commits afterward. class DeltaCatalogPhaseGate { public: - void EnterWrite(); - void ExitWrite(); - void EnterRefresh(); - void ExitRefresh(); + void EnterWrite(ClientContext &owner); + void ExitWrite(ClientContext &owner); + void EnterRefresh(ClientContext &owner); + void ExitRefresh(ClientContext &owner); private: mutex lock; std::condition_variable condition; - idx_t active_writers = 0; - idx_t active_refreshes = 0; + unordered_map active_writers; + unordered_map active_refreshes; idx_t waiting_refreshes = 0; }; @@ -56,10 +56,10 @@ class RefreshLocks { static void UnlockDelta(const string &delta_table_name); - static void EnterDeltaWrite(Catalog &catalog); - static void ExitDeltaWrite(Catalog &catalog); - static void EnterDeltaRefresh(Catalog &catalog); - static void ExitDeltaRefresh(Catalog &catalog); + static void EnterDeltaWrite(ClientContext &owner, Catalog &catalog); + static void ExitDeltaWrite(ClientContext &owner, Catalog &catalog); + static void EnterDeltaRefresh(ClientContext &owner, Catalog &catalog); + static void ExitDeltaRefresh(ClientContext &owner, Catalog &catalog); private: static std::mutex &GetViewMutex(const string &view_name); @@ -75,14 +75,15 @@ class RefreshLocks { // Write guards can be retained by transaction state and released from the commit thread; // unlike std::mutex ownership, phase-gate membership is not tied to one OS thread. class DeltaCatalogWriteGuard { + ClientContext *owner; Catalog *catalog; public: - explicit DeltaCatalogWriteGuard(Catalog &catalog_p) : catalog(&catalog_p) { - RefreshLocks::EnterDeltaWrite(*catalog); + DeltaCatalogWriteGuard(ClientContext &owner_p, Catalog &catalog_p) : owner(&owner_p), catalog(&catalog_p) { + RefreshLocks::EnterDeltaWrite(*owner, *catalog); } ~DeltaCatalogWriteGuard() { - RefreshLocks::ExitDeltaWrite(*catalog); + RefreshLocks::ExitDeltaWrite(*owner, *catalog); } DeltaCatalogWriteGuard(const DeltaCatalogWriteGuard &) = delete; DeltaCatalogWriteGuard &operator=(const DeltaCatalogWriteGuard &) = delete; @@ -91,14 +92,15 @@ class DeltaCatalogWriteGuard { }; class DeltaCatalogRefreshGuard { + ClientContext *owner; Catalog *catalog; public: - explicit DeltaCatalogRefreshGuard(Catalog &catalog_p) : catalog(&catalog_p) { - RefreshLocks::EnterDeltaRefresh(*catalog); + DeltaCatalogRefreshGuard(ClientContext &owner_p, Catalog &catalog_p) : owner(&owner_p), catalog(&catalog_p) { + RefreshLocks::EnterDeltaRefresh(*owner, *catalog); } ~DeltaCatalogRefreshGuard() { - RefreshLocks::ExitDeltaRefresh(*catalog); + RefreshLocks::ExitDeltaRefresh(*owner, *catalog); } DeltaCatalogRefreshGuard(const DeltaCatalogRefreshGuard &) = delete; DeltaCatalogRefreshGuard &operator=(const DeltaCatalogRefreshGuard &) = delete; @@ -169,8 +171,11 @@ class TryViewLockGuard { class TransactionalMVLockState : public ClientContextState { public: static TransactionalMVLockState &Get(ClientContext &context); + static optional_ptr TryGet(ClientContext &context); - void Acquire(const vector &view_names, const vector &delta_table_names); + void Acquire(const vector &view_names, const vector &delta_table_names, + const vector &source_catalogs = {}); + bool OwnsRefreshDelta(Catalog &catalog, const string &delta_table_name) const; void TransactionCommit(MetaTransaction &transaction, ClientContext &context) override; void TransactionRollback(MetaTransaction &transaction, ClientContext &context) override; @@ -180,8 +185,11 @@ class TransactionalMVLockState : public ClientContextState { unordered_set locked_views; unordered_set locked_delta_tables; + unordered_set locked_catalogs; vector> view_guards; vector> delta_guards; + vector> catalog_guards; + ClientContext *owner = nullptr; }; } // namespace duckdb diff --git a/src/rules/transactional_delta_capture.cpp b/src/rules/transactional_delta_capture.cpp index 1257c684..0c82e446 100644 --- a/src/rules/transactional_delta_capture.cpp +++ b/src/rules/transactional_delta_capture.cpp @@ -31,7 +31,7 @@ static constexpr const char *DELTA_WRITE_STATE_KEY = "openivm_transactional_delt class TransactionalDeltaWriteState : public ClientContextState { public: - void Acquire(Catalog &catalog) { + void Acquire(ClientContext &context, Catalog &catalog) { lock_guard guard(lock); if (catalog_guard) { if (&catalog != locked_catalog) { @@ -41,7 +41,7 @@ class TransactionalDeltaWriteState : public ClientContextState { } return; } - catalog_guard = make_uniq(catalog); + catalog_guard = make_uniq(context, catalog); locked_catalog = &catalog; } @@ -70,9 +70,13 @@ class TransactionalDeltaWriteState : public ClientContextState { unique_ptr catalog_guard; }; -static void EnterDeltaWritePhase(ClientContext &context, Catalog &catalog) { +static void EnterDeltaWritePhase(ClientContext &context, Catalog &catalog, const string &delta_table_name) { + auto refresh_state = TransactionalMVLockState::TryGet(context); + if (refresh_state && refresh_state->OwnsRefreshDelta(catalog, delta_table_name)) { + return; + } auto state = context.registered_state->GetOrCreate(DELTA_WRITE_STATE_KEY); - state->Acquire(catalog); + state->Acquire(context, catalog); } class TransactionalDeltaAppendState { @@ -108,7 +112,7 @@ class TransactionalDeltaAppender { if (state.entered_write_phase) { return; } - EnterDeltaWritePhase(context, delta_table.catalog); + EnterDeltaWritePhase(context, delta_table.catalog, delta_table.name); state.entered_write_phase = true; } diff --git a/src/upsert/refresh.cpp b/src/upsert/refresh.cpp index 68a70837..297c5921 100644 --- a/src/upsert/refresh.cpp +++ b/src/upsert/refresh.cpp @@ -117,7 +117,6 @@ static bool RefreshViewLocked(ClientContext &context, const string &view_catalog const string &attached_db_schema_name, bool skip_empty_refresh) { RefreshProfiler profiler(context, vn); auto lock_start = std::chrono::steady_clock::now(); - ViewLockGuard view_guard(vn); Connection probe_con(*context.db.get()); RefreshMetadata probe_meta(probe_con); auto delta_sources = probe_meta.GetDeltaSources(vn, view_catalog_name, view_schema_name); @@ -139,8 +138,10 @@ static bool RefreshViewLocked(ClientContext &context, const string &view_catalog source_catalog_names.end()); vector> delta_catalog_guards; for (auto &catalog_name : source_catalog_names) { - delta_catalog_guards.push_back(make_uniq(Catalog::GetCatalog(context, catalog_name))); + delta_catalog_guards.push_back( + make_uniq(context, Catalog::GetCatalog(context, catalog_name))); } + ViewLockGuard view_guard(vn); // Acquire delta-table locks in sorted order to serialize parallel refreshes that // share base tables (e.g. mv_A and mv_B both reading STOCK → both write to // `delta_STOCK` inside their transactions → "Conflict on tuple deletion!" when @@ -602,6 +603,20 @@ static string BuildTransactionalRefreshViewSQL(ClientContext &context, Connectio } string TransactionalRefreshQuery(ClientContext &context, const FunctionParameters ¶meters) { + if (context.transaction.IsAutoCommit()) { + // TODO: Replace query-pragma expansion with a native refresh operator/table + // function that owns compilation, execution, and lock lifetime in one caller + // transaction. Query pragmas are expanded through a preprocessing transaction; + // its transaction-end callback can release ClientContextState locks before the + // returned multi-statement program has fully completed. That boundary permits a + // concurrent parent/child refresh to enter early and produce an MVCC update + // conflict. Until refresh has a native execution boundary, retain the established + // locked helper executor for autocommit calls. Explicit caller transactions use + // the program below so their DML, MV changes, metadata, and rollback remain atomic. + UpsertDeltaQueriesLocked(context, parameters); + return "SELECT true AS Success"; + } + string view_catalog_name; string view_schema_name; string attached_db_catalog_name; @@ -669,8 +684,10 @@ string TransactionalRefreshQuery(ClientContext &context, const FunctionParameter string program; unordered_set seen; + vector ordered_nodes; vector lock_views; vector lock_deltas; + vector lock_catalogs; for (auto &node : refresh_order) { if (!seen.insert(node).second) { continue; @@ -680,13 +697,27 @@ string TransactionalRefreshQuery(ClientContext &context, const FunctionParameter throw NotImplementedException( "Transactional native refresh cannot include cross-catalog dependent view '%s'", node); } + ordered_nodes.push_back(node); lock_views.push_back(node); - auto node_deltas = metadata.GetDeltaTables(node); - lock_deltas.insert(lock_deltas.end(), node_deltas.begin(), node_deltas.end()); + auto node_sources = metadata.GetDeltaSources(node, location.catalog_name, location.schema_name); + for (auto &source : node_sources) { + lock_deltas.push_back(source.table_name); + if (source.catalog_type != "ducklake") { + lock_catalogs.push_back(&Catalog::GetCatalog(context, source.catalog_name)); + } + } if (metadata.HasDownstreamViews(node)) { lock_deltas.push_back(SqlUtils::DeltaName(node)); } + } + // GenerateRefreshSQL consults the caller context and can establish its + // transaction snapshot. Serialize first so a waiter compiles against state + // committed by the preceding refresh rather than carrying a stale snapshot + // through the lock wait. + TransactionalMVLockState::Get(context).Acquire(lock_views, lock_deltas, lock_catalogs); + for (auto &node : ordered_nodes) { + auto location = ResolveViewLocation(metadata_con, node, view_catalog_name, view_schema_name); string hook_sql; string hook_mode; auto hooks = metadata_con.Query("SELECT hook_sql, mode FROM openivm_refresh_hooks" @@ -709,7 +740,6 @@ string TransactionalRefreshQuery(ClientContext &context, const FunctionParameter program += hook_sql + ";\n"; } } - TransactionalMVLockState::Get(context).Acquire(lock_views, lock_deltas); program += "SELECT true AS Success"; return program; } From 270dd597fa74fb1d3d5e5a7f0bb9e1b79b13a011 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 23 Jul 2026 11:38:11 +0200 Subject: [PATCH 07/36] Continue cascades past empty refresh nodes --- docs/codebase-audit-2026-07-22.md | 6 +- src/upsert/refresh.cpp | 44 +++---- test/sql/pipeline.test | 203 ++++++++++++++++++++++++++++++ test/sql/refresh_hooks.test | 112 +++++++++++++++++ 4 files changed, 340 insertions(+), 25 deletions(-) diff --git a/docs/codebase-audit-2026-07-22.md b/docs/codebase-audit-2026-07-22.md index 2c7d583a..31fdc8f6 100644 --- a/docs/codebase-audit-2026-07-22.md +++ b/docs/codebase-audit-2026-07-22.md @@ -25,9 +25,9 @@ maintenance path. A bug must not be hidden by weakening a test or silently chang DuckDB query-pragma preprocessing ends a transaction before its returned program completes; `refresh.cpp` records the native-operator follow-up. DuckLake replacement materializes data and auxiliary state under unpublished staging names before publishing them. -- [ ] **Continue cascade traversal when the current node has no source deltas.** The root empty-delta fast path returns - before visiting downstream nodes. A child can have independent source deltas or pending parent-delta rows after an earlier - failed cascade. Model `node skipped` separately from `graph traversal complete` and checkpoint each node independently. +- [x] **Continue cascade traversal when the current node has no source deltas.** Empty-delta detection now skips only the + current node; it cannot terminate traversal of the downstream DAG. Regression coverage exercises both a pending parent + delta left by a cascade-off refresh and independent child-source DML, including the hook-aware empty-node path. - [ ] **Use NULL-safe affected-window partition matching.** `IN` does not select a NULL partition even though SQL window partitioning groups NULL values. Join the affected-key relation with `IS NOT DISTINCT FROM` for every partition key. - [ ] **Preserve `SUM` NULL semantics.** Weighted SUM alone cannot distinguish numeric zero from no non-NULL inputs. Persist diff --git a/src/upsert/refresh.cpp b/src/upsert/refresh.cpp index 297c5921..95113d27 100644 --- a/src/upsert/refresh.cpp +++ b/src/upsert/refresh.cpp @@ -112,7 +112,7 @@ static bool TrySkipEmptyRefresh(ClientContext &context, RefreshMetadata &metadat // Generate and execute refresh SQL for a single view under its per-view lock. // When openivm_adaptive_refresh is on, also computes a cost estimate before execution // and records execution history for the learned cost model. -static bool RefreshViewLocked(ClientContext &context, const string &view_catalog_name, const string &view_schema_name, +static void RefreshViewLocked(ClientContext &context, const string &view_catalog_name, const string &view_schema_name, const string &vn, bool cross_system, const string &attached_db_catalog_name, const string &attached_db_schema_name, bool skip_empty_refresh) { RefreshProfiler profiler(context, vn); @@ -171,7 +171,7 @@ static bool RefreshViewLocked(ClientContext &context, const string &view_catalog attached_db_catalog_name, attached_db_schema_name, &delta_activity)) { profiler.AddTotal(); profiler.Flush(*context.db.get()); - return true; + return; } if (!delta_activity.active_delta_table_names.empty() || delta_activity.requires_full_refresh) { precomputed_delta_activity = &delta_activity; @@ -363,7 +363,7 @@ static bool RefreshViewLocked(ClientContext &context, const string &view_catalog } profiler.AddTotal(); profiler.Flush(*context.db.get()); - return false; + return; } catch (...) { // Ensure the transaction is rolled back before we propagate the exception. // This covers the case where Query() itself threw (vs returning HasError) — @@ -544,30 +544,30 @@ void UpsertDeltaQueriesLocked(ClientContext &context, const FunctionParameters & // Hook-bearing refreshes keep the old pre-hook empty skip semantics. Hook-free refreshes // compute the same delta activity under the view lock and reuse it during SQL generation. - if (has_refresh_hook && TrySkipEmptyRefresh(context, metadata, con, view_catalog_name, view_schema_name, view_name, - attached_db_catalog_name, attached_db_schema_name, nullptr)) { - return; - } - - if (!hook_sql.empty() && hook_mode == "before") { - auto hr = con.Query(hook_sql); - if (hr->HasError()) { - Printer::Print("Warning: before-hook for '" + view_name + "' failed: " + hr->GetError()); + bool skip_current_node = + has_refresh_hook && TrySkipEmptyRefresh(context, metadata, con, view_catalog_name, view_schema_name, view_name, + attached_db_catalog_name, attached_db_schema_name, nullptr); + if (!skip_current_node) { + if (!hook_sql.empty() && hook_mode == "before") { + auto hr = con.Query(hook_sql); + if (hr->HasError()) { + Printer::Print("Warning: before-hook for '" + view_name + "' failed: " + hr->GetError()); + } } - } - if (hook_mode != "replace") { - if (RefreshViewLocked(context, view_catalog_name, view_schema_name, view_name, cross_system, - attached_db_catalog_name, attached_db_schema_name, !has_refresh_hook)) { - return; + if (hook_mode != "replace") { + RefreshViewLocked(context, view_catalog_name, view_schema_name, view_name, cross_system, + attached_db_catalog_name, attached_db_schema_name, !has_refresh_hook); } - } - if (!hook_sql.empty() && (hook_mode == "after" || hook_mode == "replace")) { - auto hr = con.Query(hook_sql); - if (hr->HasError()) { - Printer::Print("Warning: " + hook_mode + "-hook for '" + view_name + "' failed: " + hr->GetError()); + if (!hook_sql.empty() && (hook_mode == "after" || hook_mode == "replace")) { + auto hr = con.Query(hook_sql); + if (hr->HasError()) { + Printer::Print("Warning: " + hook_mode + "-hook for '" + view_name + "' failed: " + hr->GetError()); + } } + } else { + OPENIVM_DEBUG_PRINT("[UPSERT] Skipped refresh node '%s'; continuing cascade traversal\n", view_name.c_str()); } // Downstream cascade: refresh dependents after diff --git a/test/sql/pipeline.test b/test/sql/pipeline.test index 7d298c0c..efb7c8df 100644 --- a/test/sql/pipeline.test +++ b/test/sql/pipeline.test @@ -752,6 +752,209 @@ SELECT COUNT(*) FROM ( ---- 0 +# ============================================================ +# Empty current node must not stop downstream cascade traversal +# ============================================================ + +statement ok +SET openivm_cascade_refresh = 'downstream'; + +statement ok +CREATE TABLE cascade_skip_root_src (id INT, amount INT); + +statement ok +CREATE TABLE cascade_skip_side_src (id INT, bonus INT); + +statement ok +INSERT INTO cascade_skip_root_src VALUES (1, 10), (2, 20); + +statement ok +INSERT INTO cascade_skip_side_src VALUES (1, 1), (2, 2); + +statement ok +CREATE MATERIALIZED VIEW cascade_skip_parent AS + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id; + +statement ok +CREATE MATERIALIZED VIEW cascade_skip_child AS + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM cascade_skip_parent p + JOIN cascade_skip_side_src s USING (id) + GROUP BY p.id; + +# Leave a parent delta pending for the child, then invoke a downstream cascade +# when the parent's own source delta is empty. +statement ok +SET openivm_cascade_refresh = 'off'; + +statement ok +INSERT INTO cascade_skip_root_src VALUES (3, 30), (4, 40); + +statement ok +UPDATE cascade_skip_root_src SET amount = amount + 5 WHERE id IN (1, 3); + +statement ok +DELETE FROM cascade_skip_root_src WHERE id IN (2, 4); + +statement ok +PRAGMA refresh('cascade_skip_parent'); + +query I +SELECT COUNT(*) FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + EXCEPT ALL + SELECT id, total FROM cascade_skip_parent +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, total FROM cascade_skip_parent + EXCEPT ALL + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id +); +---- +0 + +statement ok +SET openivm_cascade_refresh = 'downstream'; + +statement ok +PRAGMA refresh('cascade_skip_parent'); + +query I +SELECT COUNT(*) FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + EXCEPT ALL + SELECT id, total FROM cascade_skip_parent +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, total FROM cascade_skip_parent + EXCEPT ALL + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + ) p + JOIN cascade_skip_side_src s USING (id) + GROUP BY p.id + EXCEPT ALL + SELECT id, score, cnt FROM cascade_skip_child +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, score, cnt FROM cascade_skip_child + EXCEPT ALL + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + ) p + JOIN cascade_skip_side_src s USING (id) + GROUP BY p.id +); +---- +0 + +# The child can also have independent source changes while the parent has none. +# Batch conflicting DML before one refresh: inserted rows are updated, and one +# inserted row is deleted before the cascade. +statement ok +INSERT INTO cascade_skip_side_src VALUES (3, 3), (4, 4); + +statement ok +UPDATE cascade_skip_side_src SET bonus = bonus + 10 WHERE id IN (1, 3); + +statement ok +DELETE FROM cascade_skip_side_src WHERE id IN (2, 4); + +statement ok +PRAGMA refresh('cascade_skip_parent'); + +query I +SELECT COUNT(*) FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + EXCEPT ALL + SELECT id, total FROM cascade_skip_parent +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, total FROM cascade_skip_parent + EXCEPT ALL + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + ) p + JOIN cascade_skip_side_src s USING (id) + GROUP BY p.id + EXCEPT ALL + SELECT id, score, cnt FROM cascade_skip_child +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, score, cnt FROM cascade_skip_child + EXCEPT ALL + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + ) p + JOIN cascade_skip_side_src s USING (id) + GROUP BY p.id +); +---- +0 + +statement ok +SET openivm_cascade_refresh = 'upstream'; + # Insert more data statement ok INSERT INTO pipe_up VALUES ('a', 5), ('b', 20); diff --git a/test/sql/refresh_hooks.test b/test/sql/refresh_hooks.test index 60a5ae17..9bbba68f 100644 --- a/test/sql/refresh_hooks.test +++ b/test/sql/refresh_hooks.test @@ -134,3 +134,115 @@ query I SELECT COUNT(*) FROM hook_log; ---- 0 + +# ============================================================ +# Test 5: An empty hook-bearing node does not stop its cascade +# ============================================================ + +statement ok +SET openivm_cascade_refresh = 'downstream'; + +statement ok +CREATE TABLE hook_cascade_root_src (id INT, amount INT); + +statement ok +CREATE TABLE hook_cascade_side_src (id INT, bonus INT); + +statement ok +INSERT INTO hook_cascade_root_src VALUES (1, 10), (2, 20), (3, 30); + +statement ok +INSERT INTO hook_cascade_side_src VALUES (1, 1), (2, 2); + +statement ok +CREATE MATERIALIZED VIEW hook_cascade_parent AS + SELECT id, SUM(amount) AS total + FROM hook_cascade_root_src + GROUP BY id; + +statement ok +CREATE MATERIALIZED VIEW hook_cascade_child AS + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM hook_cascade_parent p + JOIN hook_cascade_side_src s USING (id) + GROUP BY p.id; + +statement ok +INSERT INTO openivm_refresh_hooks VALUES ( + 'hook_cascade_parent', + 'INSERT INTO hook_log VALUES(''empty-parent hook ran'')', + 'before' +); + +# Only the child's independent source changes. The parent's pre-hook empty +# semantics still skip its hook, but traversal must refresh the child. +statement ok +INSERT INTO hook_cascade_side_src VALUES (3, 3), (4, 4); + +statement ok +UPDATE hook_cascade_side_src SET bonus = bonus + 10 WHERE id IN (1, 3); + +statement ok +DELETE FROM hook_cascade_side_src WHERE id IN (2, 4); + +statement ok +PRAGMA refresh('hook_cascade_parent'); + +query I +SELECT COUNT(*) FROM ( + SELECT id, SUM(amount) AS total + FROM hook_cascade_root_src + GROUP BY id + EXCEPT ALL + SELECT id, total FROM hook_cascade_parent +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, total FROM hook_cascade_parent + EXCEPT ALL + SELECT id, SUM(amount) AS total + FROM hook_cascade_root_src + GROUP BY id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM ( + SELECT id, SUM(amount) AS total + FROM hook_cascade_root_src + GROUP BY id + ) p + JOIN hook_cascade_side_src s USING (id) + GROUP BY p.id + EXCEPT ALL + SELECT id, score, cnt FROM hook_cascade_child +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, score, cnt FROM hook_cascade_child + EXCEPT ALL + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM ( + SELECT id, SUM(amount) AS total + FROM hook_cascade_root_src + GROUP BY id + ) p + JOIN hook_cascade_side_src s USING (id) + GROUP BY p.id +); +---- +0 + +query I +SELECT COUNT(*) FROM hook_log; +---- +0 From fee5c524ab73a691a29be0bb29cd575901b4cdf9 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 23 Jul 2026 13:20:35 +0200 Subject: [PATCH 08/36] Use NULL-safe window partition refresh --- benchmark/queries/tpcc/ducklake_0318.sql | 12 ++ docs/codebase-audit-2026-07-22.md | 6 +- src/include/upsert/refresh_compiler.hpp | 1 - src/upsert/refresh_compiler_aux.cpp | 128 ++++++++++++------ src/upsert/refresh_window.cpp | 88 +++++------- test/sql/window.test | 107 +++++++++++++++ .../window_running_aggregate_incremental.test | 10 +- 7 files changed, 251 insertions(+), 101 deletions(-) create mode 100644 benchmark/queries/tpcc/ducklake_0318.sql diff --git a/benchmark/queries/tpcc/ducklake_0318.sql b/benchmark/queries/tpcc/ducklake_0318.sql new file mode 100644 index 00000000..4c007290 --- /dev/null +++ b/benchmark/queries/tpcc/ducklake_0318.sql @@ -0,0 +1,12 @@ +-- {"operators": "DUCKLAKE,WINDOW", "complexity": "low", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE", "ducklake": true, "openivm_verified": true} +SELECT + OL_W_ID AS warehouse_id, + OL_D_ID AS district_id, + OL_O_ID AS order_id, + OL_NUMBER AS line_number, + OL_DELIVERY_D AS delivery_at, + ROW_NUMBER() OVER ( + PARTITION BY OL_DELIVERY_D + ORDER BY OL_W_ID, OL_D_ID, OL_O_ID, OL_NUMBER + ) AS delivery_row_number +FROM dl.ORDER_LINE; diff --git a/docs/codebase-audit-2026-07-22.md b/docs/codebase-audit-2026-07-22.md index 31fdc8f6..95ffb8ce 100644 --- a/docs/codebase-audit-2026-07-22.md +++ b/docs/codebase-audit-2026-07-22.md @@ -28,8 +28,10 @@ maintenance path. A bug must not be hidden by weakening a test or silently chang - [x] **Continue cascade traversal when the current node has no source deltas.** Empty-delta detection now skips only the current node; it cannot terminate traversal of the downstream DAG. Regression coverage exercises both a pending parent delta left by a cascade-off refresh and independent child-source DML, including the hook-aware empty-node path. -- [ ] **Use NULL-safe affected-window partition matching.** `IN` does not select a NULL partition even though SQL window - partitioning groups NULL values. Join the affected-key relation with `IS NOT DISTINCT FROM` for every partition key. +- [x] **Use NULL-safe affected-window partition matching.** Affected-key refresh now uses hash-semi-joinable + `IS NOT DISTINCT FROM` predicates for standard, cascade, running-window, and DuckLake paths. Regression coverage includes + batched conflicting DML on single and composite NULL partitions plus a DuckLake TPCC window whose delivery timestamp moves + from the NULL partition to a non-NULL partition. - [ ] **Preserve `SUM` NULL semantics.** Weighted SUM alone cannot distinguish numeric zero from no non-NULL inputs. Persist a hidden `COUNT(sum_argument)` and render NULL when that count reaches zero. - [ ] **Use an unconditional row count for group existence.** `COUNT(nullable_expression)=0` does not mean a group is empty. diff --git a/src/include/upsert/refresh_compiler.hpp b/src/include/upsert/refresh_compiler.hpp index 322c7a03..2859d890 100644 --- a/src/include/upsert/refresh_compiler.hpp +++ b/src/include/upsert/refresh_compiler.hpp @@ -53,7 +53,6 @@ string CompileWindowRecompute(const string &view_name, const string &view_query_ const string &catalog_prefix = "", const vector &partition_columns = {}, const vector &partition_delta_specs = {}, bool emit_cascade_delta = false, const string &affected_keys_sql = "", - const string &affected_key_cols = "", const string &affected_key_tuple = "", const vector &column_names = {}, bool running_window_incremental = false); string CompileFullRecompute(const string &view_name, const string &view_query_sql, const string &catalog_prefix = ""); diff --git a/src/upsert/refresh_compiler_aux.cpp b/src/upsert/refresh_compiler_aux.cpp index fbf3c0d5..a1fe3c0d 100644 --- a/src/upsert/refresh_compiler_aux.cpp +++ b/src/upsert/refresh_compiler_aux.cpp @@ -51,6 +51,41 @@ static string PartitionOutputColumn(const string &input) { return StripIdentifierQuotes(pos == string::npos ? input : input.substr(0, pos)); } +static vector PartitionOutputColumns(const vector &partition_columns) { + vector output_columns; + output_columns.reserve(partition_columns.size()); + for (auto &partition_column : partition_columns) { + output_columns.push_back(PartitionOutputColumn(partition_column)); + } + return output_columns; +} + +static string BuildAffectedTableFilter(const vector &columns, const string &outer_alias, + const string &affected_table) { + string match = SqlUtils::BuildNullSafeMatch(columns, "openivm_aff", outer_alias); + return "EXISTS (SELECT 1 FROM " + affected_table + " openivm_aff WHERE " + match + ")"; +} + +static string BuildDeltaAffectedFilter(const vector &partition_delta_specs, + const string &delta_where, const string &outer_alias) { + string filter; + for (idx_t i = 0; i < partition_delta_specs.size(); i++) { + if (i > 0) { + filter += " OR "; + } + const auto &spec = partition_delta_specs[i]; + string output_col = SqlUtils::QuoteIdentifier(spec.output_column); + string source_col = SqlUtils::QuoteIdentifier(spec.source_column); + string delta_table = + spec.delta_table_sql.empty() ? SqlUtils::QuoteIdentifier(spec.delta_table) : spec.delta_table_sql; + string affected_keys = + "SELECT DISTINCT " + source_col + " AS " + output_col + " FROM " + delta_table + delta_where; + filter += "EXISTS (SELECT 1 FROM (" + affected_keys + ") openivm_aff WHERE " + outer_alias + "." + output_col + + " IS NOT DISTINCT FROM openivm_aff." + output_col + ")"; + } + return filter; +} + static vector SplitTopLevelComma(const string &input) { vector parts; idx_t start = 0; @@ -710,14 +745,16 @@ static string BuildRunningWindowSuffixRefreshSQL(const string &view_name, const string key_match_dt_fk = SqlUtils::BuildNullSafeMatch(vector {plan.partition_column}, "dt", "fk"); string key_match_b_m = "b." + part_q + " IS NOT DISTINCT FROM m." + part_q; string key_match_d_fk = SqlUtils::BuildNullSafeMatch(vector {plan.partition_column}, "d", "fk"); + string affected_data_filter = + BuildAffectedTableFilter(vector {plan.partition_column}, "dt", affected_table); string sql; sql += "CREATE OR REPLACE TEMP TABLE " + affected_table + " AS\nSELECT DISTINCT " + QualifiedColumn("d", plan.partition_column) + " AS " + part_q + "\nFROM " + delta_q + " d\nWHERE " + delta_positive + ";\n\n"; - sql += "CREATE OR REPLACE TEMP TABLE " + bounds_table + " AS\nWITH old_max AS (\n SELECT " + part_q + ", MAX(" + - order_q + ") AS openivm_old_max_order FROM " + data_table + " WHERE " + part_q + " IN (SELECT " + part_q + - " FROM " + affected_table + ") GROUP BY " + part_q + "\n), delta_min AS (\n SELECT " + + sql += "CREATE OR REPLACE TEMP TABLE " + bounds_table + " AS\nWITH old_max AS (\n SELECT dt." + part_q + + ", MAX(dt." + order_q + ") AS openivm_old_max_order FROM " + data_table + " dt WHERE " + + affected_data_filter + " GROUP BY dt." + part_q + "\n), delta_min AS (\n SELECT " + QualifiedColumn("d", plan.partition_column) + " AS " + part_q + ", MIN(" + QualifiedColumn("d", plan.order_column) + ") AS openivm_delta_min_order\n FROM " + delta_q + " d\n WHERE " + delta_positive + "\n GROUP BY " + QualifiedColumn("d", plan.partition_column) + @@ -754,18 +791,29 @@ static string BuildRunningWindowSuffixRefreshSQL(const string &view_name, const " ORDER BY " + QualifiedColumn("dt", plan.order_column) + " DESC) AS openivm_rn\n FROM " + data_table + " dt\n JOIN " + fast_table + " fk ON " + key_match_dt_fk + "\n) openivm_state_ranked\nWHERE openivm_rn = 1;\n\n"; - string fallback_filter = part_q + " IN (SELECT " + part_q + " FROM " + fallback_table + ")"; + string fallback_keys = "SELECT " + part_q + " FROM " + fallback_table; + string fallback_target_match = + SqlUtils::BuildNullSafeMatch(vector {plan.partition_column}, "openivm_aff", "openivm_target"); + string fallback_recompute_match = + SqlUtils::BuildNullSafeMatch(vector {plan.partition_column}, "openivm_aff", "openivm_recompute"); if (emit_cascade_delta) { - sql += "CREATE OR REPLACE TEMP TABLE " + old_temp_table + " AS\nSELECT * FROM " + data_table + "\nWHERE " + - fallback_filter + ";\n\n"; + string fallback_old_filter = + BuildAffectedTableFilter(vector {plan.partition_column}, "openivm_old", fallback_table); + string fallback_new_filter = + BuildAffectedTableFilter(vector {plan.partition_column}, "openivm_recompute", fallback_table); + string fallback_delete_filter = + BuildAffectedTableFilter(vector {plan.partition_column}, "openivm_target", fallback_table); + sql += "CREATE OR REPLACE TEMP TABLE " + old_temp_table + " AS\nSELECT * FROM " + data_table + + " openivm_old\nWHERE " + fallback_old_filter + ";\n\n"; sql += "CREATE OR REPLACE TEMP TABLE " + new_temp_table + " AS\nSELECT * FROM (" + view_query_sql + - ") openivm_recompute\nWHERE " + fallback_filter + ";\n\n"; - sql += "DELETE FROM " + data_table + " WHERE " + fallback_filter + ";\n"; + ") openivm_recompute\nWHERE " + fallback_new_filter + ";\n\n"; + sql += "DELETE FROM " + data_table + " AS openivm_target WHERE " + fallback_delete_filter + ";\n"; sql += "INSERT INTO " + data_table + "\nSELECT * FROM " + new_temp_table + ";\n"; sql += "\n" + BuildSignedMultisetDeltaInsertSQL(delta_table, old_temp_table, new_temp_table); } else { - sql += BuildDeleteInsertRefreshSQL(data_table, view_query_sql, "openivm_recompute", fallback_filter, - fallback_filter); + sql += + BuildAffectedKeyRefreshSQL(data_table, view_query_sql, fallback_keys, "openivm_target", "openivm_recompute", + "openivm_aff", fallback_target_match, fallback_recompute_match); } auto emit_column_names = plan.output_columns.empty() ? visible_column_names : plan.output_columns; @@ -1548,10 +1596,9 @@ string CompileFilteredGroupCount(const string &view_name, const string &aux_tabl string CompileWindowRecompute(const string &view_name, const string &view_query_sql, const string &delta_ts_filter, const string &catalog_prefix, const vector &partition_columns, const vector &partition_delta_specs, bool emit_cascade_delta, - const string &affected_keys_sql, const string &affected_key_cols, - const string &affected_key_tuple, const vector &column_names, + const string &affected_keys_sql, const vector &column_names, bool running_window_incremental) { - bool have_affected_keys = !affected_keys_sql.empty() && !affected_key_cols.empty() && !affected_key_tuple.empty(); + bool have_affected_keys = !affected_keys_sql.empty(); if (!have_affected_keys && (partition_columns.empty() || partition_delta_specs.empty())) { return CompileFullRecompute(view_name, view_query_sql, catalog_prefix); } @@ -1566,25 +1613,7 @@ string CompileWindowRecompute(const string &view_name, const string &view_query_ string data_table = catalog_prefix + SqlUtils::QuoteIdentifier(IncrementalTableNames::DataTableName(view_name)); string delta_where = delta_ts_filter.empty() ? "" : " WHERE " + delta_ts_filter; string affected_temp_table = SqlUtils::QuoteIdentifier("openivm_affected_" + view_name); - - string affected_filter; - if (have_affected_keys) { - affected_filter = - affected_key_tuple + " IN (SELECT " + affected_key_cols + " FROM " + affected_temp_table + ")"; - } else { - for (size_t i = 0; i < partition_delta_specs.size(); i++) { - if (i > 0) { - affected_filter += " OR "; - } - const auto &spec = partition_delta_specs[i]; - string output_col = SqlUtils::QuoteIdentifier(spec.output_column); - string source_col = SqlUtils::QuoteIdentifier(spec.source_column); - string delta_table = - spec.delta_table_sql.empty() ? SqlUtils::QuoteIdentifier(spec.delta_table) : spec.delta_table_sql; - affected_filter += - output_col + " IN (SELECT DISTINCT " + source_col + " FROM " + delta_table + delta_where + ")"; - } - } + auto output_columns = PartitionOutputColumns(partition_columns); OPENIVM_DEBUG_PRINT( "[CompileWindowRecompute] Partition columns: %zu, delta specs: %zu, lineage keys: %s, cascade delta: %s\n", @@ -1592,15 +1621,17 @@ string CompileWindowRecompute(const string &view_name, const string &view_query_ emit_cascade_delta ? "enabled" : "disabled"); if (!emit_cascade_delta) { if (!have_affected_keys) { - return BuildDeleteInsertRefreshSQL(data_table, view_query_sql, "openivm_recompute", affected_filter, - affected_filter); + string target_filter = BuildDeltaAffectedFilter(partition_delta_specs, delta_where, "openivm_target"); + string recompute_filter = BuildDeltaAffectedFilter(partition_delta_specs, delta_where, "openivm_recompute"); + return "DELETE FROM " + data_table + " AS openivm_target WHERE " + target_filter + ";\n" + "INSERT INTO " + + data_table + "\nSELECT * FROM (" + view_query_sql + ") openivm_recompute\nWHERE " + + recompute_filter + ";\n"; } - string sql; - sql += "CREATE OR REPLACE TEMP TABLE " + affected_temp_table + " AS\n" + affected_keys_sql + ";\n\n"; - sql += BuildDeleteInsertRefreshSQL(data_table, view_query_sql, "openivm_recompute", affected_filter, - affected_filter); - sql += "DROP TABLE IF EXISTS " + affected_temp_table + ";\n"; - return sql; + string target_match = SqlUtils::BuildNullSafeMatch(output_columns, "openivm_aff", "openivm_target"); + string recompute_match = SqlUtils::BuildNullSafeMatch(output_columns, "openivm_aff", "openivm_recompute"); + return BuildAffectedKeyRefreshSQL(data_table, view_query_sql, affected_keys_sql, "openivm_target", + "openivm_recompute", "openivm_aff", target_match, recompute_match, + affected_temp_table); } string delta_table = catalog_prefix + SqlUtils::QuoteIdentifier(SqlUtils::DeltaName(view_name)); @@ -1610,11 +1641,20 @@ string CompileWindowRecompute(const string &view_name, const string &view_query_ if (have_affected_keys) { sql += "CREATE OR REPLACE TEMP TABLE " + affected_temp_table + " AS\n" + affected_keys_sql + ";\n\n"; } - sql += "CREATE OR REPLACE TEMP TABLE " + old_temp_table + " AS\nSELECT * FROM " + data_table + "\nWHERE " + - affected_filter + ";\n\n"; + string old_filter = have_affected_keys + ? BuildAffectedTableFilter(output_columns, "openivm_old", affected_temp_table) + : BuildDeltaAffectedFilter(partition_delta_specs, delta_where, "openivm_old"); + string recompute_filter = have_affected_keys + ? BuildAffectedTableFilter(output_columns, "openivm_recompute", affected_temp_table) + : BuildDeltaAffectedFilter(partition_delta_specs, delta_where, "openivm_recompute"); + string target_filter = have_affected_keys + ? BuildAffectedTableFilter(output_columns, "openivm_target", affected_temp_table) + : BuildDeltaAffectedFilter(partition_delta_specs, delta_where, "openivm_target"); + sql += "CREATE OR REPLACE TEMP TABLE " + old_temp_table + " AS\nSELECT * FROM " + data_table + + " openivm_old\nWHERE " + old_filter + ";\n\n"; sql += "CREATE OR REPLACE TEMP TABLE " + new_temp_table + " AS\nSELECT * FROM (" + view_query_sql + - ") openivm_recompute\nWHERE " + affected_filter + ";\n\n"; - sql += "DELETE FROM " + data_table + " WHERE " + affected_filter + ";\n"; + ") openivm_recompute\nWHERE " + recompute_filter + ";\n\n"; + sql += "DELETE FROM " + data_table + " AS openivm_target WHERE " + target_filter + ";\n"; sql += "INSERT INTO " + data_table + "\nSELECT * FROM " + new_temp_table + ";\n"; sql += "\n" + BuildSignedMultisetDeltaInsertSQL(delta_table, old_temp_table, new_temp_table); if (have_affected_keys) { diff --git a/src/upsert/refresh_window.cpp b/src/upsert/refresh_window.cpp index eea7717f..e77b4807 100644 --- a/src/upsert/refresh_window.cpp +++ b/src/upsert/refresh_window.cpp @@ -200,17 +200,17 @@ static string BuildStandardLookupChangedKeysSQL(RefreshMetadata &metadata, const enum class LineageAffectedKeysResult : uint8_t { UNAVAILABLE, AVAILABLE, UNSAFE }; -static LineageAffectedKeysResult BuildLineageStandardAffectedKeysSQL( - RefreshMetadata &metadata, Connection &con, const string &view_name, const vector &delta_table_names, - const vector &partition_cols, const string &delta_ts_filter, const string &view_catalog_name, - const string &view_schema_name, const string &attached_db_catalog_name, const string &attached_db_schema_name, - string &affected_keys_sql, string &key_cols, string &key_tuple) { +static LineageAffectedKeysResult +BuildLineageStandardAffectedKeysSQL(RefreshMetadata &metadata, Connection &con, const string &view_name, + const vector &delta_table_names, const vector &partition_cols, + const string &delta_ts_filter, const string &view_catalog_name, + const string &view_schema_name, const string &attached_db_catalog_name, + const string &attached_db_schema_name, string &affected_keys_sql) { if (partition_cols.size() != 1) { return LineageAffectedKeysResult::UNAVAILABLE; } auto parsed = SplitPartitionSpec(partition_cols[0]); - key_cols = SqlUtils::QuoteIdentifier(parsed.first); - key_tuple = key_cols; + string key_cols = SqlUtils::QuoteIdentifier(parsed.first); vector lineage_ops; if (!metadata.GetWindowPartitionLineage(view_name, lineage_ops)) { @@ -366,23 +366,24 @@ static string BuildDuckLakeLookupChangedKeysSQL(const DuckLakeWindowSourceSpec & return "(" + current_lookup + " UNION ALL " + old_lookup + ")"; } -static string BuildAffectedPartitionFilter(const vector &partition_cols, const string &affected_table) { - string key_cols; - string key_tuple; - for (size_t i = 0; i < partition_cols.size(); i++) { - if (i > 0) { - key_cols += ", "; - key_tuple += ", "; - } - auto parsed = SplitPartitionSpec(partition_cols[i]); - string output_col = KeywordHelper::WriteOptionallyQuoted(parsed.first); - key_cols += output_col; - key_tuple += output_col; - } - if (partition_cols.size() == 1) { - return key_tuple + " IN (SELECT " + key_cols + " FROM " + affected_table + ")"; +static vector PartitionOutputColumns(const vector &partition_cols) { + vector output_columns; + output_columns.reserve(partition_cols.size()); + for (auto &partition_col : partition_cols) { + output_columns.push_back(SplitPartitionSpec(partition_col).first); } - return "(" + key_tuple + ") IN (SELECT " + key_cols + " FROM " + affected_table + ")"; + return output_columns; +} + +static string BuildAffectedPartitionRefreshSQL(const string &data_table, const string &view_query_sql, + const string &affected_keys_sql, const string &affected_temp_table, + const vector &partition_cols) { + auto output_columns = PartitionOutputColumns(partition_cols); + string target_match = SqlUtils::BuildNullSafeMatch(output_columns, "openivm_aff", "openivm_target"); + string recompute_match = SqlUtils::BuildNullSafeMatch(output_columns, "openivm_aff", "openivm_recompute"); + return BuildAffectedKeyRefreshSQL(data_table, view_query_sql, affected_keys_sql, "openivm_target", + "openivm_recompute", "openivm_aff", target_match, recompute_match, + affected_temp_table); } static bool BuildLineageDuckLakeAffectedKeysSQL(RefreshMetadata &metadata, Connection &con, const string &view_name, @@ -390,13 +391,12 @@ static bool BuildLineageDuckLakeAffectedKeysSQL(RefreshMetadata &metadata, Conne const vector &partition_cols, const string &view_catalog_name, const string &view_schema_name, const string &attached_db_catalog_name, const string &attached_db_schema_name, string &affected_keys_sql, - string &key_cols, string &key_tuple) { + string &key_cols) { if (partition_cols.size() != 1) { return false; } auto parsed = SplitPartitionSpec(partition_cols[0]); key_cols = SqlUtils::QuoteIdentifier(parsed.first); - key_tuple = key_cols; vector specs; if (!BuildDuckLakeWindowSourceSpecs(metadata, con, view_name, delta_table_names, view_catalog_name, @@ -489,12 +489,10 @@ static string BuildSingleSourceDuckLakeWindowRefresh(RefreshMetadata &metadata, string deletions = "SELECT " + affected_select + " FROM " + SqlUtils::DuckLakeTableFunction("ducklake_table_deletions", loc.catalog_name, loc.schema_name, loc.table_name, old_snap, current_snap); - string affected_filter = BuildAffectedPartitionFilter(partition_cols, qtemp_affected); - string upsert_query = "CREATE TEMP TABLE " + qtemp_affected + " AS SELECT DISTINCT " + affected_cols + " FROM ((" + - insertions + ") UNION ALL (" + deletions + ")) openivm_changed_partitions;\n"; - upsert_query += - BuildDeleteInsertRefreshSQL(data_table, view_query_sql, "openivm_recompute", affected_filter, affected_filter); - upsert_query += "DROP TABLE " + qtemp_affected + ";\n"; + string affected_keys = "SELECT DISTINCT " + affected_cols + " FROM ((" + insertions + ") UNION ALL (" + deletions + + ")) openivm_changed_partitions"; + string upsert_query = + BuildAffectedPartitionRefreshSQL(data_table, view_query_sql, affected_keys, qtemp_affected, partition_cols); OPENIVM_DEBUG_PRINT("[UPSERT] Compiling upsert for type: WINDOW_PARTITION (DuckLake change-feed, %zu " "partition cols, old_snap=%ld, current_snap=%ld)\n", partition_cols.size(), (long)old_snap, (long)current_snap); @@ -509,24 +507,19 @@ static string BuildMultiSourceDuckLakeWindowRefresh(RefreshMetadata &metadata, C const string &attached_db_catalog_name, const string &attached_db_schema_name) { string key_cols; - string key_tuple; string affected_keys; if (BuildLineageDuckLakeAffectedKeysSQL(metadata, con, view_name, delta_table_names, partition_cols, view_catalog_name, view_schema_name, attached_db_catalog_name, - attached_db_schema_name, affected_keys, key_cols, key_tuple)) { + attached_db_schema_name, affected_keys, key_cols)) { string temp_affected = string(openivm::TEMP_TABLE_PREFIX) + "affected_" + view_name; string qtemp_affected = KeywordHelper::WriteOptionallyQuoted(temp_affected); - string affected_filter = key_tuple + " IN (SELECT " + key_cols + " FROM " + qtemp_affected + ")"; OPENIVM_DEBUG_PRINT("[UPSERT] Compiling upsert for type: WINDOW_PARTITION (DuckLake lineage change-feed, %zu " "sources)\n", delta_table_names.size()); // Conservative lineage can over-include partitions, but must cover every changed source. // If lineage is incomplete, the full logical view diff below preserves correctness. - string upsert_query = "CREATE OR REPLACE TEMP TABLE " + qtemp_affected + " AS " + affected_keys + ";\n"; - upsert_query += BuildDeleteInsertRefreshSQL(data_table, view_query_sql, "openivm_recompute", affected_filter, - affected_filter); - upsert_query += "DROP TABLE IF EXISTS " + qtemp_affected + ";\n"; - return upsert_query; + return BuildAffectedPartitionRefreshSQL(data_table, view_query_sql, affected_keys, qtemp_affected, + partition_cols); } key_cols.clear(); @@ -547,17 +540,13 @@ static string BuildMultiSourceDuckLakeWindowRefresh(RefreshMetadata &metadata, C string temp_affected = string(openivm::TEMP_TABLE_PREFIX) + "affected_" + view_name; string qtemp_affected = KeywordHelper::WriteOptionallyQuoted(temp_affected); string fallback_affected_keys = "SELECT DISTINCT " + key_cols + " FROM (" + changed_rows + ") openivm_changed_rows"; - string affected_filter = BuildAffectedPartitionFilter(partition_cols, qtemp_affected); OPENIVM_DEBUG_PRINT("[UPSERT] Compiling upsert for type: WINDOW_PARTITION (DuckLake view-diff, %zu " "partition cols, %zu sources)\n", partition_cols.size(), delta_table_names.size()); // Materialize the affected partition keys once; otherwise DuckDB/DuckLake repeats the // full view diff independently for DELETE and INSERT. - string upsert_query = "CREATE OR REPLACE TEMP TABLE " + qtemp_affected + " AS " + fallback_affected_keys + ";\n"; - upsert_query += - BuildDeleteInsertRefreshSQL(data_table, view_query_sql, "openivm_recompute", affected_filter, affected_filter); - upsert_query += "DROP TABLE IF EXISTS " + qtemp_affected + ";\n"; - return upsert_query; + return BuildAffectedPartitionRefreshSQL(data_table, view_query_sql, fallback_affected_keys, qtemp_affected, + partition_cols); } string BuildWindowPartitionRefresh(RefreshMetadata &metadata, Connection &con, const string &view_name, @@ -573,8 +562,6 @@ string BuildWindowPartitionRefresh(RefreshMetadata &metadata, Connection &con, c bool any_ducklake = AnyDuckLakeSource(metadata, view_name, delta_table_names); bool safe_for_snapdiff = IsSafeForDuckLakeSnapshotDiff(partition_cols, column_names, any_ducklake); string affected_keys_sql; - string affected_key_cols; - string affected_key_tuple; bool have_lineage_affected_keys = false; if (safe_for_snapdiff && delta_table_names.size() == 1) { @@ -594,8 +581,7 @@ string BuildWindowPartitionRefresh(RefreshMetadata &metadata, Connection &con, c } auto lineage_result = BuildLineageStandardAffectedKeysSQL( metadata, con, view_name, delta_table_names, partition_cols, delta_ts_filter, view_catalog_name, - view_schema_name, attached_db_catalog_name, attached_db_schema_name, affected_keys_sql, affected_key_cols, - affected_key_tuple); + view_schema_name, attached_db_catalog_name, attached_db_schema_name, affected_keys_sql); if (lineage_result == LineageAffectedKeysResult::UNSAFE) { OPENIVM_DEBUG_PRINT("[UPSERT] WINDOW_PARTITION lineage is unsafe for '%s' — full recompute fallback\n", view_name.c_str()); @@ -624,8 +610,8 @@ string BuildWindowPartitionRefresh(RefreshMetadata &metadata, Connection &con, c } } return CompileWindowRecompute(view_name, view_query_sql, delta_ts_filter, internal_catalog_prefix, partition_cols, - partition_delta_specs, emit_cascade_delta, affected_keys_sql, affected_key_cols, - affected_key_tuple, running_window_column_names, running_window_incremental); + partition_delta_specs, emit_cascade_delta, affected_keys_sql, + running_window_column_names, running_window_incremental); } } // namespace duckdb diff --git a/test/sql/window.test b/test/sql/window.test index e3313746..8af55c78 100644 --- a/test/sql/window.test +++ b/test/sql/window.test @@ -545,3 +545,110 @@ SELECT count(*) FROM ( ); ---- 0 + +# ========================================== +# Regression: NULL partition keys use window equality semantics +# ========================================== + +statement ok +CREATE TABLE w_null_part (id INT, grp VARCHAR, val INT); + +statement ok +INSERT INTO w_null_part VALUES + (1, NULL, 10), + (2, NULL, 20), + (3, 'a', 5); + +statement ok +CREATE MATERIALIZED VIEW mv_null_part AS +SELECT id, grp, val, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY val, id) AS rn +FROM w_null_part; + +# Batch conflicting changes in the NULL partition before one refresh. +statement ok +INSERT INTO w_null_part VALUES + (4, NULL, 15), + (5, NULL, 30); + +statement ok +UPDATE w_null_part SET val = 25 WHERE id = 1; + +statement ok +DELETE FROM w_null_part WHERE id IN (2, 5); + +statement ok +PRAGMA refresh('mv_null_part'); + +query I +SELECT count(*) FROM ( + SELECT id, grp, val, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY val, id) AS rn + FROM w_null_part + EXCEPT ALL + SELECT * FROM mv_null_part +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM mv_null_part + EXCEPT ALL + SELECT id, grp, val, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY val, id) AS rn + FROM w_null_part +); +---- +0 + +# Composite partition keys must also match mixed-NULL tuples. +statement ok +CREATE TABLE w_null_comp (id INT, dept VARCHAR, team VARCHAR, val INT); + +statement ok +INSERT INTO w_null_comp VALUES + (1, 'eng', NULL, 10), + (2, 'eng', NULL, 20), + (3, NULL, 'ops', 30), + (4, 'hr', 'rec', 40); + +statement ok +CREATE MATERIALIZED VIEW mv_null_comp AS +SELECT id, dept, team, val, + ROW_NUMBER() OVER (PARTITION BY dept, team ORDER BY val, id) AS rn +FROM w_null_comp; + +statement ok +INSERT INTO w_null_comp VALUES + (5, 'eng', NULL, 15), + (6, NULL, 'ops', 35), + (7, 'eng', NULL, 50); + +statement ok +UPDATE w_null_comp SET val = 25 WHERE id = 1; + +statement ok +DELETE FROM w_null_comp WHERE id IN (2, 7); + +statement ok +PRAGMA refresh('mv_null_comp'); + +query I +SELECT count(*) FROM ( + SELECT id, dept, team, val, + ROW_NUMBER() OVER (PARTITION BY dept, team ORDER BY val, id) AS rn + FROM w_null_comp + EXCEPT ALL + SELECT * FROM mv_null_comp +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM mv_null_comp + EXCEPT ALL + SELECT id, dept, team, val, + ROW_NUMBER() OVER (PARTITION BY dept, team ORDER BY val, id) AS rn + FROM w_null_comp +); +---- +0 diff --git a/test/sql/window_running_aggregate_incremental.test b/test/sql/window_running_aggregate_incremental.test index 0c4a26c4..70b26501 100644 --- a/test/sql/window_running_aggregate_incremental.test +++ b/test/sql/window_running_aggregate_incremental.test @@ -18,7 +18,9 @@ INSERT INTO wrai_sales VALUES (1, 'a', 1, 10), (2, 'a', 2, 5), (3, 'b', 1, 7), - (4, 'c', 1, 3); + (4, 'c', 1, 3), + (11, NULL, 1, 4), + (12, NULL, 2, 6); statement ok CREATE MATERIALIZED VIEW wrai_sum_mv AS @@ -34,7 +36,8 @@ statement ok INSERT INTO wrai_sales VALUES (5, 'a', 3, 8), (6, 'b', 2, 9), - (7, 'c', 2, 4); + (7, 'c', 2, 4), + (13, NULL, 3, 2); query I SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_run_fast_wrai_sum_mv%' @@ -94,7 +97,8 @@ statement ok INSERT INTO wrai_sales VALUES (8, 'a', 0, 11), (9, 'b', 3, 2), - (10, 'd', 1, 6); + (10, 'd', 1, 6), + (14, NULL, 0, 13); statement ok PRAGMA refresh('wrai_sum_mv'); From 0e0c55234b3d481809b80f395b2665f7518efa36 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 23 Jul 2026 15:47:08 +0200 Subject: [PATCH 09/36] Preserve NULL semantics for incremental SUM --- benchmark/queries/ducklake_0251.sql | 5 + docs/codebase-audit-2026-07-22.md | 5 +- src/core/ivm_view_classifier.cpp | 9 +- src/core/parser.cpp | 7 + src/core/plan_rewrite.cpp | 5 +- src/core/plan_rewrite_aggregates.cpp | 132 +++++++++ src/include/core/openivm_constants.hpp | 4 + src/include/core/plan_rewrite_internal.hpp | 2 + src/upsert/refresh_compiler.cpp | 84 +++++- test/sql/aggregate.test | 307 +++++++++++++++++++++ test/sql/ducklake_aggregate.test | 77 ++++++ 11 files changed, 621 insertions(+), 16 deletions(-) create mode 100644 benchmark/queries/ducklake_0251.sql diff --git a/benchmark/queries/ducklake_0251.sql b/benchmark/queries/ducklake_0251.sql new file mode 100644 index 00000000..405d411e --- /dev/null +++ b/benchmark/queries/ducklake_0251.sql @@ -0,0 +1,5 @@ +-- {"operators": "AGGREGATE,CASE", "complexity": "medium", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": true, "tables": "ORDER_LINE", "ducklake": true} +SELECT OL_W_ID, OL_D_ID, OL_O_ID, OL_NUMBER, + SUM(CASE WHEN OL_DELIVERY_D IS NULL THEN OL_AMOUNT ELSE NULL END) AS pending_amount +FROM dl.ORDER_LINE +GROUP BY OL_W_ID, OL_D_ID, OL_O_ID, OL_NUMBER; diff --git a/docs/codebase-audit-2026-07-22.md b/docs/codebase-audit-2026-07-22.md index 95ffb8ce..cec54a69 100644 --- a/docs/codebase-audit-2026-07-22.md +++ b/docs/codebase-audit-2026-07-22.md @@ -32,7 +32,7 @@ maintenance path. A bug must not be hidden by weakening a test or silently chang `IS NOT DISTINCT FROM` predicates for standard, cascade, running-window, and DuckLake paths. Regression coverage includes batched conflicting DML on single and composite NULL partitions plus a DuckLake TPCC window whose delivery timestamp moves from the NULL partition to a non-NULL partition. -- [ ] **Preserve `SUM` NULL semantics.** Weighted SUM alone cannot distinguish numeric zero from no non-NULL inputs. Persist +- [x] **Preserve `SUM` NULL semantics.** Weighted SUM alone cannot distinguish numeric zero from no non-NULL inputs. Persist a hidden `COUNT(sum_argument)` and render NULL when that count reaches zero. - [ ] **Use an unconditional row count for group existence.** `COUNT(nullable_expression)=0` does not mean a group is empty. Persist a hidden `COUNT(*)` solely for deciding whether the group row must be deleted. @@ -122,6 +122,9 @@ maintenance path. A bug must not be hidden by weakening a test or silently chang base-leaf appearances, compilation work, and generated SQL size. - [ ] **Remove the fake `ConstraintCache` or make it a cache.** It repopulates a map that no read path consumes, while FK cost estimation separately parses textual constraint descriptions. +- [ ] **Audit edge-case tests and mirror them in the rewriter benchmark.** Review the SQL harness for missing boundary and + transition cases, add deterministic bidirectional `EXCEPT ALL` coverage, and add representative cases to the rewriter + benchmark so real refresh compilation and execution exercise the same semantics. ## Target refresh flow diff --git a/src/core/ivm_view_classifier.cpp b/src/core/ivm_view_classifier.cpp index 86212442..44d88772 100644 --- a/src/core/ivm_view_classifier.cpp +++ b/src/core/ivm_view_classifier.cpp @@ -418,9 +418,10 @@ static void BuildGroupColumns(DeltaViewModel &model, const CreateMVPlanFacts &fa model.group_columns = DeriveGroupColumnNames(facts, group_index, group_count, output_names); } + idx_t actual_visible_outputs = GetVisibleOutputCount(output_names, visible_output_count); if (analysis.found_delim_join && analysis.found_aggregation && model.group_columns.empty() && - output_names.size() > analysis.aggregate_types.size()) { - idx_t key_count = output_names.size() - analysis.aggregate_types.size(); + actual_visible_outputs > analysis.aggregate_types.size()) { + idx_t key_count = actual_visible_outputs - analysis.aggregate_types.size(); for (idx_t i = 0; i < key_count; i++) { if (!output_names[i].empty() && !IncrementalTableNames::IsInternalColumn(output_names[i])) { model.group_columns.push_back(output_names[i]); @@ -507,11 +508,11 @@ static void BuildGroupColumns(DeltaViewModel &model, const CreateMVPlanFacts &fa if (analysis.found_join && analysis.found_aggregation && !model.group_columns.empty()) { idx_t expected_linear_outputs = model.group_columns.size() + model.aggregate_types.size(); - if (output_names.size() > expected_linear_outputs) { + if (actual_visible_outputs > expected_linear_outputs) { AddUnique(model.strategy_reasons, DeltaStrategyReason::JOIN_AGGREGATE_PROJECTION_FALLBACK); OPENIVM_DEBUG_PRINT("[CREATE MV] Join-over-aggregate exposes %zu columns but only %zu are " "group/aggregate outputs -- using GROUP_RECOMPUTE\n", - output_names.size(), expected_linear_outputs); + actual_visible_outputs, expected_linear_outputs); } } diff --git a/src/core/parser.cpp b/src/core/parser.cpp index 7cb3a5b7..ccc3ca8d 100644 --- a/src/core/parser.cpp +++ b/src/core/parser.cpp @@ -6,6 +6,7 @@ #include "core/parser_ddl.hpp" #include "core/parser_plan_helpers.hpp" #include "core/parser_sql_extractors.hpp" +#include "core/plan_rewrite_internal.hpp" #include "core/refresh_locks.hpp" #include "core/refresh_metadata.hpp" #include "core/ivm_delta_model.hpp" @@ -401,6 +402,12 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // Strip HAVING filter from plan — data table stores all groups. // The predicate is extracted as SQL (using output aliases) for the VIEW WHERE clause. having_predicate = StripHavingFilter(select_plan, output_names); + // HAVING-only aggregates are exposed by StripHavingFilter. Inject SUM's + // non-NULL state afterward so visible, wrapped, and HAVING-only SUMs all + // use the same output-index mapping during incremental maintenance. + InjectSumNonNullCounts(context, select_plan); + PropagateHiddenAggregateColumns(select_plan); + output_names = PrepareOutputNames(select_plan.get(), select_planner.names); auto post_rewrite_facts = BuildCreateMVPlanFacts(select_plan.get(), current_catalog); stored_query_has_aggregate_filter = post_rewrite_facts.has_filter_above_aggregate; has_hidden_minmax_having = post_rewrite_facts.has_hidden_minmax_having_column; diff --git a/src/core/plan_rewrite.cpp b/src/core/plan_rewrite.cpp index 208865af..a385d337 100644 --- a/src/core/plan_rewrite.cpp +++ b/src/core/plan_rewrite.cpp @@ -946,7 +946,8 @@ static void InjectGroupCountStar(unique_ptr &plan) { /// projection's expression up through a pass-through parent projection. static bool IsHiddenAggregateAlias(const string &alias) { return alias.find(openivm::SUM_COL_PREFIX) == 0 || alias.find(openivm::COUNT_COL_PREFIX) == 0 || - alias.find(openivm::SUM_SQ_COL_PREFIX) == 0 || alias.find(openivm::SUM_SQP_COL_PREFIX) == 0; + alias.find(openivm::SUM_SQ_COL_PREFIX) == 0 || alias.find(openivm::SUM_SQP_COL_PREFIX) == 0 || + alias.find(openivm::SUM_COUNT_COL_PREFIX) == 0; } /// Propagate hidden aggregate columns (openivm_sum_*, openivm_count_*, …) added @@ -962,7 +963,7 @@ static bool IsHiddenAggregateAlias(const string &alias) { /// data table stores only the final AVG and the MERGE computes `v.avg + d.avg` /// — wrong for non-summable aggregates. Propagation lets CompileAggregateGroups /// see the hidden SUM/COUNT columns and maintain them separately. -static void PropagateHiddenAggregateColumns(unique_ptr &plan) { +void PropagateHiddenAggregateColumns(unique_ptr &plan) { for (auto &child : plan->children) { PropagateHiddenAggregateColumns(child); } diff --git a/src/core/plan_rewrite_aggregates.cpp b/src/core/plan_rewrite_aggregates.cpp index 2c1762f6..d1615ab4 100644 --- a/src/core/plan_rewrite_aggregates.cpp +++ b/src/core/plan_rewrite_aggregates.cpp @@ -9,6 +9,7 @@ #include "duckdb/optimizer/optimizer.hpp" #include "duckdb/planner/expression/bound_aggregate_expression.hpp" #include "duckdb/planner/expression/bound_case_expression.hpp" +#include "duckdb/planner/expression/bound_cast_expression.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/expression/bound_comparison_expression.hpp" #include "duckdb/planner/expression/bound_constant_expression.hpp" @@ -134,6 +135,137 @@ LogicalOperator *FindProjectionAggregateInput(unique_ptr &plan, } return nullptr; } + +void InjectSumNonNullCounts(ClientContext &context, unique_ptr &plan) { + if (!plan) { + return; + } + + // Find the linear output path down to the aggregate. ORDER BY/LIMIT/TOP_N + // and MATERIALIZED_CTE are pass-through wrappers; projections may rename or + // cast aggregate outputs and therefore need to be followed explicitly. + vector projections; + LogicalOperator *node = plan.get(); + LogicalOperator *agg_search = nullptr; + while (node) { + switch (node->type) { + case LogicalOperatorType::LOGICAL_ORDER_BY: + case LogicalOperatorType::LOGICAL_LIMIT: + case LogicalOperatorType::LOGICAL_TOP_N: + case LogicalOperatorType::LOGICAL_DISTINCT: + node = node->children.empty() ? nullptr : node->children[0].get(); + continue; + case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE: + node = node->children.size() < 2 ? nullptr : node->children[1].get(); + continue; + case LogicalOperatorType::LOGICAL_PROJECTION: + projections.push_back(&node->Cast()); + node = node->children.empty() ? nullptr : node->children[0].get(); + continue; + case LogicalOperatorType::LOGICAL_FILTER: + if (!node->children.empty() && + node->children[0]->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + agg_search = node->children[0].get(); + } + node = nullptr; + continue; + case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: + agg_search = node; + node = nullptr; + continue; + default: + node = nullptr; + continue; + } + } + if (!agg_search || projections.empty()) { + return; + } + + auto &agg = agg_search->Cast(); + auto &aggregate_projection = *projections.back(); + auto &output_projection = *projections.front(); + const idx_t original_aggregate_count = agg.expressions.size(); + + struct SumCountOutput { + idx_t projection_index; + idx_t aggregate_index; + }; + vector outputs; + for (idx_t projection_index = 0; projection_index < output_projection.expressions.size(); projection_index++) { + Expression *resolved = output_projection.expressions[projection_index].get(); + while (resolved) { + while (resolved->expression_class == ExpressionClass::BOUND_CAST) { + resolved = resolved->Cast().child.get(); + } + if (resolved->type != ExpressionType::BOUND_COLUMN_REF) { + break; + } + auto &ref = resolved->Cast(); + if (ref.binding.table_index == agg.aggregate_index) { + break; + } + LogicalProjection *referenced_projection = nullptr; + for (idx_t projection_depth = 1; projection_depth < projections.size(); projection_depth++) { + if (projections[projection_depth]->table_index == ref.binding.table_index) { + referenced_projection = projections[projection_depth]; + break; + } + } + if (!referenced_projection || ref.binding.column_index >= referenced_projection->expressions.size()) { + resolved = nullptr; + break; + } + resolved = referenced_projection->expressions[ref.binding.column_index].get(); + } + if (!resolved || resolved->type != ExpressionType::BOUND_COLUMN_REF) { + continue; + } + auto &ref = resolved->Cast(); + if (ref.binding.table_index != agg.aggregate_index || ref.binding.column_index >= original_aggregate_count) { + continue; + } + auto &aggregate_expr = agg.expressions[ref.binding.column_index]; + if (aggregate_expr->expression_class != ExpressionClass::BOUND_AGGREGATE) { + continue; + } + auto &sum = aggregate_expr->Cast(); + if (sum.function.name != "sum" || sum.IsDistinct() || sum.children.size() != 1) { + continue; + } + if (sum.alias.find(string(openivm::SUM_COL_PREFIX)) == 0 || + sum.alias.find(string(openivm::VAR_SQ_COL_PREFIX)) == 0 || + sum.alias.find(string(openivm::VAR_SQP_COL_PREFIX)) == 0) { + continue; + } + + auto count_func = BindAggregateByName(context, "count", {sum.children[0]->return_type}); + vector> count_args; + count_args.push_back(sum.children[0]->Copy()); + auto count_expr = make_uniq(std::move(count_func), std::move(count_args), nullptr, + nullptr, AggregateType::NON_DISTINCT); + count_expr->alias = string(openivm::SUM_COUNT_COL_PREFIX) + to_string(projection_index); + outputs.push_back({projection_index, agg.expressions.size()}); + agg.expressions.push_back(std::move(count_expr)); + } + if (outputs.empty()) { + return; + } + + agg.ResolveOperatorTypes(); + auto aggregate_bindings = agg_search->GetColumnBindings(); + auto aggregate_types = agg_search->types; + for (auto &output : outputs) { + idx_t output_index = agg.groups.size() + output.aggregate_index; + auto count_ref = + make_uniq(aggregate_types[output_index], aggregate_bindings[output_index]); + count_ref->alias = string(openivm::SUM_COUNT_COL_PREFIX) + to_string(output.projection_index); + aggregate_projection.expressions.push_back(std::move(count_ref)); + } + aggregate_projection.ResolveOperatorTypes(); + OPENIVM_DEBUG_PRINT("[PlanRewrite] Injected %zu SUM non-NULL counts\n", outputs.size()); +} + void RewriteDerivedAggregates(ClientContext &context, unique_ptr &plan, Optimizer &opt, bool is_top) { for (auto &child : plan->children) { RewriteDerivedAggregates(context, child, opt, false); diff --git a/src/include/core/openivm_constants.hpp b/src/include/core/openivm_constants.hpp index 8fdef1a1..9c31d0ae 100644 --- a/src/include/core/openivm_constants.hpp +++ b/src/include/core/openivm_constants.hpp @@ -37,6 +37,10 @@ constexpr const char *VAR_SQ_COL_PREFIX = "openivm_var_sq_"; // VARIANCE: no s constexpr const char *SUM_SQP_COL_PREFIX = "openivm_sum_sqp_"; // STDDEV_POP: sqrt + population denominator constexpr const char *VAR_SQP_COL_PREFIX = "openivm_var_sqp_"; // VAR_POP: no sqrt + population denominator constexpr const char *COUNT_COL_PREFIX = "openivm_count_"; +// COUNT(sum_argument) companions for user-visible SUM outputs. The suffix is +// the visible projection index, which associates the state with the exact +// bound output without inspecting SQL text or aliases. +constexpr const char *SUM_COUNT_COL_PREFIX = "openivm_nonnull_sum_count_"; // Hidden COUNT(*) injected into AGGREGATE_GROUP MVs that don't already have a // count aggregate. Tracks per-group cardinality so the cleanup can delete rows diff --git a/src/include/core/plan_rewrite_internal.hpp b/src/include/core/plan_rewrite_internal.hpp index 1e7e26a1..c354d3e2 100644 --- a/src/include/core/plan_rewrite_internal.hpp +++ b/src/include/core/plan_rewrite_internal.hpp @@ -11,6 +11,8 @@ AggregateFunction BindAggregateByName(ClientContext &context, const string &name LogicalOperator *FindProjectionAggregateInput(unique_ptr &plan, bool allow_having_filter); void RewriteDerivedAggregates(ClientContext &context, unique_ptr &plan, Optimizer &opt, bool is_top = true); +void InjectSumNonNullCounts(ClientContext &context, unique_ptr &plan); +void PropagateHiddenAggregateColumns(unique_ptr &plan); bool RewriteSafeSemiAntiDelimGets(ClientContext &context, unique_ptr &plan); } // namespace duckdb diff --git a/src/upsert/refresh_compiler.cpp b/src/upsert/refresh_compiler.cpp index 2e6dff40..f2b16c14 100644 --- a/src/upsert/refresh_compiler.cpp +++ b/src/upsert/refresh_compiler.cpp @@ -7,6 +7,7 @@ #include "upsert/refresh_internal.hpp" #include +#include namespace duckdb { @@ -329,6 +330,10 @@ static string BuildUpdatedAggregateColumn(const string &col) { return "COALESCE(v." + col + " + d." + col + ", v." + col + ", d." + col + ")"; } +static string BuildNullableSum(const string &sum_expr, const string &count_expr) { + return "CASE WHEN " + count_expr + " = 0 THEN NULL ELSE " + sum_expr + " END"; +} + static string BuildNullSafeExtremumUpdate(const string &col, const string &fn) { return "CASE WHEN v." + col + " IS NULL THEN d." + col + " WHEN d." + col + " IS NULL THEN v." + col + " ELSE " + fn + "(v." + col + ", d." + col + ") END"; @@ -496,6 +501,32 @@ static DerivedAggDecomposition DetectDerivedAggColumns(const vector &col return result; } +static unordered_map DetectSumNullCountColumns(const vector &columns) { + unordered_map result; + const string prefix = openivm::SUM_COUNT_COL_PREFIX; + for (auto &column : columns) { + if (column.size() <= prefix.size() || column.compare(0, prefix.size(), prefix) != 0) { + continue; + } + idx_t projection_index = 0; + for (idx_t i = prefix.size(); i < column.size(); i++) { + if (!std::isdigit(static_cast(column[i]))) { + throw InternalException("Invalid SUM count state column '%s'", column); + } + idx_t digit = column[i] - '0'; + if (projection_index > (std::numeric_limits::max() - digit) / 10) { + throw InternalException("Invalid SUM count state column '%s'", column); + } + projection_index = projection_index * 10 + digit; + } + if (projection_index >= columns.size()) { + throw InternalException("Invalid SUM count state column '%s'", column); + } + result[SqlUtils::QuoteIdentifier(columns[projection_index])] = SqlUtils::QuoteIdentifier(column); + } + return result; +} + string CompileAggregateGroups(const string &view_name, optional_ptr index_delta_view_catalog_entry, vector column_names, const string &view_query_sql, bool has_minmax, bool list_mode, const string &delta_ts_filter, const vector &group_column_names, @@ -567,6 +598,7 @@ string CompileAggregateGroups(const string &view_name, optional_ptrsecond)); + inserted_column_expressions[raw_column] = BuildNullableSum("d." + column, "d." + sum_count->second); + } else if (insert_only && agg_type == "min") { updated_column_expressions[raw_column] = BuildNullSafeExtremumUpdate(column, "LEAST"); } else if (insert_only && agg_type == "max") { updated_column_expressions[raw_column] = BuildNullSafeExtremumUpdate(column, "GREATEST"); @@ -986,7 +1029,8 @@ string CompileAggregateGroups(const string &view_name, optional_ptr