diff --git a/CMakeLists.txt b/CMakeLists.txt index 326cbe74..9889f8bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,6 +57,7 @@ set(EXTENSION_SOURCES src/upsert/refresh_index_regen.cpp src/rules/incremental_rewrite_rule.cpp src/rules/refresh_insert_rule.cpp + src/rules/transactional_delta_capture.cpp src/rules/schema_evolution.cpp src/rules/column_hider.cpp src/core/refresh_locks.cpp diff --git a/benchmark/queries/ducklake_0251.sql b/benchmark/queries/ducklake_0251.sql new file mode 100644 index 00000000..405d411e --- /dev/null +++ b/benchmark/queries/ducklake_0251.sql @@ -0,0 +1,5 @@ +-- {"operators": "AGGREGATE,CASE", "complexity": "medium", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": true, "tables": "ORDER_LINE", "ducklake": true} +SELECT OL_W_ID, OL_D_ID, OL_O_ID, OL_NUMBER, + SUM(CASE WHEN OL_DELIVERY_D IS NULL THEN OL_AMOUNT ELSE NULL END) AS pending_amount +FROM dl.ORDER_LINE +GROUP BY OL_W_ID, OL_D_ID, OL_O_ID, OL_NUMBER; diff --git a/benchmark/queries/tpcc/ducklake_0309.sql b/benchmark/queries/tpcc/ducklake_0309.sql index da97cb12..6ae8099d 100644 --- a/benchmark/queries/tpcc/ducklake_0309.sql +++ b/benchmark/queries/tpcc/ducklake_0309.sql @@ -1140,5 +1140,5 @@ SELECT min_balance, max_balance FROM topk_projected -ORDER BY total_amount DESC, warehouse_id ASC, district_id ASC, customer_id ASC +ORDER BY ROUND(total_amount, 10) DESC, warehouse_id ASC, district_id ASC, customer_id ASC LIMIT 25 OFFSET 2 diff --git a/benchmark/queries/tpcc/ducklake_0316.sql b/benchmark/queries/tpcc/ducklake_0316.sql new file mode 100644 index 00000000..5b954630 --- /dev/null +++ b/benchmark/queries/tpcc/ducklake_0316.sql @@ -0,0 +1,2 @@ +-- {"operators": "DUCKLAKE,OUTER_JOIN,WINDOW,CAST", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": true, "has_case": false, "tables": "CUSTOMER,OORDER", "ducklake": true, "openivm_verified": true} +SELECT c.C_W_ID AS warehouse_id, c.C_D_ID AS district_id, c.C_ID AS customer_id, o.O_ID AS order_id, ROW_NUMBER() OVER (PARTITION BY c.C_W_ID ORDER BY o.O_ID DESC NULLS LAST, c.C_D_ID, c.C_ID) AS rn FROM dl.CUSTOMER c LEFT JOIN dl.OORDER o ON c.C_W_ID = o.O_W_ID AND c.C_D_ID = o.O_D_ID AND CAST(CAST(c.C_ID AS BIGINT) AS VARCHAR) = CAST(o.O_C_ID AS VARCHAR); diff --git a/benchmark/queries/tpcc/ducklake_0317.sql b/benchmark/queries/tpcc/ducklake_0317.sql new file mode 100644 index 00000000..df0dcd33 --- /dev/null +++ b/benchmark/queries/tpcc/ducklake_0317.sql @@ -0,0 +1,2 @@ +-- {"operators": "DUCKLAKE,OUTER_JOIN,WINDOW,CAST", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": true, "has_case": false, "tables": "CUSTOMER,OORDER", "ducklake": true, "openivm_verified": true} +SELECT c.C_W_ID AS warehouse_id, c.C_D_ID AS district_id, c.C_ID AS customer_id, o.O_ID AS order_id, ROW_NUMBER() OVER (PARTITION BY c.C_W_ID ORDER BY o.O_ID DESC NULLS LAST, c.C_D_ID, c.C_ID) AS rn FROM dl.CUSTOMER c LEFT JOIN dl.OORDER o ON c.C_W_ID = o.O_W_ID AND c.C_D_ID = o.O_D_ID AND CAST(TRY_CAST(CAST(c.C_ID AS VARCHAR) AS BIGINT) AS VARCHAR) = CAST(o.O_C_ID AS VARCHAR); diff --git a/benchmark/queries/tpcc/ducklake_0318.sql b/benchmark/queries/tpcc/ducklake_0318.sql new file mode 100644 index 00000000..4c007290 --- /dev/null +++ b/benchmark/queries/tpcc/ducklake_0318.sql @@ -0,0 +1,12 @@ +-- {"operators": "DUCKLAKE,WINDOW", "complexity": "low", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE", "ducklake": true, "openivm_verified": true} +SELECT + OL_W_ID AS warehouse_id, + OL_D_ID AS district_id, + OL_O_ID AS order_id, + OL_NUMBER AS line_number, + OL_DELIVERY_D AS delivery_at, + ROW_NUMBER() OVER ( + PARTITION BY OL_DELIVERY_D + ORDER BY OL_W_ID, OL_D_ID, OL_O_ID, OL_NUMBER + ) AS delivery_row_number +FROM dl.ORDER_LINE; diff --git a/benchmark/queries/tpcc/ducklake_0319.sql b/benchmark/queries/tpcc/ducklake_0319.sql new file mode 100644 index 00000000..80292f6b --- /dev/null +++ b/benchmark/queries/tpcc/ducklake_0319.sql @@ -0,0 +1,9 @@ +-- {"operators": "DISTINCT,SUBQUERY", "complexity": "medium", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "CUSTOMER", "ducklake": true, "openivm_verified": true} +SELECT payment_count % 2 AS parity +FROM ( + SELECT distinct_payment_count AS payment_count + FROM ( + SELECT DISTINCT C_PAYMENT_CNT AS distinct_payment_count + FROM dl.CUSTOMER + ) +); diff --git a/benchmark/queries/tpcc/query_2094.sql b/benchmark/queries/tpcc/query_2094.sql index c2afbacf..bbc2630d 100644 --- a/benchmark/queries/tpcc/query_2094.sql +++ b/benchmark/queries/tpcc/query_2094.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,FILTER", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"} +-- {"operators": "ASOF_JOIN,FILTER", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"} SELECT ol.OL_W_ID, ol.OL_D_ID, ol.OL_O_ID, o.O_ENTRY_D FROM ORDER_LINE ol ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D WHERE ol.OL_AMOUNT > 0; diff --git a/benchmark/queries/tpcc/query_2095.sql b/benchmark/queries/tpcc/query_2095.sql index 1a7f5c28..32d2e803 100644 --- a/benchmark/queries/tpcc/query_2095.sql +++ b/benchmark/queries/tpcc/query_2095.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,LEFT_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"} +-- {"operators": "ASOF_JOIN,LEFT_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"} SELECT ol.OL_W_ID, ol.OL_I_ID, o.O_C_ID FROM ORDER_LINE ol ASOF LEFT JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D; diff --git a/benchmark/queries/tpcc/query_2097.sql b/benchmark/queries/tpcc/query_2097.sql index 1b9c9c07..8092a007 100644 --- a/benchmark/queries/tpcc/query_2097.sql +++ b/benchmark/queries/tpcc/query_2097.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,INNER_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER,CUSTOMER"} +-- {"operators": "ASOF_JOIN,INNER_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER,CUSTOMER", "non_incr_reason": "op:ASOF_JOIN"} SELECT c.C_W_ID, c.C_D_ID, c.C_ID, ol.OL_AMOUNT FROM ORDER_LINE ol ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D JOIN CUSTOMER c ON c.C_W_ID = o.O_W_ID AND c.C_D_ID = o.O_D_ID AND c.C_ID = o.O_C_ID; diff --git a/benchmark/queries/tpcc/query_2099.sql b/benchmark/queries/tpcc/query_2099.sql index 9dc99455..1b1c280c 100644 --- a/benchmark/queries/tpcc/query_2099.sql +++ b/benchmark/queries/tpcc/query_2099.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,CTE", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"} +-- {"operators": "ASOF_JOIN,CTE", "complexity": "high", "is_incremental": false, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"} WITH delivered AS (SELECT * FROM ORDER_LINE WHERE OL_DELIVERY_D IS NOT NULL) SELECT d.OL_W_ID, d.OL_O_ID, o.O_C_ID FROM delivered d ASOF JOIN OORDER o ON d.OL_W_ID = o.O_W_ID AND d.OL_D_ID = o.O_D_ID AND d.OL_DELIVERY_D >= o.O_ENTRY_D; diff --git a/benchmark/queries/tpcc/query_2101.sql b/benchmark/queries/tpcc/query_2101.sql index d4ef5a78..9b2c30d5 100644 --- a/benchmark/queries/tpcc/query_2101.sql +++ b/benchmark/queries/tpcc/query_2101.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,FULL_OUTER_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER,STOCK"} +-- {"operators": "ASOF_JOIN,FULL_OUTER_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER,STOCK", "non_incr_reason": "op:ASOF_JOIN"} SELECT COALESCE(s.S_W_ID, ol.OL_W_ID) AS w_id, COALESCE(s.S_I_ID, ol.OL_I_ID) AS item_id FROM ORDER_LINE ol ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D FULL OUTER JOIN STOCK s ON s.S_W_ID = ol.OL_SUPPLY_W_ID AND s.S_I_ID = ol.OL_I_ID; diff --git a/benchmark/queries/tpcc/query_2102.sql b/benchmark/queries/tpcc/query_2102.sql index 0f7bc45d..755de67e 100644 --- a/benchmark/queries/tpcc/query_2102.sql +++ b/benchmark/queries/tpcc/query_2102.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,SUBQUERY", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"} +-- {"operators": "ASOF_JOIN,SUBQUERY", "complexity": "high", "is_incremental": false, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"} SELECT * FROM (SELECT ol.OL_W_ID, ol.OL_D_ID, o.O_ENTRY_D FROM ORDER_LINE ol ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D) q WHERE q.O_ENTRY_D IS NOT NULL; diff --git a/benchmark/queries/tpcc/query_2103.sql b/benchmark/queries/tpcc/query_2103.sql index 549ed6ff..7dc6c58b 100644 --- a/benchmark/queries/tpcc/query_2103.sql +++ b/benchmark/queries/tpcc/query_2103.sql @@ -1,2 +1,2 @@ --- {"operators": "ASOF_JOIN,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"} +-- {"operators": "ASOF_JOIN,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"} SELECT ol.OL_W_ID, u.label, o.O_ENTRY_D FROM ORDER_LINE ol CROSS JOIN UNNEST(['delivery', 'entry']) AS u(label) ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D; diff --git a/benchmark/queries/tpcc/query_2114.sql b/benchmark/queries/tpcc/query_2114.sql index fa1905ab..73e749c8 100644 --- a/benchmark/queries/tpcc/query_2114.sql +++ b/benchmark/queries/tpcc/query_2114.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE", "complexity": "medium", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK"} +-- {"operators": "SAMPLE", "complexity": "medium", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "non_incr_reason": "op:SAMPLE"} SELECT S_W_ID, S_I_ID, S_QUANTITY FROM STOCK USING SAMPLE reservoir(10 ROWS) REPEATABLE (42); diff --git a/benchmark/queries/tpcc/query_2115.sql b/benchmark/queries/tpcc/query_2115.sql index 3bd115fa..a40c1566 100644 --- a/benchmark/queries/tpcc/query_2115.sql +++ b/benchmark/queries/tpcc/query_2115.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE,AGGREGATE", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE"} +-- {"operators": "SAMPLE,AGGREGATE", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE", "non_incr_reason": "op:SAMPLE"} SELECT OL_W_ID, SUM(OL_AMOUNT) AS sampled_amount FROM (SELECT * FROM ORDER_LINE USING SAMPLE reservoir(25 ROWS) REPEATABLE (7)) sampled_order_line GROUP BY OL_W_ID; diff --git a/benchmark/queries/tpcc/query_2116.sql b/benchmark/queries/tpcc/query_2116.sql index 13b26c43..b9677f7c 100644 --- a/benchmark/queries/tpcc/query_2116.sql +++ b/benchmark/queries/tpcc/query_2116.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE,WINDOW", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "openivm_verified": true} +-- {"operators": "SAMPLE,WINDOW", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "openivm_verified": true, "non_incr_reason": "op:SAMPLE"} SELECT S_W_ID, S_I_ID, S_QUANTITY, ROW_NUMBER() OVER (PARTITION BY S_W_ID ORDER BY S_QUANTITY DESC) AS sampled_rank FROM STOCK USING SAMPLE reservoir(20 ROWS) REPEATABLE (11); diff --git a/benchmark/queries/tpcc/query_2117.sql b/benchmark/queries/tpcc/query_2117.sql index 63293c5e..c000b6f4 100644 --- a/benchmark/queries/tpcc/query_2117.sql +++ b/benchmark/queries/tpcc/query_2117.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE,INNER_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ITEM,STOCK"} +-- {"operators": "SAMPLE,INNER_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ITEM,STOCK", "non_incr_reason": "op:SAMPLE"} SELECT i.I_ID, i.I_NAME, s.S_W_ID, s.S_QUANTITY FROM (SELECT * FROM STOCK USING SAMPLE reservoir(20 ROWS) REPEATABLE (13)) s JOIN ITEM i ON s.S_I_ID = i.I_ID; diff --git a/benchmark/queries/tpcc/query_2118.sql b/benchmark/queries/tpcc/query_2118.sql index 81fcd811..1cd4f687 100644 --- a/benchmark/queries/tpcc/query_2118.sql +++ b/benchmark/queries/tpcc/query_2118.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE,LEFT_JOIN,AGGREGATE", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ITEM,STOCK"} +-- {"operators": "SAMPLE,LEFT_JOIN,AGGREGATE", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ITEM,STOCK", "non_incr_reason": "op:SAMPLE"} SELECT i.I_IM_ID, COUNT(s.S_I_ID) AS sampled_stock_rows FROM ITEM i LEFT JOIN (SELECT * FROM STOCK USING SAMPLE reservoir(30 ROWS) REPEATABLE (17)) s ON i.I_ID = s.S_I_ID GROUP BY i.I_IM_ID; diff --git a/benchmark/queries/tpcc/query_2119.sql b/benchmark/queries/tpcc/query_2119.sql index e40b396d..83f2b310 100644 --- a/benchmark/queries/tpcc/query_2119.sql +++ b/benchmark/queries/tpcc/query_2119.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK"} +-- {"operators": "SAMPLE,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "non_incr_reason": "op:SAMPLE"} SELECT s.S_W_ID, u.threshold FROM (SELECT * FROM STOCK USING SAMPLE reservoir(15 ROWS) REPEATABLE (19)) s CROSS JOIN UNNEST([10, 50, 90]) AS u(threshold) WHERE s.S_QUANTITY >= u.threshold; diff --git a/benchmark/queries/tpcc/query_2121.sql b/benchmark/queries/tpcc/query_2121.sql index 732881e9..152d82a9 100644 --- a/benchmark/queries/tpcc/query_2121.sql +++ b/benchmark/queries/tpcc/query_2121.sql @@ -1,2 +1,2 @@ --- {"operators": "SAMPLE,PIVOT,AGGREGATE", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK"} +-- {"operators": "SAMPLE,PIVOT,AGGREGATE", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "non_incr_reason": "op:SAMPLE"} SELECT * FROM (PIVOT (SELECT * FROM STOCK USING SAMPLE reservoir(20 ROWS) REPEATABLE (29)) ON S_W_ID IN (1, 2, 3) USING SUM(S_QUANTITY) GROUP BY S_I_ID) p; diff --git a/benchmark/queries/tpcc/query_2122.sql b/benchmark/queries/tpcc/query_2122.sql index 7feee2ff..6a6f9539 100644 --- a/benchmark/queries/tpcc/query_2122.sql +++ b/benchmark/queries/tpcc/query_2122.sql @@ -1,2 +1,2 @@ --- {"operators": "POSITIONAL_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM"} +-- {"operators": "POSITIONAL_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM", "non_incr_reason": "join:POSITIONAL"} SELECT s.S_W_ID, s.S_I_ID, i.I_ID, i.I_NAME FROM STOCK s POSITIONAL JOIN ITEM i; diff --git a/benchmark/queries/tpcc/query_2123.sql b/benchmark/queries/tpcc/query_2123.sql index 863124f0..57c5aecb 100644 --- a/benchmark/queries/tpcc/query_2123.sql +++ b/benchmark/queries/tpcc/query_2123.sql @@ -1,2 +1,2 @@ --- {"operators": "POSITIONAL_JOIN,AGGREGATE", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM"} +-- {"operators": "POSITIONAL_JOIN,AGGREGATE", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM", "non_incr_reason": "join:POSITIONAL"} SELECT s.S_W_ID, COUNT(i.I_ID) AS paired_items, SUM(s.S_QUANTITY) AS paired_qty FROM STOCK s POSITIONAL JOIN ITEM i GROUP BY s.S_W_ID; diff --git a/benchmark/queries/tpcc/query_2124.sql b/benchmark/queries/tpcc/query_2124.sql index 803f2958..30324486 100644 --- a/benchmark/queries/tpcc/query_2124.sql +++ b/benchmark/queries/tpcc/query_2124.sql @@ -1,2 +1,2 @@ --- {"operators": "POSITIONAL_JOIN,WINDOW", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "CUSTOMER,OORDER", "openivm_verified": true} +-- {"operators": "POSITIONAL_JOIN,WINDOW", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "CUSTOMER,OORDER", "openivm_verified": true, "non_incr_reason": "join:POSITIONAL"} SELECT c.C_W_ID, c.C_ID, o.O_ID, ROW_NUMBER() OVER (ORDER BY c.C_W_ID, c.C_ID) AS pos_rank FROM CUSTOMER c POSITIONAL JOIN OORDER o; diff --git a/benchmark/queries/tpcc/query_2125.sql b/benchmark/queries/tpcc/query_2125.sql index 4a930fc1..4bb920f4 100644 --- a/benchmark/queries/tpcc/query_2125.sql +++ b/benchmark/queries/tpcc/query_2125.sql @@ -1,2 +1,2 @@ --- {"operators": "POSITIONAL_JOIN,LEFT_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM,ORDER_LINE"} +-- {"operators": "POSITIONAL_JOIN,LEFT_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM,ORDER_LINE", "non_incr_reason": "join:POSITIONAL"} SELECT s.S_W_ID, s.S_I_ID, i.I_NAME, ol.OL_AMOUNT FROM STOCK s POSITIONAL JOIN ITEM i LEFT JOIN ORDER_LINE ol ON ol.OL_I_ID = i.I_ID AND ol.OL_SUPPLY_W_ID = s.S_W_ID; diff --git a/benchmark/queries/tpcc/query_2126.sql b/benchmark/queries/tpcc/query_2126.sql index 884a3789..7a475bcf 100644 --- a/benchmark/queries/tpcc/query_2126.sql +++ b/benchmark/queries/tpcc/query_2126.sql @@ -1,2 +1,2 @@ --- {"operators": "POSITIONAL_JOIN,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM"} +-- {"operators": "POSITIONAL_JOIN,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM", "non_incr_reason": "join:POSITIONAL"} SELECT s.S_W_ID, i.I_ID, u.label FROM STOCK s POSITIONAL JOIN ITEM i CROSS JOIN UNNEST(['left_pos', 'right_pos']) AS u(label); diff --git a/benchmark/src/rewriter_benchmark.cpp b/benchmark/src/rewriter_benchmark.cpp index eae802e0..c2e71a3d 100644 --- a/benchmark/src/rewriter_benchmark.cpp +++ b/benchmark/src/rewriter_benchmark.cpp @@ -5,6 +5,7 @@ // #include "duckdb.hpp" +#include "duckdb/parser/keyword_helper.hpp" #include "duckdb/main/connection.hpp" #include "duckdb/common/printer.hpp" #include "core/openivm_extension.hpp" @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +47,55 @@ using openivm_bench::ReadAllBytes; using openivm_bench::Timestamp; using openivm_bench::WriteAllBytes; +static bool ReadAllBytesUntil(int fd, void *buf, size_t n, std::chrono::steady_clock::time_point deadline, + bool &timed_out) { + char *p = static_cast(buf); + while (n > 0) { + auto now = std::chrono::steady_clock::now(); + if (now >= deadline) { + timed_out = true; + return false; + } + + int remaining_ms = + static_cast(std::chrono::duration_cast(deadline - now).count()); + struct pollfd pfd; + pfd.fd = fd; + pfd.events = POLLIN; + pfd.revents = 0; + + int ret = poll(&pfd, 1, std::min(remaining_ms, 500)); + if (ret < 0) { + if (errno == EINTR) { + continue; + } + return false; + } + if (ret == 0) { + continue; + } + if (pfd.revents & POLLIN) { + ssize_t r = read(fd, p, n); + if (r < 0) { + if (errno == EINTR) { + continue; + } + return false; + } + if (r == 0) { + return false; + } + p += r; + n -= static_cast(r); + continue; + } + if (pfd.revents & (POLLHUP | POLLERR | POLLNVAL)) { + return false; + } + } + return true; +} + static string FormatNumber(double v) { std::ostringstream oss; oss << std::setprecision(5) << std::defaultfloat << v; @@ -476,6 +527,18 @@ static void ChildWorkerMain(int read_fd, int write_fd, const string &db_path, co duckdb::Connection con(db); con.Query("PRAGMA threads=4"); + // A few corpus queries mix TIMESTAMP columns with CURRENT_TIMESTAMP. + // DuckDB implements the required TIMESTAMPTZ conversion in ICU, so install + // it up front instead of turning a missing mid-run autoload into a query + // validation failure. + auto icu_install = con.Query("INSTALL icu"); + auto icu_load = con.Query("LOAD icu"); + if ((icu_install && icu_install->HasError()) || (icu_load && icu_load->HasError())) { + fprintf(stderr, "Warning: ICU extension unavailable; timestamp validation may fail: %s%s\n", + icu_install && icu_install->HasError() ? icu_install->GetError().c_str() : "", + icu_load && icu_load->HasError() ? icu_load->GetError().c_str() : ""); + } + // Figure out the default (native) catalog name so we can switch back // after a ducklake query. When `db_path` is a file like // `rewriter_benchmark_sf1.db`, DuckDB names the catalog @@ -487,7 +550,8 @@ static void ChildWorkerMain(int read_fd, int write_fd, const string &db_path, co native_catalog = cur->GetValue(0, 0).ToString(); } } - string native_use = "USE " + native_catalog + ".main"; + string quoted_native_catalog = duckdb::KeywordHelper::WriteOptionallyQuoted(native_catalog); + string native_use = "USE " + quoted_native_catalog + ".main"; // DuckLake support is only for the tpcc workload — tpcdi has no DuckLake variants. bool ducklake_ok = false; @@ -509,7 +573,8 @@ static void ChildWorkerMain(int read_fd, int write_fd, const string &db_path, co CreateTPCCSchema(con); for (const char *t : {"WAREHOUSE", "DISTRICT", "CUSTOMER", "ITEM", "STOCK", "OORDER", "NEW_ORDER", "ORDER_LINE", "HISTORY"}) { - con.Query(string("INSERT INTO ") + t + " SELECT * FROM " + native_catalog + ".main." + t); + con.Query(string("INSERT INTO ") + t + " SELECT * FROM " + quoted_native_catalog + + ".main." + t); } con.Query(native_use); } else { @@ -669,24 +734,35 @@ static void ChildWorkerMain(int read_fd, int write_fd, const string &db_path, co // Qualify with native_catalog so the lookup works both when the active catalog // is a DuckLake catalog (USE dl.main) and when the DB is file-based (catalog // name = filename, never "memory"). - auto check_result = con.Query("SELECT type FROM " + native_catalog + - ".openivm_views WHERE view_name = '" + mv_name + "'"); + auto check_result = con.Query("SELECT type FROM " + quoted_native_catalog + + ".main.openivm_views WHERE view_name = '" + mv_name + "'"); if (check_result && !check_result->HasError() && check_result->RowCount() > 0) { int64_t refresh_type = check_result->GetValue(0, 0).GetValue(); is_incremental = (refresh_type != 3) ? 1 : 0; + } else { + error = "OpenIVM metadata lookup failed for " + mv_name; + if (check_result && check_result->HasError()) { + error += ": " + check_result->GetError(); + } + phase_reached = PHASE_MV_CREATION_FAILED; } // Phase 3: Apply deltas - for (int d = 0; d < delta_batch_size && (size_t)delta_idx < deltas.size(); d++, delta_idx++) { - con.Query(deltas[delta_idx]); // Errors are OK (e.g. duplicate keys) + if (phase_reached == PHASE_DELTA_FAILED) { + for (int d = 0; d < delta_batch_size && (size_t)delta_idx < deltas.size(); + d++, delta_idx++) { + con.Query(deltas[delta_idx]); // Errors are OK (e.g. duplicate keys) + } + if ((size_t)delta_idx >= deltas.size()) + delta_idx = 0; + phase_reached = PHASE_REFRESH_FAILED; } - if ((size_t)delta_idx >= deltas.size()) - delta_idx = 0; - phase_reached = PHASE_REFRESH_FAILED; // Phase 4: PRAGMA refresh() start = std::chrono::steady_clock::now(); - auto refresh_result = con.Query("PRAGMA refresh('" + mv_name + "')"); + auto refresh_result = phase_reached == PHASE_REFRESH_FAILED + ? con.Query("PRAGMA refresh('" + mv_name + "')") + : nullptr; if (refresh_result && refresh_result->HasError()) { error = refresh_result->GetError(); if (IsFatalError(error)) { @@ -694,7 +770,7 @@ static void ChildWorkerMain(int read_fd, int write_fd, const string &db_path, co } else { phase_reached = PHASE_REFRESH_FAILED; } - } else { + } else if (refresh_result) { auto end_refresh = std::chrono::steady_clock::now(); time_refresh_ms = std::chrono::duration(end_refresh - start).count(); phase_reached = PHASE_VERIFY_FAILED; @@ -1019,34 +1095,51 @@ struct ForkWorker { int ret = poll(&pfd, 1, poll_ms); if (ret > 0 && (pfd.revents & POLLIN)) { - if (!ReadAllBytes(from_child_fd, &result_phase, sizeof(result_phase)) || - !ReadAllBytes(from_child_fd, &result_incremental, sizeof(result_incremental)) || - !ReadAllBytes(from_child_fd, &result_correct, sizeof(result_correct)) || - !ReadAllBytes(from_child_fd, &result_time_select_ms, sizeof(result_time_select_ms)) || - !ReadAllBytes(from_child_fd, &result_time_mv_ms, sizeof(result_time_mv_ms)) || - !ReadAllBytes(from_child_fd, &result_time_refresh_ms, sizeof(result_time_refresh_ms)) || - !ReadAllBytes(from_child_fd, &result_time_verify_ms, sizeof(result_time_verify_ms))) { + bool timed_out = false; + auto read_field = [&](void *buf, size_t n) { + return ReadAllBytesUntil(from_child_fd, buf, n, deadline, timed_out); + }; + auto finish_read_failure = [&]() { + if (timed_out) { + kill(child_pid, SIGKILL); + waitpid(child_pid, nullptr, 0); + child_pid = -1; + result_phase = PHASE_TIMEOUT; + result_error = "timeout"; + return; + } int status; waitpid(child_pid, &status, 0); child_pid = -1; result_phase = PHASE_CRASH; + if (WIFEXITED(status)) { + result_error = "child exited with code " + std::to_string(WEXITSTATUS(status)); + } else if (WIFSIGNALED(status)) { + result_error = "child killed by signal " + std::to_string(WTERMSIG(status)); + } else { + result_error = "child died while writing result"; + } + }; + + if (!read_field(&result_phase, sizeof(result_phase)) || + !read_field(&result_incremental, sizeof(result_incremental)) || + !read_field(&result_correct, sizeof(result_correct)) || + !read_field(&result_time_select_ms, sizeof(result_time_select_ms)) || + !read_field(&result_time_mv_ms, sizeof(result_time_mv_ms)) || + !read_field(&result_time_refresh_ms, sizeof(result_time_refresh_ms)) || + !read_field(&result_time_verify_ms, sizeof(result_time_verify_ms))) { + finish_read_failure(); return; } uint32_t err_len = 0; - if (!ReadAllBytes(from_child_fd, &err_len, sizeof(err_len))) { - int status; - waitpid(child_pid, &status, 0); - child_pid = -1; - result_phase = PHASE_CRASH; + if (!read_field(&err_len, sizeof(err_len))) { + finish_read_failure(); return; } if (err_len > 0) { result_error.resize(err_len); - if (!ReadAllBytes(from_child_fd, &result_error[0], err_len)) { - int status; - waitpid(child_pid, &status, 0); - child_pid = -1; - result_phase = PHASE_CRASH; + if (!read_field(&result_error[0], err_len)) { + finish_read_failure(); return; } } diff --git a/docs/codebase-audit-2026-07-22.md b/docs/codebase-audit-2026-07-22.md new file mode 100644 index 00000000..443abd82 --- /dev/null +++ b/docs/codebase-audit-2026-07-22.md @@ -0,0 +1,165 @@ +# OpenIVM codebase audit and remediation plan + +Date: 2026-07-22 + +This audit covers the current working tree, including the in-progress nonlocal refresh changes. Findings were checked +from correctness, reliability, performance, simplicity, and code-hygiene perspectives. Confirmed bugs were reproduced +with real materialized views and refreshes rather than inferred only from static analysis. + +The remediation rule is correctness first: an incrementalizable query must retain a correct incremental or affected-domain +maintenance path. A bug must not be hidden by weakening a test or silently changing the view to `FULL_REFRESH`. + +## P0: correctness and transaction safety + +- [x] **Capture delta rows in the base DML transaction.** Delta capture is now a streaming logical/physical operator that + appends through the caller's `ClientContext`, so base DML and delta rows share one commit or rollback. A resolved + catalog-level lock prevents refresh from advancing its watermark past an uncommitted delta. Regression coverage includes + rollback, constraint failure, defaults, generated columns, `RETURNING`, and an uncommitted DML/refresh race. +- [x] **Capture actual `ON CONFLICT` outcomes.** Conflict clauses lower to `LogicalMergeInto`; a transparent physical action + decorator now captures only the INSERT, UPDATE, and DELETE rows selected by DuckDB after match and action predicates are + resolved. Updates emit the required delete-old/insert-new pair, volatile defaults are evaluated once, `RETURNING` is + preserved, and all writes remain in the base DML transaction. The same boundary also covers native `MERGE` actions. +- [x] **Make MV lifecycle and native refresh transactional.** Native CREATE, REPLACE, DROP, ALTER, and refresh inside an + explicit caller transaction now mutate the user view, backing state, metadata, and deltas atomically and retain their + OpenIVM locks through commit or rollback. Autocommit refresh temporarily retains the established locked executor because + DuckDB query-pragma preprocessing ends a transaction before its returned program completes; `refresh.cpp` records the + native-operator follow-up. DuckLake replacement materializes data and auxiliary state under unpublished staging names + before publishing them. +- [x] **Continue cascade traversal when the current node has no source deltas.** Empty-delta detection now skips only the + current node; it cannot terminate traversal of the downstream DAG. Regression coverage exercises both a pending parent + delta left by a cascade-off refresh and independent child-source DML, including the hook-aware empty-node path. +- [x] **Use NULL-safe affected-window partition matching.** Affected-key refresh now uses hash-semi-joinable + `IS NOT DISTINCT FROM` predicates for standard, cascade, running-window, and DuckLake paths. Regression coverage includes + batched conflicting DML on single and composite NULL partitions plus a DuckLake TPCC window whose delivery timestamp moves + from the NULL partition to a non-NULL partition. +- [x] **Preserve `SUM` NULL semantics.** Weighted SUM alone cannot distinguish numeric zero from no non-NULL inputs. Persist + a hidden `COUNT(sum_argument)` and render NULL when that count reaches zero. +- [x] **Use an unconditional row count for group existence.** `COUNT(nullable_expression)=0` does not mean a group is empty. + Grouped incremental aggregates persist a hidden `COUNT(*)` solely for deciding whether the group row must be deleted, + including aggregates below `ORDER BY`, `LIMIT`, and `TOP_N` wrappers. +- [ ] **Delete the heuristic group-measure update fast path.** `refresh_group_measure.cpp` infers aggregate lineage from SQL + text, output-alias substrings, LPTS formatting, and value strings. A reproduced update to input `v` changed unrelated + aliases `revenue` and `savings`. Use the existing affected-group recompute path until bound aggregate lineage can prove a + safe fast path. +- [ ] **Validate the persisted `RefreshType` ABI.** Unknown values are raw-cast and can pass through refresh without applying + data changes while still advancing the watermark. Give every persisted value an explicit ordinal and decode it through a + single validating migration function; every dispatch must reject unknown values. +- [ ] **Give one refresh-node API ownership of hooks.** Scheduled hooks currently run twice, empty-delta skipping can suppress + replace hooks, cascaded nodes bypass hooks, and hook errors are swallowed. Hooks, skip decisions, refresh, and error + propagation must have exactly-once semantics. +- [ ] **Make crash recovery a durable state transition.** Some full-recompute paths bypass the in-progress journal, while + recovery clears the journal before recovery succeeds. Persist an attempt/checkpoint before every data mutation and clear it + only with successful post-refresh metadata. +- [ ] **Do not consume DuckLake structural-change identity before refresh succeeds.** Source-identity probing currently + persists a new table ID immediately. Carry it as pending activity and commit it with the successful snapshot/watermark. + +## P1: fail-closed metadata and authoritative planning + +- [ ] **Distinguish metadata error, absence, and empty state.** Several getters convert query errors into `{}`, `false`, or a + fallback strategy. Refresh must fail closed on schema/query errors; `optional` should mean genuinely absent metadata only. +- [ ] **Reject missing strategy contracts instead of falling through.** Missing DISTINCT or semi/anti metadata currently + falls into unrelated compiler cases. Validate the selected strategy and all required metadata once before compilation. +- [ ] **Remove runtime type guessing from window lineage.** Bare-name `information_schema` lookup with `LIMIT 1` is ambiguous + across schemas/catalogs, and coercing both comparison sides to VARCHAR changes bound comparison semantics. Persist exact + bound cast chains and source identity at CREATE time. +- [ ] **Replace window partition metadata mini-parsing with a typed contract.** Partition columns, source expressions, casts, + and lookup edges are reconstructed from independently encoded strings at refresh time. Persist one versioned structured + representation, validate it atomically, and pass the decoded object through affected-partition compilation. +- [ ] **Make source identity fully qualified.** DML classification, metadata lookup, locking, cleanup, and dependency handling + rely too heavily on bare table or view names. Use catalog, schema, and stable object identity. +- [ ] **Give one abstraction ownership of source and delta resolution.** Standard tables, DuckLake snapshots, chained views, + legacy metadata, and attached catalogs are resolved in several compiler branches with slightly different fallback rules. + Load and validate each source once into an immutable qualified descriptor, including its delta relation and snapshot range. +- [ ] **Centralize complete per-view DROP cleanup.** Hooks, refresh history/profile rows, dependency rows, and matcher state can + survive DROP and be inherited by a newly created MV of the same name. +- [ ] **Preserve quoted output identifiers.** Planner output names are sanitized for internal use and then reused where the + physical schema requires the original SQL identifier. Quoted names containing punctuation can fail during MV creation. +- [ ] **Classify only after required lineage is valid.** Window lineage can demote a finalized model to `FULL_REFRESH` after + node maintenance and update semantics have already been derived, leaving contradictory model state. +- [ ] **Finish or reverse the `CURRENT_DIFF_RECOMPUTE` removal coherently.** Deterministic SAMPLE, POSITIONAL, and ASOF shapes + are demoted to unconditional full refresh while stale nonlocal strategies, documentation, and benchmark paths remain. + Preserve a typed affected/current-diff recompute path where correctness permits it; delete truly dead strategies. +- [ ] **Strip computed top-k wrappers from stored aggregate state.** CREATE only moves a root `TOP_N` or + `LIMIT -> ORDER_BY` into the user-facing view. Plans such as + `PROJECTION -> TOP_N -> PROJECTION -> AGGREGATE` therefore initialize a limited backing table, while refresh strips the + top-k operator and applies unbounded deltas. Resolve the bound ordering expressions through the projection path, keep the + backing state unbounded, and apply the computed `ORDER BY`/`LIMIT` only in the user-facing view. + +## P2: remove SQL-text hacks and duplicate abstractions + +- [ ] **Render Spark SQL by dialect instead of global regex replacement.** Whole-program replacement mutates literals, + comments, and quoted identifiers, while early full-refresh paths bypass conversion entirely. Render casts, timestamps, and + NULL-safe equality through dialect-aware builders/LPTS. +- [ ] **Replace affected-group source substitution with plan-node substitution.** Plain occurrence replacement can modify + literals, comments, CTE references, and longer identifiers. Replace the exact `LogicalGet` occurrence before serialization. +- [ ] **Delete the workload-specific TPC-DI left-join shortcut.** General refresh code hardcodes `fact_market_history`, three + source names, and `sk_company_id`. Retain the generic correct path unless typed lineage proves a generic optimization. +- [ ] **Replace `parser_sql_extractors.cpp` with bound-plan extraction.** Its approximately 1,250 lines duplicate tokenization, + quote handling, clause parsing, alias rewriting, and parenthesis tracking for DISTINCT, filtered aggregates, and semi/anti + metadata already represented by bound logical operators and expressions. +- [ ] **Delete the independent refresh SQL mini-parsers.** Aggregate-filter stripping, SCD2 predicate injection, running-window + parsing, and LPTS CTE parsing belong in logical-plan/AST construction, not four inconsistent scanners of emitted SQL. +- [ ] **Make the delta IR authoritative or remove its diagnostic-only graph.** Per-node maintenance, update semantics, generic + affected domains, lineage facts, and auxiliary-state annotations are populated and logged but do not select compilation. + Keeping them beside parallel `RefreshType`/metadata dispatch creates drift. +- [ ] **Remove the nonfunctional view-matching product surface until it has behavior.** Candidate lookup, canonicalization, + and predicate implication are stubs, but settings, metadata columns, logs, CMake targets, and estimator APIs remain public. + Keep the FK constraint functionality that is actually used. +- [ ] **Replace handwritten JSON and pipe-delimited lineage encoding.** Important semantic metadata is manually scanned and + malformed fields can be skipped while keeping an incomplete lineage arm. Use typed normalized rows or a versioned structured + serializer and reject malformed records as a whole. +- [ ] **Split CREATE and refresh into typed phase programs.** `PlanFunction` and `GenerateRefreshSQL` each span roughly a + thousand lines. Use immutable inputs and results such as `ViewDefinition`, `RefreshMetadataSnapshot`, `PendingDeltaBatch`, + `RefreshStrategy`, and `RefreshProgram`; do not introduce a class hierarchy. +- [ ] **Replace boolean/argument soup with strategy input/result structs.** Compiler functions with 17 parameters, output + booleans, and implicit reclassification hide invariants and make call-order mistakes likely. +- [ ] **Use tagged CREATE operations instead of magic strings.** Cleanup, profiling, schema derivation, and executable SQL are + currently mixed in one string vector and reparsed by prefixes and tab-delimited fields. +- [ ] **Consolidate duplicated profiling and NULL-safe predicate helpers.** `CreateMVProfiler` and `RefreshProfiler` + independently implement IDs, retention, step collection, and persistence; SQL utilities contain parallel NULL-safe + builders. Put the shared lifecycle and storage policy in one small profiler component. + +## P3: refresh performance and repeated work + +- [ ] **Remove exact base-table counting from normal join compilation.** Standard join refresh performs `COUNT(*)` for every + base table on every refresh after delta activity has already been computed. Use activity/statistics or remove the heuristic. +- [ ] **Avoid repeated parse/bind/plan/model reconstruction.** Refresh planning currently reparses and optimizes the stored + query multiple times. Persist a versioned create-time typed model and rebuild only for migration or invalidation. +- [ ] **Load metadata once.** `GenerateRefreshSQL` issues many independent queries for one view and sometimes reloads the same + field. Load one immutable snapshot; keep writes and recovery checkpoints separate. +- [ ] **Consolidate projection deletes once.** The projection path builds the same grouped/ranked net delta independently for + DELETE and INSERT. Materialize or share the consolidated batch. +- [ ] **Carry delta activity through the full refresh.** Activity checks, join-term selection, max timestamp lookup, and cleanup + repeatedly scan the same delta tables. Capture count/delete/watermark information once per transaction. Explicit caller + transactions currently remain conservative because query-pragma preprocessing cannot observe the caller's local delta + rows or retain a reliable activity signal; solve this at the planned native refresh operator boundary rather than with + process-global transaction registries. +- [x] **Share repeated LEFT JOIN transition-count subplans deliberately.** The join compiler now builds one explicit + materialized transition-key CTE per nullable source/key and gives each inclusion-exclusion term a freshly bound CTE + reference. This avoids the unsafe blanket common-subplan optimizer while preserving term-local bindings and delta + semantics. +- [ ] **Price actual join work in the adaptive model.** The model counts exponential terms but underprices copied plan nodes, + base-leaf appearances, compilation work, and generated SQL size. +- [ ] **Remove the fake `ConstraintCache` or make it a cache.** It repopulates a map that no read path consumes, while FK cost + estimation separately parses textual constraint descriptions. +- [ ] **Audit edge-case tests and mirror them in the rewriter benchmark.** Review the SQL harness for missing boundary and + transition cases, add deterministic bidirectional `EXCEPT ALL` coverage, and add representative cases to the rewriter + benchmark so real refresh compilation and execution exercise the same semantics. +- [ ] **Add test-only concurrency barriers.** Several SQL concurrency regressions still coordinate with bounded sleeps. + Introduce a deterministic handshake that can prove a writer or refresh owns or is waiting for the mutation gate without + exposing a production pragma, then replace the timing assumptions in lifecycle tests. + +## Target refresh flow + +At CREATE time, bind once and persist a versioned typed view definition, delta plan, output schema, exact source identities, +and concrete strategy metadata. + +At refresh time: + +1. Acquire the graph/node transaction and locks. +2. Load one immutable metadata snapshot. +3. Capture one pending delta batch and its watermark. +4. Choose one explicit correctness-preserving strategy; use cost only between valid strategies. +5. Compile a structured refresh program containing data, cascade, cleanup, and checkpoint phases. +6. Execute native work atomically; render SQL once only at a genuine external-dialect boundary. +7. Commit data, cursor/snapshot, source identity, and recovery state together. diff --git a/docs/internals/concurrency.md b/docs/internals/concurrency.md index 24903159..642153f1 100644 --- a/docs/internals/concurrency.md +++ b/docs/internals/concurrency.md @@ -1,29 +1,29 @@ # Concurrency -## Refresh serialization +## Mutation serialization -Each materialized view has a per-view mutex. When `PRAGMA refresh('view_name')` runs, it -acquires the view's lock before generating or executing any SQL. This prevents two -concurrent refresh calls from applying overlapping deltas to the same view. +OpenIVM serializes tracked source-table writes, refreshes, and materialized-view +lifecycle operations through one database-wide mutation gate. An explicit transaction +retains the gate until commit or rollback. Helper connections use the same logical +owner, making the gate re-entrant even when DuckDB executes work on another thread. -The [automatic refresh daemon](../refresh/automatic-refresh.md) uses `TryLockView()` — -if the view is already being refreshed, the daemon skips it and retries at the next -interval. - -## Delta table safety - -Each delta table has a per-delta-table mutex. The insert rule acquires the delta lock -when writing DML-triggered rows into a delta table. This prevents concurrent INSERTs -from interleaving delta rows in a way that breaks timestamp ordering. +This coarse boundary prevents refresh/write and parent/child refresh races without a +multi-lock hierarchy. Unrelated OpenIVM mutations in the same database also serialize; +ordinary reads remain concurrent. The [automatic refresh daemon](../refresh/automatic-refresh.md) +waits behind an active mutation and refreshes once it acquires the gate. ## Snapshot isolation -Refresh executes SQL through a separate `Connection` from the user's session. DuckDB -provides snapshot isolation per transaction, so: +Autocommit refresh executes through a locked helper connection. Refresh inside an +explicit transaction compiles metadata through a helper but executes the generated +program in the caller transaction, so transaction-local DML and MV lifecycle changes +remain visible and atomic. The mutation gate prevents another tracked writer or +refresh from changing OpenIVM state while the refresh is active. DuckDB snapshot +isolation additionally ensures: - The refresh reads a consistent snapshot of base tables and delta tables -- Concurrent DML by other connections does not affect the in-progress refresh -- Delta rows written by concurrent DML after the refresh's snapshot are not seen +- The refresh sees transaction-local changes made before it acquired its snapshot +- Non-OpenIVM activity cannot change the refresh's visible snapshot For DuckLake tables, the snapshot is determined by the `DuckLakeFunctionInfo::snapshot_id` bound at plan time. `AT VERSION` pinning reads exactly the state at that snapshot. @@ -45,14 +45,13 @@ Each `(view, base_table)` pair tracks two timestamps in `openivm_delta_tables`: 4. We set `last_update = now()` (which is *less than* this row's ts). 5. The next refresh's filter `ts >= last_update` includes this row again → double-application → MV drift. -Anchoring `last_update` to the maximum timestamp we *actually* processed eliminates the gap: the next refresh's filter excludes everything we've seen and includes everything we haven't. See `src/upsert/refresh.cpp:1370–1403` for the implementation. +Anchoring `last_update` to the maximum timestamp we *actually* processed eliminates the gap: the next refresh's filter excludes everything we've seen and includes everything we haven't. See `GenerateRefreshSQL()` in `src/upsert/refresh_sql.cpp` for the implementation. -## Lock hierarchy +## Locking | Lock | Scope | Held during | Used by | |---|---|---|---| -| View mutex | Per view name | Entire refresh cycle | `PRAGMA refresh()`, refresh daemon | -| Delta mutex | Per delta table name | Delta row insertion | Insert rule (DML triggers) | -| Map mutex | Global (static) | Mutex map lookup | Internal — protects the mutex maps | +| Mutation gate | Per DuckDB database instance | Entire explicit transaction or autocommit OpenIVM mutation | Delta capture, refresh, lifecycle DDL | +| Map mutex | Global (static) | Mutation-gate lookup | Internal — protects the gate map | -All locks are non-recursive mutexes. The view mutex is the outermost lock. +Transactional lock state retains the mutation guard through commit or rollback. diff --git a/docs/internals/cost_model.md b/docs/internals/cost_model.md index 75503d42..378ba71b 100644 --- a/docs/internals/cost_model.md +++ b/docs/internals/cost_model.md @@ -121,7 +121,6 @@ match the refresh path that actually runs: affected groups. - `WINDOW_PARTITION` prices delta scanning plus the affected partition fraction of the base scan. -- `CURRENT_DIFF_RECOMPUTE` prices the same compute and replace work as full recompute. - `DISTINCT_INCREMENTAL` adds aux-state maintenance over affected distinct tuples. - `SEMI_ANTI_RECOMPUTE` prices aux-state/domain recompute for supported semi/anti projection shapes. diff --git a/docs/internals/parser.md b/docs/internals/parser.md index 01a1686d..271f67c5 100644 --- a/docs/internals/parser.md +++ b/docs/internals/parser.md @@ -64,7 +64,13 @@ The IVM compatibility checker validates the entire plan tree, flagging unsupport ## Generated DDL -The parser produces a sequence of DDL statements executed during the bind phase: +The parser produces a sequence of DDL statements, but does not mutate the catalog +during bind. Native-catalog lifecycle statements are rendered as a SQL program and +execute in the caller transaction. A later lifecycle or refresh statement in that +same transaction replays only the affected view's uncommitted metadata into +constrained temporary shadow tables for compilation. DuckLake and other +cross-catalog lifecycles use staged execution because DuckDB cannot commit writes to +two attached catalogs in one transaction. 1. **System tables**: `CREATE TABLE IF NOT EXISTS openivm_views (...)` and `openivm_delta_tables (...)`. 2. **Metadata inserts**: Registers the view name, query string, type, and source table mappings. @@ -82,6 +88,8 @@ Stores one row per materialized view. | Column | Type | Description | |---|---|---| | `view_name` | `VARCHAR` (PK) | Name of the materialized view. | +| `view_catalog` | `VARCHAR` | Catalog containing the user-facing view. | +| `view_schema` | `VARCHAR` | Schema containing the user-facing view. | | `sql_string` | `VARCHAR` | The original SELECT query defining the view. | | `type` | `TINYINT` | View classification (see IVM compatibility classification above). | | `has_minmax` | `BOOLEAN` | Whether the view uses MIN/MAX or another aggregate shape that may need group-recompute. | @@ -99,6 +107,7 @@ Stores one row per materialized view. | `distinct_aux_meta_json` | `VARCHAR` | JSON metadata for DISTINCT aux-state maintenance. | | `semi_anti_aux_meta_json` | `VARCHAR` | JSON metadata for SEMI/ANTI aux-state maintenance. | | `lineage_json` | `VARCHAR` | JSON lineage metadata for window and projection-key refresh paths. | +| `leftjoin_secondary_meta_json` | `VARCHAR` | Structured source/key identities for supported LEFT JOIN aggregate correction deltas. | | `signature_hash`, `canonical_plan_blob`, `output_columns_json`, `predicate_summary_json`, `fd_summary_json`, `nullified_columns_json` | Mixed | View-matching metadata. These stay NULL unless view matching is enabled. | Example content: diff --git a/docs/refresh/automatic-refresh.md b/docs/refresh/automatic-refresh.md index 881c6392..5e61de04 100644 --- a/docs/refresh/automatic-refresh.md +++ b/docs/refresh/automatic-refresh.md @@ -27,7 +27,7 @@ The refresh daemon is a background `std::thread` started at extension load. It: 1. Wakes every 30 seconds 2. Queries `openivm_views` for views with `refresh_interval IS NOT NULL` 3. For each view where `now() - last_update >= interval`: calls `PRAGMA refresh('view_name')` -4. Skips views that are already being refreshed (via `TryLockView`) +4. Waits for any active OpenIVM mutation, then refreshes the due view The daemon holds a non-owning reference to the database and exits cleanly when the database is destroyed. @@ -80,11 +80,11 @@ This adds two small UPDATE statements per refresh cycle (one before, one after t ## Concurrency -Automatic refresh uses the same per-view locking as manual `PRAGMA refresh()`: +Automatic refresh uses the same database-wide mutation gate as manual `PRAGMA refresh()`: - **Reads during refresh**: always safe (DuckDB MVCC — readers see a consistent snapshot) -- **Two concurrent refreshes of the same view**: serialized by a per-view mutex. The daemon skips views that are locked (e.g., by a manual PRAGMA), and manual PRAGMAs wait for the daemon to finish. -- **DML during refresh**: safe. A per-delta-table mutex in the insert rule prevents delta writes from racing with the refresh's timestamp logic. +- **Concurrent refreshes**: serialized by the mutation gate. The daemon and manual refreshes wait for the active mutation to finish. +- **Tracked DML during refresh**: serialized by the same gate, preventing delta writes from racing with refresh bookkeeping. ## Configuration diff --git a/docs/refresh/refresh-strategies.md b/docs/refresh/refresh-strategies.md index 565fb050..58d2b47c 100644 --- a/docs/refresh/refresh-strategies.md +++ b/docs/refresh/refresh-strategies.md @@ -88,7 +88,7 @@ Returns a single row: |---|---:|---:|---:|---:|---| | incremental | 1200.0 | 50000.0 | 1200.0 | 50000.0 | false | -- `decision`: the selected strategy. Values include `incremental`, `group_recompute`, `window_partition`, `current_diff_recompute`, `distinct_incremental`, `semi_anti_recompute`, and `full`. +- `decision`: the selected strategy. Values include `incremental`, `group_recompute`, `window_partition`, `distinct_incremental`, `semi_anti_recompute`, and `full`. - `incremental_cost`: estimated cost of the selected non-full strategy. For affected-domain strategies, this is not a pure delta-only cost. - `recompute_cost`: estimated cost of a full DELETE + INSERT. - `incremental_predicted_ms`: learned prediction for the selected non-full strategy, or the static estimate before calibration. diff --git a/src/core/ivm_delta_model.cpp b/src/core/ivm_delta_model.cpp index afdf1245..08210f6c 100644 --- a/src/core/ivm_delta_model.cpp +++ b/src/core/ivm_delta_model.cpp @@ -641,10 +641,8 @@ void PopulateDeltaViewModelLineage(DeltaViewModel &model, const CreateMVPlanFact if (analysis.found_asof_join && (!has_lineage || AsofWindowPartitionReadsRightSideDirectly(direct_lineage_ops, model) || !WindowLineageCoversAllSources(model.window_lineage_ops, facts))) { - model.type = RefreshType::CURRENT_DIFF_RECOMPUTE; + model.type = RefreshType::FULL_REFRESH; model.window_lineage_ops.clear(); - AddUnique(model.features, DeltaModelFeature::CURRENT_DIFF_RECOMPUTE); - AddUnique(model.strategy_reasons, DeltaStrategyReason::ASOF_CURRENT_DIFF_RECOMPUTE); ValidateDeltaViewModelInvariants(model); return; } diff --git a/src/core/ivm_view_classifier.cpp b/src/core/ivm_view_classifier.cpp index 177a9056..269dec7a 100644 --- a/src/core/ivm_view_classifier.cpp +++ b/src/core/ivm_view_classifier.cpp @@ -307,6 +307,8 @@ static void BuildUnsupportedReasons(DeltaViewModel &model, const CreateMVPlanFac } static void BuildModelFeatures(DeltaViewModel &model, const PlanAnalysis &analysis, const DeltaViewModelInput &input) { + const bool has_nonredundant_distinct = + analysis.found_distinct && (!input.has_top_level_redundant_distinct || input.facts->has_descendant_distinct); if (!model.unsupported_reasons.empty()) { AddUnique(model.features, DeltaModelFeature::FULL_ONLY); } @@ -353,7 +355,7 @@ static void BuildModelFeatures(DeltaViewModel &model, const PlanAnalysis &analys AddUnique(model.features, DeltaModelFeature::WINDOW_AFFECTED_PARTITION); } if (!model.HasFeature(DeltaModelFeature::FULL_ONLY)) { - if (input.distinct_aux_candidate) { + if (input.distinct_aux_candidate && has_nonredundant_distinct) { AddUnique(model.features, DeltaModelFeature::DISTINCT_STATEFUL); } if (input.semi_anti_aux_candidate) { @@ -391,6 +393,26 @@ static void BuildUpdateSemantics(DeltaViewModel &model, const PlanAnalysis &anal } } +static bool IsTransparentDistinctWrapper(LogicalOperatorType type) { + return type == LogicalOperatorType::LOGICAL_CREATE_TABLE || type == LogicalOperatorType::LOGICAL_FILTER || + type == LogicalOperatorType::LOGICAL_ORDER_BY || type == LogicalOperatorType::LOGICAL_LIMIT || + type == LogicalOperatorType::LOGICAL_TOP_N; +} + +static bool HasOuterProjectionOverDistinct(const CreateMVPlanFacts &facts) { + auto *node = facts.root; + bool found_projection = false; + while (node && node->children.size() == 1) { + if (node->type == LogicalOperatorType::LOGICAL_PROJECTION) { + found_projection = true; + } else if (!IsTransparentDistinctWrapper(node->type)) { + break; + } + node = node->children[0].get(); + } + return found_projection && node && node->type == LogicalOperatorType::LOGICAL_DISTINCT; +} + static void BuildGroupColumns(DeltaViewModel &model, const CreateMVPlanFacts &facts, const vector &output_names, idx_t visible_output_count) { const auto &analysis = facts.analysis; @@ -411,16 +433,22 @@ static void BuildGroupColumns(DeltaViewModel &model, const CreateMVPlanFacts &fa model.group_columns.size()); } } else if (model.distinct_at_top) { - model.group_columns = analysis.aggregate_columns; - } else if (analysis.found_distinct && analysis.aggregate_columns.empty()) { - model.group_columns = analysis.aggregate_columns; + AddVisibleGroupNames(model.group_columns, output_names); + } else if (analysis.found_distinct && !analysis.found_aggregation && HasOuterProjectionOverDistinct(facts)) { + AddVisibleGroupNames(model.group_columns, output_names); + if (!model.group_columns.empty()) { + AddUnique(model.strategy_reasons, DeltaStrategyReason::INNER_DISTINCT_PROJECTION_RECOMPUTE); + OPENIVM_DEBUG_PRINT("[CREATE MV] Inner DISTINCT below an outer projection -- using " + "GROUP_RECOMPUTE\n"); + } } else if (group_count > 0 && group_index != DConstants::INVALID_INDEX) { model.group_columns = DeriveGroupColumnNames(facts, group_index, group_count, output_names); } + idx_t actual_visible_outputs = GetVisibleOutputCount(output_names, visible_output_count); if (analysis.found_delim_join && analysis.found_aggregation && model.group_columns.empty() && - output_names.size() > analysis.aggregate_types.size()) { - idx_t key_count = output_names.size() - analysis.aggregate_types.size(); + actual_visible_outputs > analysis.aggregate_types.size()) { + idx_t key_count = actual_visible_outputs - analysis.aggregate_types.size(); for (idx_t i = 0; i < key_count; i++) { if (!output_names[i].empty() && !IncrementalTableNames::IsInternalColumn(output_names[i])) { model.group_columns.push_back(output_names[i]); @@ -507,11 +535,11 @@ static void BuildGroupColumns(DeltaViewModel &model, const CreateMVPlanFacts &fa if (analysis.found_join && analysis.found_aggregation && !model.group_columns.empty()) { idx_t expected_linear_outputs = model.group_columns.size() + model.aggregate_types.size(); - if (output_names.size() > expected_linear_outputs) { + if (actual_visible_outputs > expected_linear_outputs) { AddUnique(model.strategy_reasons, DeltaStrategyReason::JOIN_AGGREGATE_PROJECTION_FALLBACK); OPENIVM_DEBUG_PRINT("[CREATE MV] Join-over-aggregate exposes %zu columns but only %zu are " "group/aggregate outputs -- using GROUP_RECOMPUTE\n", - output_names.size(), expected_linear_outputs); + actual_visible_outputs, expected_linear_outputs); } } @@ -527,30 +555,26 @@ static void BuildGroupColumns(DeltaViewModel &model, const CreateMVPlanFacts &fa } static void SelectRefreshType(DeltaViewModel &model, const PlanAnalysis &analysis, const DeltaViewModelInput &input) { + const bool has_nonredundant_distinct = + analysis.found_distinct && (!input.has_top_level_redundant_distinct || input.facts->has_descendant_distinct); auto has_argminmax = std::any_of(analysis.aggregate_types.begin(), analysis.aggregate_types.end(), [](const string &agg_type) { return agg_type == "arg_min" || agg_type == "arg_max"; }); - auto select_current_diff = [&](DeltaStrategyReason reason) { - model.type = RefreshType::CURRENT_DIFF_RECOMPUTE; - AddUnique(model.features, DeltaModelFeature::CURRENT_DIFF_RECOMPUTE); - AddUnique(model.strategy_reasons, reason); - }; if (input.has_unsupported_incremental_construct) { model.type = RefreshType::FULL_REFRESH; } else if (!analysis.incremental_compatible) { model.type = RefreshType::FULL_REFRESH; model.warn_unsupported_incremental = true; } else if (analysis.found_sample) { - select_current_diff(DeltaStrategyReason::SAMPLE_CURRENT_DIFF_RECOMPUTE); + model.type = RefreshType::FULL_REFRESH; } else if (analysis.found_positional_join) { - select_current_diff(DeltaStrategyReason::POSITIONAL_CURRENT_DIFF_RECOMPUTE); + model.type = RefreshType::FULL_REFRESH; } else if (analysis.found_asof_join && analysis.found_window && !model.window_partition_columns.empty()) { model.type = RefreshType::WINDOW_PARTITION; } else if (analysis.found_asof_join && analysis.found_aggregation && !model.group_columns.empty()) { model.type = RefreshType::GROUP_RECOMPUTE; - AddUnique(model.strategy_reasons, DeltaStrategyReason::ASOF_CURRENT_DIFF_RECOMPUTE); } else if (analysis.found_asof_join) { - select_current_diff(DeltaStrategyReason::ASOF_CURRENT_DIFF_RECOMPUTE); + model.type = RefreshType::FULL_REFRESH; } else if (analysis.found_window) { model.type = RefreshType::WINDOW_PARTITION; } else if (analysis.found_grouping_sets) { @@ -564,10 +588,18 @@ static void SelectRefreshType(DeltaViewModel &model, const PlanAnalysis &analysi model.type = RefreshType::GROUP_RECOMPUTE; } else if (analysis.found_filtered_list) { model.type = RefreshType::FULL_REFRESH; + } else if (input.has_computed_sum_aggregate_projection) { + model.type = model.group_columns.empty() ? RefreshType::FULL_REFRESH : RefreshType::GROUP_RECOMPUTE; + } else if (analysis.found_distinct && !analysis.found_union_distinct && model.distinct_at_top && + analysis.found_aggregation) { + // DISTINCT over multiple aggregate result rows is a second non-linear + // aggregation level. The outer DISTINCT key is an aggregate value, not + // a source group key that affected-group recompute can recover. + model.type = RefreshType::FULL_REFRESH; } else if (analysis.found_count_distinct && !model.group_columns.empty()) { model.type = input.count_distinct_aux_candidate ? RefreshType::COUNT_DISTINCT_INCREMENTAL : RefreshType::GROUP_RECOMPUTE; - } else if (analysis.found_distinct && !model.distinct_at_top && analysis.found_aggregation) { + } else if (has_nonredundant_distinct && !model.distinct_at_top && analysis.found_aggregation) { model.type = model.HasFeature(DeltaModelFeature::DISTINCT_STATEFUL) ? RefreshType::DISTINCT_INCREMENTAL : RefreshType::GROUP_RECOMPUTE; } else if (model.union_distinct_over_agg && !model.group_columns.empty()) { @@ -608,7 +640,8 @@ static void SelectGroupRecomputeAffectedMode(DeltaViewModel &model, const DeltaV if ((input.facts && input.facts->analysis.found_asof_join) || input.stored_query_has_top_k || aggregate_filter_join || (input.stored_query_has_aggregate_filter && input.has_ducklake_source)) { model.group_recompute_affected_mode = GroupRecomputeAffectedMode::CURRENT_DIFF; - } else if (HasStrategyReason(model, DeltaStrategyReason::JOIN_AGGREGATE_PROJECTION_FALLBACK)) { + } else if (HasStrategyReason(model, DeltaStrategyReason::JOIN_AGGREGATE_PROJECTION_FALLBACK) || + HasStrategyReason(model, DeltaStrategyReason::OUTER_JOIN_PRESERVED_TABLE_FUNCTION_RECOMPUTE)) { model.group_recompute_affected_mode = GroupRecomputeAffectedMode::CURRENT_DIFF; } else if (input.stored_query_has_aggregate_filter) { model.group_recompute_affected_mode = GroupRecomputeAffectedMode::SOURCE_DELTA_RELAX_AGGREGATE_FILTER; @@ -659,12 +692,10 @@ const char *DeltaStrategyReasonName(DeltaStrategyReason reason) { return "SEMI_ANTI_AGGREGATE_GROUP_FALLBACK"; case DeltaStrategyReason::OUTER_JOIN_AGGREGATE_RECOMPUTE: return "OUTER_JOIN_AGGREGATE_RECOMPUTE"; - case DeltaStrategyReason::ASOF_CURRENT_DIFF_RECOMPUTE: - return "ASOF_CURRENT_DIFF_RECOMPUTE"; - case DeltaStrategyReason::SAMPLE_CURRENT_DIFF_RECOMPUTE: - return "SAMPLE_CURRENT_DIFF_RECOMPUTE"; - case DeltaStrategyReason::POSITIONAL_CURRENT_DIFF_RECOMPUTE: - return "POSITIONAL_CURRENT_DIFF_RECOMPUTE"; + case DeltaStrategyReason::OUTER_JOIN_PRESERVED_TABLE_FUNCTION_RECOMPUTE: + return "OUTER_JOIN_PRESERVED_TABLE_FUNCTION_RECOMPUTE"; + case DeltaStrategyReason::INNER_DISTINCT_PROJECTION_RECOMPUTE: + return "INNER_DISTINCT_PROJECTION_RECOMPUTE"; default: return "UNKNOWN"; } @@ -702,8 +733,6 @@ const char *DeltaModelFeatureName(DeltaModelFeature feature) { return "SAMPLE_GLOBAL_RECOMPUTE"; case DeltaModelFeature::POSITIONAL_GLOBAL_RECOMPUTE: return "POSITIONAL_GLOBAL_RECOMPUTE"; - case DeltaModelFeature::CURRENT_DIFF_RECOMPUTE: - return "CURRENT_DIFF_RECOMPUTE"; case DeltaModelFeature::FULL_ONLY: return "FULL_ONLY"; default: @@ -859,22 +888,30 @@ idx_t DeltaViewModel::LineageEntryCount() const { return count; } -bool IsDistinctAtTop(const PlanAnalysis &analysis, const vector &output_names) { - if (!analysis.found_distinct || analysis.aggregate_columns.empty() || output_names.empty()) { +bool IsDistinctAtTop(const CreateMVPlanFacts &facts, const vector &output_names) { + if (!facts.analysis.found_distinct) { return false; } - unordered_set output_lc; - for (auto &name : output_names) { - output_lc.insert(StringUtil::Lower(name)); + auto *node = facts.root; + while (node && node->children.size() == 1) { + if (!IsTransparentDistinctWrapper(node->type)) { + break; + } + node = node->children[0].get(); } - - for (auto &target : analysis.aggregate_columns) { - if (!output_lc.count(StringUtil::Lower(target))) { - return false; + if (node && node->type == LogicalOperatorType::LOGICAL_DISTINCT) { + return true; + } + if (!node || node->type != LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + return false; + } + for (auto &name : output_names) { + if (name == openivm::DISTINCT_COUNT_COL) { + return true; } } - return true; + return false; } DeltaViewModel BuildDeltaViewModel(const DeltaViewModelInput &input) { @@ -893,7 +930,10 @@ DeltaViewModel BuildDeltaViewModel(const DeltaViewModelInput &input) { } model.has_minmax_metadata = analysis.found_minmax || analysis.found_count_distinct || analysis.found_list; - model.distinct_at_top = IsDistinctAtTop(analysis, output_names); + model.distinct_at_top = IsDistinctAtTop(facts, output_names); + if (input.has_top_level_redundant_distinct) { + model.distinct_at_top = false; + } BuildGroupColumns(model, facts, output_names, input.visible_output_count); if ((analysis.found_left_join || analysis.found_full_outer) && analysis.found_aggregation && @@ -903,6 +943,12 @@ DeltaViewModel BuildDeltaViewModel(const DeltaViewModelInput &input) { OPENIVM_DEBUG_PRINT("[CREATE MV] LEFT/OUTER JOIN aggregate with computed aggregate or projection wrapper -- " "using group-recompute metadata\n"); } + if (analysis.found_aggregation && OuterJoinPreservedSideHasTableFunction(facts)) { + model.has_minmax_metadata = true; + AddUnique(model.strategy_reasons, DeltaStrategyReason::OUTER_JOIN_PRESERVED_TABLE_FUNCTION_RECOMPUTE); + OPENIVM_DEBUG_PRINT("[CREATE MV] LEFT/RIGHT JOIN aggregate with a table function on the preserved side -- " + "using current-diff group recompute\n"); + } if (analysis.found_full_outer) { model.full_outer_join_cols = ExtractFullOuterJoinMetadata(facts); diff --git a/src/core/parser.cpp b/src/core/parser.cpp index f5b24ec6..7eb8a3fe 100644 --- a/src/core/parser.cpp +++ b/src/core/parser.cpp @@ -6,6 +6,8 @@ #include "core/parser_ddl.hpp" #include "core/parser_plan_helpers.hpp" #include "core/parser_sql_extractors.hpp" +#include "core/plan_rewrite_internal.hpp" +#include "core/refresh_locks.hpp" #include "core/refresh_metadata.hpp" #include "core/ivm_delta_model.hpp" #include "core/ivm_view_classifier.hpp" @@ -15,10 +17,14 @@ #include "upsert/refresh_compiler.hpp" #include "duckdb/common/printer.hpp" #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp" +#include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" #include "duckdb/main/client_config.hpp" #include "duckdb/main/client_data.hpp" +#include "duckdb/main/database_manager.hpp" #include "duckdb/main/settings.hpp" #include "duckdb/parser/parser.hpp" +#include "duckdb/parser/qualified_name.hpp" +#include "duckdb/parser/statement/drop_statement.hpp" #include "duckdb/planner/expression/bound_aggregate_expression.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/operator/logical_top_n.hpp" @@ -32,6 +38,36 @@ namespace duckdb { +struct MaterializedViewTarget { + string catalog_name; + string schema_name; + string view_name; + bool qualified; +}; + +static MaterializedViewTarget ResolveMaterializedViewTarget(ClientContext &context, const string &target_name) { + auto components = QualifiedName::ParseComponents(target_name); + if (components.empty() || components.size() > 3) { + throw ParserException("Invalid materialized-view target '%s'", target_name); + } + auto &default_entry = ClientData::Get(context).catalog_search_path->GetDefault(); + string default_catalog = + default_entry.catalog.empty() ? DatabaseManager::GetDefaultDatabase(context) : default_entry.catalog; + string default_schema = default_entry.schema.empty() ? DEFAULT_SCHEMA : default_entry.schema; + if (components.size() == 1) { + return {default_catalog, default_schema, components[0], false}; + } + if (components.size() == 2) { + auto attached = DatabaseManager::Get(context).GetDatabase(context, components[0]); + if (attached) { + string schema_name = StringUtil::CIEquals(default_catalog, components[0]) ? default_schema : DEFAULT_SCHEMA; + return {components[0], schema_name, components[1], true}; + } + return {default_catalog, components[0], components[1], true}; + } + return {components[0], components[1], components[2], true}; +} + static vector BuildGroupRecomputeSourceOccurrences(const CreateMVPlanFacts &facts) { vector occurrences; @@ -126,6 +162,20 @@ static bool MatchesPatternCI(const string &text, idx_t pos, const string &patter return true; } +static bool RelationExists(ClientContext &context, const string &catalog_name, const string &schema_name, + const string &relation_name) { + QueryErrorContext error_context; + for (auto type : {CatalogType::TABLE_ENTRY, CatalogType::VIEW_ENTRY}) { + auto entry = + Catalog::GetEntry(context, catalog_name, schema_name, EntryLookupInfo(type, relation_name, error_context), + OnEntryNotFound::RETURN_NULL); + if (entry) { + return true; + } + } + return false; +} + static bool HasIdentifierBoundary(const string &text, idx_t pos, idx_t len) { bool left_ok = pos == 0 || !IsIdentifierTokenChar(text[pos - 1]); idx_t end = pos + len; @@ -243,12 +293,20 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // Handle ALTER MATERIALIZED VIEW — just execute the metadata UPDATE if (!parse_data_ref.alter_sql.empty()) { - auto r = con.Query(parse_data_ref.alter_sql); - if (r->HasError()) { - throw CatalogException("Failed to alter materialized view: " + r->GetError()); + auto target = ResolveMaterializedViewTarget(context, parse_data_ref.target_name); + auto metadata_table = SqlUtils::FullName(default_db, default_schema, openivm::VIEWS_TABLE); + auto target_filter = "view_name = '" + SqlUtils::EscapeValue(target.view_name) + + "' AND COALESCE(view_catalog, '" + SqlUtils::EscapeValue(default_db) + "') = '" + + SqlUtils::EscapeValue(target.catalog_name) + "' AND COALESCE(view_schema, '" + + string(DEFAULT_SCHEMA) + "') = '" + SqlUtils::EscapeValue(target.schema_name) + "'"; + auto tracked = con.Query("SELECT 1 FROM " + metadata_table + " WHERE " + target_filter); + if (tracked->HasError() || tracked->RowCount() == 0) { + throw CatalogException("Materialized view '%s' does not exist in OpenIVM metadata", + parse_data_ref.target_name); } - // Return via the DDL executor with no DDL to run (the UPDATE already executed) - ConfigureDDLExecutorResult(result); + result.parameters.push_back(Value("UPDATE " + metadata_table + " SET refresh_interval = " + + parse_data_ref.alter_sql + " WHERE " + target_filter)); + ConfigureDDLExecutorResult(result, DDLExecutionMode::CALLER_TRANSACTION); return result; } @@ -258,7 +316,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC ForwardPacSettingsIfLoaded(context, con); auto name_resolution_start = create_profile_now(); - auto full_view_name = SqlUtils::ExtractTableName(statement->query); + auto full_view_name = parse_data_ref.target_name; // Keep the user's raw AS-query as the source of truth for original-SQL fallback. // Do not recover this from DuckDB's parsed QueryNode::ToString(): that path is a // best-effort pretty-printer and has segfaulted on set-operation query nodes with @@ -266,26 +324,14 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // for supported logical plans; this string is only the safe fallback input. auto original_view_query = SqlUtils::ExtractViewQuery(statement->query); - // Split catalog-qualified name (e.g. "dl.mv_totals") into prefix and bare name. - string view_catalog_prefix; // e.g. "dl." or "" for default catalog - string view_name; // bare name without catalog, e.g. "mv_totals" - string view_target_catalog = current_catalog; - string view_target_schema = current_schema; - auto dot_pos = full_view_name.rfind('.'); - if (dot_pos != string::npos) { - auto raw_prefix = full_view_name.substr(0, dot_pos); - view_catalog_prefix = SqlUtils::QuoteQualifiedPrefix(raw_prefix + "."); - auto schema_dot_pos = raw_prefix.rfind('.'); - if (schema_dot_pos != string::npos) { - view_target_catalog = raw_prefix.substr(0, schema_dot_pos); - view_target_schema = raw_prefix.substr(schema_dot_pos + 1); - } else { - view_target_catalog = raw_prefix; - view_target_schema = current_schema.empty() ? "main" : current_schema; - } - view_name = full_view_name.substr(dot_pos + 1); + auto target = ResolveMaterializedViewTarget(context, full_view_name); + string view_catalog_prefix; + string view_name = target.view_name; + string view_target_catalog = target.catalog_name; + string view_target_schema = target.schema_name; + if (target.qualified) { + view_catalog_prefix = SqlUtils::QualifiedPrefix(view_target_catalog, view_target_schema); } else { - view_name = full_view_name; // When the MV name is unqualified but the session is in a non-default catalog // (e.g. USE dl.main), explicitly qualify so data/view tables land in dl rather // than the physical default. Metadata tables (unqualified) stay in the physical @@ -294,18 +340,33 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC view_catalog_prefix = SqlUtils::QualifiedPrefix(current_catalog, current_schema); } } + if (view_target_catalog.empty()) { + view_target_catalog = default_db; + } + if (view_target_schema.empty()) { + view_target_schema = default_schema; + } RefreshMetadata metadata(con); bool target_is_ducklake = metadata.IsDuckLakeCatalog(view_target_catalog); string internal_catalog_prefix = view_catalog_prefix; + string internal_target_catalog = view_target_catalog; + string internal_target_schema = view_target_schema; // Native MVs created from another active catalog keep OpenIVM state in the physical // default DB. DuckLake-targeted MVs store their data/delta tables in DuckLake so // initial materialization follows the same storage path as DuckLake CTAS. - if (!target_is_ducklake && !view_catalog_prefix.empty() && default_db != "memory") { + if (!target_is_ducklake && !view_catalog_prefix.empty() && default_db != "memory" && + view_target_catalog != default_db) { internal_catalog_prefix = SqlUtils::QualifiedPrefix(default_db, default_schema); + internal_target_catalog = default_db; + internal_target_schema = default_schema; } string data_table = IncrementalTableNames::DataTableName(view_name); string qdt = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(data_table); string qvn = view_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(view_name); + bool staged_cross_catalog_replace = target_is_ducklake && parse_data_ref.is_replace; + string staged_data_table = "openivm_stage_" + view_name; + string staged_qdt = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(staged_data_table); + string initial_load_target = staged_cross_catalog_replace ? staged_qdt : qdt; string view_query = original_view_query; // will be overwritten by LPTS for DDL string top_k_suffix; // ORDER BY … LIMIT k, appended to the CREATE VIEW string top_k_order_suffix; // ORDER BY only, used when fallback stored SQL already applied LIMIT @@ -327,12 +388,15 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // Fail before registering cleanup DDL. Otherwise a duplicate CREATE attempt // can fail on the pre-existing backing table and then cleanup would drop the // original MV's user-facing view/data table. - if (RelationExists(con, qvn) || RelationExists(con, qdt)) { + if (RelationExists(context, view_target_catalog, view_target_schema, view_name) || + RelationExists(context, internal_target_catalog, internal_target_schema, data_table)) { throw CatalogException("Table with name \"" + view_name + "\" already exists!"); } } - // Use con for planning — sees all committed state from previous bind-phase DDL + // Plan through the caller context so objects created earlier in the same + // transaction are visible. The helper connection remains for committed + // metadata probes and cross-catalog DDL preparation only. con.BeginTransaction(); // GetTableNames binds the query internally. For MV queries that DuckDB's binder // can't evaluate out-of-context (e.g. multi-column `(a, b) IN (SELECT x, y FROM t)` @@ -351,7 +415,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // Plan the full CREATE TABLE AS SELECT statement (for plan walking) auto full_plan_start = create_profile_now(); - Planner planner(*con.context); + Planner planner(context); planner.CreatePlan(statement->Copy()); auto plan = std::move(planner.plan); add_create_profile_step("create_compile_full_plan", full_plan_start); @@ -369,14 +433,21 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC bool has_hidden_minmax_having = false; bool has_computed_minmax_aggregate_projection = false; DerivedAggregateOutputInfo derived_aggregate_outputs; + bool has_computed_sum_aggregate_projection = false; { auto select_parse_plan_start = create_profile_now(); Parser select_parser; select_parser.ParseQuery(original_view_query); - Planner select_planner(*con.context); + Planner select_planner(context); select_planner.CreatePlan(std::move(select_parser.statements[0])); auto select_plan = std::move(select_planner.plan); visible_output_count = select_planner.names.size(); + for (auto &name : select_planner.names) { + if (StringUtil::CIEquals(name, openivm::MULTIPLICITY_COL) || + StringUtil::CIEquals(name, openivm::TIMESTAMP_COL)) { + throw BinderException("Materialized-view output uses reserved OpenIVM column '%s'", name); + } + } add_create_profile_step("create_compile_select_plan", select_parse_plan_start); // Inline CTEs without running the full optimizer, which can reshape plans @@ -393,10 +464,17 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // Strip HAVING filter from plan — data table stores all groups. // The predicate is extracted as SQL (using output aliases) for the VIEW WHERE clause. having_predicate = StripHavingFilter(select_plan, output_names); + // HAVING-only aggregates are exposed by StripHavingFilter. Inject SUM's + // non-NULL state afterward so visible, wrapped, and HAVING-only SUMs all + // use the same output-index mapping during incremental maintenance. + InjectSumNonNullCounts(context, select_plan); + PropagateHiddenAggregateColumns(select_plan); + output_names = PrepareOutputNames(select_plan.get(), select_planner.names); auto post_rewrite_facts = BuildCreateMVPlanFacts(select_plan.get(), current_catalog); stored_query_has_aggregate_filter = post_rewrite_facts.has_filter_above_aggregate; has_hidden_minmax_having = post_rewrite_facts.has_hidden_minmax_having_column; has_computed_minmax_aggregate_projection = post_rewrite_facts.has_computed_minmax_aggregate_projection; + has_computed_sum_aggregate_projection = post_rewrite_facts.has_computed_sum_aggregate_projection; // Keep data tables unlimited/unordered; apply ORDER BY/LIMIT in the user-facing view. { @@ -460,7 +538,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // CREATE MATERIALIZED VIEW always stores the view body in DuckDB's own dialect. // Refresh-time target dialects are selected per CompileFacts. SqlDialect dialect = SqlDialect::DUCKDB; - auto ast = LogicalPlanToAst(*con.context, select_plan, dialect); + auto ast = LogicalPlanToAst(context, select_plan, dialect); auto cte_list = AstToCteList(*ast, dialect); view_query = cte_list->ToQuery(true, output_names); if (!view_query.empty() && view_query.back() == ';') { @@ -560,8 +638,10 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC model_input.stored_query_has_top_k = stored_query_retains_top_k; model_input.has_hidden_minmax_having = has_hidden_minmax_having; model_input.has_computed_minmax_aggregate_projection = has_computed_minmax_aggregate_projection; + model_input.has_computed_sum_aggregate_projection = has_computed_sum_aggregate_projection; + model_input.has_top_level_redundant_distinct = facts.has_top_level_redundant_distinct; model_input.has_ducklake_source = HasDuckLakeSourceForModel(facts, table_names, target_is_ducklake); - const bool distinct_at_top = IsDistinctAtTop(analysis, output_names); + const bool distinct_at_top = IsDistinctAtTop(facts, output_names) && !facts.has_top_level_redundant_distinct; // Populated by ExtractInnerDistinct when classified as DISTINCT_INCREMENTAL. vector distinct_extracted_cols; @@ -866,6 +946,22 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC model_input.semi_anti_aux_candidate = &semi_anti_aux_candidate; } auto view_model = BuildDeltaViewModel(model_input); + string leftjoin_secondary_sql; + vector leftjoin_preserved_cols; + vector leftjoin_inner_tables, leftjoin_inner_keys, leftjoin_pres_tables, leftjoin_pres_keys; + if (view_model.type == RefreshType::AGGREGATE_GROUP && facts.analysis.found_left_join) { + leftjoin_secondary_sql = BuildLeftJoinSecondaryDeltaSQL( + context, facts, output_names, view_name, leftjoin_preserved_cols, internal_catalog_prefix, + leftjoin_inner_tables, leftjoin_inner_keys, leftjoin_pres_tables, leftjoin_pres_keys); + if (leftjoin_secondary_sql.empty()) { + // The aggregate MERGE requires the Larson & Zhou correction for null-padded row + // transitions. An incomplete correction is not optional: recompute only the groups + // reached from source deltas instead of silently applying incorrect arithmetic. + OPENIVM_DEBUG_PRINT("[CREATE MV] LEFT JOIN secondary-delta unsupported; using GROUP_RECOMPUTE\n"); + view_model.type = RefreshType::GROUP_RECOMPUTE; + view_model.group_recompute_affected_mode = GroupRecomputeAffectedMode::SOURCE_DELTA; + } + } auto lineage_start = create_profile_now(); PopulateDeltaViewModelLineage(view_model, facts, output_names); string lineage_json = BuildDeltaViewModelLineageJson(view_model); @@ -947,7 +1043,12 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC vector cleanup_ddl; vector metadata_ddl; vector aux_metadata_ddl; + vector> staged_aux_tables; + unordered_map aux_state_targets; auto add_cleanup = [&](const string &query) { + if (staged_cross_catalog_replace) { + return; + } cleanup_ddl.push_back(string(OPENIVM_DDL_CLEANUP_PREFIX) + query); }; auto add_profile_marker = [&](const string &step_name, const string &detail = string()) { @@ -957,6 +1058,24 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC ddl.push_back(string(OPENIVM_DDL_PROFILE_RECORD_PREFIX) + view_name + "\t" + step_name + "\t" + to_string(duration_ms) + "\t" + detail); }; + auto get_aux_state_target = [&](const string &aux_table) { + auto existing = aux_state_targets.find(aux_table); + if (existing != aux_state_targets.end()) { + return existing->second; + } + auto published_target = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(aux_table); + if (!staged_cross_catalog_replace) { + aux_state_targets.emplace(aux_table, published_target); + return published_target; + } + auto staged_target = + internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted("openivm_stage_" + aux_table); + ddl.push_back("drop table if exists " + staged_target); + cleanup_ddl.push_back(string(OPENIVM_DDL_CLEANUP_PREFIX) + "DROP TABLE IF EXISTS " + staged_target); + staged_aux_tables.emplace_back(staged_target, published_target); + aux_state_targets.emplace(aux_table, staged_target); + return staged_target; + }; for (const auto &step : create_profile_steps) { add_profile_record(step.step_name, step.duration_ms, step.detail); } @@ -965,7 +1084,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC "; lpts_fallback=" + string(lpts_fallback ? "true" : "false")); AppendCreateMVSystemTablesDDL(ddl, view_name, parse_data_ref.is_replace); - if (parse_data_ref.is_replace) { + if (parse_data_ref.is_replace && !staged_cross_catalog_replace) { add_profile_marker("create_mv_replace_cleanup"); string qvn_drop = view_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(view_name); string qdt_drop = internal_catalog_prefix + @@ -1003,19 +1122,21 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC } } - metadata_ddl.push_back("insert or replace into " + string(openivm::VIEWS_TABLE) + - " (view_name, sql_string, type, has_minmax, has_left_join, has_join, last_update, " - "refresh_interval, refresh_in_progress, group_columns, aggregate_types, " - "having_predicate, group_recompute_affected_mode, " - "group_recompute_source_occurrences_json, has_full_outer, " - "full_outer_join_cols) values ('" + - view_name + "', '" + SqlUtils::EscapeSingleQuotes(view_query) + "', " + - to_string((int)refresh_type) + ", " + (has_minmax_metadata ? "true" : "false") + ", " + - (analysis.found_left_join ? "true" : "false") + ", " + - (analysis.found_join ? "true" : "false") + ", now(), " + refresh_val + ", false, " + - group_cols_val + ", " + agg_types_val + ", " + having_val + ", " + group_recompute_mode_val + - ", " + group_recompute_source_occurrences_val + ", " + - (analysis.found_full_outer ? "true" : "false") + ", " + full_outer_join_cols_val + ")"); + metadata_ddl.push_back( + "insert or replace into " + string(openivm::VIEWS_TABLE) + + " (view_name, view_catalog, view_schema, sql_string, type, has_minmax, has_left_join, " + "has_join, last_update, " + "refresh_interval, refresh_in_progress, group_columns, aggregate_types, " + "having_predicate, group_recompute_affected_mode, " + "group_recompute_source_occurrences_json, has_full_outer, " + "full_outer_join_cols) values ('" + + view_name + "', '" + SqlUtils::EscapeSingleQuotes(view_target_catalog) + "', '" + + SqlUtils::EscapeSingleQuotes(view_target_schema) + "', '" + SqlUtils::EscapeSingleQuotes(view_query) + "', " + + to_string((int)refresh_type) + ", " + (has_minmax_metadata ? "true" : "false") + ", " + + (analysis.found_left_join ? "true" : "false") + ", " + (analysis.found_join ? "true" : "false") + ", now(), " + + refresh_val + ", false, " + group_cols_val + ", " + agg_types_val + ", " + having_val + ", " + + group_recompute_mode_val + ", " + group_recompute_source_occurrences_val + ", " + + (analysis.found_full_outer ? "true" : "false") + ", " + full_outer_join_cols_val + ")"); if (!lineage_json.empty()) { aux_metadata_ddl.push_back(BuildUpdateViewJsonSQL("lineage_json", lineage_json, view_name)); @@ -1066,10 +1187,10 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC if (view_model.HasDistinctAux()) { add_profile_marker("create_mv_distinct_aux"); const auto &meta = view_model.distinct_aux; - string aux_target = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table); + string aux_target = get_aux_state_target(meta.aux_table); string aux_create = BuildDistinctAuxStateCreateSQL(aux_target, meta.cols, meta.source_exprs, "(" + meta.input_sql + ")", "", - /*replace=*/false); + /*replace=*/parse_data_ref.is_replace && !staged_cross_catalog_replace); ddl.push_back(aux_create); add_cleanup("DROP TABLE IF EXISTS " + internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table)); @@ -1081,10 +1202,11 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC add_profile_marker("create_mv_count_distinct_aux"); const auto &meta = view_model.count_distinct_aux; string source_table = QualifyCreateSourceTable(meta.source, current_catalog, current_schema, default_db); - string aux_target = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table); + string aux_target = get_aux_state_target(meta.aux_table); string aux_create = BuildCountDistinctAuxStateCreateSQL(aux_target, source_table, meta.group_cols, meta.group_source_exprs, - meta.distinct_col, meta.distinct_expr, meta.filter, /*replace=*/false); + meta.distinct_col, meta.distinct_expr, meta.filter, + /*replace=*/parse_data_ref.is_replace && !staged_cross_catalog_replace); ddl.push_back(aux_create); add_cleanup("DROP TABLE IF EXISTS " + internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table)); @@ -1097,10 +1219,10 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC const auto &req = view_model.filtered_group_count_aux; const auto &meta = req.meta; string source_table = QualifyCreateSourceTable(req.create_source, current_catalog, current_schema, default_db); - string aux_target = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table); + string aux_target = get_aux_state_target(meta.aux_table); string aux_create = BuildFilteredGroupCountAuxStateCreateSQL( aux_target, source_table, meta.group_col, meta.sum_col, meta.source_group_expr, meta.source_sum_expr, - /*replace=*/false); + /*replace=*/parse_data_ref.is_replace && !staged_cross_catalog_replace); ddl.push_back(aux_create); add_cleanup("DROP TABLE IF EXISTS " + internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table)); @@ -1115,10 +1237,11 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC QualifyCreateSourceTable(meta.left_table, current_catalog, current_schema, default_db); string right_source_table = QualifyCreateSourceTable(meta.right_table, current_catalog, current_schema, default_db); - string aux_target = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table); + string aux_target = get_aux_state_target(meta.aux_table); string aux_create = BuildSemiAntiAuxStateCreateSQL( aux_target, left_source_table, meta.left_alias, right_source_table, meta.right_alias, meta.predicate, - meta.post_filter, meta.right_filter, meta.left_cols, meta.left_exprs, /*replace=*/false, meta.null_aware, + meta.post_filter, meta.right_filter, meta.left_cols, meta.left_exprs, + /*replace=*/parse_data_ref.is_replace && !staged_cross_catalog_replace, meta.null_aware, meta.null_aware_right_expr); ddl.push_back(aux_create); add_cleanup("DROP TABLE IF EXISTS " + internal_catalog_prefix + @@ -1127,6 +1250,21 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC BuildUpdateViewJsonSQL("semi_anti_aux_meta_json", RefreshMetadata::SemiAntiAuxMetaToJson(meta), view_name)); } + // LEFT JOIN pipeline secondary-delta (Larson & Zhou): generate the secondary-delta INSERT once at CREATE + // time and store it; refresh appends it between the primary-delta INSERT and the MERGE. + if (!leftjoin_secondary_sql.empty()) { + add_profile_marker("create_mv_leftjoin_secondary"); + RefreshMetadata::LeftJoinSecondaryMeta sm; + sm.sql = leftjoin_secondary_sql; + sm.preserved_cols = leftjoin_preserved_cols; + sm.inner_tables = leftjoin_inner_tables; + sm.inner_keys = leftjoin_inner_keys; + sm.pres_tables = leftjoin_pres_tables; + sm.pres_keys = leftjoin_pres_keys; + aux_metadata_ddl.push_back(BuildUpdateViewJsonSQL("leftjoin_secondary_meta_json", + RefreshMetadata::LeftJoinSecondaryMetaToJson(sm), view_name)); + } + const auto &source_table_info = facts.source_table_info; const auto &dl_table_info = facts.ducklake_table_info; // keyed by lowercased name @@ -1213,7 +1351,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC if (!current_catalog.empty() && current_catalog != default_db) { con.Query("USE " + current_catalog_schema); } - string initial_load_statement = "CREATE TABLE " + qdt + " AS " + view_query; + string initial_load_statement = "CREATE TABLE " + initial_load_target + " AS " + view_query; string diagnostic; diagnostic += "\n[OpenIVM initial-load diagnostic]\n"; diagnostic += "view_name: " + view_name + "\n"; @@ -1234,7 +1372,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC false, diagnostic); } if (SqlUtils::GetBoolSetting(context, "openivm_explain_initial_load_only", false)) { - ConfigureDDLExecutorResult(result); + ConfigureDDLExecutorResult(result, DDLExecutionMode::CALLER_TRANSACTION); return result; } } @@ -1247,13 +1385,28 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC if (!current_catalog.empty() && current_catalog != default_db) { ddl.push_back("use " + current_catalog_schema); } + if (staged_cross_catalog_replace) { + ddl.push_back("drop table if exists " + staged_qdt); + cleanup_ddl.push_back(string(OPENIVM_DDL_CLEANUP_PREFIX) + "DROP TABLE IF EXISTS " + staged_qdt); + } if (view_model.HasSemiAntiAux()) { const auto &meta = view_model.semi_anti_aux; - string aux_target = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(meta.aux_table); - ddl.push_back(BuildSemiAntiInitialDataSQL(qdt, aux_target, meta.join_type, meta.left_cols, meta.output_cols, - meta.null_aware, meta.null_aware_left_col)); + string aux_target = get_aux_state_target(meta.aux_table); + ddl.push_back(BuildSemiAntiInitialDataSQL(initial_load_target, aux_target, meta.join_type, meta.left_cols, + meta.output_cols, meta.null_aware, meta.null_aware_left_col)); } else { - ddl.push_back("create table " + qdt + " as " + view_query); + ddl.push_back("create table " + initial_load_target + " as " + view_query); + } + if (staged_cross_catalog_replace) { + // DuckDB cannot make the DuckLake objects and native metadata atomic + // together. Materialize the expensive replacement under an unpublished + // name first; CREATE OR REPLACE publishes it only after the query succeeds. + for (auto &entry : staged_aux_tables) { + ddl.push_back("create or replace table " + entry.second + " as select * from " + entry.first); + ddl.push_back("drop table " + entry.first); + } + ddl.push_back("create or replace table " + qdt + " as select * from " + staged_qdt); + ddl.push_back("drop table " + staged_qdt); } if (!view_catalog_prefix.empty()) { // Keep the same connection after a DuckLake CTAS. Reopening here can force @@ -1299,10 +1452,12 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC } string view_tail = having_where + top_k_view_suffix; if (internal_cols.empty()) { - ddl.push_back("create view " + qvn + " as select * from " + qdt + view_tail); + ddl.push_back(string(staged_cross_catalog_replace ? "create or replace view " : "create view ") + qvn + + " as select * from " + qdt + view_tail); } else { - ddl.push_back("create view " + qvn + " as select * exclude (" + SqlUtils::JoinQuotedColumns(internal_cols) + - ") from " + qdt + view_tail); + ddl.push_back(string(staged_cross_catalog_replace ? "create or replace view " : "create view ") + qvn + + " as select * exclude (" + SqlUtils::JoinQuotedColumns(internal_cols) + ") from " + qdt + + view_tail); } } @@ -1356,7 +1511,7 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // Delta table for the MV — based on the DATA table (has all columns) add_profile_marker("create_mv_mv_delta_table"); string qdv = internal_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(SqlUtils::DeltaName(view_name)); - ddl.push_back(string(OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX) + qdv + "\t" + qdt); + ddl.push_back(BuildCreateDeltaFromDataOperation(qdv, qdt, staged_cross_catalog_replace)); add_cleanup("DROP TABLE IF EXISTS " + qdv); // --- Index DDL (for aggregate group queries) --- @@ -1382,6 +1537,12 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC // Record source-table metadata only after physical MV objects exist. If a later // DuckLake publish fails, the DDL executor removes these rows before the retry. add_profile_marker("create_mv_source_metadata", "rows=" + to_string(source_metadata_ddl.size())); + if (staged_cross_catalog_replace) { + ddl.push_back("DELETE FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name = '" + + SqlUtils::EscapeSingleQuotes(view_name) + "'"); + ddl.push_back("DELETE FROM " + string(openivm::HISTORY_TABLE) + " WHERE view_name = '" + + SqlUtils::EscapeSingleQuotes(view_name) + "'"); + } ddl.insert(ddl.end(), source_metadata_ddl.begin(), source_metadata_ddl.end()); add_cleanup("DELETE FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name = '" + SqlUtils::EscapeSingleQuotes(view_name) + "'"); @@ -1467,7 +1628,314 @@ MaterializedViewParserExtension::PlanFunction(ParserExtensionInfo *info, ClientC } // Return DDL executor table function - ConfigureDDLExecutorResult(result); + bool caller_transactional_ddl = + !target_is_ducklake && (view_catalog_prefix.empty() || view_target_catalog == default_db); + ConfigureDDLExecutorResult(result, caller_transactional_ddl ? DDLExecutionMode::CALLER_TRANSACTION + : DDLExecutionMode::STAGED_CROSS_CATALOG); return result; } + +string MaterializedViewLifecycleQuery(ClientContext &context, const FunctionParameters ¶meters) { + auto query = StringValue::Get(parameters.values[0]); + auto parse_result = MaterializedViewParserExtension::ParseFunction(nullptr, query); + if (parse_result.type != ParserExtensionResultType::PARSE_SUCCESSFUL) { + throw ParserException("OpenIVM could not parse the materialized-view lifecycle statement"); + } + auto view_name = dynamic_cast(*parse_result.parse_data).target_name; + auto target = ResolveMaterializedViewTarget(context, view_name); + auto lock_view_name = target.view_name; + auto plan_result = + MaterializedViewParserExtension::PlanFunction(nullptr, context, std::move(parse_result.parse_data)); + if (plan_result.function.name == OPENIVM_TRANSACTIONAL_DDL_FUNCTION) { + if (!lock_view_name.empty()) { + TransactionalMVLockState::Get(context).AcquireMutationLock(); + } + if (!context.transaction.IsAutoCommit()) { + TransactionalMVMetadataState::Get(context).Register(context, plan_result.parameters, lock_view_name); + } + return RenderTransactionalDDL(context, plan_result.parameters); + } + ExecuteStagedDDL(context, plan_result.parameters); + return "SELECT true AS \"MATERIALIZED VIEW CREATION\""; +} + +static void AppendTrackedViewDropProgram(ClientContext &context, RefreshMetadata &metadata, const string &view_name, + const RefreshMetadata::StoredViewLocation &location, string &program, + vector &sources, bool cascade, + OnEntryNotFound if_not_found) { + QueryErrorContext error_context; + auto data_name = IncrementalTableNames::DataTableName(view_name); + string data_ref; + auto view_entry = Catalog::GetEntry(context, location.catalog_name, location.schema_name, + EntryLookupInfo(CatalogType::VIEW_ENTRY, view_name, error_context), + OnEntryNotFound::RETURN_NULL); + if (view_entry) { + data_ref = SqlUtils::FindTableReference(view_entry->Cast().sql, data_name); + } + string internal_prefix = SqlUtils::QualifiedPrefix(location.catalog_name, location.schema_name); + if (!data_ref.empty()) { + auto separator = data_ref.rfind('.'); + internal_prefix = separator == string::npos ? "" : data_ref.substr(0, separator + 1); + } else { + data_ref = internal_prefix + KeywordHelper::WriteOptionallyQuoted(data_name); + } + + DropInfo view_drop; + view_drop.type = CatalogType::VIEW_ENTRY; + view_drop.catalog = location.catalog_name; + view_drop.schema = location.schema_name; + view_drop.name = view_name; + view_drop.cascade = cascade; + view_drop.if_not_found = if_not_found; + program += BuildDropViewStatement(view_drop) + ";\n"; + program += "DROP TABLE IF EXISTS " + data_ref + ";\n"; + program += "DROP TABLE IF EXISTS " + internal_prefix + + KeywordHelper::WriteOptionallyQuoted(SqlUtils::DeltaName(view_name)) + ";\n"; + program += "DELETE FROM openivm_refresh_hooks WHERE view_name = '" + SqlUtils::EscapeValue(view_name) + "';\n"; + program += "DELETE FROM " + string(openivm::MV_DEPS_TABLE) + " WHERE parent_view = '" + + SqlUtils::EscapeValue(view_name) + "' OR child_view = '" + SqlUtils::EscapeValue(view_name) + "';\n"; + program += "DELETE FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name = '" + + SqlUtils::EscapeValue(view_name) + "';\n"; + program += "DELETE FROM " + string(openivm::VIEWS_TABLE) + " WHERE view_name = '" + + SqlUtils::EscapeValue(view_name) + "';\n"; + auto view_sources = metadata.GetDeltaSources(view_name, location.catalog_name, location.schema_name); + sources.insert(sources.end(), view_sources.begin(), view_sources.end()); +} + +static void AppendUnusedSourceDropProgram(Connection &con, const vector &sources, + const string &excluded_views, string &program) { + unordered_set checked_sources; + for (auto &source : sources) { + if (source.catalog_type == "ducklake") { + continue; + } + auto identity = source.catalog_name + "\n" + source.schema_name + "\n" + source.table_name; + if (!checked_sources.insert(identity).second) { + continue; + } + auto remaining = con.Query( + "SELECT count(*) FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE table_name = '" + + SqlUtils::EscapeValue(source.table_name) + "' AND COALESCE(source_catalog, '" + + SqlUtils::EscapeValue(source.catalog_name) + "') = '" + SqlUtils::EscapeValue(source.catalog_name) + + "' AND COALESCE(source_schema, '" + SqlUtils::EscapeValue(source.schema_name) + "') = '" + + SqlUtils::EscapeValue(source.schema_name) + "' AND view_name NOT IN (" + excluded_views + ")"); + if (remaining->HasError()) { + throw CatalogException("OpenIVM could not verify delta-table consumers for '%s': %s", source.table_name, + remaining->GetError()); + } + if (remaining->RowCount() > 0 && remaining->GetValue(0, 0).GetValue() == 0) { + program += "DROP TABLE IF EXISTS " + + SqlUtils::FullName(source.catalog_name, source.schema_name, source.table_name) + ";\n"; + } + } +} + +static string ExecuteAutocommitDropProgram(Connection &con, const string &program) { + con.BeginTransaction(); + try { + auto result = con.Query(program); + if (result->HasError()) { + throw CatalogException("OpenIVM DROP cleanup failed: %s", result->GetError()); + } + con.Commit(); + } catch (std::exception &) { + try { + con.Rollback(); + } catch (std::exception &) { + } + throw; + } + return "SELECT true AS Success"; +} + +static string BuildCascadeDropTableProgram(ClientContext &context, DropInfo &drop_info) { + unique_ptr autocommit_guard; + if (context.transaction.IsAutoCommit()) { + autocommit_guard = make_uniq(context); + } else { + TransactionalMVLockState::Get(context).AcquireMutationLock(); + } + + QueryErrorContext error_context; + auto table_entry = Catalog::GetEntry(context, drop_info.catalog, drop_info.schema, + EntryLookupInfo(CatalogType::TABLE_ENTRY, drop_info.name, error_context), + OnEntryNotFound::RETURN_NULL); + if (table_entry) { + drop_info.catalog = table_entry->ParentCatalog().GetName(); + drop_info.schema = table_entry->ParentSchema().name; + } + auto &default_entry = ClientData::Get(context).catalog_search_path->GetDefault(); + if (drop_info.catalog.empty()) { + drop_info.catalog = + default_entry.catalog.empty() ? DatabaseManager::GetDefaultDatabase(context) : default_entry.catalog; + } + if (drop_info.schema.empty()) { + drop_info.schema = default_entry.schema.empty() ? DEFAULT_SCHEMA : default_entry.schema; + } + + Connection con(*context.db); + if (auto metadata_state = TransactionalMVMetadataState::TryGet(context)) { + metadata_state->Apply(con); + } + auto dependent_rows = + con.Query("SELECT DISTINCT view_name FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE table_name = '" + + SqlUtils::EscapeValue(SqlUtils::DeltaName(drop_info.name)) + "' AND COALESCE(source_catalog, '" + + SqlUtils::EscapeValue(drop_info.catalog) + "') = '" + SqlUtils::EscapeValue(drop_info.catalog) + + "' AND COALESCE(source_schema, '" + SqlUtils::EscapeValue(drop_info.schema) + "') = '" + + SqlUtils::EscapeValue(drop_info.schema) + "' ORDER BY view_name"); + if (dependent_rows->HasError()) { + throw CatalogException("OpenIVM could not resolve materialized views depending on '%s': %s", drop_info.name, + dependent_rows->GetError()); + } + if (dependent_rows->RowCount() == 0) { + auto program = BuildDropTableStatement(drop_info) + ";\n"; + return context.transaction.IsAutoCommit() ? ExecuteAutocommitDropProgram(con, program) : program; + } + + RefreshMetadata metadata(con); + vector dependent_views; + unordered_set seen; + for (idx_t row = 0; row < dependent_rows->RowCount(); row++) { + auto direct_view = dependent_rows->GetValue(0, row).ToString(); + auto downstream = metadata.GetDownstreamViewsStrict(direct_view); + for (auto it = downstream.rbegin(); it != downstream.rend(); ++it) { + if (seen.insert(*it).second) { + dependent_views.push_back(*it); + } + } + if (seen.insert(direct_view).second) { + dependent_views.push_back(std::move(direct_view)); + } + } + + string excluded_views; + for (auto &view_name : dependent_views) { + if (!excluded_views.empty()) { + excluded_views += ", "; + } + excluded_views += "'" + SqlUtils::EscapeValue(view_name) + "'"; + } + + string program; + vector sources; + for (auto &view_name : dependent_views) { + auto location = metadata.GetStoredViewLocation(view_name); + AppendTrackedViewDropProgram(context, metadata, view_name, location, program, sources, true, + OnEntryNotFound::RETURN_NULL); + } + + AppendUnusedSourceDropProgram(con, sources, excluded_views, program); + program += BuildDropTableStatement(drop_info) + ";\n"; + + if (context.transaction.IsAutoCommit()) { + return ExecuteAutocommitDropProgram(con, program); + } else { + auto &state = TransactionalMVMetadataState::Get(context); + state.RegisterSQL(program, dependent_views.front()); + for (idx_t index = 1; index < dependent_views.size(); index++) { + state.IncludeView(dependent_views[index]); + } + } + return program; +} + +string MaterializedViewDropQuery(ClientContext &context, const FunctionParameters ¶meters) { + auto query = StringValue::Get(parameters.values[0]); + ParserOptions options = context.GetParserOptions(); + options.extensions = nullptr; + Parser parser(options); + parser.ParseQuery(query); + if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::DROP_STATEMENT) { + throw InternalException("OpenIVM DROP rewrite expected one DROP statement"); + } + auto &drop = parser.statements[0]->Cast(); + if (drop.info->type == CatalogType::TABLE_ENTRY && drop.info->cascade) { + return BuildCascadeDropTableProgram(context, *drop.info); + } + if (drop.info->type != CatalogType::VIEW_ENTRY) { + throw InternalException("OpenIVM DROP rewrite expected DROP VIEW"); + } + + string catalog_name = drop.info->catalog; + string schema_name = drop.info->schema; + QueryErrorContext error_context; + auto view_entry = Catalog::GetEntry(context, catalog_name, schema_name, + EntryLookupInfo(CatalogType::VIEW_ENTRY, drop.info->name, error_context), + OnEntryNotFound::RETURN_NULL); + if (view_entry) { + catalog_name = view_entry->ParentCatalog().GetName(); + schema_name = view_entry->ParentSchema().name; + } + // INVALID_CATALOG and INVALID_SCHEMA are both "", so these are emptiness checks; spell them that way + // (clang-tidy readability-container-size-empty). + if (catalog_name.empty() || schema_name.empty()) { + auto &default_entry = ClientData::Get(context).catalog_search_path->GetDefault(); + if (catalog_name.empty()) { + catalog_name = default_entry.catalog; + } + if (schema_name.empty()) { + schema_name = default_entry.schema.empty() ? DEFAULT_SCHEMA : default_entry.schema; + } + } + if (catalog_name.empty()) { + catalog_name = Catalog::GetSystemCatalog(context).GetName(); + } + if (schema_name.empty()) { + schema_name = DEFAULT_SCHEMA; + } + drop.info->catalog = catalog_name; + drop.info->schema = schema_name; + + string program = BuildDropViewStatement(*drop.info) + ";\n"; + string data_table_name = IncrementalTableNames::DataTableName(drop.info->name); + string data_table_ref; + if (view_entry) { + auto &view = view_entry->Cast(); + data_table_ref = SqlUtils::FindTableReference(view.sql, data_table_name); + } + Connection con(*context.db); + if (auto metadata_state = TransactionalMVMetadataState::TryGet(context)) { + metadata_state->Apply(con); + } + auto tracked = con.Query("SELECT view_catalog, view_schema FROM " + string(openivm::VIEWS_TABLE) + + " WHERE view_name = '" + SqlUtils::EscapeValue(drop.info->name) + "'"); + bool legacy_identity = tracked->HasError(); + if (legacy_identity) { + tracked = con.Query("SELECT 1 FROM " + string(openivm::VIEWS_TABLE) + " WHERE view_name = '" + + SqlUtils::EscapeValue(drop.info->name) + "'"); + } + if (data_table_ref.empty() || tracked->HasError() || tracked->RowCount() == 0) { + return program; + } + if (!legacy_identity && !tracked->GetValue(0, 0).IsNull() && !tracked->GetValue(1, 0).IsNull()) { + if (!StringUtil::CIEquals(tracked->GetValue(0, 0).ToString(), catalog_name) || + !StringUtil::CIEquals(tracked->GetValue(1, 0).ToString(), schema_name)) { + return program; + } + } else { + // Rows created before target identity was persisted can only be cleaned + // through the default search-path location. A qualified same-named view + // elsewhere is not sufficient proof of ownership. + auto &default_entry = ClientData::Get(context).catalog_search_path->GetDefault(); + string default_schema = default_entry.schema.empty() ? DEFAULT_SCHEMA : default_entry.schema; + if (!StringUtil::CIEquals(default_entry.catalog, catalog_name) || + !StringUtil::CIEquals(default_schema, schema_name)) { + return program; + } + } + + RefreshMetadata metadata(con); + program.clear(); + vector delta_sources; + RefreshMetadata::StoredViewLocation location {catalog_name, schema_name}; + AppendTrackedViewDropProgram(context, metadata, drop.info->name, location, program, delta_sources, + drop.info->cascade, drop.info->if_not_found); + auto excluded_view = "'" + SqlUtils::EscapeValue(drop.info->name) + "'"; + AppendUnusedSourceDropProgram(con, delta_sources, excluded_view, program); + TransactionalMVLockState::Get(context).AcquireMutationLock(); + if (!context.transaction.IsAutoCommit()) { + TransactionalMVMetadataState::Get(context).RegisterSQL(program, drop.info->name); + } + return program; +} } // namespace duckdb diff --git a/src/core/parser_create_mv_helpers.cpp b/src/core/parser_create_mv_helpers.cpp index cb3542f4..5ffc9191 100644 --- a/src/core/parser_create_mv_helpers.cpp +++ b/src/core/parser_create_mv_helpers.cpp @@ -34,7 +34,8 @@ void AppendCreateMVSystemTablesDDL(vector &ddl, const string &view_name, // Matcher metadata columns (signature_hash..nullified_columns_json) stay // NULL unless openivm_enable_view_matching=true; populated by Stage I wiring. ddl.push_back("create table if not exists " + string(openivm::VIEWS_TABLE) + - " (view_name varchar primary key, sql_string varchar, type tinyint," + " (view_name varchar primary key, view_catalog varchar default null," + " view_schema varchar default null, sql_string varchar, type tinyint," " has_minmax boolean default false, has_left_join boolean default false," " has_join boolean default false," " last_update timestamp, refresh_interval bigint default null," @@ -65,10 +66,13 @@ void AppendCreateMVSystemTablesDDL(vector &ddl, const string &view_name, AddColumnIfNotExists(ddl, openivm::VIEWS_TABLE, "count_distinct_aux_meta_json varchar default null"); AddColumnIfNotExists(ddl, openivm::VIEWS_TABLE, "semi_anti_aux_meta_json varchar default null"); AddColumnIfNotExists(ddl, openivm::VIEWS_TABLE, "lineage_json varchar default null"); + AddColumnIfNotExists(ddl, openivm::VIEWS_TABLE, "leftjoin_secondary_meta_json varchar default null"); AddColumnIfNotExists(ddl, openivm::VIEWS_TABLE, "has_join boolean default null"); AddColumnIfNotExists(ddl, openivm::VIEWS_TABLE, "group_recompute_affected_mode varchar default null"); AddColumnIfNotExists(ddl, openivm::VIEWS_TABLE, "group_recompute_source_occurrences_json varchar default null"); AddColumnIfNotExists(ddl, openivm::VIEWS_TABLE, "derived_aggregate_outputs_json varchar default null"); + AddColumnIfNotExists(ddl, openivm::VIEWS_TABLE, "view_catalog varchar default null"); + AddColumnIfNotExists(ddl, openivm::VIEWS_TABLE, "view_schema varchar default null"); if (!is_replace) { string escaped_view_name = SqlUtils::EscapeSingleQuotes(view_name); string escaped_data_table = SqlUtils::EscapeSingleQuotes(IncrementalTableNames::DataTableName(view_name)); diff --git a/src/core/parser_ddl.cpp b/src/core/parser_ddl.cpp index 88889573..7efe44af 100644 --- a/src/core/parser_ddl.cpp +++ b/src/core/parser_ddl.cpp @@ -2,11 +2,18 @@ #include "core/openivm_constants.hpp" #include "core/openivm_debug.hpp" +#include "core/refresh_locks.hpp" #include "core/sql_utils.hpp" +#include "duckdb/catalog/catalog.hpp" #include "duckdb/common/printer.hpp" #include "duckdb/function/table_function.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" +#include "duckdb/parser/parser.hpp" +#include "duckdb/parser/statement/drop_statement.hpp" +#include "duckdb/planner/binder.hpp" #include +#include #include #include @@ -14,6 +21,131 @@ namespace duckdb { namespace { +static const vector &TransactionalMetadataTables() { + static const vector tables = {openivm::VIEWS_TABLE, "openivm_refresh_hooks", openivm::DELTA_TABLES_TABLE, + openivm::HISTORY_TABLE, openivm::PROFILE_TABLE, openivm::MV_DEPS_TABLE}; + return tables; +} + +static const vector &MetadataStatementPrefixes() { + static const vector prefixes = { + "create table if not exists ", + "alter table ", + "insert or replace into ", + "insert or ignore into ", + "insert into ", + "update ", + "delete from ", + }; + return prefixes; +} + +static bool HasMetadataTarget(const string &statement, const string &prefix, const string &table) { + if (!StringUtil::StartsWith(statement, prefix)) { + return false; + } + auto target = statement.substr(prefix.size()); + auto table_name = StringUtil::Lower(table); + return StringUtil::StartsWith(target, table_name) && + (target.size() == table_name.size() || std::isspace(static_cast(target[table_name.size()])) || + target[table_name.size()] == '('); +} + +static bool TargetsMetadataTable(const string &statement, const string &table) { + auto trimmed = statement; + StringUtil::Trim(trimmed); + auto lower = StringUtil::Lower(trimmed); + for (auto &prefix : MetadataStatementPrefixes()) { + if (HasMetadataTarget(lower, prefix, table)) { + return true; + } + } + return false; +} + +static bool IsReplayableMetadataStatement(const string &statement) { + auto trimmed = statement; + StringUtil::Trim(trimmed); + auto lower = StringUtil::Lower(trimmed); + for (auto &prefix : MetadataStatementPrefixes()) { + for (auto &table : TransactionalMetadataTables()) { + if (HasMetadataTarget(lower, prefix, table)) { + return true; + } + } + } + return false; +} + +static string MakeTemporaryMetadataDDL(const string &statement) { + auto lower = StringUtil::Lower(statement); + const string prefix = "create table if not exists "; + if (!StringUtil::StartsWith(lower, prefix)) { + return statement; + } + return "create temp table if not exists " + statement.substr(prefix.size()); +} + +static bool IsMetadataSchemaStatement(const string &statement) { + auto trimmed = statement; + StringUtil::Trim(trimmed); + auto lower = StringUtil::Lower(trimmed); + return StringUtil::StartsWith(lower, "create table") || StringUtil::StartsWith(lower, "alter table"); +} + +static bool MatchesNowCall(const string &statement, idx_t offset) { + static const string now_call = "now()"; + if (offset + now_call.size() > statement.size()) { + return false; + } + for (idx_t i = 0; i < now_call.size(); i++) { + if (StringUtil::CharacterToLower(statement[offset + i]) != now_call[i]) { + return false; + } + } + return true; +} + +static string StabilizeTransactionalMetadata(const string &statement, const string ×tamp_sql) { + string result; + result.reserve(statement.size() + timestamp_sql.size()); + bool in_single_quote = false; + bool in_double_quote = false; + for (idx_t offset = 0; offset < statement.size();) { + char current = statement[offset]; + if (current == '\'' && !in_double_quote) { + result.push_back(current); + if (in_single_quote && offset + 1 < statement.size() && statement[offset + 1] == '\'') { + result.push_back(statement[offset + 1]); + offset += 2; + continue; + } + in_single_quote = !in_single_quote; + offset++; + continue; + } + if (current == '"' && !in_single_quote) { + result.push_back(current); + if (in_double_quote && offset + 1 < statement.size() && statement[offset + 1] == '"') { + result.push_back(statement[offset + 1]); + offset += 2; + continue; + } + in_double_quote = !in_double_quote; + offset++; + continue; + } + if (!in_single_quote && !in_double_quote && MatchesNowCall(statement, offset)) { + result += timestamp_sql; + offset += 5; + continue; + } + result.push_back(current); + offset++; + } + return result; +} + struct CreateMVProfileStep { int32_t step_order; string step_name; @@ -160,16 +292,26 @@ struct DeltaSchemaDDL { idx_t column_count = 0; }; -void ParseCreateDeltaFromDataPayload(const string &payload, string &delta_table, string &data_table) { +void ParseCreateDeltaFromDataPayload(const string &payload, bool &replace, string &delta_table, string &data_table) { auto first = payload.find('\t'); if (first == string::npos) { return; } - delta_table = payload.substr(0, first); - data_table = payload.substr(first + 1); + auto second = payload.find('\t', first + 1); + if (second == string::npos) { + return; + } + auto mode = payload.substr(0, first); + if (mode != "create" && mode != "replace") { + return; + } + replace = mode == "replace"; + delta_table = payload.substr(first + 1, second - first - 1); + data_table = payload.substr(second + 1); } -DeltaSchemaDDL BuildCreateDeltaFromDataSQL(Connection &conn, const string &delta_table, const string &data_table) { +DeltaSchemaDDL BuildCreateDeltaFromDataSQL(Connection &conn, const string &delta_table, const string &data_table, + bool replace) { auto described = conn.Query("DESCRIBE SELECT * FROM " + data_table); if (described->HasError()) { throw CatalogException("Could not derive IVM delta schema from data table '" + data_table + @@ -209,7 +351,8 @@ DeltaSchemaDDL BuildCreateDeltaFromDataSQL(Connection &conn, const string &delta columns.push_back(string(openivm::MULTIPLICITY_COL) + " INTEGER DEFAULT 1"); columns.push_back(string(openivm::TIMESTAMP_COL) + " TIMESTAMP DEFAULT now()"); DeltaSchemaDDL result; - result.sql = "create table if not exists " + delta_table + " (" + StringUtil::Join(columns, ", ") + ")"; + result.sql = string(replace ? "create or replace table " : "create table if not exists ") + delta_table + " (" + + StringUtil::Join(columns, ", ") + ")"; result.column_count = described->RowCount(); return result; } @@ -220,6 +363,9 @@ void ExecuteDDL(ClientContext &context, const vector &ddl) { } auto &db = DatabaseInstance::GetDatabase(context); auto conn = make_uniq(db); + auto &helper_lock_state = TransactionalMVLockState::Get(*conn->context); + helper_lock_state.SetMutationOwner(&context); + MutationLockGuard mutation_guard(db, &context); bool suspended_autocommit_transaction = false; auto restore_outer_transaction = [&]() { if (suspended_autocommit_transaction && !context.transaction.HasActiveTransaction()) { @@ -317,17 +463,18 @@ void ExecuteDDL(ClientContext &context, const vector &ddl) { } if (StringUtil::StartsWith(q, OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX)) { flush_pending(); + bool replace = false; string delta_table; string data_table; - ParseCreateDeltaFromDataPayload(q.substr(strlen(OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX)), delta_table, - data_table); + ParseCreateDeltaFromDataPayload(q.substr(strlen(OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX)), replace, + delta_table, data_table); if (delta_table.empty() || data_table.empty()) { fail_ddl("malformed delta-schema payload"); } auto ddl_start = std::chrono::steady_clock::now(); DeltaSchemaDDL derived; try { - derived = BuildCreateDeltaFromDataSQL(*conn, delta_table, data_table); + derived = BuildCreateDeltaFromDataSQL(*conn, delta_table, data_table, replace); } catch (std::exception &ex) { profiler.AddStep(current_profile_step, ddl_start, current_profile_detail + "; delta_schema_derivation_failed=true"); @@ -382,9 +529,465 @@ void DDLExecutorExecuteFunction(ClientContext &context, TableFunctionInput &data } // namespace -void ConfigureDDLExecutorResult(ParserExtensionPlanResult &result) { +TransactionalMVMetadataState &TransactionalMVMetadataState::Get(ClientContext &context) { + return *context.registered_state->GetOrCreate("openivm_transactional_mv_metadata"); +} + +optional_ptr TransactionalMVMetadataState::TryGet(ClientContext &context) { + return context.registered_state->Get("openivm_transactional_mv_metadata"); +} + +void TransactionalMVMetadataState::Register(ClientContext &context, const vector ¶meters, + const string &view_name) { + view_names.insert(view_name); + auto transaction_timestamp = context.transaction.ActiveTransaction().GetCurrentTransactionStartTimestamp(); + // Delta-table DEFAULT now() is transaction-stable. Compile from one + // microsecond before it so rows inserted into a newly created MV delta + // table during this same transaction satisfy the compiler's strict `>` + // watermark predicate. + transaction_timestamp -= 1; + auto timestamp_sql = Value::TIMESTAMP(transaction_timestamp).ToSQLString() + "::TIMESTAMP"; + for (auto ¶meter : parameters) { + auto statement = parameter.GetValue(); + if (IsReplayableMetadataStatement(statement)) { + statements.push_back(StabilizeTransactionalMetadata(statement, timestamp_sql)); + } + } +} + +void TransactionalMVMetadataState::RegisterSQL(const string &sql, const string &view_name) { + view_names.insert(view_name); + for (auto &statement : StringUtil::Split(sql, ";\n")) { + if (IsReplayableMetadataStatement(statement)) { + statements.push_back(std::move(statement)); + } + } +} + +void TransactionalMVMetadataState::IncludeView(const string &view_name) { + view_names.insert(view_name); +} + +void TransactionalMVMetadataState::Apply(Connection &connection) const { + if (statements.empty()) { + return; + } + string metadata_catalog; + auto current_database = connection.Query("SELECT current_database()"); + if (!current_database->HasError() && current_database->RowCount() > 0 && + !current_database->GetValue(0, 0).IsNull()) { + metadata_catalog = current_database->GetValue(0, 0).ToString(); + } + auto durable_table = [&](const string &table) { + return metadata_catalog.empty() ? "main." + table : SqlUtils::FullName(metadata_catalog, DEFAULT_SCHEMA, table); + }; + // Build constrained TEMP schemas first. CREATE TABLE AS would discard the + // primary keys required by INSERT OR REPLACE metadata operations. + for (auto &statement : statements) { + if (!IsMetadataSchemaStatement(statement)) { + continue; + } + auto result = connection.Query(MakeTemporaryMetadataDDL(statement)); + if (result->HasError()) { + throw CatalogException("Could not reconstruct transaction-local OpenIVM metadata schema: %s", + result->GetError()); + } + } + auto seed_table = [&](const string &table, const string &filter) { + bool has_constrained_schema = false; + for (auto &statement : statements) { + auto lower = StringUtil::Lower(statement); + if (StringUtil::StartsWith(lower, "create table") && lower.find(StringUtil::Lower(table)) != string::npos) { + has_constrained_schema = true; + break; + } + } + if (has_constrained_schema) { + // Seed the constrained shadow with committed rows. A missing durable + // table is expected for the first MV in a database. + connection.Query("INSERT OR IGNORE INTO " + table + " BY NAME SELECT * FROM " + durable_table(table) + + filter); + } else { + // Less central metadata (for example optional dependency state) may + // be initialized outside the lifecycle program. Still shadow it so + // replay can never mutate the durable helper-connection catalog. + connection.Query("CREATE TEMP TABLE IF NOT EXISTS " + table + " AS SELECT * FROM " + durable_table(table) + + filter); + } + }; + auto replay_statement = [&](const string &statement) { + auto result = connection.Query(MakeTemporaryMetadataDDL(statement)); + if (result->HasError()) { + throw CatalogException("Could not reconstruct transaction-local OpenIVM metadata: %s", result->GetError()); + } + }; + + // Cascade dependencies are recorded both explicitly for view matching and + // implicitly when a view consumes another view's delta/data table. Seed and + // replay all three relation tables first so the closure reflects this + // transaction's CREATE/REPLACE/DROP statements. + const vector dependency_tables = { + openivm::VIEWS_TABLE, + openivm::DELTA_TABLES_TABLE, + openivm::MV_DEPS_TABLE, + }; + for (auto &table : dependency_tables) { + seed_table(table, ""); + } + for (auto &statement : statements) { + if (IsMetadataSchemaStatement(statement)) { + continue; + } + for (auto &table : dependency_tables) { + if (TargetsMetadataTable(statement, table)) { + replay_statement(statement); + break; + } + } + } + unordered_set included_views = view_names; + unordered_map> dependency_graph; + idx_t dependency_edge_count = 0; + auto add_dependency = [&](const string &parent, const string &child) { + dependency_graph[parent].push_back(child); + dependency_graph[child].push_back(parent); + dependency_edge_count++; + }; + auto dependencies = connection.Query("SELECT parent_view, child_view FROM " + string(openivm::MV_DEPS_TABLE)); + if (!dependencies->HasError()) { + for (idx_t row = 0; row < dependencies->RowCount(); row++) { + add_dependency(dependencies->GetValue(0, row).ToString(), dependencies->GetValue(1, row).ToString()); + } + } + unordered_set registered_views; + auto views = connection.Query("SELECT view_name FROM " + string(openivm::VIEWS_TABLE)); + if (!views->HasError()) { + for (idx_t row = 0; row < views->RowCount(); row++) { + registered_views.insert(views->GetValue(0, row).ToString()); + } + } + auto delta_dependencies = + connection.Query("SELECT view_name, table_name FROM " + string(openivm::DELTA_TABLES_TABLE)); + if (!delta_dependencies->HasError()) { + static const string delta_prefix(openivm::DELTA_PREFIX); + static const string data_prefix(openivm::DATA_TABLE_PREFIX); + for (idx_t row = 0; row < delta_dependencies->RowCount(); row++) { + auto child = delta_dependencies->GetValue(0, row).ToString(); + auto table = delta_dependencies->GetValue(1, row).ToString(); + string parent; + if (StringUtil::StartsWith(table, delta_prefix)) { + parent = table.substr(delta_prefix.size()); + } else if (StringUtil::StartsWith(table, data_prefix)) { + parent = table.substr(data_prefix.size()); + } + if (!parent.empty() && registered_views.count(parent)) { + add_dependency(parent, child); + } + } + } + vector pending_views(included_views.begin(), included_views.end()); + for (idx_t pending_index = 0; pending_index < pending_views.size(); pending_index++) { + auto neighbors = dependency_graph.find(pending_views[pending_index]); + if (neighbors == dependency_graph.end()) { + continue; + } + for (auto &neighbor : neighbors->second) { + if (included_views.insert(neighbor).second) { + pending_views.push_back(neighbor); + } + } + } + OPENIVM_DEBUG_PRINT("[TRANSACTIONAL METADATA] Seed views=%zu, dependency edges=%zu, closure views=%zu\n", + registered_views.size(), dependency_edge_count, included_views.size()); + string view_filter; + vector sorted_included_views(included_views.begin(), included_views.end()); + std::sort(sorted_included_views.begin(), sorted_included_views.end()); + for (auto &view_name : sorted_included_views) { + if (!view_filter.empty()) { + view_filter += ", "; + } + view_filter += "'" + SqlUtils::EscapeValue(view_name) + "'"; + } + if (!view_filter.empty()) { + connection.Query("DELETE FROM " + string(openivm::VIEWS_TABLE) + " WHERE view_name NOT IN (" + view_filter + + ")"); + connection.Query("DELETE FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name NOT IN (" + + view_filter + ")"); + connection.Query("DELETE FROM " + string(openivm::MV_DEPS_TABLE) + " WHERE parent_view NOT IN (" + view_filter + + ") OR child_view NOT IN (" + view_filter + ")"); + } + for (auto &table : TransactionalMetadataTables()) { + if (std::find(dependency_tables.begin(), dependency_tables.end(), table) != dependency_tables.end()) { + continue; + } + string filter = view_filter.empty() ? string() : " WHERE view_name IN (" + view_filter + ")"; + seed_table(table, filter); + } + for (auto &statement : statements) { + if (IsMetadataSchemaStatement(statement)) { + continue; + } + bool already_replayed = false; + for (auto &table : dependency_tables) { + if (TargetsMetadataTable(statement, table)) { + already_replayed = true; + break; + } + } + if (already_replayed) { + continue; + } + replay_statement(statement); + } +} + +void TransactionalMVMetadataState::TransactionCommit(MetaTransaction &transaction, ClientContext &context) { + Clear(); +} + +void TransactionalMVMetadataState::TransactionRollback(MetaTransaction &transaction, ClientContext &context) { + Clear(); +} + +void TransactionalMVMetadataState::Clear() { + statements.clear(); + view_names.clear(); +} + +string BuildCreateDeltaFromDataOperation(const string &delta_table, const string &data_table, bool replace) { + return string(OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX) + (replace ? "replace\t" : "create\t") + delta_table + + "\t" + data_table; +} + +string BuildDropViewStatement(const DropInfo &drop_info) { + return "SELECT * FROM openivm_execute_drop_view('" + SqlUtils::EscapeValue(drop_info.catalog) + "', '" + + SqlUtils::EscapeValue(drop_info.schema) + "', '" + SqlUtils::EscapeValue(drop_info.name) + "', " + + (drop_info.cascade ? "true" : "false") + ", " + + (drop_info.if_not_found == OnEntryNotFound::RETURN_NULL ? "true" : "false") + ")"; +} + +string BuildDropTableStatement(const DropInfo &drop_info) { + return "SELECT * FROM openivm_execute_drop_table('" + SqlUtils::EscapeValue(drop_info.catalog) + "', '" + + SqlUtils::EscapeValue(drop_info.schema) + "', '" + SqlUtils::EscapeValue(drop_info.name) + "', " + + (drop_info.cascade ? "true" : "false") + ", " + + (drop_info.if_not_found == OnEntryNotFound::RETURN_NULL ? "true" : "false") + ")"; +} + +struct DropViewBindData : public TableFunctionData { + DropInfo info; +}; + +struct DropViewGlobalState : public GlobalTableFunctionState { + bool finished = false; +}; + +static unique_ptr BindDropEntry(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names, + CatalogType type) { + if (input.inputs.size() != 5) { + throw InternalException("OpenIVM DROP executor expected five arguments"); + } + auto result = make_uniq(); + result->info.type = type; + result->info.catalog = StringValue::Get(input.inputs[0]); + result->info.schema = StringValue::Get(input.inputs[1]); + result->info.name = StringValue::Get(input.inputs[2]); + result->info.cascade = BooleanValue::Get(input.inputs[3]); + result->info.if_not_found = + BooleanValue::Get(input.inputs[4]) ? OnEntryNotFound::RETURN_NULL : OnEntryNotFound::THROW_EXCEPTION; + if (!input.binder) { + throw InternalException("OpenIVM DROP executor requires a binder"); + } + auto &catalog = Catalog::GetCatalog(context, result->info.catalog); + input.binder->GetStatementProperties().RegisterDBModify(catalog, context, + DatabaseModificationType::DROP_CATALOG_ENTRY); + return_types.push_back(LogicalType::BOOLEAN); + names.emplace_back("Success"); + return std::move(result); +} + +unique_ptr BindDropView(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + return BindDropEntry(context, input, return_types, names, CatalogType::VIEW_ENTRY); +} + +unique_ptr BindDropTable(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names) { + return BindDropEntry(context, input, return_types, names, CatalogType::TABLE_ENTRY); +} + +unique_ptr InitDropView(ClientContext &context, TableFunctionInitInput &input) { + return make_uniq(); +} + +void ExecuteDropView(ClientContext &context, TableFunctionInput &input, DataChunk &output) { + auto &state = input.global_state->Cast(); + if (state.finished) { + return; + } + auto &bind_data = input.bind_data->Cast(); + auto &catalog = Catalog::GetCatalog(context, bind_data.info.catalog); + DropInfo info(bind_data.info); + catalog.DropEntry(context, info); + output.SetValue(0, 0, Value::BOOLEAN(true)); + output.SetCardinality(1); + state.finished = true; +} + +string RenderTransactionalDDL(ClientContext &context, const vector ¶meters) { + struct ProfileRow { + string view_name; + string step_name; + int64_t duration_ms; + string detail; + }; + struct PendingProfileMarker { + string view_name; + string step_name; + string detail; + idx_t statement_start; + }; + + string sql; + idx_t logical_statement_count = 0; + auto append_statement = [&](const string &statement, bool logical_statement = true) { + if (statement.empty()) { + return; + } + sql += statement; + if (statement.back() != ';') { + sql += ";"; + } + sql += "\n"; + if (logical_statement) { + logical_statement_count++; + } + }; + Value profile_value; + bool profile_enabled = context.TryGetCurrentSetting("openivm_profile_refresh", profile_value) && + !profile_value.IsNull() && BooleanValue::Get(profile_value); + vector profile_rows; + unique_ptr pending_profile; + auto finish_profile_marker = [&]() { + if (!pending_profile) { + return; + } + auto statement_count = logical_statement_count - pending_profile->statement_start; + auto detail = pending_profile->detail; + if (!detail.empty()) { + detail += "; "; + } + detail += + "statements=" + to_string(statement_count) + "; transactional_program=true; duration_not_measured=true"; + profile_rows.push_back({pending_profile->view_name, pending_profile->step_name, 0, std::move(detail)}); + pending_profile.reset(); + }; + for (auto ¶meter : parameters) { + auto statement = parameter.GetValue(); + if (statement.empty() || StringUtil::StartsWith(statement, OPENIVM_DDL_CLEANUP_PREFIX)) { + continue; + } + if (StringUtil::StartsWith(statement, OPENIVM_DDL_PROFILE_RECORD_PREFIX)) { + if (profile_enabled) { + string view_name; + string step_name; + string detail; + int64_t duration_ms = 0; + ParseCreateMVProfileRecord(statement.substr(strlen(OPENIVM_DDL_PROFILE_RECORD_PREFIX)), view_name, + step_name, duration_ms, detail); + profile_rows.push_back({std::move(view_name), std::move(step_name), duration_ms, std::move(detail)}); + } + continue; + } + if (StringUtil::StartsWith(statement, OPENIVM_DDL_PROFILE_PREFIX)) { + if (profile_enabled) { + finish_profile_marker(); + string view_name; + string step_name; + string detail; + ParseCreateMVProfileMarker(statement.substr(strlen(OPENIVM_DDL_PROFILE_PREFIX)), view_name, step_name, + detail); + pending_profile = make_uniq(PendingProfileMarker { + std::move(view_name), std::move(step_name), std::move(detail), logical_statement_count}); + } + continue; + } + if (StringUtil::StartsWith(statement, OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX)) { + bool replace = false; + string delta_table; + string data_table; + ParseCreateDeltaFromDataPayload(statement.substr(strlen(OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX)), + replace, delta_table, data_table); + if (delta_table.empty() || data_table.empty()) { + throw InternalException("Malformed OpenIVM delta-schema operation"); + } + append_statement(string(replace ? "CREATE OR REPLACE TABLE " : "CREATE TABLE ") + delta_table + + " AS SELECT *, 1::INTEGER AS " + string(openivm::MULTIPLICITY_COL) + + ", now()::TIMESTAMP AS " + string(openivm::TIMESTAMP_COL) + " FROM " + data_table + + " LIMIT 0"); + append_statement( + "ALTER TABLE " + delta_table + " ALTER " + string(openivm::MULTIPLICITY_COL) + " SET DEFAULT 1", false); + append_statement("ALTER TABLE " + delta_table + " ALTER " + string(openivm::TIMESTAMP_COL) + + " SET DEFAULT now()", + false); + continue; + } + + try { + Parser parser; + parser.ParseQuery(statement); + if (parser.statements.size() == 1 && parser.statements[0]->type == StatementType::DROP_STATEMENT) { + auto &drop = parser.statements[0]->Cast(); + if (drop.info->type == CatalogType::VIEW_ENTRY) { + append_statement(BuildDropViewStatement(*drop.info)); + continue; + } + } + } catch (std::exception &) { + // The normal DuckDB parser will produce the authoritative error when the + // transactional program is executed. + } + append_statement(statement); + } + if (profile_enabled) { + finish_profile_marker(); + if (!profile_rows.empty()) { + auto view_name = profile_rows.front().view_name; + profile_rows.push_back( + {view_name, "create_mv_total", 0, "transactional_program=true; duration_not_measured=true"}); + auto now = std::chrono::steady_clock::now().time_since_epoch(); + auto refresh_id = view_name + "_create_tx_" + + to_string(std::chrono::duration_cast(now).count()); + for (idx_t step_order = 0; step_order < profile_rows.size(); step_order++) { + auto &row = profile_rows[step_order]; + append_statement("INSERT OR REPLACE INTO " + string(openivm::PROFILE_TABLE) + + " (refresh_id, view_name, step_order, step_name, duration_ms, detail) VALUES ('" + + SqlUtils::EscapeValue(refresh_id) + "', '" + SqlUtils::EscapeValue(row.view_name) + + "', " + to_string(step_order) + ", '" + SqlUtils::EscapeValue(row.step_name) + + "', " + to_string(row.duration_ms) + ", '" + SqlUtils::EscapeValue(row.detail) + + "')", + false); + } + } + } + append_statement("SELECT true AS \"MATERIALIZED VIEW CREATION\""); + return sql; +} + +void ExecuteStagedDDL(ClientContext &context, const vector ¶meters) { + vector ddl; + ddl.reserve(parameters.size()); + for (auto ¶meter : parameters) { + ddl.push_back(parameter.GetValue()); + } + ExecuteDDL(context, ddl); +} + +void ConfigureDDLExecutorResult(ParserExtensionPlanResult &result, DDLExecutionMode mode) { result.function = TableFunction("openivm_ddl_executor", {}, DDLExecutorExecuteFunction, DDLExecutorBindFunction, DDLExecutorInitFunction); + result.function.name = + mode == DDLExecutionMode::CALLER_TRANSACTION ? OPENIVM_TRANSACTIONAL_DDL_FUNCTION : OPENIVM_STAGED_DDL_FUNCTION; result.requires_valid_transaction = true; result.return_type = StatementReturnType::QUERY_RESULT; } diff --git a/src/core/parser_parse.cpp b/src/core/parser_parse.cpp index ebb84865..0efec8fc 100644 --- a/src/core/parser_parse.cpp +++ b/src/core/parser_parse.cpp @@ -3,12 +3,56 @@ #include "core/openivm_constants.hpp" #include "core/openivm_debug.hpp" #include "core/sql_utils.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" #include "duckdb/parser/parser.hpp" +#include "duckdb/parser/qualified_name.hpp" +#include "duckdb/parser/statement/drop_statement.hpp" +#include "duckdb/parser/statement/pragma_statement.hpp" #include namespace duckdb { +static unique_ptr BuildInternalPragma(const string &name, const string &query) { + auto statement = make_uniq(); + statement->info->name = name; + statement->info->parameters.push_back(make_uniq(Value(query))); + return std::move(statement); +} + +ParserOverrideResult MaterializedViewParserExtension::OverrideFunction(ParserExtensionInfo *info, const string &query, + ParserOptions &options) { + try { + auto extension_result = ParseFunction(info, query); + if (extension_result.type == ParserExtensionResultType::PARSE_SUCCESSFUL) { + vector> statements; + statements.push_back(BuildInternalPragma("openivm_materialized_view_lifecycle", query)); + return ParserOverrideResult(std::move(statements)); + } + + // DuckDB parses these statements natively, so the regular parser-extension + // fallback never sees them. Route tracked-view drops and cascading source + // drops through OpenIVM so their cleanup uses the caller transaction. + ParserOptions native_options = options; + native_options.extensions = nullptr; + Parser parser(native_options); + parser.ParseQuery(query); + if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::DROP_STATEMENT) { + return ParserOverrideResult(); + } + auto &drop = parser.statements[0]->Cast(); + if (drop.info->type != CatalogType::VIEW_ENTRY && + (drop.info->type != CatalogType::TABLE_ENTRY || !drop.info->cascade)) { + return ParserOverrideResult(); + } + vector> statements; + statements.push_back(BuildInternalPragma("openivm_materialized_view_drop", query)); + return ParserOverrideResult(std::move(statements)); + } catch (std::exception &ex) { + return ParserOverrideResult(ex); + } +} + ParserExtensionParseResult MaterializedViewParserExtension::ParseFunction(ParserExtensionInfo *info, const string &query) { auto query_lower = SqlUtils::SQLToLowercase(StringUtil::Replace(query, ";", "")); @@ -21,35 +65,38 @@ ParserExtensionParseResult MaterializedViewParserExtension::ParseFunction(Parser // Handle ALTER MATERIALIZED VIEW SET REFRESH EVERY '' | SET REFRESH MANUAL if (StringUtil::Contains(query_lower, "alter materialized view")) { - std::regex alter_re("alter\\s+materialized\\s+view\\s+(\"(?:[^\"]+)\"|[a-zA-Z0-9_.]+)\\s+set\\s+refresh\\s+(" - "every\\s+'([^']+)'|manual)", + const string identifier = "(?:\"(?:[^\"]|\"\")*\"|[a-zA-Z_][a-zA-Z0-9_$]*)"; + const string qualified_identifier = identifier + "(?:\\s*\\.\\s*" + identifier + "){0,2}"; + std::regex alter_re("^alter\\s+materialized\\s+view\\s+(" + qualified_identifier + + ")\\s+set\\s+refresh\\s+(every\\s+'([^']+)'|manual)$", std::regex::icase); std::smatch match; - if (!std::regex_search(query_lower, match, alter_re)) { + if (!std::regex_match(query_lower, match, alter_re)) { throw ParserException("Invalid ALTER MATERIALIZED VIEW syntax. " "Expected: ALTER MATERIALIZED VIEW SET REFRESH EVERY '' " "or ALTER MATERIALIZED VIEW SET REFRESH MANUAL"); } string alter_view_name = match[1].str(); - if (alter_view_name.size() >= 2 && alter_view_name.front() == '"' && alter_view_name.back() == '"') { - alter_view_name = alter_view_name.substr(1, alter_view_name.size() - 2); + auto name_components = QualifiedName::ParseComponents(alter_view_name); + if (name_components.empty()) { + throw ParserException("Invalid materialized-view target '%s'", alter_view_name); } string refresh_type = StringUtil::Lower(match[2].str()); - string update_sql; + string alter_value; if (refresh_type == "manual") { - update_sql = "UPDATE " + string(openivm::VIEWS_TABLE) + " SET refresh_interval = NULL WHERE view_name = '" + - SqlUtils::EscapeSingleQuotes(alter_view_name) + "'"; + alter_value = "NULL"; } else { int64_t interval = SqlUtils::ParseRefreshInterval(match[3].str()); - update_sql = "UPDATE " + string(openivm::VIEWS_TABLE) + " SET refresh_interval = " + to_string(interval) + - " WHERE view_name = '" + SqlUtils::EscapeSingleQuotes(alter_view_name) + "'"; + alter_value = to_string(interval); } // Pass the UPDATE SQL through MaterializedViewParseData; PlanFunction will execute it Parser alter_parser; alter_parser.ParseQuery("SELECT 1"); auto parse_data = make_uniq_base(std::move(alter_parser.statements[0])); - dynamic_cast(*parse_data).alter_sql = update_sql; + auto &materialized_view_data = dynamic_cast(*parse_data); + materialized_view_data.alter_sql = alter_value; + materialized_view_data.target_name = alter_view_name; return ParserExtensionParseResult(std::move(parse_data)); } @@ -84,7 +131,9 @@ ParserExtensionParseResult MaterializedViewParserExtension::ParseFunction(Parser auto parse_data = make_uniq_base(std::move(p.statements[0]), refresh_interval); - dynamic_cast(*parse_data).is_replace = is_replace; + auto &materialized_view_data = dynamic_cast(*parse_data); + materialized_view_data.is_replace = is_replace; + materialized_view_data.target_name = SqlUtils::ExtractTableName(query_lower); return ParserExtensionParseResult(std::move(parse_data)); } diff --git a/src/core/parser_plan_helpers.cpp b/src/core/parser_plan_helpers.cpp index c903b204..8c85197b 100644 --- a/src/core/parser_plan_helpers.cpp +++ b/src/core/parser_plan_helpers.cpp @@ -9,11 +9,13 @@ #include "duckdb/optimizer/cte_inlining.hpp" #include "duckdb/optimizer/optimizer.hpp" #include "duckdb/parser/keyword_helper.hpp" +#include "duckdb/parser/constraints/not_null_constraint.hpp" #include "duckdb/planner/expression/bound_aggregate_expression.hpp" #include "duckdb/planner/expression/bound_cast_expression.hpp" #include "duckdb/planner/expression/bound_window_expression.hpp" #include "duckdb/planner/expression_iterator.hpp" #include "duckdb/planner/operator/logical_cteref.hpp" +#include "duckdb/planner/operator/logical_distinct.hpp" #include "duckdb/planner/operator/logical_join.hpp" #include "duckdb/planner/operator/logical_materialized_cte.hpp" #include "duckdb/planner/operator/logical_set_operation.hpp" @@ -230,6 +232,35 @@ bool OuterJoinAggregateNeedsRecompute(const CreateMVPlanFacts &facts, idx_t grou return false; } +static bool ContainsTableFunction(LogicalOperator &op) { + if (op.type == LogicalOperatorType::LOGICAL_GET && !op.Cast().GetTable().get()) { + return true; + } + for (auto &child : op.children) { + if (ContainsTableFunction(*child)) { + return true; + } + } + return false; +} + +bool OuterJoinPreservedSideHasTableFunction(const CreateMVPlanFacts &facts) { + for (auto *join : facts.comparison_joins) { + idx_t preserved_child; + if (join->join_type == JoinType::LEFT) { + preserved_child = 0; + } else if (join->join_type == JoinType::RIGHT) { + preserved_child = 1; + } else { + continue; + } + if (join->children.size() > preserved_child && ContainsTableFunction(*join->children[preserved_child])) { + return true; + } + } + return false; +} + static void AddGetFacts(LogicalGet &get, const string ¤t_catalog, CreateMVPlanFacts &facts, unordered_map &next_occurrence) { facts.gets_by_index[get.table_index] = &get; @@ -253,9 +284,6 @@ static void AddGetFacts(LogicalGet &get, const string ¤t_catalog, CreateMV } auto &info = get.function.function_info->Cast(); string lc = StringUtil::Lower(info.table_name); - if (facts.ducklake_table_info.find(lc) != facts.ducklake_table_info.end()) { - return; - } string cat = info.table.ParentCatalog().GetName(); if (cat.empty()) { if (current_catalog.empty()) { @@ -263,6 +291,18 @@ static void AddGetFacts(LogicalGet &get, const string ¤t_catalog, CreateMV } cat = current_catalog; } + auto existing = facts.ducklake_table_info.find(lc); + if (existing != facts.ducklake_table_info.end()) { + if (!StringUtil::CIEquals(existing->second.catalog_name, cat) || + !StringUtil::CIEquals(existing->second.schema_name, info.table.schema.name) || + existing->second.table_id != static_cast(info.table_id.index)) { + throw NotImplementedException( + "DuckLake materialized views cannot reference different source tables with the same unqualified " + "name '%s'; rename one source before creating the materialized view", + info.table_name); + } + return; + } DuckLakeSourceTableInfo source_info; source_info.table_name = info.table_name; source_info.catalog_name = cat; @@ -431,6 +471,49 @@ static bool ExpressionReferencesMinMaxAggregate(Expression &expr, return found; } +static bool ExpressionReferencesUserSumAggregate(Expression &expr, const CreateMVPlanFacts &facts, + const unordered_map &aggregates, + idx_t depth = 0) { + if (depth > 16) { + return false; + } + if (expr.expression_class == ExpressionClass::BOUND_COLUMN_REF) { + auto &column_ref = expr.Cast(); + auto aggregate_it = aggregates.find(column_ref.binding.table_index); + if (aggregate_it != aggregates.end()) { + auto &aggregate = *aggregate_it->second; + if (column_ref.binding.column_index >= aggregate.expressions.size()) { + return false; + } + auto &aggregate_expr = aggregate.expressions[column_ref.binding.column_index]; + if (aggregate_expr->expression_class != ExpressionClass::BOUND_AGGREGATE) { + return false; + } + auto &bound_aggregate = aggregate_expr->Cast(); + return bound_aggregate.function.name == "sum" && + !IncrementalTableNames::IsInternalColumn(bound_aggregate.alias); + } + auto projection_it = facts.projections_by_index.find(column_ref.binding.table_index); + if (projection_it == facts.projections_by_index.end()) { + return false; + } + auto &projection = *projection_it->second; + if (column_ref.binding.column_index >= projection.expressions.size()) { + return false; + } + return ExpressionReferencesUserSumAggregate(*projection.expressions[column_ref.binding.column_index], facts, + aggregates, depth + 1); + } + + bool found = false; + ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { + if (!found && ExpressionReferencesUserSumAggregate(child, facts, aggregates, depth + 1)) { + found = true; + } + }); + return found; +} + static void FinalizeCreateMVPlanFacts(CreateMVPlanFacts &facts) { unordered_map aggregates; for (auto *aggregate : facts.aggregates) { @@ -441,6 +524,14 @@ static void FinalizeCreateMVPlanFacts(CreateMVPlanFacts &facts) { } for (auto *projection : facts.projections) { for (auto &expr : projection->expressions) { + Expression *unwrapped = expr.get(); + while (unwrapped->expression_class == ExpressionClass::BOUND_CAST) { + unwrapped = unwrapped->Cast().child.get(); + } + if (unwrapped->expression_class != ExpressionClass::BOUND_COLUMN_REF && + ExpressionReferencesUserSumAggregate(*unwrapped, facts, aggregates)) { + facts.has_computed_sum_aggregate_projection = true; + } if (expr->expression_class == ExpressionClass::BOUND_COLUMN_REF && IsHiddenHavingColumn(expr->alias)) { auto &column_ref = expr->Cast(); if (IsMinMaxAggregateColumn(column_ref, aggregates)) { @@ -513,11 +604,6 @@ static bool ResolvesToGroupBinding(idx_t table_index, idx_t column_index, idx_t return false; } -bool RelationExists(Connection &con, const string &qualified_name) { - auto result = con.Query("SELECT * FROM " + qualified_name + " LIMIT 0"); - return !result->HasError(); -} - static string ProjectionOutputName(const unique_ptr &expr, idx_t expr_index, const vector &output_names, const BoundColumnRefExpression &bcr) { if (!expr->alias.empty()) { @@ -530,14 +616,26 @@ static string ProjectionOutputName(const unique_ptr &expr, idx_t exp return bcr.GetName(); } -static BoundColumnRefExpression *GetColumnRefThroughCasts(Expression *expr) { +static BoundColumnRefExpression *GetColumnRefThroughCasts(Expression *expr, string *cast_type = nullptr) { + if (cast_type) { + cast_type->clear(); + } + vector cast_specs; while (expr && expr->expression_class == ExpressionClass::BOUND_CAST) { auto &cast = expr->Cast(); + if (cast_type) { + cast_specs.push_back(SqlUtils::BuildCastSpec(expr->return_type.ToString(), cast.try_cast)); + } expr = cast.child.get(); } if (!expr || expr->type != ExpressionType::BOUND_COLUMN_REF) { return nullptr; } + if (cast_type) { + for (auto it = cast_specs.rbegin(); it != cast_specs.rend(); it++) { + *cast_type = SqlUtils::ComposeCastSpecs(*it, *cast_type); + } + } return &expr->Cast(); } @@ -742,6 +840,25 @@ static bool GetLogicalGetColumnName(LogicalGet &get, idx_t column_index, string return false; } +static bool GetLogicalGetColumnType(LogicalGet &get, idx_t column_index, LogicalType &type) { + if (column_index < get.returned_types.size()) { + type = get.returned_types[column_index]; + return true; + } + if (get.GetTable().get()) { + auto &ids = get.GetColumnIds(); + if (column_index < ids.size()) { + auto base_idx = ids[column_index].GetPrimaryIndex(); + auto &cols = get.GetTable().get()->GetColumns(); + if (base_idx < cols.LogicalColumnCount()) { + type = cols.GetColumn(LogicalIndex(base_idx)).Type(); + return true; + } + } + } + return false; +} + static bool ResolveBindingToGetColumn(ColumnBinding binding, const CreateMVPlanFacts &facts, LogicalGet *&get, string &column) { idx_t table_index = binding.table_index; @@ -794,6 +911,53 @@ static bool ResolveBindingToOccurrenceRef(ColumnBinding binding, const CreateMVP return true; } +static bool ResolveBindingToOccurrenceRefWithCast(ColumnBinding binding, const CreateMVPlanFacts &facts, + OccurrenceColumnRef &out, string &cast_type, + const LogicalType &expression_type) { + idx_t table_index = binding.table_index; + idx_t column_index = binding.column_index; + for (int depth = 0; depth < 16; depth++) { + auto get_it = facts.gets_by_index.find(table_index); + if (get_it != facts.gets_by_index.end()) { + auto *get = get_it->second; + if (!GetLogicalGetColumnName(*get, column_index, out.column)) { + return false; + } + auto occurrence_it = facts.occurrence_by_index.find(get->table_index); + if (occurrence_it == facts.occurrence_by_index.end()) { + return false; + } + out.table = occurrence_it->second.table; + out.occurrence = occurrence_it->second.occurrence; + LogicalType base_type; + if (cast_type.empty() && GetLogicalGetColumnType(*get, column_index, base_type) && + !(base_type == expression_type)) { + cast_type = expression_type.ToString(); + } + return true; + } + auto proj_it = facts.projections_by_index.find(table_index); + if (proj_it == facts.projections_by_index.end()) { + return false; + } + auto *proj = proj_it->second; + if (column_index >= proj->expressions.size()) { + return false; + } + string projection_cast; + auto *next = GetColumnRefThroughCasts(proj->expressions[column_index].get(), &projection_cast); + if (!next) { + return false; + } + if (!projection_cast.empty()) { + cast_type = SqlUtils::ComposeCastSpecs(cast_type, projection_cast); + } + table_index = next->binding.table_index; + column_index = next->binding.column_index; + } + return false; +} + static void AddJoinEdgesFromFacts(CreateMVPlanFacts &facts) { for (auto *join : facts.comparison_joins) { if (join->join_type != JoinType::INNER) { @@ -823,10 +987,122 @@ static void AddJoinEdgesFromFacts(CreateMVPlanFacts &facts) { } } +bool ProducesAtMostOneRow(LogicalOperator &node) { + LogicalOperator *current = &node; + while ((current->type == LogicalOperatorType::LOGICAL_PROJECTION || + current->type == LogicalOperatorType::LOGICAL_FILTER) && + !current->children.empty()) { + current = current->children[0].get(); + } + if (current->type != LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + return false; + } + auto &aggregate = current->Cast(); + return aggregate.groups.empty() && aggregate.grouping_sets.size() <= 1; +} + +static bool ResolveBindingToGroupKey(const ColumnBinding &binding, + const unordered_map &projections, + const LogicalAggregate &aggregate, idx_t &group_index, idx_t depth = 0) { + if (depth > 16) { + return false; + } + if (binding.table_index == aggregate.group_index) { + if (binding.column_index >= aggregate.groups.size()) { + return false; + } + group_index = binding.column_index; + return true; + } + auto projection = projections.find(binding.table_index); + if (projection == projections.end() || binding.column_index >= projection->second->expressions.size()) { + return false; + } + auto &expression = projection->second->expressions[binding.column_index]; + if (expression->expression_class != ExpressionClass::BOUND_COLUMN_REF) { + return false; + } + auto &column = expression->Cast(); + return ResolveBindingToGroupKey(column.binding, projections, aggregate, group_index, depth + 1); +} + +bool IsRedundantDistinctOverGroupKeys(LogicalOperator &node) { + if (node.type != LogicalOperatorType::LOGICAL_DISTINCT || node.children.empty()) { + return false; + } + auto &distinct = node.Cast(); + if (distinct.distinct_type != DistinctType::DISTINCT || distinct.order_by) { + return false; + } + + auto *output = node.children[0].get(); + auto *current = output; + unordered_map projections; + while (current && current->children.size() == 1 && + (current->type == LogicalOperatorType::LOGICAL_PROJECTION || + current->type == LogicalOperatorType::LOGICAL_FILTER)) { + if (current->type == LogicalOperatorType::LOGICAL_PROJECTION) { + auto &projection = current->Cast(); + projections[projection.table_index] = &projection; + } + current = current->children[0].get(); + } + if (!current || current->type != LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + return false; + } + auto &aggregate = current->Cast(); + if (aggregate.groups.empty() || aggregate.grouping_sets.size() > 1) { + return false; + } + if (!aggregate.grouping_sets.empty() && aggregate.grouping_sets[0].size() != aggregate.groups.size()) { + return false; + } + + auto output_bindings = output->GetColumnBindings(); + if (output_bindings.size() != aggregate.groups.size()) { + return false; + } + vector seen_group(aggregate.groups.size(), false); + for (auto &binding : output_bindings) { + idx_t group_index; + if (!ResolveBindingToGroupKey(binding, projections, aggregate, group_index) || seen_group[group_index]) { + return false; + } + seen_group[group_index] = true; + } + return true; +} + +static bool HasDistinctOperator(const LogicalOperator &node) { + if (node.type == LogicalOperatorType::LOGICAL_DISTINCT) { + return true; + } + for (auto &child : node.children) { + if (HasDistinctOperator(*child)) { + return true; + } + } + return false; +} + CreateMVPlanFacts BuildCreateMVPlanFacts(LogicalOperator *plan, const string ¤t_catalog) { CreateMVPlanFacts facts; facts.root = plan; facts.analysis = AnalyzePlan(plan); + LogicalOperator *top = plan; + while (top && top->children.size() == 1 && + (top->type == LogicalOperatorType::LOGICAL_CREATE_TABLE || + top->type == LogicalOperatorType::LOGICAL_PROJECTION || top->type == LogicalOperatorType::LOGICAL_FILTER || + top->type == LogicalOperatorType::LOGICAL_ORDER_BY || top->type == LogicalOperatorType::LOGICAL_LIMIT || + top->type == LogicalOperatorType::LOGICAL_TOP_N)) { + top = top->children[0].get(); + } + facts.has_top_level_redundant_distinct = + top && top->type == LogicalOperatorType::LOGICAL_DISTINCT && !top->children.empty() && + (ProducesAtMostOneRow(*top->children[0]) || IsRedundantDistinctOverGroupKeys(*top)); + if (facts.has_top_level_redundant_distinct) { + facts.has_descendant_distinct = HasDistinctOperator(*top->children[0]); + } unordered_map next_occurrence; CollectCreateMVPlanFacts(plan, current_catalog, facts, next_occurrence, false, false); FinalizeCreateMVPlanFacts(facts); @@ -981,13 +1257,12 @@ bool BuildLeftJoinKeySource(const CreateMVPlanFacts &facts, RefreshMetadata::Lef if (!join || join->join_type != JoinType::LEFT || join->conditions.empty()) { continue; } - auto *preserved_key = join->conditions[0].left.get(); - if (!preserved_key || preserved_key->expression_class != ExpressionClass::BOUND_COLUMN_REF) { + auto *preserved_key = GetColumnRefThroughCasts(join->conditions[0].left.get()); + if (!preserved_key) { return false; } - auto &column_ref = preserved_key->Cast(); OccurrenceColumnRef source; - if (!ResolveBindingToOccurrenceRef(column_ref.binding, facts, source)) { + if (!ResolveBindingToOccurrenceRef(preserved_key->binding, facts, source)) { return false; } out.table = source.table; @@ -1085,15 +1360,31 @@ struct WindowLineageOp { string source_table; idx_t source_occurrence = 0; string source_col; + string source_cast; string lookup_table; idx_t lookup_occurrence = 0; string lookup_col; + string lookup_cast; string lookup_output_col; + string lookup_output_cast; }; struct WindowLookupEdge { OccurrenceColumnRef lookup_join; + string lookup_cast; OccurrenceColumnRef changed_join; + string changed_cast; +}; + +struct WindowEquivalenceEdge { + OccurrenceColumnRef first; + string first_cast; + OccurrenceColumnRef second; + string second_cast; + + bool HasCast() const { + return !first_cast.empty() || !second_cast.empty(); + } }; static void CollectLeafColumnBindings(Expression *expr, vector &bindings) { @@ -1155,7 +1446,7 @@ static bool ResolveBindingToOccurrenceRefs(ColumnBinding binding, const CreateMV } static void CollectInnerJoinEdgesOccurrence(LogicalOperator *op, const CreateMVPlanFacts &facts, - vector> &edges) { + vector &edges) { if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) { auto &join = op->Cast(); @@ -1164,15 +1455,18 @@ static void CollectInnerJoinEdgesOccurrence(LogicalOperator *op, const CreateMVP if (cond.comparison != ExpressionType::COMPARE_EQUAL) { continue; } - auto *left = GetColumnRefThroughCasts(cond.left.get()); - auto *right = GetColumnRefThroughCasts(cond.right.get()); + string left_cast; + string right_cast; + auto *left = GetColumnRefThroughCasts(cond.left.get(), &left_cast); + auto *right = GetColumnRefThroughCasts(cond.right.get(), &right_cast); if (!left || !right) { continue; } OccurrenceColumnRef lref, rref; - if (ResolveBindingToOccurrenceRef(left->binding, facts, lref) && - ResolveBindingToOccurrenceRef(right->binding, facts, rref)) { - edges.emplace_back(std::move(lref), std::move(rref)); + if (ResolveBindingToOccurrenceRefWithCast(left->binding, facts, lref, left_cast, left->return_type) && + ResolveBindingToOccurrenceRefWithCast(right->binding, facts, rref, right_cast, + right->return_type)) { + edges.push_back({std::move(lref), std::move(left_cast), std::move(rref), std::move(right_cast)}); } } } @@ -1187,35 +1481,39 @@ static void CollectWindowLookupEdges(LogicalOperator *op, const CreateMVPlanFact if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op->type == LogicalOperatorType::LOGICAL_ASOF_JOIN) { auto &join = op->Cast(); - auto add_lookup_edge = [&](OccurrenceColumnRef lookup_ref, OccurrenceColumnRef changed_ref) { - edges.push_back({std::move(lookup_ref), std::move(changed_ref)}); + auto add_lookup_edge = [&](OccurrenceColumnRef lookup_ref, string lookup_cast, OccurrenceColumnRef changed_ref, + string changed_cast) { + edges.push_back( + {std::move(lookup_ref), std::move(lookup_cast), std::move(changed_ref), std::move(changed_cast)}); }; for (auto &cond : join.conditions) { if (cond.comparison != ExpressionType::COMPARE_EQUAL) { continue; } - auto *left = GetColumnRefThroughCasts(cond.left.get()); - auto *right = GetColumnRefThroughCasts(cond.right.get()); + string left_cast; + string right_cast; + auto *left = GetColumnRefThroughCasts(cond.left.get(), &left_cast); + auto *right = GetColumnRefThroughCasts(cond.right.get(), &right_cast); if (!left || !right) { continue; } OccurrenceColumnRef lref, rref; - if (!ResolveBindingToOccurrenceRef(left->binding, facts, lref) || - !ResolveBindingToOccurrenceRef(right->binding, facts, rref)) { + if (!ResolveBindingToOccurrenceRefWithCast(left->binding, facts, lref, left_cast, left->return_type) || + !ResolveBindingToOccurrenceRefWithCast(right->binding, facts, rref, right_cast, right->return_type)) { continue; } switch (join.join_type) { case JoinType::INNER: - add_lookup_edge(lref, rref); - add_lookup_edge(rref, lref); + add_lookup_edge(lref, left_cast, rref, right_cast); + add_lookup_edge(rref, right_cast, lref, left_cast); break; case JoinType::LEFT: - add_lookup_edge(lref, rref); - add_lookup_edge(rref, lref); + add_lookup_edge(lref, left_cast, rref, right_cast); + add_lookup_edge(rref, right_cast, lref, left_cast); break; case JoinType::RIGHT: - add_lookup_edge(rref, lref); - add_lookup_edge(lref, rref); + add_lookup_edge(rref, right_cast, lref, left_cast); + add_lookup_edge(lref, left_cast, rref, right_cast); break; default: break; @@ -1237,7 +1535,8 @@ static void CollectWindowPartitionRefs(LogicalOperator *op, const CreateMVPlanFa } auto &win_expr = expr->Cast(); for (auto &part : win_expr.partitions) { - auto *bcr = GetColumnRefThroughCasts(part.get()); + string partition_cast; + auto *bcr = GetColumnRefThroughCasts(part.get(), &partition_cast); if (!bcr) { continue; } @@ -1259,6 +1558,7 @@ static void CollectWindowPartitionRefs(LogicalOperator *op, const CreateMVPlanFa op.source_table = ref.table; op.source_occurrence = ref.occurrence; op.source_col = ref.column; + op.source_cast = partition_cast; direct_ops.push_back(std::move(op)); } } @@ -1278,9 +1578,11 @@ static bool SameRef(const OccurrenceColumnRef &a, const OccurrenceColumnRef &b) static bool SameOp(const WindowLineageOp &a, const WindowLineageOp &b) { return a.kind == b.kind && StringUtil::CIEquals(a.output_col, b.output_col) && StringUtil::CIEquals(a.source_table, b.source_table) && a.source_occurrence == b.source_occurrence && - StringUtil::CIEquals(a.source_col, b.source_col) && StringUtil::CIEquals(a.lookup_table, b.lookup_table) && - a.lookup_occurrence == b.lookup_occurrence && StringUtil::CIEquals(a.lookup_col, b.lookup_col) && - StringUtil::CIEquals(a.lookup_output_col, b.lookup_output_col); + StringUtil::CIEquals(a.source_col, b.source_col) && StringUtil::CIEquals(a.source_cast, b.source_cast) && + StringUtil::CIEquals(a.lookup_table, b.lookup_table) && a.lookup_occurrence == b.lookup_occurrence && + StringUtil::CIEquals(a.lookup_col, b.lookup_col) && StringUtil::CIEquals(a.lookup_cast, b.lookup_cast) && + StringUtil::CIEquals(a.lookup_output_col, b.lookup_output_col) && + StringUtil::CIEquals(a.lookup_output_cast, b.lookup_output_cast); } static void AddUniqueLineageOp(vector &ops, WindowLineageOp op) { @@ -1310,9 +1612,12 @@ static RefreshMetadata::WindowPartitionLineageOp ToMetadataWindowLineageOp(const metadata_op.output_col = op.output_col; metadata_op.source = op.source_table; metadata_op.source_col = op.source_col; + metadata_op.source_cast = op.source_cast; metadata_op.lookup = op.lookup_table; metadata_op.lookup_col = op.lookup_col; + metadata_op.lookup_cast = op.lookup_cast; metadata_op.lookup_out = op.lookup_output_col; + metadata_op.lookup_out_cast = op.lookup_output_cast; return metadata_op; } @@ -1342,7 +1647,7 @@ bool BuildWindowPartitionLineageOps(const CreateMVPlanFacts &facts, const vector } } - vector> edges; + vector edges; CollectInnerJoinEdgesOccurrence(plan, facts, edges); vector lookup_edges; CollectWindowLookupEdges(plan, facts, lookup_edges); @@ -1357,6 +1662,9 @@ bool BuildWindowPartitionLineageOps(const CreateMVPlanFacts &facts, const vector changed = false; vector next_ops = partition_ops; for (auto &edge : edges) { + if (edge.HasCast()) { + continue; + } for (auto &direct : partition_ops) { OccurrenceColumnRef direct_ref; direct_ref.table = direct.source_table; @@ -1380,6 +1688,7 @@ bool BuildWindowPartitionLineageOps(const CreateMVPlanFacts &facts, const vector equivalent.source_table = other.table; equivalent.source_occurrence = other.occurrence; equivalent.source_col = other.column; + equivalent.source_cast = direct.source_cast; AddUniqueLineageOp(next_ops, std::move(equivalent)); changed = true; } @@ -1409,10 +1718,13 @@ bool BuildWindowPartitionLineageOps(const CreateMVPlanFacts &facts, const vector lookup.source_table = edge.changed_join.table; lookup.source_occurrence = edge.changed_join.occurrence; lookup.source_col = edge.changed_join.column; + lookup.source_cast = edge.changed_cast; lookup.lookup_table = direct.source_table; lookup.lookup_occurrence = direct.source_occurrence; lookup.lookup_col = edge.lookup_join.column; + lookup.lookup_cast = edge.lookup_cast; lookup.lookup_output_col = direct.source_col; + lookup.lookup_output_cast = direct.source_cast; AddUniqueLineageOp(ops, std::move(lookup)); } } @@ -1772,4 +2084,461 @@ void ForwardPacSettingsIfLoaded(ClientContext &context, Connection &con) { } } +static bool SubtreeContainsComparisonJoin(LogicalOperator *op) { + if (!op) { + return false; + } + if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || op->type == LogicalOperatorType::LOGICAL_ANY_JOIN || + op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN) { + return true; + } + for (auto &child : op->children) { + if (SubtreeContainsComparisonJoin(child.get())) { + return true; + } + } + return false; +} + +static void CollectSubtreeTableIndices(LogicalOperator *op, std::set &out) { + if (!op) { + return; + } + if (op->type == LogicalOperatorType::LOGICAL_GET) { + out.insert(op->Cast().table_index); + } + for (auto &child : op->children) { + CollectSubtreeTableIndices(child.get(), out); + } +} + +// Which base tables read as NULL in a null-padded row of join `oj`. +// +// It is not just oj's own inner side: in a chain c ⟕ o ⟕ l ⟕ s, when a line disappears the supplier +// joined to that line's key disappears too, and anything joined to the supplier after it, and so on. +// So start from oj's inner subtree and close transitively over any join whose condition touches an +// already-NULL table, adding that join's inner side. Aggregates over these tables contribute 0 to the +// null-padded row; aggregates over the preserved side still contribute. +static std::set ComputeNullTablesForLevel(const CreateMVPlanFacts &facts, LogicalComparisonJoin *oj) { + std::set null_tables; + if (!oj || oj->children.size() != 2) { + return null_tables; + } + CollectSubtreeTableIndices(oj->children[1].get(), null_tables); + bool changed = true; + while (changed) { + changed = false; + for (auto *other : facts.comparison_joins) { + if (other == oj || other->join_type != JoinType::LEFT || other->children.size() != 2) { + continue; + } + std::set other_inner; + CollectSubtreeTableIndices(other->children[1].get(), other_inner); + bool already_null = true; + for (auto idx : other_inner) { + if (!null_tables.count(idx)) { + already_null = false; + break; + } + } + if (already_null || other_inner.empty()) { + continue; + } + bool touches_null = false; + for (auto &cond : other->conditions) { + for (auto *side : {cond.left.get(), cond.right.get()}) { + auto *cref = GetColumnRefThroughCasts(side); + if (!cref) { + continue; + } + LogicalGet *g = nullptr; + string c; + if (ResolveBindingToGetColumn(cref->binding, facts, g, c) && g && + null_tables.count(g->table_index)) { + touches_null = true; + break; + } + } + if (touches_null) { + break; + } + } + if (touches_null) { + for (auto idx : other_inner) { + null_tables.insert(idx); + } + changed = true; + } + } + } + return null_tables; +} + +// Secondary delta for ONE LEFT JOIN level. `level` indexes the emitted placeholders, and +// `null_tables` is the set of base tables that read NULL in this level's null-padded row (see +// ComputeNullTablesForLevel) which decides each aggregate's contribution. +static bool IsUnfilteredGetSubtree(LogicalOperator *op) { + while (op && op->type == LogicalOperatorType::LOGICAL_PROJECTION && op->children.size() == 1) { + op = op->children[0].get(); + } + return op && op->type == LogicalOperatorType::LOGICAL_GET && op->Cast().table_filters.filters.empty(); +} + +static bool ContainsCardinalityChangingFilter(LogicalOperator *op) { + if (!op) { + return false; + } + if (op->type == LogicalOperatorType::LOGICAL_FILTER) { + return true; + } + if (op->type == LogicalOperatorType::LOGICAL_GET && !op->Cast().table_filters.filters.empty()) { + return true; + } + for (auto &child : op->children) { + if (ContainsCardinalityChangingFilter(child.get())) { + return true; + } + } + return false; +} + +static bool ColumnIsNotNull(LogicalGet &get, const string &column_name) { + auto table = get.GetTable(); + if (!table.get() || !table.get()->ColumnExists(column_name)) { + return false; + } + auto column_index = table.get()->GetColumn(column_name).Logical(); + for (auto &constraint : table.get()->GetConstraints()) { + if (constraint->type == ConstraintType::NOT_NULL && + constraint->Cast().index == column_index) { + return true; + } + } + return false; +} + +static string BuildLeftJoinSecondaryForLevel(ClientContext &context, const CreateMVPlanFacts &facts, + const vector &output_names, const string &view_name, + LogicalComparisonJoin *oj, size_t level, + const std::set &null_tables, vector &preserved_cols, + const string &delta_view_catalog_prefix, string &out_inner_table, + string &out_inner_key, string &out_pres_table, string &out_pres_key) { + preserved_cols.clear(); + if (facts.aggregates.size() != 1) { + return ""; + } + if (!oj || oj->join_type != JoinType::LEFT || oj->conditions.size() != 1 || oj->children.size() != 2) { + return ""; + } + // Binder can flatten a filtered preserved-side subquery by lifting its + // LogicalFilter above the join. Inspect the complete view plan as well as + // the local children; secondary transition counts are valid only when no + // additional cardinality predicate exists anywhere in this aggregate view. + if (ContainsCardinalityChangingFilter(facts.root)) { + return ""; + } + // __newc below counts rows directly in the inner base table. A filter or any other + // cardinality-changing inner subtree (including a right-only ON predicate pushed below + // the join) would make that count differ from the actual join matches. + if (!IsUnfilteredGetSubtree(oj->children[1].get())) { + return ""; + } + auto &condition = oj->conditions[0]; + if (condition.comparison != ExpressionType::COMPARE_EQUAL || + condition.left->expression_class != ExpressionClass::BOUND_COLUMN_REF || + condition.right->expression_class != ExpressionClass::BOUND_COLUMN_REF) { + return ""; + } + // Every LEFT JOIN level needs the secondary, including a SINGLE left join whose preserved side is a + // bare table. The old comment here claimed the primary delta already emits the null-padded + // reappearance for that case; it does not. The primary emits only a retraction of the previously + // matched row, and the MERGE's gating then synthesises the user-visible NULL/0 values. That leaves + // openivm_count_star at 0 for a group that still has a NULL-padded output row, which makes it + // impossible to tell "this group still exists, unmatched" from "this group is gone" -- and so + // emptied groups could never be cleaned up. Emitting the reappearance restores that distinction. + auto &agg = *facts.aggregates[0]; + size_t G = agg.groups.size(); + if (G == 0 || output_names.size() < G + agg.expressions.size()) { + return ""; + } + // Deepest-join keys: condition.left = preserved side, condition.right = inner (null) side. + auto &pres_ref = condition.left->Cast(); + auto &inner_ref = condition.right->Cast(); + ColumnBinding key_pres = pres_ref.binding; + LogicalGet *inner_get = nullptr; + string inner_col; + if (!ResolveBindingToGetColumn(inner_ref.binding, facts, inner_get, inner_col) || !inner_get || + !inner_get->GetTable().get()) { + return ""; + } + idx_t inner_tidx = inner_get->table_index; + string inner_table = inner_get->GetTable().get()->name; + auto sti = facts.source_table_info.find(inner_table); + if (sti == facts.source_table_info.end()) { + return ""; + } + string cat = sti->second.catalog_name; + string sch = sti->second.schema_name; + string qual; + if (!cat.empty()) { + qual += KeywordHelper::WriteOptionallyQuoted(cat) + "."; + } + if (!sch.empty()) { + qual += KeywordHelper::WriteOptionallyQuoted(sch) + "."; + } + string inner_base = qual + KeywordHelper::WriteOptionallyQuoted(inner_table); + string qcol = KeywordHelper::WriteOptionallyQuoted(inner_col); + + // The correction below assumes a dangling (null-padded) row for this key is ALREADY materialized + // in the view -- true only if the preserved-side row (e.g. the order) existed before this refresh + // batch. If the preserved-side row is itself brand new in this same batch (e.g. a freshly inserted + // order that immediately gets its first line), no dangling row was ever materialized, and applying + // this correction would spuriously subtract a row that never existed. Resolve the preserved side's + // own base table/column so we can gate on "did this key exist before this batch's delta." + LogicalGet *pres_get = nullptr; + string pres_col; + if (!ResolveBindingToGetColumn(key_pres, facts, pres_get, pres_col) || !pres_get || !pres_get->GetTable().get()) { + return ""; + } + string pres_table = pres_get->GetTable().get()->name; + if (pres_get->table_index == inner_tidx || StringUtil::CIEquals(pres_table, inner_table)) { + // Self-referencing deepest join (preserved and inner sides are the same base table): the + // old_pres_count/old_count arithmetic below assumes two distinct tables with independent + // delta tracking. Bail rather than risk generating ambiguous or incorrect SQL for this rare + // shape. + return ""; + } + auto pres_sti = facts.source_table_info.find(pres_table); + if (pres_sti == facts.source_table_info.end()) { + return ""; + } + string pres_qual; + if (!pres_sti->second.catalog_name.empty()) { + pres_qual += KeywordHelper::WriteOptionallyQuoted(pres_sti->second.catalog_name) + "."; + } + if (!pres_sti->second.schema_name.empty()) { + pres_qual += KeywordHelper::WriteOptionallyQuoted(pres_sti->second.schema_name) + "."; + } + string pres_base = pres_qual + KeywordHelper::WriteOptionallyQuoted(pres_table); + string pres_qcol = KeywordHelper::WriteOptionallyQuoted(pres_col); + + // Classify each aggregate output column's contribution to a null-padded row of the deepest join. + // output_names layout: [group cols (G)] [visible aggregates (agg.expressions, in order)] [hidden helpers]. + // Visible: resolve the arg's base table — inner-side count/sum -> 0, preserved-side count -> 1 + // (preserved-side sum needs the value: unsupported for now -> bail). count_star (no arg) -> 1. + // Hidden: openivm_count_star -> 1, openivm_match_count / openivm_nonnull_sum_count* -> 0 (all inner-derived). + size_t num_agg = output_names.size() - G; + vector contribs; + for (size_t j = 0; j < num_agg; j++) { + if (j < agg.expressions.size()) { + auto &e = agg.expressions[j]; + if (e->expression_class != ExpressionClass::BOUND_AGGREGATE) { + return ""; + } + auto &ba = e->Cast(); + string fn = StringUtil::Lower(ba.function.name); + if (ba.children.empty()) { + contribs.push_back("1"); // count_star: null-padded row still counts + continue; + } + auto *cref = GetColumnRefThroughCasts(ba.children[0].get()); + if (!cref) { + return ""; + } + LogicalGet *g = nullptr; + string c; + bool resolved = ResolveBindingToGetColumn(cref->binding, facts, g, c); + bool is_inner = resolved && g && null_tables.count(g->table_index) > 0; + if (fn == "count") { + if (!is_inner && (!resolved || !g || !ColumnIsNotNull(*g, c))) { + return ""; + } + contribs.push_back(is_inner ? "0" : "1"); + if (!is_inner) { + // Preserved-side COUNT: the MERGE must NOT gate it by the (inner-side) match_count. + preserved_cols.push_back(output_names[G + j]); + } + } else if (fn == "sum") { + if (is_inner) { + contribs.push_back("0"); + } else { + return ""; // preserved-side SUM needs the value; defer to a later generalization + } + } else { + return ""; + } + } else { + // Hidden helper columns are internally generated with fixed openivm_* names (never + // user-chosen), so an exact/prefix match is safe and tighter than a bare substring search. + const string &col = output_names[G + j]; + if (StringUtil::CIEquals(col, "openivm_count_star")) { + contribs.push_back("1"); + } else if (StringUtil::CIEquals(col, "openivm_match_count") || + StringUtil::StartsWith(col, "openivm_nonnull_sum_count")) { + contribs.push_back("0"); + } else { + return ""; + } + } + } + + // Build the preserved-side subquery X via LPTS, naming the join key "__k" and each group col "__g". + auto x_plan = oj->children[0]->Copy(context); + x_plan->ResolveOperatorTypes(); + auto x_bindings = x_plan->GetColumnBindings(); + vector group_bindings; + for (auto &ge : agg.groups) { + auto *gr = GetColumnRefThroughCasts(ge.get()); + if (!gr) { + return ""; + } + group_bindings.push_back(gr->binding); + } + vector x_names(x_bindings.size()); + int key_pos = -1; + vector group_pos(G, -1); + // A column can carry only ONE alias in the emitted column list. When the join key IS also a group + // column (e.g. GROUP BY the same column the deepest join uses -- common in star schemas), the group + // alias wins and no "__k" column exists, so referencing X."__k" failed to bind. Track the alias the + // key actually ended up with and use that instead of assuming "__k". + string key_alias = "__k"; + for (size_t i = 0; i < x_bindings.size(); i++) { + x_names[i] = "__x" + std::to_string(i); + if (x_bindings[i] == key_pres) { + x_names[i] = "__k"; + key_pos = static_cast(i); + } + for (size_t gi = 0; gi < G; gi++) { + if (x_bindings[i] == group_bindings[gi]) { + x_names[i] = "__g" + std::to_string(gi); + group_pos[gi] = static_cast(i); + if (key_pos == static_cast(i)) { + key_alias = x_names[i]; + } + } + } + } + if (key_pos < 0) { + return ""; + } + for (size_t gi = 0; gi < G; gi++) { + if (group_pos[gi] < 0) { + return ""; + } + } + string x_sql; + SqlDialect dialect = SqlDialect::DUCKDB; + auto ast = LogicalPlanToAst(context, x_plan, dialect); + auto cte_list = AstToCteList(*ast, dialect); + x_sql = cte_list->ToQuery(true, x_names); + if (!x_sql.empty() && x_sql.back() == ';') { + x_sql.pop_back(); + } + if (x_sql.empty()) { + return ""; + } + + // delta_mv columns: group cols + aggregate cols + openivm_multiplicity (matches the primary INSERT). + string col_list; + string sel; + for (size_t gi = 0; gi < G; gi++) { + col_list += KeywordHelper::WriteOptionallyQuoted(output_names[gi]) + ", "; + sel += "X.\"__g" + std::to_string(gi) + "\", "; + } + for (size_t j = 0; j < contribs.size(); j++) { + col_list += KeywordHelper::WriteOptionallyQuoted(output_names[G + j]) + ", "; + sel += contribs[j] + ", "; + } + col_list += "openivm_multiplicity"; + sel += "CASE WHEN (__newc - __t.dsum) > 0 AND __newc = 0 THEN 1 ELSE -1 END"; + + out_inner_table = inner_table; + out_inner_key = inner_col; + out_pres_table = pres_table; + out_pres_key = pres_col; + + // Must carry the same catalog prefix the delta table was created with. Unqualified works only when + // the MV's state lives in the default catalog; a DuckLake-backed view's delta table is + // dl.main.openivm_delta_, and an unqualified INSERT there fails the whole refresh. + string dmv = delta_view_catalog_prefix + KeywordHelper::WriteOptionallyQuoted(SqlUtils::DeltaName(view_name)); + // The two delta row sources are placeholders resolved at refresh: a regular table reads its + // openivm_delta_, a DuckLake table reads ducklake_table_insertions/deletions between + // snapshots (whose IDs only exist at refresh time). Both substitutions yield columns (__k, __m). + string inner_delta_src = + string(openivm::LJSEC_INNER_DELTA_PREFIX) + std::to_string(level) + string(openivm::LJSEC_PLACEHOLDER_SUFFIX); + string pres_delta_src = + string(openivm::LJSEC_PRES_DELTA_PREFIX) + std::to_string(level) + string(openivm::LJSEC_PLACEHOLDER_SUFFIX); + // The correlated LATERAL form is deliberate: rewriting these as pre-aggregated CTEs restricted to + // the delta keys measured ~40% SLOWER at TPC-H SF50 (DuckDB already decorrelates them well). + string sql = "INSERT INTO " + dmv + " (" + col_list + ")\nSELECT " + sel + + "\nFROM (SELECT __k, SUM(__m) AS dsum FROM " + inner_delta_src + " GROUP BY __k) __t\nJOIN (" + x_sql + + ") X ON X.\"" + key_alias + "\" = __t.__k,\nLATERAL (SELECT (SELECT COUNT(*) FROM " + inner_base + + " ib WHERE ib." + qcol + " = __t.__k) AS __newc) __n,\nLATERAL (SELECT (SELECT COUNT(*) FROM " + + pres_base + " pb WHERE pb." + pres_qcol + " = __t.__k) - COALESCE((SELECT SUM(__m) FROM " + + pres_delta_src + + " __pd WHERE __pd.__k = __t.__k), 0) AS __old_pres_count) __op" + "\nWHERE (__newc > 0) <> ((__newc - __t.dsum) > 0) AND __old_pres_count > 0;"; + return sql; +} + +// Emit a secondary delta for EVERY LEFT JOIN level whose preserved side is itself a join. +// +// A chain of N tables has N-1 LEFT JOIN levels, and a disappearing match at ANY of them can strand a +// preserved-side count. Correcting only the outermost level (the previous behaviour) left +// COUNT(intermediate_key) undercounted for 4+ table chains: with c ⟕ o ⟕ l ⟕ s, deleting an order's +// last line dropped that order from COUNT(o.oid) because the `⟕ l` level had no correction. +// +// The per-level SQL fragments are concatenated and the two source identities per level are stored as +// index-aligned arrays, so refresh can resolve each level's placeholders independently (a regular +// table reads its delta table, a DuckLake table reads snapshot insertions/deletions). +string BuildLeftJoinSecondaryDeltaSQL(ClientContext &context, const CreateMVPlanFacts &facts, + const vector &output_names, const string &view_name, + vector &preserved_cols, const string &delta_view_catalog_prefix, + vector &out_inner_tables, vector &out_inner_keys, + vector &out_pres_tables, vector &out_pres_keys) { + preserved_cols.clear(); + string combined_sql; + vector inner_tables, inner_keys, pres_tables, pres_keys; + size_t level = 0; + for (auto *oj : facts.comparison_joins) { + if (!oj || oj->join_type != JoinType::LEFT || oj->conditions.empty() || oj->children.size() != 2) { + continue; + } + // Every LEFT JOIN level gets a secondary, including the innermost one whose preserved side is a + // bare table -- see the note in BuildLeftJoinSecondaryForLevel for why the primary delta does + // NOT cover that case. + auto null_tables = ComputeNullTablesForLevel(facts, oj); + vector level_preserved; + string it, ik, pt, pk; + string level_sql = + BuildLeftJoinSecondaryForLevel(context, facts, output_names, view_name, oj, level, null_tables, + level_preserved, delta_view_catalog_prefix, it, ik, pt, pk); + if (level_sql.empty()) { + // An unsupported level makes the whole correction incomplete, and a partial correction is + // worse than none (it would double-count the levels it does cover). Bail entirely. + preserved_cols.clear(); + return ""; + } + combined_sql += (combined_sql.empty() ? "" : "\n") + level_sql; + inner_tables.push_back(it); + inner_keys.push_back(ik); + pres_tables.push_back(pt); + pres_keys.push_back(pk); + for (auto &c : level_preserved) { + if (std::find(preserved_cols.begin(), preserved_cols.end(), c) == preserved_cols.end()) { + preserved_cols.push_back(c); + } + } + level++; + } + if (combined_sql.empty()) { + return ""; + } + out_inner_tables = std::move(inner_tables); + out_inner_keys = std::move(inner_keys); + out_pres_tables = std::move(pres_tables); + out_pres_keys = std::move(pres_keys); + return combined_sql; +} + } // namespace duckdb diff --git a/src/core/plan_rewrite.cpp b/src/core/plan_rewrite.cpp index 208865af..4f8269c9 100644 --- a/src/core/plan_rewrite.cpp +++ b/src/core/plan_rewrite.cpp @@ -2,6 +2,7 @@ #include "core/openivm_constants.hpp" #include "core/openivm_debug.hpp" +#include "core/parser_plan_helpers.hpp" #include "core/plan_rewrite_internal.hpp" #include "core/sql_utils.hpp" #include "duckdb/catalog/catalog.hpp" @@ -41,18 +42,36 @@ namespace duckdb { /// Replace LOGICAL_DISTINCT with LOGICAL_AGGREGATE + COUNT(*). -static void RewriteDistinct(ClientContext &context, Binder &binder, unique_ptr &node) { +/// Returns whether a DISTINCT state aggregate was added at the top level. +static bool RewriteDistinct(ClientContext &context, Binder &binder, unique_ptr &node, + bool is_top_level) { if (node->type != LogicalOperatorType::LOGICAL_DISTINCT) { - for (auto &child : node->children) { - RewriteDistinct(context, binder, child); + bool added_top_level_state = false; + for (idx_t child_idx = 0; child_idx < node->children.size(); child_idx++) { + bool child_is_top_level = + is_top_level && node->type == LogicalOperatorType::LOGICAL_PROJECTION && child_idx == 0; + added_top_level_state = RewriteDistinct(context, binder, node->children[child_idx], child_is_top_level) || + added_top_level_state; } - return; + return added_top_level_state; } auto &distinct = node->Cast(); if (node->children.empty()) { OPENIVM_DEBUG_PRINT("[PlanRewrite] DISTINCT has no children — skipping\n"); - return; + return false; + } + if (ProducesAtMostOneRow(*node->children[0])) { + OPENIVM_DEBUG_PRINT("[PlanRewrite] Removed redundant DISTINCT over scalar aggregate\n"); + node = std::move(node->children[0]); + RewriteDistinct(context, binder, node, false); + return false; + } + if (IsRedundantDistinctOverGroupKeys(*node)) { + OPENIVM_DEBUG_PRINT("[PlanRewrite] Removed redundant DISTINCT over aggregate group keys\n"); + node = std::move(node->children[0]); + RewriteDistinct(context, binder, node, false); + return false; } auto &child = node->children[0]; child->ResolveOperatorTypes(); @@ -88,6 +107,7 @@ static void RewriteDistinct(ClientContext &context, Binder &binder, unique_ptrgroups.size()); node = std::move(agg_node); + return is_top_level; } static bool IsSemiAntiJoinType(JoinType join_type) { @@ -861,6 +881,27 @@ void FoldConstantScalarSubqueries(ClientContext &context, unique_ptrResolveOperatorTypes(); } +static ColumnBinding AppendProjectionPassthrough(LogicalProjection &proj, const ColumnBinding &binding, + const LogicalType &type, const string &alias) { + auto passthrough = make_uniq(type, binding); + passthrough->alias = alias; + proj.expressions.push_back(std::move(passthrough)); + proj.ResolveOperatorTypes(); + auto bindings = proj.GetColumnBindings(); + if (bindings.empty()) { + throw InternalException("OpenIVM: projection produced no bindings after appending hidden passthrough"); + } + return bindings.back(); +} + +static void PropagateHiddenBindingThroughProjectionPath(vector &projection_path, + ColumnBinding binding, LogicalType type, const string &alias) { + for (auto it = projection_path.rbegin(); it != projection_path.rend(); ++it) { + binding = AppendProjectionPassthrough(**it, binding, type, alias); + type = (*it)->types.back(); + } +} + /// Inject a hidden COUNT(*) (alias `openivm_count_star`) into AGGREGATE_GROUP /// aggregates that don't already have a reliable total-row-count aggregate. /// @@ -876,12 +917,40 @@ void FoldConstantScalarSubqueries(ClientContext &context, unique_ptr &plan) { - // Only inject at the top of the plan — the AGGREGATE_GROUP compile path only - // runs when the MV root is PROJECTION → [FILTER] → AGGREGATE. Inner aggregates - // under a UNION/INTERSECT/EXCEPT or subquery are handled by different compile - // paths (often FULL_REFRESH) that would be broken by extra columns. - auto *agg_search = FindProjectionAggregateInput(plan, true); - if (!agg_search) { + // Only inject into the aggregate on the top-level output path. ORDER BY/LIMIT/TOP_N + // preserve the projection schema; inner aggregates under joins or set operations + // are handled by other refresh strategies. + vector projection_path; + LogicalOperator *node = plan.get(); + LogicalOperator *agg_search = nullptr; + while (node) { + switch (node->type) { + case LogicalOperatorType::LOGICAL_ORDER_BY: + case LogicalOperatorType::LOGICAL_LIMIT: + case LogicalOperatorType::LOGICAL_TOP_N: + node = node->children.size() == 1 ? node->children[0].get() : nullptr; + continue; + case LogicalOperatorType::LOGICAL_PROJECTION: + projection_path.push_back(&node->Cast()); + node = node->children.size() == 1 ? node->children[0].get() : nullptr; + continue; + case LogicalOperatorType::LOGICAL_FILTER: + if (node->children.size() == 1 && + node->children[0]->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + agg_search = node->children[0].get(); + } + node = nullptr; + continue; + case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: + agg_search = node; + node = nullptr; + continue; + default: + node = nullptr; + continue; + } + } + if (!agg_search || projection_path.empty()) { return; } auto &agg = agg_search->Cast(); @@ -932,11 +1001,8 @@ static void InjectGroupCountStar(unique_ptr &plan) { agg.ResolveOperatorTypes(); ColumnBinding count_binding(agg.aggregate_index, new_agg_idx); - auto count_pt = make_uniq(count_type, count_binding); - count_pt->alias = openivm::COUNT_STAR_COL; - auto &proj = plan->Cast(); - proj.expressions.push_back(std::move(count_pt)); - proj.ResolveOperatorTypes(); + PropagateHiddenBindingThroughProjectionPath(projection_path, count_binding, count_type, openivm::COUNT_STAR_COL); + plan->ResolveOperatorTypes(); OPENIVM_DEBUG_PRINT("[PlanRewrite] Injected openivm_count_star for AGGREGATE_GROUP\n"); } @@ -946,7 +1012,8 @@ static void InjectGroupCountStar(unique_ptr &plan) { /// projection's expression up through a pass-through parent projection. static bool IsHiddenAggregateAlias(const string &alias) { return alias.find(openivm::SUM_COL_PREFIX) == 0 || alias.find(openivm::COUNT_COL_PREFIX) == 0 || - alias.find(openivm::SUM_SQ_COL_PREFIX) == 0 || alias.find(openivm::SUM_SQP_COL_PREFIX) == 0; + alias.find(openivm::SUM_SQ_COL_PREFIX) == 0 || alias.find(openivm::SUM_SQP_COL_PREFIX) == 0 || + alias.find(openivm::SUM_COUNT_COL_PREFIX) == 0; } /// Propagate hidden aggregate columns (openivm_sum_*, openivm_count_*, …) added @@ -962,7 +1029,7 @@ static bool IsHiddenAggregateAlias(const string &alias) { /// data table stores only the final AVG and the MERGE computes `v.avg + d.avg` /// — wrong for non-summable aggregates. Propagation lets CompileAggregateGroups /// see the hidden SUM/COUNT columns and maintain them separately. -static void PropagateHiddenAggregateColumns(unique_ptr &plan) { +void PropagateHiddenAggregateColumns(unique_ptr &plan) { for (auto &child : plan->children) { PropagateHiddenAggregateColumns(child); } @@ -1006,27 +1073,6 @@ static void PropagateHiddenAggregateColumns(unique_ptr &plan) { } } -static ColumnBinding AppendProjectionPassthrough(LogicalProjection &proj, const ColumnBinding &binding, - const LogicalType &type, const string &alias) { - auto passthrough = make_uniq(type, binding); - passthrough->alias = alias; - proj.expressions.push_back(std::move(passthrough)); - proj.ResolveOperatorTypes(); - auto bindings = proj.GetColumnBindings(); - if (bindings.empty()) { - throw InternalException("OpenIVM: projection produced no bindings after appending hidden passthrough"); - } - return bindings.back(); -} - -static void PropagateHiddenBindingThroughProjectionPath(vector &projection_path, - ColumnBinding binding, LogicalType type, const string &alias) { - for (auto it = projection_path.rbegin(); it != projection_path.rend(); ++it) { - binding = AppendProjectionPassthrough(**it, binding, type, alias); - type = (*it)->types.back(); - } -} - struct OuterJoinBindings { bool found = false; bool is_full_outer = false; @@ -1356,9 +1402,9 @@ static void RewritePassAggregateFilters(PlanRewriteContext &rewrite_context) { } static void RewritePassDistinct(PlanRewriteContext &rewrite_context) { - bool had_distinct = HasTopLevelDistinct(rewrite_context.plan); - RewriteDistinct(rewrite_context.context, rewrite_context.binder, rewrite_context.plan); - if (had_distinct) { + bool added_top_level_state = RewriteDistinct(rewrite_context.context, rewrite_context.binder, rewrite_context.plan, + HasTopLevelDistinct(rewrite_context.plan)); + if (added_top_level_state) { rewrite_context.planner_names.push_back(openivm::DISTINCT_COUNT_COL); } } diff --git a/src/core/plan_rewrite_aggregates.cpp b/src/core/plan_rewrite_aggregates.cpp index 2c1762f6..d1615ab4 100644 --- a/src/core/plan_rewrite_aggregates.cpp +++ b/src/core/plan_rewrite_aggregates.cpp @@ -9,6 +9,7 @@ #include "duckdb/optimizer/optimizer.hpp" #include "duckdb/planner/expression/bound_aggregate_expression.hpp" #include "duckdb/planner/expression/bound_case_expression.hpp" +#include "duckdb/planner/expression/bound_cast_expression.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/expression/bound_comparison_expression.hpp" #include "duckdb/planner/expression/bound_constant_expression.hpp" @@ -134,6 +135,137 @@ LogicalOperator *FindProjectionAggregateInput(unique_ptr &plan, } return nullptr; } + +void InjectSumNonNullCounts(ClientContext &context, unique_ptr &plan) { + if (!plan) { + return; + } + + // Find the linear output path down to the aggregate. ORDER BY/LIMIT/TOP_N + // and MATERIALIZED_CTE are pass-through wrappers; projections may rename or + // cast aggregate outputs and therefore need to be followed explicitly. + vector projections; + LogicalOperator *node = plan.get(); + LogicalOperator *agg_search = nullptr; + while (node) { + switch (node->type) { + case LogicalOperatorType::LOGICAL_ORDER_BY: + case LogicalOperatorType::LOGICAL_LIMIT: + case LogicalOperatorType::LOGICAL_TOP_N: + case LogicalOperatorType::LOGICAL_DISTINCT: + node = node->children.empty() ? nullptr : node->children[0].get(); + continue; + case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE: + node = node->children.size() < 2 ? nullptr : node->children[1].get(); + continue; + case LogicalOperatorType::LOGICAL_PROJECTION: + projections.push_back(&node->Cast()); + node = node->children.empty() ? nullptr : node->children[0].get(); + continue; + case LogicalOperatorType::LOGICAL_FILTER: + if (!node->children.empty() && + node->children[0]->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + agg_search = node->children[0].get(); + } + node = nullptr; + continue; + case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY: + agg_search = node; + node = nullptr; + continue; + default: + node = nullptr; + continue; + } + } + if (!agg_search || projections.empty()) { + return; + } + + auto &agg = agg_search->Cast(); + auto &aggregate_projection = *projections.back(); + auto &output_projection = *projections.front(); + const idx_t original_aggregate_count = agg.expressions.size(); + + struct SumCountOutput { + idx_t projection_index; + idx_t aggregate_index; + }; + vector outputs; + for (idx_t projection_index = 0; projection_index < output_projection.expressions.size(); projection_index++) { + Expression *resolved = output_projection.expressions[projection_index].get(); + while (resolved) { + while (resolved->expression_class == ExpressionClass::BOUND_CAST) { + resolved = resolved->Cast().child.get(); + } + if (resolved->type != ExpressionType::BOUND_COLUMN_REF) { + break; + } + auto &ref = resolved->Cast(); + if (ref.binding.table_index == agg.aggregate_index) { + break; + } + LogicalProjection *referenced_projection = nullptr; + for (idx_t projection_depth = 1; projection_depth < projections.size(); projection_depth++) { + if (projections[projection_depth]->table_index == ref.binding.table_index) { + referenced_projection = projections[projection_depth]; + break; + } + } + if (!referenced_projection || ref.binding.column_index >= referenced_projection->expressions.size()) { + resolved = nullptr; + break; + } + resolved = referenced_projection->expressions[ref.binding.column_index].get(); + } + if (!resolved || resolved->type != ExpressionType::BOUND_COLUMN_REF) { + continue; + } + auto &ref = resolved->Cast(); + if (ref.binding.table_index != agg.aggregate_index || ref.binding.column_index >= original_aggregate_count) { + continue; + } + auto &aggregate_expr = agg.expressions[ref.binding.column_index]; + if (aggregate_expr->expression_class != ExpressionClass::BOUND_AGGREGATE) { + continue; + } + auto &sum = aggregate_expr->Cast(); + if (sum.function.name != "sum" || sum.IsDistinct() || sum.children.size() != 1) { + continue; + } + if (sum.alias.find(string(openivm::SUM_COL_PREFIX)) == 0 || + sum.alias.find(string(openivm::VAR_SQ_COL_PREFIX)) == 0 || + sum.alias.find(string(openivm::VAR_SQP_COL_PREFIX)) == 0) { + continue; + } + + auto count_func = BindAggregateByName(context, "count", {sum.children[0]->return_type}); + vector> count_args; + count_args.push_back(sum.children[0]->Copy()); + auto count_expr = make_uniq(std::move(count_func), std::move(count_args), nullptr, + nullptr, AggregateType::NON_DISTINCT); + count_expr->alias = string(openivm::SUM_COUNT_COL_PREFIX) + to_string(projection_index); + outputs.push_back({projection_index, agg.expressions.size()}); + agg.expressions.push_back(std::move(count_expr)); + } + if (outputs.empty()) { + return; + } + + agg.ResolveOperatorTypes(); + auto aggregate_bindings = agg_search->GetColumnBindings(); + auto aggregate_types = agg_search->types; + for (auto &output : outputs) { + idx_t output_index = agg.groups.size() + output.aggregate_index; + auto count_ref = + make_uniq(aggregate_types[output_index], aggregate_bindings[output_index]); + count_ref->alias = string(openivm::SUM_COUNT_COL_PREFIX) + to_string(output.projection_index); + aggregate_projection.expressions.push_back(std::move(count_ref)); + } + aggregate_projection.ResolveOperatorTypes(); + OPENIVM_DEBUG_PRINT("[PlanRewrite] Injected %zu SUM non-NULL counts\n", outputs.size()); +} + void RewriteDerivedAggregates(ClientContext &context, unique_ptr &plan, Optimizer &opt, bool is_top) { for (auto &child : plan->children) { RewriteDerivedAggregates(context, child, opt, false); diff --git a/src/core/refresh_daemon.cpp b/src/core/refresh_daemon.cpp index 4d2588ca..9c774766 100644 --- a/src/core/refresh_daemon.cpp +++ b/src/core/refresh_daemon.cpp @@ -2,7 +2,6 @@ #include "core/openivm_debug.hpp" #include "core/refresh_metadata.hpp" -#include "core/refresh_locks.hpp" #include "core/sql_utils.hpp" #include "duckdb/common/printer.hpp" #include "duckdb/main/connection.hpp" @@ -35,6 +34,11 @@ void RefreshDaemon::Stop() { started_ = false; } +void RefreshDaemon::Wake() { + wake_requested_ = true; + cv_.notify_all(); +} + RefreshDaemon::~RefreshDaemon() { Stop(); } @@ -59,11 +63,13 @@ void RefreshDaemon::Run() { while (!shutdown_.load()) { { std::unique_lock lock(cv_mutex_); - cv_.wait_for(lock, std::chrono::seconds(WAKE_INTERVAL_SECONDS), [this] { return shutdown_.load(); }); + cv_.wait_for(lock, std::chrono::seconds(WAKE_INTERVAL_SECONDS), + [this] { return shutdown_.load() || wake_requested_.load(); }); } if (shutdown_.load()) { break; } + wake_requested_ = false; try { Connection con(*db_); @@ -151,8 +157,8 @@ void RefreshDaemon::Run() { // Check if the view is due for refresh if (!sv.last_update.empty()) { - auto elapsed_result = - con.Query("SELECT EXTRACT(EPOCH FROM (now() - '" + sv.last_update + "'::TIMESTAMP))"); + auto elapsed_result = con.Query("SELECT EXTRACT(EPOCH FROM (now()::TIMESTAMP - '" + sv.last_update + + "'::TIMESTAMP))"); if (!elapsed_result->HasError() && elapsed_result->RowCount() > 0 && !elapsed_result->GetValue(0, 0).IsNull()) { auto elapsed_seconds = elapsed_result->GetValue(0, 0).GetValue(); @@ -162,16 +168,6 @@ void RefreshDaemon::Run() { } } - // Quick check if the view is already being refreshed (non-blocking). - { - TryViewLockGuard view_lock(sv.view_name); - if (!view_lock.OwnsLock()) { - OPENIVM_DEBUG_PRINT("[REFRESH DAEMON] Skipping '%s' — refresh already in progress\n", - sv.view_name.c_str()); - continue; - } - } - { std::lock_guard guard(refreshing_mutex_); currently_refreshing_ = sv.view_name; @@ -180,42 +176,16 @@ void RefreshDaemon::Run() { OPENIVM_DEBUG_PRINT("[REFRESH DAEMON] Refreshing '%s'\n", sv.view_name.c_str()); auto before = std::chrono::steady_clock::now(); + bool refresh_succeeded = false; try { Connection refresh_con(*db_); - - // Check for refresh hooks - auto hook_r = - refresh_con.Query("SELECT hook_sql, mode FROM openivm_refresh_hooks WHERE view_name = '" + - SqlUtils::EscapeValue(sv.view_name) + "'"); - string hook_sql; - string hook_mode; - if (!hook_r->HasError() && hook_r->RowCount() > 0) { - hook_sql = hook_r->GetValue(0, 0).ToString(); - hook_mode = StringUtil::Lower(hook_r->GetValue(1, 0).ToString()); - } - - if (!hook_sql.empty() && hook_mode == "before") { - auto hr = refresh_con.Query(hook_sql); - if (hr->HasError()) { - Printer::Print("Warning: before-hook for '" + sv.view_name + "' failed: " + hr->GetError()); - } - } - - if (hook_mode != "replace") { - auto result = - refresh_con.Query("PRAGMA refresh('" + SqlUtils::EscapeValue(sv.view_name) + "')"); - if (result->HasError()) { - Printer::Print("Warning: auto-refresh of '" + sv.view_name + - "' failed: " + result->GetError()); - } - } - - if (!hook_sql.empty() && (hook_mode == "after" || hook_mode == "replace")) { - auto hr = refresh_con.Query(hook_sql); - if (hr->HasError()) { - Printer::Print("Warning: " + hook_mode + "-hook for '" + sv.view_name + - "' failed: " + hr->GetError()); - } + auto result = refresh_con.Query( + "PRAGMA refresh_options('" + SqlUtils::EscapeValue(sv.catalog_name) + "', '" + + SqlUtils::EscapeValue(sv.schema_name) + "', '" + SqlUtils::EscapeValue(sv.view_name) + "')"); + if (result->HasError()) { + Printer::Print("Warning: auto-refresh of '" + sv.view_name + "' failed: " + result->GetError()); + } else { + refresh_succeeded = true; } } catch (std::exception &e) { Printer::Print("Warning: auto-refresh of '" + sv.view_name + "' failed: " + string(e.what())); @@ -225,6 +195,9 @@ void RefreshDaemon::Run() { std::lock_guard guard(refreshing_mutex_); currently_refreshing_.clear(); } + if (!refresh_succeeded) { + continue; + } // Mark this view and any cascaded views as done for this cycle refreshed_this_cycle.insert(sv.view_name); diff --git a/src/core/refresh_locks.cpp b/src/core/refresh_locks.cpp index b0dbff70..d900ab77 100644 --- a/src/core/refresh_locks.cpp +++ b/src/core/refresh_locks.cpp @@ -1,47 +1,89 @@ #include "core/refresh_locks.hpp" +#include "core/openivm_debug.hpp" namespace duckdb { std::mutex RefreshLocks::map_mutex_; -std::unordered_map> RefreshLocks::view_mutexes_; -std::unordered_map> RefreshLocks::delta_mutexes_; +std::unordered_map> RefreshLocks::mutation_gates_; -std::mutex &RefreshLocks::GetViewMutex(const string &view_name) { - std::lock_guard guard(map_mutex_); - auto &entry = view_mutexes_[view_name]; - if (!entry) { - entry = duckdb::unique_ptr(new std::mutex()); +void MutationGate::Lock(const void *owner) { + std::unique_lock guard(lock); + if (active_owner == owner) { + depth++; + return; } - return *entry; + condition.wait(guard, [&]() { return active_owner == nullptr; }); + active_owner = owner; + depth = 1; } -std::mutex &RefreshLocks::GetDeltaMutex(const string &delta_table_name) { +void MutationGate::Unlock(const void *owner) { + std::lock_guard guard(lock); + if (active_owner != owner || depth == 0) { + D_ASSERT(false); + return; + } + depth--; + if (depth == 0) { + active_owner = nullptr; + condition.notify_one(); + } +} + +MutationGate &RefreshLocks::GetMutationGate(DatabaseInstance &db) { std::lock_guard guard(map_mutex_); - auto &entry = delta_mutexes_[delta_table_name]; + auto &entry = mutation_gates_[&db]; if (!entry) { - entry = duckdb::unique_ptr(new std::mutex()); + entry = make_uniq(); } return *entry; } -void RefreshLocks::LockView(const string &view_name) { - GetViewMutex(view_name).lock(); +void RefreshLocks::LockMutation(DatabaseInstance &db, const void *owner) { + GetMutationGate(db).Lock(owner); +} + +void RefreshLocks::UnlockMutation(DatabaseInstance &db, const void *owner) { + GetMutationGate(db).Unlock(owner); } -bool RefreshLocks::TryLockView(const string &view_name) { - return GetViewMutex(view_name).try_lock(); +TransactionalMVLockState &TransactionalMVLockState::Get(ClientContext &context) { + auto state = context.registered_state->GetOrCreate("openivm_transactional_mv_locks"); + state->owner = &context; + if (!state->mutation_owner) { + state->mutation_owner = &context; + } + return *state; +} + +void TransactionalMVLockState::AcquireMutationLock() { + if (!owner) { + throw InternalException("OpenIVM transactional lock state has no owning client context"); + } + if (!mutation_guard) { + mutation_guard = make_uniq(DatabaseInstance::GetDatabase(*owner), mutation_owner); + OPENIVM_DEBUG_PRINT("[LOCK] acquired database mutation lock owner=%p\n", static_cast(owner)); + } +} + +void TransactionalMVLockState::SetMutationOwner(const void *owner_token) { + if (mutation_guard) { + throw InternalException("OpenIVM cannot change mutation ownership after acquiring the mutation lock"); + } + mutation_owner = owner_token; } -void RefreshLocks::UnlockView(const string &view_name) { - GetViewMutex(view_name).unlock(); +void TransactionalMVLockState::TransactionCommit(MetaTransaction &transaction, ClientContext &context) { + Release(); } -void RefreshLocks::LockDelta(const string &delta_table_name) { - GetDeltaMutex(delta_table_name).lock(); +void TransactionalMVLockState::TransactionRollback(MetaTransaction &transaction, ClientContext &context) { + Release(); } -void RefreshLocks::UnlockDelta(const string &delta_table_name) { - GetDeltaMutex(delta_table_name).unlock(); +void TransactionalMVLockState::Release() { + OPENIVM_DEBUG_PRINT("[LOCK] release database mutation lock owner=%p\n", static_cast(owner)); + mutation_guard.reset(); } } // namespace duckdb diff --git a/src/core/refresh_metadata.cpp b/src/core/refresh_metadata.cpp index 68a5a6e7..a56e7269 100644 --- a/src/core/refresh_metadata.cpp +++ b/src/core/refresh_metadata.cpp @@ -29,9 +29,22 @@ RefreshType RefreshMetadata::GetViewType(const string &view_name) { auto result = con.Query("SELECT type FROM " + string(openivm::VIEWS_TABLE) + " WHERE view_name = '" + SqlUtils::EscapeValue(view_name) + "'"); if (result->HasError() || result->RowCount() == 0) { + if (result->HasError()) { + auto locus = con.Query("SELECT current_database(), current_schema()"); + auto locus_text = !locus->HasError() && locus->RowCount() > 0 + ? " (connection locus " + locus->GetValue(0, 0).ToString() + "." + + locus->GetValue(1, 0).ToString() + ")" + : ""; + throw ParserException("Could not read IVM metadata for materialized view '%s'%s: %s", view_name, locus_text, + result->GetError()); + } throw ParserException("Materialized view '%s' does not exist in IVM metadata.", view_name); } - return static_cast(result->GetValue(0, 0).GetValue()); + auto raw_type = result->GetValue(0, 0).GetValue(); + if (raw_type == static_cast(openivm::LEGACY_CURRENT_DIFF_RECOMPUTE_TYPE)) { + return RefreshType::FULL_REFRESH; + } + return static_cast(raw_type); } bool RefreshMetadata::HasMinMax(const string &view_name) { @@ -122,6 +135,54 @@ RefreshMetadata::SourceLocation RefreshMetadata::GetSourceLocation(const string return loc; } +RefreshMetadata::StoredViewLocation RefreshMetadata::GetStoredViewLocation(const string &view_name, + const string &fallback_catalog, + const string &fallback_schema) { + StoredViewLocation loc {fallback_catalog, fallback_schema}; + auto result = con.Query("SELECT view_catalog, view_schema FROM " + string(openivm::VIEWS_TABLE) + + " WHERE view_name = '" + SqlUtils::EscapeValue(view_name) + "'"); + if (!result->HasError() && result->RowCount() > 0) { + if (!result->GetValue(0, 0).IsNull()) { + loc.catalog_name = result->GetValue(0, 0).ToString(); + } + if (!result->GetValue(1, 0).IsNull()) { + loc.schema_name = result->GetValue(1, 0).ToString(); + } + } + if (loc.catalog_name.empty()) { + auto current = con.Query("SELECT current_database()"); + if (!current->HasError() && current->RowCount() > 0 && !current->GetValue(0, 0).IsNull()) { + loc.catalog_name = current->GetValue(0, 0).ToString(); + } + } + if (loc.schema_name.empty()) { + loc.schema_name = DEFAULT_SCHEMA; + } + return loc; +} + +vector RefreshMetadata::GetDeltaSources(const string &view_name, + const string &fallback_catalog, + const string &fallback_schema) { + auto result = con.Query("SELECT table_name, catalog_type, source_catalog, source_schema FROM " + + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name = '" + + SqlUtils::EscapeValue(view_name) + "'"); + vector sources; + if (result->HasError()) { + return sources; + } + for (idx_t row = 0; row < result->RowCount(); row++) { + DeltaSource source; + source.table_name = result->GetValue(0, row).ToString(); + source.catalog_type = result->GetValue(1, row).IsNull() ? "duckdb" : result->GetValue(1, row).ToString(); + source.catalog_name = + result->GetValue(2, row).IsNull() ? fallback_catalog : result->GetValue(2, row).ToString(); + source.schema_name = result->GetValue(3, row).IsNull() ? fallback_schema : result->GetValue(3, row).ToString(); + sources.push_back(std::move(source)); + } + return sources; +} + string RefreshMetadata::ResolveDeltaQualifiedName(const string &view_name, const string &delta_table_name, const string &fallback_catalog, const string &fallback_schema) { auto loc = GetSourceLocation(view_name, delta_table_name, fallback_catalog, fallback_schema); @@ -226,7 +287,7 @@ vector RefreshMetadata::GetUpstreamViews(const string &view_name) { return result; // topological order: ancestors first } -vector RefreshMetadata::GetDownstreamViews(const string &view_name) { +static vector GetDownstreamViewsInternal(Connection &con, const string &view_name, bool throw_on_error) { // Find all reachable views that depend on delta_ or openivm_data_ as a source. // DuckLake chained MVs use openivm_data_* (data table) instead of delta_* (delta table). // @@ -241,6 +302,10 @@ vector RefreshMetadata::GetDownstreamViews(const string &view_name) { auto dependents = con.Query("SELECT DISTINCT view_name FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE table_name = '" + SqlUtils::EscapeValue(delta_name) + "' OR table_name = '" + SqlUtils::EscapeValue(data_name) + "' ORDER BY view_name"); + if (dependents->HasError() && throw_on_error) { + throw CatalogException("OpenIVM could not resolve downstream dependencies for '%s': %s", view_name, + dependents->GetError()); + } if (!dependents->HasError()) { for (size_t i = 0; i < dependents->RowCount(); i++) { string dep = dependents->GetValue(0, i).ToString(); @@ -256,9 +321,24 @@ vector RefreshMetadata::GetDownstreamViews(const string &view_name) { }; collect(view_name); std::reverse(result.begin(), result.end()); + OPENIVM_DEBUG_PRINT("[CASCADE] View '%s' has %zu downstream refresh nodes\n", view_name.c_str(), result.size()); return result; // topological order: downstream parents before fan-in children } +vector RefreshMetadata::GetDownstreamViews(const string &view_name) { + return GetDownstreamViewsInternal(con, view_name, false); +} + +vector RefreshMetadata::GetDownstreamViewsStrict(const string &view_name) { + return GetDownstreamViewsInternal(con, view_name, true); +} + +bool RefreshMetadata::HasDownstreamViews(const string &view_name) { + auto result = con.Query("SELECT 1 FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE table_name = '" + + SqlUtils::EscapeValue(SqlUtils::DeltaName(view_name)) + "' LIMIT 1"); + return !result->HasError() && result->RowCount() > 0; +} + vector RefreshMetadata::GetGroupColumns(const string &view_name) { auto result = con.Query("SELECT group_columns FROM " + string(openivm::VIEWS_TABLE) + " WHERE view_name = '" + SqlUtils::EscapeValue(view_name) + "'"); @@ -332,7 +412,10 @@ int64_t RefreshMetadata::GetRefreshInterval(const string &view_name) { } vector RefreshMetadata::GetScheduledViews() { - auto result = con.Query("SELECT v.view_name, v.refresh_interval, " + auto result = con.Query("SELECT v.view_name, COALESCE(v.view_catalog, current_database()), " + "COALESCE(v.view_schema, '" + + string(DEFAULT_SCHEMA) + + "'), v.refresh_interval, " "(SELECT MIN(d.last_update) FROM " + string(openivm::DELTA_TABLES_TABLE) + " d WHERE d.view_name = v.view_name) AS last_update " @@ -346,8 +429,10 @@ vector RefreshMetadata::GetScheduledViews() { for (size_t i = 0; i < result->RowCount(); i++) { ScheduledView sv; sv.view_name = result->GetValue(0, i).ToString(); - sv.interval_seconds = result->GetValue(1, i).GetValue(); - sv.last_update = result->GetValue(2, i).IsNull() ? "" : result->GetValue(2, i).ToString(); + sv.catalog_name = result->GetValue(1, i).IsNull() ? "" : result->GetValue(1, i).ToString(); + sv.schema_name = result->GetValue(2, i).IsNull() ? DEFAULT_SCHEMA : result->GetValue(2, i).ToString(); + sv.interval_seconds = result->GetValue(3, i).GetValue(); + sv.last_update = result->GetValue(4, i).IsNull() ? "" : result->GetValue(4, i).ToString(); views.push_back(sv); } } @@ -359,11 +444,12 @@ void RefreshMetadata::SetRefreshInProgress(const string &view_name, bool in_prog (in_progress ? "true" : "false") + " WHERE view_name = '" + SqlUtils::EscapeValue(view_name) + "'"); } -string RefreshMetadata::BuildDeltaCleanupSQL(const string &target, const string &metadata_key) { +string RefreshMetadata::BuildDeltaCleanupSQL(const string &target, const string &metadata_key, + const string &delta_metadata_table) { string qtarget = target.find('.') == string::npos ? KeywordHelper::WriteOptionallyQuoted(target) : target; + auto metadata_table = delta_metadata_table.empty() ? string(openivm::DELTA_TABLES_TABLE) : delta_metadata_table; return "DELETE FROM " + qtarget + " WHERE " + string(openivm::TIMESTAMP_COL) + " < (SELECT MIN(last_update) FROM " + - string(openivm::DELTA_TABLES_TABLE) + " WHERE table_name = '" + SqlUtils::EscapeValue(metadata_key) + - "');\n"; + metadata_table + " WHERE table_name = '" + SqlUtils::EscapeValue(metadata_key) + "');\n"; } // --- DuckLake support --- @@ -459,8 +545,10 @@ RefreshMetadata::DuckLakeSourceIdentity RefreshMetadata::ResolveDuckLakeSourceId } string RefreshMetadata::BuildDuckLakeRefreshMetadataSQL(const string &view_name, const string &table_name, - const string &snapshot_expr) { - return "UPDATE " + string(openivm::DELTA_TABLES_TABLE) + " SET last_snapshot_id = " + snapshot_expr + + const string &snapshot_expr, + const string &delta_metadata_table) { + auto metadata_table = delta_metadata_table.empty() ? string(openivm::DELTA_TABLES_TABLE) : delta_metadata_table; + return "UPDATE " + metadata_table + " SET last_snapshot_id = " + snapshot_expr + ", last_update = now(), last_refresh_ts = now() WHERE view_name = '" + SqlUtils::EscapeValue(view_name) + "' AND table_name = '" + SqlUtils::EscapeValue(table_name) + "';\n"; } @@ -918,12 +1006,15 @@ bool RefreshMetadata::GetWindowPartitionLineage(const string &view_name, vector< !ExtractJsonString(object, "source_col", op.source_col)) { continue; } + ExtractJsonString(object, "source_cast", op.source_cast); if (op.kind == "lookup") { if (!ExtractJsonString(object, "lookup", op.lookup) || !ExtractJsonString(object, "lookup_col", op.lookup_col) || !ExtractJsonString(object, "lookup_out", op.lookup_out)) { continue; } + ExtractJsonString(object, "lookup_cast", op.lookup_cast); + ExtractJsonString(object, "lookup_out_cast", op.lookup_out_cast); } else if (op.kind != "direct") { continue; } @@ -942,10 +1033,19 @@ string RefreshMetadata::WindowPartitionLineageToJson(const vectorHasError() || result->RowCount() == 0 || result->GetValue(0, 0).IsNull()) { + return false; + } + string json = result->GetValue(0, 0).ToString(); + if (json.empty()) { + return false; + } + string kind; + if (!ExtractJsonString(json, "kind", kind) || kind != "leftjoin_secondary") { + return false; + } + auto extract_array_or_legacy_csv = [&](const string &array_key, const string &legacy_key, vector &values) { + if (ExtractJsonStringArray(json, array_key, values)) { + return; + } + string legacy; + if (ExtractJsonString(json, legacy_key, legacy) && !legacy.empty()) { + values = StringUtil::Split(legacy, ','); + } + }; + extract_array_or_legacy_csv("preserved_cols", "preserved_cols", out.preserved_cols); + extract_array_or_legacy_csv("inner_tables", "inner_table", out.inner_tables); + extract_array_or_legacy_csv("inner_keys", "inner_key", out.inner_keys); + extract_array_or_legacy_csv("pres_tables", "pres_table", out.pres_tables); + extract_array_or_legacy_csv("pres_keys", "pres_key", out.pres_keys); + return ExtractJsonString(json, "sql", out.sql); +} + +string RefreshMetadata::LeftJoinSecondaryMetaToJson(const LeftJoinSecondaryMeta &meta) { + return "{\"kind\":\"leftjoin_secondary\",\"sql\":" + SqlUtils::JsonQuote(meta.sql) + + ",\"preserved_cols\":" + SqlUtils::JsonArray(meta.preserved_cols) + + ",\"inner_tables\":" + SqlUtils::JsonArray(meta.inner_tables) + + ",\"inner_keys\":" + SqlUtils::JsonArray(meta.inner_keys) + + ",\"pres_tables\":" + SqlUtils::JsonArray(meta.pres_tables) + + ",\"pres_keys\":" + SqlUtils::JsonArray(meta.pres_keys) + "}"; +} + vector RefreshMetadata::ExpectedDistinctAuxColumns(const DistinctAuxMeta &meta) { auto expected = meta.cols; expected.push_back("_count"); diff --git a/src/core/sql_utils.cpp b/src/core/sql_utils.cpp index 06d22825..56b86e4e 100644 --- a/src/core/sql_utils.cpp +++ b/src/core/sql_utils.cpp @@ -11,6 +11,9 @@ namespace duckdb { +static constexpr const char *CAST_EXPRESSION_PREFIX = "openivm_expr:"; +static constexpr const char *CAST_COLUMN_PLACEHOLDER = "{openivm_column}"; + static bool IsIdentifierChar(char c) { return std::isalnum(static_cast(c)) || c == '_'; } @@ -43,6 +46,40 @@ static string TrimSQLFragment(const string &input) { return input.substr(start, end - start); } +string SqlUtils::BuildCastSpec(const string &target_type, bool try_cast) { + if (!try_cast) { + return target_type; + } + return string(CAST_EXPRESSION_PREFIX) + "TRY_CAST(" + CAST_COLUMN_PLACEHOLDER + " AS " + target_type + ")"; +} + +string SqlUtils::ApplyCastSpec(const string &column_expression, const string &cast_spec) { + if (cast_spec.empty()) { + return column_expression; + } + if (!StringUtil::StartsWith(cast_spec, CAST_EXPRESSION_PREFIX)) { + return "CAST(" + column_expression + " AS " + cast_spec + ")"; + } + string expression = cast_spec.substr(strlen(CAST_EXPRESSION_PREFIX)); + auto placeholder = expression.find(CAST_COLUMN_PLACEHOLDER); + if (placeholder == string::npos) { + throw InternalException("OpenIVM cast expression is missing its column placeholder"); + } + expression.replace(placeholder, strlen(CAST_COLUMN_PLACEHOLDER), column_expression); + return expression; +} + +string SqlUtils::ComposeCastSpecs(const string &outer_cast_spec, const string &inner_cast_spec) { + if (outer_cast_spec.empty()) { + return inner_cast_spec; + } + if (inner_cast_spec.empty()) { + return outer_cast_spec; + } + string inner_expression = ApplyCastSpec(CAST_COLUMN_PLACEHOLDER, inner_cast_spec); + return string(CAST_EXPRESSION_PREFIX) + ApplyCastSpec(inner_expression, outer_cast_spec); +} + static bool ReadCreateTargetName(const string &sql, const string &object_keyword, string &out) { string lower = StringUtil::Lower(sql); size_t pos = lower.find("create"); @@ -702,6 +739,17 @@ string SqlUtils::BuildAllNullPredicate(const vector &columns) { return result; } +string SqlUtils::BuildAnyNullPredicate(const vector &columns, const string &prefix) { + string result; + for (idx_t i = 0; i < columns.size(); i++) { + if (i > 0) { + result += " OR "; + } + result += prefix + QuoteIdentifier(columns[i]) + " IS NULL"; + } + return result; +} + string SqlUtils::BuildNullSafeMatch(const vector &columns, const string &lhs_alias, const string &rhs_alias) { string result; for (idx_t i = 0; i < columns.size(); i++) { @@ -727,8 +775,25 @@ string SqlUtils::BuildNullSafeKeyPredicate(const vector &columns, const return result; } -string SqlUtils::BuildFullRecomputeSQL(const string &data_table, const string &view_query_sql) { - return "DELETE FROM " + data_table + ";\n" + "INSERT INTO " + data_table + " " + view_query_sql + ";\n"; +string SqlUtils::BuildFullRecomputeSQL(const string &data_table, const string &view_query_sql, + const vector &unique_keys, const string &temp_table) { + if (unique_keys.empty() || temp_table.empty()) { + return "DELETE FROM " + data_table + ";\n" + "INSERT INTO " + data_table + " " + view_query_sql + ";\n"; + } + // Safe form for data tables carrying a UNIQUE index. `DELETE FROM t; INSERT INTO t ...` re-inserts + // keys deleted earlier in the same transaction, and DuckDB's on-disk unique index still reports + // those keys as present -- raising a spurious "Duplicate key ... violates unique constraint". + // Materialize the new contents once, delete only the keys that are gone, then upsert the rest, so + // no non-NULL key is deleted and re-inserted in the same transaction. NULL-containing keys do not + // conflict in a UNIQUE index, so remove them explicitly before INSERT OR REPLACE to avoid duplicates. + string keep_match = BuildNullSafeKeyPredicate(unique_keys, "openivm_new.", "openivm_old."); + string nullable_key = BuildAnyNullPredicate(unique_keys, "openivm_old."); + string sql = "CREATE OR REPLACE TEMP TABLE " + temp_table + " AS " + view_query_sql + ";\n"; + sql += "DELETE FROM " + data_table + " AS openivm_old\nWHERE (" + nullable_key + + ")\n OR NOT EXISTS (\n SELECT 1 FROM " + temp_table + " AS openivm_new WHERE " + keep_match + "\n);\n"; + sql += "INSERT OR REPLACE INTO " + data_table + " SELECT * FROM " + temp_table + ";\n"; + sql += "DROP TABLE IF EXISTS " + temp_table + ";\n"; + return sql; } string SqlUtils::ReplaceAllOccurrences(string haystack, const string &needle, const string &replacement) { diff --git a/src/delta/delta_helpers.cpp b/src/delta/delta_helpers.cpp index 54d0c5d1..8d9c3c20 100644 --- a/src/delta/delta_helpers.cpp +++ b/src/delta/delta_helpers.cpp @@ -2,6 +2,7 @@ #include "core/openivm_constants.hpp" #include "core/openivm_debug.hpp" +#include "core/parser_ddl.hpp" #include "core/plan_rewrite_internal.hpp" #include "core/sql_utils.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" @@ -126,6 +127,9 @@ static DeltaGetResult CreateDuckLakeDeltaNode(ClientContext &context, Binder &bi // Get last snapshot from IVM metadata. Uses a separate connection because // the optimizer holds a lock on the main context during plan rewriting. Connection con(*context.db); + if (auto metadata_state = TransactionalMVMetadataState::TryGet(context)) { + metadata_state->Apply(con); + } auto snap_result = con.Query("SELECT last_snapshot_id FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name = '" + SqlUtils::EscapeValue(view_name) + "' AND table_name = '" + SqlUtils::EscapeValue(table_name) + "'"); @@ -345,6 +349,9 @@ DeltaGetResult CreateDeltaGetNode(ClientContext &context, Binder &binder, Logica // Timestamp filter Connection con(*context.db); + if (auto metadata_state = TransactionalMVMetadataState::TryGet(context)) { + metadata_state->Apply(con); + } con.SetAutoCommit(false); auto timestamp_query = "select last_update from " + string(openivm::DELTA_TABLES_TABLE) + " where view_name = '" + SqlUtils::EscapeValue(view_name) + "' and table_name = '" + diff --git a/src/delta/operators/ducklake_join.cpp b/src/delta/operators/ducklake_join.cpp index 647d72a3..cfba3cc2 100644 --- a/src/delta/operators/ducklake_join.cpp +++ b/src/delta/operators/ducklake_join.cpp @@ -11,10 +11,14 @@ #include "duckdb/main/connection.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/operator/logical_filter.hpp" +#include "duckdb/planner/operator/logical_join.hpp" #include "duckdb/planner/operator/logical_projection.hpp" #include "storage/ducklake_scan.hpp" #include "upsert/refresh_internal.hpp" +#include + namespace duckdb { struct DuckLakeJoinColumnRef { @@ -22,6 +26,140 @@ struct DuckLakeJoinColumnRef { string column_name; }; +static bool IsJoinOperator(LogicalOperatorType type) { + return type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN || type == LogicalOperatorType::LOGICAL_CROSS_PRODUCT || + type == LogicalOperatorType::LOGICAL_ANY_JOIN; +} + +static bool CollectDuckLakeJoinLeaves(LogicalOperator *node, vector &path, vector &leaves, + bool is_right_of_left, string &fallback_reason) { + if (IsJoinOperator(node->type)) { + auto *join = dynamic_cast(node); + bool left_is_nullable = join && (join->join_type == JoinType::RIGHT || join->join_type == JoinType::OUTER); + bool right_is_nullable = join && (join->join_type == JoinType::LEFT || join->join_type == JoinType::OUTER); + for (size_t child_idx = 0; child_idx < node->children.size(); child_idx++) { + path.push_back(child_idx); + bool child_is_nullable = is_right_of_left || (child_idx == 0 ? left_is_nullable : right_is_nullable); + if (!CollectDuckLakeJoinLeaves(node->children[child_idx].get(), path, leaves, child_is_nullable, + fallback_reason)) { + return false; + } + path.pop_back(); + } + return true; + } + if (node->type == LogicalOperatorType::LOGICAL_PROJECTION || node->type == LogicalOperatorType::LOGICAL_FILTER) { + if (node->children.size() != 1) { + fallback_reason = node->GetName() + " does not have exactly one child"; + return false; + } + path.push_back(0); + bool result = + CollectDuckLakeJoinLeaves(node->children[0].get(), path, leaves, is_right_of_left, fallback_reason); + path.pop_back(); + return result; + } + if (node->type != LogicalOperatorType::LOGICAL_GET) { + fallback_reason = "unsupported wrapper " + node->GetName(); + return false; + } + auto *get = dynamic_cast(node); + if (!get || get->function.name != "ducklake_scan" || !get->function.function_info) { + fallback_reason = "non-DuckLake scan " + node->GetName(); + return false; + } + leaves.push_back({path, get, node, is_right_of_left}); + return true; +} + +bool TryCollectDuckLakeJoinLeaves(LogicalOperator *node, vector &leaves, string &fallback_reason) { + leaves.clear(); + fallback_reason.clear(); + vector path; + if (!CollectDuckLakeJoinLeaves(node, path, leaves, false, fallback_reason)) { + leaves.clear(); + return false; + } + if (leaves.empty()) { + fallback_reason = "no DuckLake scans found"; + return false; + } + OPENIVM_DEBUG_PRINT("[DuckLakeJoin] Flattened leaf count: %zu\n", leaves.size()); + return true; +} + +static void AddDuckLakeLeafColumnRefs(LogicalOperator *root, const JoinLeafInfo &leaf, size_t leaf_index, + unordered_map &column_refs) { + vector ancestors; + ancestors.reserve(leaf.path.size()); + LogicalOperator *node = root; + for (auto child_idx : leaf.path) { + if (child_idx >= node->children.size()) { + throw InternalException("DuckLakeJoin: leaf path child %llu is out of bounds", + static_cast(child_idx)); + } + ancestors.push_back(node); + node = node->children[child_idx].get(); + } + + auto *get = leaf.get; + if (!get) { + return; + } + unordered_map visible_columns; + auto bindings = get->GetColumnBindings(); + auto &column_ids = get->GetColumnIds(); + for (idx_t output_idx = 0; output_idx < bindings.size(); output_idx++) { + idx_t column_id_idx = output_idx; + if (!get->projection_ids.empty()) { + if (output_idx >= get->projection_ids.size()) { + continue; + } + column_id_idx = get->projection_ids[output_idx]; + } + if (column_id_idx >= column_ids.size() || column_ids[column_id_idx].IsVirtualColumn()) { + continue; + } + visible_columns[DeltaJoinBindingKey(bindings[output_idx])] = get->GetColumnName(column_ids[column_id_idx]); + } + + auto record_visible = [&]() { + for (auto &entry : visible_columns) { + column_refs[entry.first] = {leaf_index, entry.second}; + } + }; + record_visible(); + + for (size_t depth = leaf.path.size(); depth-- > 0;) { + auto *parent = ancestors[depth]; + unordered_map parent_columns; + if (parent->type == LogicalOperatorType::LOGICAL_PROJECTION) { + auto &projection = parent->Cast(); + auto parent_bindings = parent->GetColumnBindings(); + idx_t count = std::min(projection.expressions.size(), parent_bindings.size()); + for (idx_t expr_idx = 0; expr_idx < count; expr_idx++) { + ColumnBinding child_binding; + if (!TryGetDeltaJoinColumnRef(*projection.expressions[expr_idx], child_binding)) { + continue; + } + auto child_entry = visible_columns.find(DeltaJoinBindingKey(child_binding)); + if (child_entry != visible_columns.end()) { + parent_columns[DeltaJoinBindingKey(parent_bindings[expr_idx])] = child_entry->second; + } + } + } else { + for (auto &binding : parent->GetColumnBindings()) { + auto child_entry = visible_columns.find(DeltaJoinBindingKey(binding)); + if (child_entry != visible_columns.end()) { + parent_columns[DeltaJoinBindingKey(binding)] = child_entry->second; + } + } + } + visible_columns = std::move(parent_columns); + record_visible(); + } +} + static string DuckLakeQualifiedTable(const string &catalog, const string &schema, const string &table_name, int64_t snapshot_id) { string result = SqlUtils::QuoteIdentifier(catalog) + "." + SqlUtils::QuoteIdentifier(schema) + "." + @@ -65,44 +203,128 @@ static bool DuckLakeDeltaKeyHasMatch(Connection &con, const string &catalog, con return result->GetValue(0, 0).GetValue(); } -// ============================================================================ -// PinToOldSnapshot: set a DuckLake scan to read the table at last_snapshot_id -// ============================================================================ +static bool PathStartsWith(const vector &path, const vector &prefix) { + return path.size() >= prefix.size() && std::equal(prefix.begin(), prefix.end(), path.begin()); +} + +static void DemoteOuterJoinsForLeaf(LogicalOperator *node, const vector &leaf_path, vector &path) { + if (auto *join = dynamic_cast(node)) { + bool left_has_delta = false; + bool right_has_delta = false; + path.push_back(0); + left_has_delta = PathStartsWith(leaf_path, path); + path.pop_back(); + path.push_back(1); + right_has_delta = PathStartsWith(leaf_path, path); + path.pop_back(); -/// Walk the subtree and pin any DuckLake scan with the given table_index to -/// the old snapshot. LPTS detects the historical snapshot and emits AT VERSION. -static void PinToOldSnapshot(LogicalOperator &op, idx_t table_index, idx_t old_snapshot_id) { - if (op.type == LogicalOperatorType::LOGICAL_GET) { - auto &get = op.Cast(); - if (get.table_index == table_index && get.function.name == "ducklake_scan" && get.function.function_info) { - auto &func_info = get.function.function_info->Cast(); - func_info.snapshot.snapshot_id = old_snapshot_id; - OPENIVM_DEBUG_PRINT("[DuckLakeJoin] Pinned table_index=%lu to old snapshot %lu\n", - (unsigned long)table_index, (unsigned long)old_snapshot_id); + if ((join->join_type == JoinType::LEFT && right_has_delta) || + (join->join_type == JoinType::RIGHT && left_has_delta) || + (join->join_type == JoinType::OUTER && (left_has_delta || right_has_delta))) { + join->join_type = JoinType::INNER; } } - for (auto &child : op.children) { - PinToOldSnapshot(*child, table_index, old_snapshot_id); + for (size_t child_idx = 0; child_idx < node->children.size(); child_idx++) { + path.push_back(child_idx); + DemoteOuterJoinsForLeaf(node->children[child_idx].get(), leaf_path, path); + path.pop_back(); } } +static void DemoteOuterJoinsForLeaf(LogicalOperator *node, const vector &leaf_path) { + vector path; + DemoteOuterJoinsForLeaf(node, leaf_path, path); +} + +static idx_t FindBindingPosition(LogicalOperator &op, const ColumnBinding &binding, const char *context_label) { + auto bindings = op.GetColumnBindings(); + for (idx_t binding_idx = 0; binding_idx < bindings.size(); binding_idx++) { + if (bindings[binding_idx] == binding) { + return binding_idx; + } + } + throw InternalException("%s: multiplicity binding %s is not exposed by %s", context_label, + binding.ToString().c_str(), op.GetName().c_str()); +} + +static ColumnBinding PropagateMultiplicityThroughPath(unique_ptr &term, + const vector &leaf_path, ColumnBinding mul_binding) { + vector ancestors; + ancestors.reserve(leaf_path.size()); + LogicalOperator *node = term.get(); + for (size_t depth = 0; depth < leaf_path.size(); depth++) { + if (leaf_path[depth] >= node->children.size()) { + throw InternalException("DuckLakeJoin: leaf path child %llu out of bounds at depth %llu", + static_cast(leaf_path[depth]), static_cast(depth)); + } + ancestors.push_back(node); + node = node->children[leaf_path[depth]].get(); + } + + for (size_t depth = leaf_path.size(); depth-- > 0;) { + auto *parent = ancestors[depth]; + size_t child_side = leaf_path[depth]; + auto &child = *parent->children[child_side]; + idx_t mul_idx = FindBindingPosition(child, mul_binding, "DuckLakeJoin"); + + if (parent->type == LogicalOperatorType::LOGICAL_PROJECTION) { + auto &projection = parent->Cast(); + projection.expressions.push_back(make_uniq(LogicalType::INTEGER, mul_binding)); + mul_binding = ColumnBinding(projection.table_index, projection.expressions.size() - 1); + continue; + } + if (parent->type == LogicalOperatorType::LOGICAL_FILTER) { + auto &filter = parent->Cast(); + if (!filter.projection_map.empty() && std::find(filter.projection_map.begin(), filter.projection_map.end(), + mul_idx) == filter.projection_map.end()) { + filter.projection_map.push_back(mul_idx); + } + continue; + } + if (auto *join = dynamic_cast(parent)) { + auto &projection_map = child_side == 0 ? join->left_projection_map : join->right_projection_map; + if (!projection_map.empty() && + std::find(projection_map.begin(), projection_map.end(), mul_idx) == projection_map.end()) { + projection_map.push_back(mul_idx); + } + continue; + } + throw InternalException("DuckLakeJoin: unsupported ancestor %s in flattened path", parent->GetName()); + } + return mul_binding; +} + // ============================================================================ // BuildDuckLakeJoinTerms: N-term telescoping delta product // ============================================================================ vector> BuildDuckLakeJoinTerms(DeltaOperatorInput input, ClientContext &context, Binder &binder, const vector &leaves, - bool has_left_join) { + bool has_left_join, bool flattened_leaves) { size_t N = leaves.size(); vector> terms; - // Collect last_snapshot_id for each leaf upfront (one query per table). + // Collect last_snapshot_id for all leaves upfront in one metadata query. Connection con(*context.db); vector old_snapshots(N); vector current_snapshots(N, -1); vector table_catalogs(N); vector table_schemas(N); vector table_names(N); + unordered_map stored_snapshots; + auto snapshot_result = con.Query("SELECT table_name, last_snapshot_id FROM " + string(openivm::DELTA_TABLES_TABLE) + + " WHERE view_name = '" + SqlUtils::EscapeValue(input.context.view) + "'"); + if (snapshot_result->HasError()) { + throw Exception(ExceptionType::CATALOG, "IVM: could not read DuckLake snapshot metadata for view '" + + input.context.view + "': " + snapshot_result->GetError()); + } + for (idx_t row = 0; row < snapshot_result->RowCount(); row++) { + if (snapshot_result->GetValue(0, row).IsNull() || snapshot_result->GetValue(1, row).IsNull()) { + continue; + } + stored_snapshots[StringUtil::Lower(snapshot_result->GetValue(0, row).ToString())] = + snapshot_result->GetValue(1, row).GetValue(); + } for (size_t i = 0; i < N; i++) { auto *get = leaves[i].get ? leaves[i].get : FindGetInSubtree(leaves[i].node); D_ASSERT(get); @@ -111,14 +333,12 @@ vector> BuildDuckLakeJoinTerms(DeltaOperatorInput in table_catalogs[i] = table_ref->ParentCatalog().GetName(); table_schemas[i] = table_ref->schema.name; table_names[i] = table_name; - auto snap_result = con.Query("SELECT last_snapshot_id FROM " + string(openivm::DELTA_TABLES_TABLE) + - " WHERE view_name = '" + SqlUtils::EscapeValue(input.context.view) + - "' AND table_name = '" + SqlUtils::EscapeValue(table_name) + "'"); - if (snap_result->HasError() || snap_result->RowCount() == 0 || snap_result->GetValue(0, 0).IsNull()) { + auto stored_snapshot = stored_snapshots.find(StringUtil::Lower(table_name)); + if (stored_snapshot == stored_snapshots.end()) { throw Exception(ExceptionType::CATALOG, "IVM: no snapshot ID recorded for DuckLake table '" + table_name + "' in view '" + input.context.view + "'"); } - old_snapshots[i] = snap_result->GetValue(0, 0).GetValue(); + old_snapshots[i] = stored_snapshot->second; if (get->function.name == "ducklake_scan" && get->function.function_info) { auto &func_info = get->function.function_info->Cast(); current_snapshots[i] = static_cast(func_info.snapshot.snapshot_id); @@ -132,9 +352,32 @@ vector> BuildDuckLakeJoinTerms(DeltaOperatorInput in // last_snapshot_id != current_snapshot because another table changed. Probe table-level changes before // building the term so unchanged tables do not force a full plan copy/rewrite. vector empty_table_delta(N, false); + vector activity_known(N, false); if (skip_empty_enabled) { + auto compile_facts = openivm::CompileFactsContextSlot::Get(context); + size_t reused_activity_count = 0; + for (size_t i = 0; i < N; i++) { + for (auto &entry : compile_facts.delta_shape) { + if (!StringUtil::CIEquals(SqlUtils::LastIdentifierPart(entry.first), table_names[i])) { + continue; + } + if (StringUtil::CIEquals(entry.second, "UNCHANGED")) { + empty_table_delta[i] = true; + activity_known[i] = true; + } else if (StringUtil::CIEquals(entry.second, "INSERT_ONLY") || + StringUtil::CIEquals(entry.second, "MIXED")) { + activity_known[i] = true; + } + reused_activity_count += activity_known[i] ? 1 : 0; + break; + } + } + OPENIVM_DEBUG_PRINT("[DuckLakeJoin] Reused source activity for %zu/%zu leaves\n", reused_activity_count, N); RefreshMetadata metadata(con); for (size_t i = 0; i < N; i++) { + if (activity_known[i]) { + continue; + } if (current_snapshots[i] < 0) { continue; } @@ -185,28 +428,13 @@ vector> BuildDuckLakeJoinTerms(DeltaOperatorInput in if (!get) { continue; } - auto bindings = get->GetColumnBindings(); - auto &column_ids = get->GetColumnIds(); - idx_t count = std::min(bindings.size(), column_ids.size()); - for (idx_t col_idx = 0; col_idx < count; col_idx++) { - if (column_ids[col_idx].IsVirtualColumn()) { - continue; - } - column_refs[DeltaJoinBindingKey(bindings[col_idx])] = {i, get->GetColumnName(column_ids[col_idx])}; - } - auto leaf_bindings = leaves[i].node->GetColumnBindings(); - idx_t leaf_count = std::min(leaf_bindings.size(), count); - for (idx_t col_idx = 0; col_idx < leaf_count; col_idx++) { - if (column_ids[col_idx].IsVirtualColumn()) { - continue; - } - column_refs[DeltaJoinBindingKey(leaf_bindings[col_idx])] = {i, get->GetColumnName(column_ids[col_idx])}; - } + AddDuckLakeLeafColumnRefs(input.plan.get(), leaves[i], i, column_refs); } CollectDeltaJoinKeyProbes(input.plan.get(), column_refs, key_probes); } - OPENIVM_DEBUG_PRINT("[DuckLakeJoin] Building N-term telescoping delta terms (%zu leaves)\n", N); + OPENIVM_DEBUG_PRINT("[DuckLakeJoin] Building N-term telescoping delta terms (%zu leaves, flattened=%s)\n", N, + flattened_leaves ? "true" : "false"); for (size_t i = 0; i < N; i++) { // Skip term if this table has no changes since last refresh. @@ -243,28 +471,40 @@ vector> BuildDuckLakeJoinTerms(DeltaOperatorInput in // Re-collect leaves from the copied plan (pointers change after Copy). vector term_leaves; - CollectJoinLeaves(term.get(), {}, term_leaves); + if (flattened_leaves) { + string fallback_reason; + if (!TryCollectDuckLakeJoinLeaves(term.get(), term_leaves, fallback_reason)) { + throw InternalException("DuckLakeJoin: copied plan no longer supports flattening: %s", + fallback_reason.c_str()); + } + } else { + CollectJoinLeaves(term.get(), {}, term_leaves); + } D_ASSERT(term_leaves.size() == N); LogicalOperator *term_root = term.get(); - // For LEFT JOINs: demote to INNER when only right-side leaves have deltas. + // Demote only the outer joins whose NULL-supplying subtree contains this + // term's delta. Preserved joins elsewhere in a left-deep star must remain + // outer joins so unmatched rows continue to flow to later dimensions. if (has_left_join) { - if (!leaves[i].is_right_of_left_join) { - // Delta is on left side — keep LEFT JOIN semantics - } else { - // Delta is only on the right side — demote to INNER + if (flattened_leaves) { + DemoteOuterJoinsForLeaf(term.get(), term_leaves[i].path); + } else if (leaves[i].is_right_of_left_join) { DemoteLeftJoins(term.get()); } } // Replace leaf[i] with its delta scan. ColumnBinding mul_binding; - if (term_leaves[i].get) { + if (flattened_leaves || term_leaves[i].get) { // Simple GET leaf — replace directly. DeltaGetResult delta_result = CreateDeltaGetNode(context, binder, term_leaves[i].get, input.context.view); mul_binding = delta_result.mul_binding; GetNodeAtPath(term, term_leaves[i].path) = std::move(delta_result.node); + if (flattened_leaves) { + mul_binding = PropagateMultiplicityThroughPath(term, term_leaves[i].path, mul_binding); + } } else { // GET wrapped in projections/filters — rewrite the entire subtree. auto &subtree_ref = GetNodeAtPath(term, term_leaves[i].path); @@ -272,7 +512,9 @@ vector> BuildDuckLakeJoinTerms(DeltaOperatorInput in mul_binding = rewritten.mul_binding; subtree_ref = std::move(rewritten.op); } - UpdateParentProjectionMap(term, term_leaves[i], mul_binding); + if (!flattened_leaves) { + UpdateParentProjectionMap(term, term_leaves[i], mul_binding); + } // Telescoping: pin leaves j > i to old snapshot (AT VERSION). // Leaves j < i stay at current state (already the default). @@ -315,6 +557,7 @@ vector> BuildDuckLakeJoinTerms(DeltaOperatorInput in OPENIVM_DEBUG_PRINT("[DuckLakeJoin] Term %zu: delta on leaf %zu, %zu leaves pinned to old\n", i, i, N - i - 1); } + OPENIVM_DEBUG_PRINT("[DuckLakeJoin] Active N-term count: %zu/%zu\n", terms.size(), N); return terms; } diff --git a/src/delta/operators/join.cpp b/src/delta/operators/join.cpp index ea4d6288..9ce532c1 100644 --- a/src/delta/operators/join.cpp +++ b/src/delta/operators/join.cpp @@ -4,6 +4,7 @@ #include "delta/operators/join_key_probe.hpp" #include "core/openivm_constants.hpp" #include "core/openivm_debug.hpp" +#include "core/plan_rewrite_internal.hpp" #include "core/sql_utils.hpp" #include "upsert/refresh_index_regen.hpp" #include "match/constraint_cache.hpp" @@ -13,14 +14,21 @@ #include "duckdb/parser/constraints/foreign_key_constraint.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/function/function_binder.hpp" +#include "duckdb/planner/expression/bound_aggregate_expression.hpp" +#include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/planner/expression/bound_operator_expression.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/expression/bound_comparison_expression.hpp" +#include "duckdb/planner/expression/bound_conjunction_expression.hpp" #include "duckdb/planner/expression/bound_constant_expression.hpp" #include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/operator/logical_aggregate.hpp" #include "duckdb/planner/operator/logical_any_join.hpp" #include "duckdb/planner/operator/logical_comparison_join.hpp" #include "duckdb/planner/operator/logical_cteref.hpp" +#include "duckdb/planner/operator/logical_filter.hpp" #include "duckdb/planner/operator/logical_join.hpp" +#include "duckdb/planner/operator/logical_materialized_cte.hpp" #include "duckdb/planner/operator/logical_projection.hpp" #include @@ -393,6 +401,389 @@ static void DemoteLeftJoinsForMask(LogicalOperator *node, const vectorGetColumnBindings(); + for (idx_t i = 0; i < bindings.size(); i++) { + if (bindings[i] == binding) { + out_pos = i; + return true; + } + } + return false; + } + if (node->type == LogicalOperatorType::LOGICAL_PROJECTION && !node->children.empty()) { + auto &projection = node->Cast(); + auto bindings = node->GetColumnBindings(); + idx_t count = std::min(bindings.size(), projection.expressions.size()); + for (idx_t i = 0; i < count; i++) { + if (bindings[i] != binding) { + continue; + } + ColumnBinding child_binding; + if (!TryGetDeltaJoinColumnRef(*projection.expressions[i], child_binding)) { + return false; + } + return ResolveKeyToGetPosition(node->children[0].get(), child_binding, target_get, out_pos); + } + return false; + } + if (node->children.size() == 1) { + auto bindings = node->GetColumnBindings(); + auto child_bindings = node->children[0]->GetColumnBindings(); + idx_t count = std::min(bindings.size(), child_bindings.size()); + for (idx_t i = 0; i < count; i++) { + if (bindings[i] == binding) { + return ResolveKeyToGetPosition(node->children[0].get(), child_bindings[i], target_get, out_pos); + } + } + } + return false; +} + +struct TransitioningKeySet { + unique_ptr node; + ColumnBinding key_binding; +}; + +struct TransitioningKeyCTEDefinition { + string name; + idx_t cte_index; + unique_ptr node; + vector types; + vector names; +}; + +// Build the set of DISTINCT key values (from base_get's column at key_pos) whose match-count within +// base_get's own rows ACTUALLY transitions across zero between old (pre-delta) and new (current, +// post-delta) state -- i.e. old_count>0 && new_count==0, or old_count==0 && new_count>0. A key merely +// appearing in the delta is NOT sufficient: for a 1:many relationship (e.g. one customer with many +// orders) a single changed order must not suppress the customer's row when other, unchanged orders +// still match. Returns nullptr if unsupported (caller must then skip the optimization, not guess). +static unique_ptr BuildTransitioningKeySetImpl(ClientContext &context, Binder &binder, + LogicalGet *base_get, idx_t key_pos, + const string &view_name) { + auto delta_result = CreateDeltaGetNode(context, binder, base_get, view_name); + auto delta_renumbered = renumber_and_rebind_subtree(std::move(delta_result.node), binder); + auto delta_bindings = delta_renumbered.op->GetColumnBindings(); + auto delta_types = delta_renumbered.op->types; + if (delta_bindings.empty() || key_pos >= delta_bindings.size() - 1) { + return nullptr; + } + idx_t mul_pos = delta_bindings.size() - 1; // CreateDeltaGetNode/CompactDeltaNode appends multiplicity last. + ColumnBinding delta_key_binding = delta_bindings[key_pos]; + ColumnBinding delta_mul_binding = delta_bindings[mul_pos]; + LogicalType key_type = delta_types[key_pos]; + LogicalType mul_type = delta_types[mul_pos]; + + // Restrict the current-state count to keys present in this delta before + // aggregating. Without this join, every inclusion-exclusion term hashes every + // key in the nullable base table even when only one key changed. + auto affected_delta = renumber_and_rebind_subtree(delta_renumbered.op->Copy(context), binder); + auto affected_bindings = affected_delta.op->GetColumnBindings(); + if (key_pos >= affected_bindings.size()) { + return nullptr; + } + auto affected_group_index = binder.GenerateTableIndex(); + auto affected_aggregate_index = binder.GenerateTableIndex(); + auto affected_keys = + make_uniq(affected_group_index, affected_aggregate_index, vector>()); + affected_keys->groups.push_back(make_uniq(key_type, affected_bindings[key_pos])); + affected_keys->group_stats.push_back(make_uniq(BaseStatistics::CreateUnknown(key_type))); + GroupingSet affected_grouping_set; + affected_grouping_set.insert(0); + affected_keys->grouping_sets.push_back(std::move(affected_grouping_set)); + affected_keys->children.push_back(std::move(affected_delta.op)); + affected_keys->ResolveOperatorTypes(); + ColumnBinding affected_key_binding = affected_keys->GetColumnBindings()[0]; + + // Fresh scan of the SAME base table (current/post-delta state), filtered to + // affected keys and grouped by key with COUNT(*). + auto base_copy_op = base_get->Copy(context); + auto base_renumbered = renumber_and_rebind_subtree(std::move(base_copy_op), binder); + auto base_scan_bindings = base_renumbered.op->GetColumnBindings(); + if (key_pos >= base_scan_bindings.size()) { + return nullptr; + } + ColumnBinding base_key_source_binding = base_scan_bindings[key_pos]; + auto affected_condition = make_uniq( + ExpressionType::COMPARE_EQUAL, make_uniq(key_type, base_key_source_binding), + make_uniq(key_type, affected_key_binding)); + auto affected_base = + LogicalComparisonJoin::CreateJoin(context, JoinType::INNER, JoinRefType::REGULAR, std::move(base_renumbered.op), + std::move(affected_keys), std::move(affected_condition)); + affected_base->ResolveOperatorTypes(); + + auto group_index = binder.GenerateTableIndex(); + auto aggregate_index = binder.GenerateTableIndex(); + auto count_func = BindAggregateByName(context, "count_star", {}); + auto count_expr = make_uniq(std::move(count_func), vector>(), + nullptr, nullptr, AggregateType::NON_DISTINCT); + count_expr->alias = "current_count"; + vector> aggregates; + aggregates.push_back(std::move(count_expr)); + auto base_agg = make_uniq(group_index, aggregate_index, std::move(aggregates)); + base_agg->groups.push_back(make_uniq(key_type, base_key_source_binding)); + base_agg->group_stats.push_back(make_uniq(BaseStatistics::CreateUnknown(key_type))); + GroupingSet base_grouping_set; + base_grouping_set.insert(0); + base_agg->grouping_sets.push_back(std::move(base_grouping_set)); + base_agg->children.push_back(std::move(affected_base)); + base_agg->ResolveOperatorTypes(); + auto base_agg_bindings = base_agg->GetColumnBindings(); + ColumnBinding base_key_binding = base_agg_bindings[0]; + ColumnBinding base_count_binding = base_agg_bindings[1]; + LogicalType count_type = base_agg->types[1]; + + // LEFT JOIN: delta (left) LEFT JOIN base_agg (right) ON key. LEFT so a key with new_count=0 (no + // rows left in base_agg's GROUP BY at all, e.g. all matches deleted) still appears, with a NULL + // current_count treated as 0 below. + auto join_cond = make_uniq( + ExpressionType::COMPARE_EQUAL, make_uniq(key_type, delta_key_binding), + make_uniq(key_type, base_key_binding)); + auto joined = + LogicalComparisonJoin::CreateJoin(context, JoinType::LEFT, JoinRefType::REGULAR, std::move(delta_renumbered.op), + std::move(base_agg), std::move(join_cond)); + joined->ResolveOperatorTypes(); + + // new_count = COALESCE(current_count, 0); old_count = new_count - net_multiplicity; + // keep only keys where (old_count>0) != (new_count>0) -- an actual 0<->>0 transition. + // COALESCE is not a catalog scalar function -- it's a bound operator expression. + FunctionBinder fbinder(binder); + auto build_new_count = [&]() -> unique_ptr { + auto coalesce_expr = make_uniq(ExpressionType::OPERATOR_COALESCE, count_type); + coalesce_expr->children.push_back(make_uniq(count_type, base_count_binding)); + coalesce_expr->children.push_back(make_uniq(Value::BIGINT(0))); + return coalesce_expr; + }; + auto mul_as_count_type = BoundCastExpression::AddCastToType( + context, make_uniq(mul_type, delta_mul_binding), count_type); + vector> sub_args; + sub_args.push_back(build_new_count()); + sub_args.push_back(std::move(mul_as_count_type)); + ErrorData sub_err; + auto old_count_expr = fbinder.BindScalarFunction(DEFAULT_SCHEMA, "-", std::move(sub_args), sub_err, true); + if (!old_count_expr) { + throw InternalException("DeltaJoin: failed to bind '-' for transition check: %s", sub_err.RawMessage()); + } + // Only the DOWNWARD transition (old>0, new=0) is a phantom-NULL-pad risk here: the "other" side + // row is LEFT JOINed against base_get's CURRENT (already-merged) state, so a key that just LOST + // its last match reads as unmatched in "current" even though it was genuinely matched pre-batch -- + // that phantom dangling row must be excluded (the higher-order term supplies the real removal row + // instead). A key that just GAINED its first match (old=0, new>0) is the opposite: "current" + // already reflects that real, new match, so this term's row IS the correct contribution and must + // NOT be excluded -- excluding it would drop the row entirely, since no other term re-adds it. + auto old_gt_zero = + make_uniq(ExpressionType::COMPARE_GREATERTHAN, std::move(old_count_expr), + make_uniq(Value::BIGINT(0))); + auto new_eq_zero = make_uniq(ExpressionType::COMPARE_EQUAL, build_new_count(), + make_uniq(Value::BIGINT(0))); + auto transition_expr = make_uniq(ExpressionType::CONJUNCTION_AND, + std::move(old_gt_zero), std::move(new_eq_zero)); + auto filter = make_uniq(std::move(transition_expr)); + filter->children.push_back(std::move(joined)); + filter->ResolveOperatorTypes(); + + // Project just the key column. + vector> proj_exprs; + proj_exprs.push_back(make_uniq(key_type, delta_key_binding)); + auto proj_index = binder.GenerateTableIndex(); + auto projection = make_uniq(proj_index, std::move(proj_exprs)); + projection->children.push_back(std::move(filter)); + projection->ResolveOperatorTypes(); + ColumnBinding final_key_binding = projection->GetColumnBindings()[0]; + + auto result = make_uniq(); + result->node = std::move(projection); + result->key_binding = final_key_binding; + return result; +} + +static unique_ptr +GetTransitioningKeySetRef(ClientContext &context, Binder &binder, LogicalGet *base_get, idx_t key_pos, + size_t leaf_index, const string &view_name, + map, idx_t> &transition_cte_indexes, + vector &transition_ctes) { + auto cache_key = make_pair(leaf_index, key_pos); + auto existing = transition_cte_indexes.find(cache_key); + idx_t definition_index; + if (existing == transition_cte_indexes.end()) { + auto transitioning_keys = BuildTransitioningKeySetImpl(context, binder, base_get, key_pos, view_name); + if (!transitioning_keys) { + return nullptr; + } + D_ASSERT(transitioning_keys->node->types.size() == 1); + TransitioningKeyCTEDefinition definition; + definition.cte_index = binder.GenerateTableIndex(); + definition.name = "openivm_transition_keys_" + to_string(definition.cte_index); + definition.types = transitioning_keys->node->types; + definition.names = {"openivm_transition_key"}; + definition.node = std::move(transitioning_keys->node); + definition_index = transition_ctes.size(); + transition_ctes.push_back(std::move(definition)); + transition_cte_indexes[cache_key] = definition_index; + OPENIVM_DEBUG_PRINT("[DeltaJoin] Materialized transition-key CTE for leaf=%zu key=%zu\n", leaf_index, key_pos); + } else { + definition_index = existing->second; + } + auto &definition = transition_ctes[definition_index]; + auto ref_table_index = binder.GenerateTableIndex(); + auto ref = make_uniq(ref_table_index, definition.cte_index, definition.types, definition.names); + ref->ResolveOperatorTypes(); + auto result = make_uniq(); + result->key_binding = ref->GetColumnBindings()[0]; + result->node = std::move(ref); + return result; +} + +// After DemoteLeftJoinsForMask, a LEFT/RIGHT join may remain un-demoted (kept as an outer join) +// because its null-supplying side has no delta leaf in THIS mask — so it reads that side as +// "current" state. If that null-supplying leaf independently has ANY pending delta (from this +// same refresh, just not part of this term's mask), "current" silently mixes old and new state +// for the dangling-tuple decision: a preserved-side row whose matching child rows are ALSO being +// deleted in this same batch gets a spurious extra dangling row here, on top of the correct +// removal already produced by the higher-order term that covers {this leaf, that leaf} together +// (double-count). This is the outer-join analogue of the classic T_old-vs-T_new join delta +// problem — LEFT JOIN's NULL-padding is a non-linear threshold function (unlike inner join's +// bilinear product), so it cannot be decomposed by inclusion-exclusion the way ordinary joins +// can; instead (matching Larson & Zhou / DBSP's semijoin-count treatment), we must exclude keys +// that are themselves transitioning, since those are exclusively owned by the term(s) that +// include this leaf's delta bit. Guard: anti-join the null-supplying leaf's current scan against +// its own delta table on the join key, excluding any key present there. Only applies when the +// null-supplying side is a single, unwrapped base-table leaf directly under the join (bails +// silently otherwise, matching this file's existing unsupported-shape convention). +static void GuardKeptOuterJoinsForMaskRec(ClientContext &context, Binder &binder, LogicalOperator *node, + const vector &leaves, uint64_t leaf_has_delta_mask, + const string &view_name, bool portable_anti_guard, + map, idx_t> &transition_cte_indexes, + vector &transition_ctes, + vector &path) { + if (node->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + auto *j = dynamic_cast(node); + if (j && (j->join_type == JoinType::LEFT || j->join_type == JoinType::RIGHT) && !j->conditions.empty()) { + idx_t null_side_child = j->join_type == JoinType::LEFT ? 1 : 0; + // leaves[]/path matching is ONLY used to find null_leaf_idx (a structural, path-based + // lookup unaffected by renumbering) for the leaf_has_delta_mask check below. The actual + // resolution below must use j->children[null_side_child] directly -- that subtree lives + // in `term`'s OWN freshly-renumbered copy, whereas leaves[i].node/.get are stale pointers + // into the ORIGINAL (pre-copy) plan and carry different table indices entirely. + path.push_back(null_side_child); + size_t null_leaf_idx = SIZE_MAX; + for (size_t i = 0; i < leaves.size(); i++) { + if (leaves[i].path == path) { + null_leaf_idx = i; + break; + } + } + path.pop_back(); + idx_t other_child = 1 - null_side_child; + LogicalOperator *current_null_side_node = j->children[null_side_child].get(); + LogicalGet *current_null_side_get = FindGetInSubtree(current_null_side_node); + if (current_null_side_get && null_leaf_idx != SIZE_MAX && (leaf_has_delta_mask & (1ULL << null_leaf_idx))) { + auto &cond = j->conditions[0]; + // null_key_expr references the null-supplying side (used to size/position the Δ-scan probe + // we build below). other_key_expr references the OTHER (mask-driven, preserved) side -- the + // side that must actually be filtered. A RIGHT/LEFT join always outputs every row of its + // preserved side (matched or NULL-padded); excluding rows from the null-supplying side's + // scan cannot prevent a dangling row, since "no match found" is exactly what happens + // regardless. What must be excluded is the OTHER side's row itself: if its key ALSO has a + // pending delta on the null-supplying side, this term must neither match nor dangling-pad + // it -- that key is handled entirely by the higher-order term that includes both leaves. + auto &null_key_expr = j->join_type == JoinType::LEFT ? cond.right : cond.left; + auto &other_key_expr = j->join_type == JoinType::LEFT ? cond.left : cond.right; + BoundColumnRefExpression *key_expr = + null_key_expr->expression_class == ExpressionClass::BOUND_COLUMN_REF + ? &null_key_expr->Cast() + : nullptr; + idx_t key_pos = DConstants::INVALID_INDEX; + if (key_expr) { + ResolveKeyToGetPosition(current_null_side_node, key_expr->binding, current_null_side_get, key_pos); + } + if (key_expr && key_pos != DConstants::INVALID_INDEX && + other_key_expr->expression_class == ExpressionClass::BOUND_COLUMN_REF) { + // Build the set of keys whose match-count on the null-supplying side ACTUALLY + // transitions across zero (old>0,new=0 or old=0,new>0). A key merely appearing in + // the delta is NOT enough to exclude it: for a 1:many relationship (e.g. one + // customer, many orders) a single changed order must not suppress the customer's + // row when the customer still has OTHER, unchanged matches. Only a true 0<->>0 + // transition means "this key's presence here is fully owned by the higher-order + // term" -- matching the same match-count-transition principle as the secondary-delta + // fix, just applied here to avoid a double-count instead of to add a missing row. + auto transitioning_keys = + GetTransitioningKeySetRef(context, binder, current_null_side_get, key_pos, null_leaf_idx, + view_name, transition_cte_indexes, transition_ctes); + if (transitioning_keys) { + auto &other_bcr = other_key_expr->Cast(); + auto left_expr = make_uniq(other_bcr.return_type, other_bcr.binding); + auto right_expr = + make_uniq(other_bcr.return_type, transitioning_keys->key_binding); + auto anti_condition = make_uniq( + ExpressionType::COMPARE_EQUAL, std::move(left_expr), std::move(right_expr)); + auto &other_subtree = j->children[other_child]; + if (portable_anti_guard) { + auto output_count = other_subtree->GetColumnBindings().size(); + auto mark_join = LogicalComparisonJoin::CreateJoin( + context, JoinType::MARK, JoinRefType::REGULAR, std::move(other_subtree), + std::move(transitioning_keys->node), std::move(anti_condition)); + auto &mark = mark_join->Cast(); + mark.mark_index = binder.GenerateTableIndex(); + mark.convert_mark_to_semi = false; + mark_join->ResolveOperatorTypes(); + + auto mark_ref = make_uniq(LogicalType::BOOLEAN, + ColumnBinding(mark.mark_index, 0)); + auto keep_unmatched = make_uniq( + ExpressionType::COMPARE_DISTINCT_FROM, std::move(mark_ref), + make_uniq(Value::BOOLEAN(true))); + auto filter = make_uniq(std::move(keep_unmatched)); + for (idx_t output_idx = 0; output_idx < output_count; output_idx++) { + filter->projection_map.push_back(output_idx); + } + filter->children.push_back(std::move(mark_join)); + filter->ResolveOperatorTypes(); + other_subtree = std::move(filter); + OPENIVM_DEBUG_PRINT( + "[DeltaJoin] Rendered transition-key exclusion as portable MARK filter\n"); + } else { + auto anti_join = LogicalComparisonJoin::CreateJoin( + context, JoinType::ANTI, JoinRefType::REGULAR, std::move(other_subtree), + std::move(transitioning_keys->node), std::move(anti_condition)); + anti_join->ResolveOperatorTypes(); + other_subtree = std::move(anti_join); + } + OPENIVM_DEBUG_PRINT("[DeltaJoin] Guarded kept outer join: excluded rows whose key " + "match-count transitions across zero via leaf %zu's delta\n", + null_leaf_idx); + } + } + } + } + } + for (size_t ci = 0; ci < node->children.size(); ci++) { + path.push_back(ci); + GuardKeptOuterJoinsForMaskRec(context, binder, node->children[ci].get(), leaves, leaf_has_delta_mask, view_name, + portable_anti_guard, transition_cte_indexes, transition_ctes, path); + path.pop_back(); + } +} + +static void GuardKeptOuterJoinsForMask(ClientContext &context, Binder &binder, LogicalOperator *node, + const vector &leaves, uint64_t leaf_has_delta_mask, + const string &view_name, bool portable_anti_guard, + map, idx_t> &transition_cte_indexes, + vector &transition_ctes) { + vector path; + GuardKeptOuterJoinsForMaskRec(context, binder, node, leaves, leaf_has_delta_mask, view_name, portable_anti_guard, + transition_cte_indexes, transition_ctes, path); +} + void AppendMultiplicityToAncestorProjectionMaps(unique_ptr &term, const vector &leaf_path, const ColumnBinding &mul_binding, const char *context_label, bool preserve_constant_sibling_child_outputs, idx_t fallback_mul_idx) { @@ -860,10 +1251,10 @@ static uint64_t ComputeFactsInsertOnlyMask(const openivm::CompileFacts &facts, c // ============================================================================ // BuildInclusionExclusionTerms: create 2^N - 1 delta terms // ============================================================================ -static vector> BuildInclusionExclusionTerms(DeltaOperatorInput input, - ClientContext &context, Binder &binder, - const vector &leaves, - bool has_left_join) { +static vector> +BuildInclusionExclusionTerms(DeltaOperatorInput input, ClientContext &context, Binder &binder, + const vector &leaves, bool has_left_join, + vector &transition_ctes) { size_t N = leaves.size(); vector> terms; @@ -963,6 +1354,7 @@ static vector> BuildInclusionExclusionTerms(DeltaOpe } uint64_t pruned_count = 0; + map, idx_t> transition_cte_indexes; OPENIVM_DEBUG_PRINT("[DeltaJoin] Building inclusion-exclusion terms (%lu total, skip_bits=%lu, empty_mask=%lu)\n", (unsigned long)total_terms, (unsigned long)skip_bits, (unsigned long)empty_mask); for (uint64_t mask = 1; mask < (1ULL << N); mask++) { @@ -1054,6 +1446,19 @@ static vector> BuildInclusionExclusionTerms(DeltaOpe } } + // Guard kept (un-demoted) outer joins AFTER delta leaves are replaced: the mask-driven side + // (e.g. Δ(P1) via CompileCopiedSubtree) must already be its final compiled form before we wrap + // it in an anti-join -- doing this earlier corrupts the leaf-replacement step above, which would + // otherwise try to compute a delta of our anti-join wrapper instead of the original subtree. + if (has_left_join) { + uint64_t leaf_has_delta_mask = (~delta_status.empty_mask) & total_terms; + if (leaf_has_delta_mask) { + bool portable_anti_guard = compile_facts.target_dialect != SqlDialect::DUCKDB; + GuardKeptOuterJoinsForMask(context, binder, term.get(), leaves, leaf_has_delta_mask, input.context.view, + portable_anti_guard, transition_cte_indexes, transition_ctes); + } + } + term->ResolveOperatorTypes(); // Build projection: original columns + combined multiplicity @@ -1161,12 +1566,36 @@ DeltaPlanFragment CompileJoinDelta(DeltaOperatorInput input) { D_ASSERT(types.size() == original_bindings.size()); types.emplace_back(input.mul_type); - // 3. Build terms — use DuckLake N-term path when all leaves are DuckLake scans - // Check if all leaves are DuckLake scans AND N-term telescoping is enabled. + // 3. Build terms — use DuckLake N-term path when all leaves are DuckLake scans. + // The DuckLake collector looks through transparent wrappers so each physical + // source contributes one term even when a projection wraps a preserved join. bool all_ducklake = true; + bool flattened_ducklake = false; + vector ducklake_leaves; + string ducklake_fallback_reason; if (!SqlUtils::GetBoolSetting(context, "openivm_ducklake_nterm", true)) { all_ducklake = false; // forced to inclusion-exclusion + ducklake_fallback_reason = "openivm_ducklake_nterm is disabled"; } else { + if (input.context.model.type == RefreshType::SIMPLE_PROJECTION && + TryCollectDuckLakeJoinLeaves(input.plan.get(), ducklake_leaves, ducklake_fallback_reason)) { + bool has_wrapped_leaf = ducklake_leaves.size() != leaves.size(); + if (!has_wrapped_leaf) { + for (auto &leaf : leaves) { + if (!leaf.get) { + has_wrapped_leaf = true; + break; + } + } + } + if (has_wrapped_leaf) { + flattened_ducklake = true; + leaves = std::move(ducklake_leaves); + N = leaves.size(); + } + } else if (input.context.model.type != RefreshType::SIMPLE_PROJECTION) { + ducklake_fallback_reason = "refresh type is outside SIMPLE_PROJECTION scope"; + } for (size_t i = 0; i < N; i++) { auto *get = GetLeafScan(leaves[i]); if (!get || get->function.name != "ducklake_scan") { @@ -1175,14 +1604,32 @@ DeltaPlanFragment CompileJoinDelta(DeltaOperatorInput input) { } } } + if (!flattened_ducklake && !ducklake_fallback_reason.empty()) { + OPENIVM_DEBUG_PRINT("[DuckLakeJoin] Flattening fallback: %s\n", ducklake_fallback_reason.c_str()); + } + if (N > openivm::MAX_JOIN_TABLES) { + throw NotImplementedException("IVM not supported for joins with more than 16 tables"); + } LogDeltaOperatorStrategy(input, all_ducklake ? DeltaOperatorStrategy::JOIN_DUCKLAKE_N_TERM : DeltaOperatorStrategy::JOIN_INCLUSION_EXCLUSION); - auto terms = all_ducklake ? BuildDuckLakeJoinTerms(input, context, binder, leaves, has_left_join) - : BuildInclusionExclusionTerms(input, context, binder, leaves, has_left_join); + vector transition_ctes; + auto terms = all_ducklake + ? BuildDuckLakeJoinTerms(input, context, binder, leaves, has_left_join, flattened_ducklake) + : BuildInclusionExclusionTerms(input, context, binder, leaves, has_left_join, transition_ctes); // 4. UNION ALL auto result = AssembleJoinUnionAll(terms, types, binder); + for (auto definition = transition_ctes.rbegin(); definition != transition_ctes.rend(); definition++) { + result = make_uniq(definition->name, definition->cte_index, definition->types.size(), + std::move(definition->node), std::move(result), + CTEMaterialize::CTE_MATERIALIZE_ALWAYS); + result->ResolveOperatorTypes(); + } + if (!transition_ctes.empty()) { + OPENIVM_DEBUG_PRINT("[DeltaJoin] Shared %zu transition-key CTEs across inclusion-exclusion terms\n", + transition_ctes.size()); + } // 5. Rebind parent references ColumnBinding new_mul_binding = ReplaceJoinOutputBindings(original_bindings, result, *input.root); diff --git a/src/delta/operators/nonlocal.cpp b/src/delta/operators/nonlocal.cpp index 1c823e2c..7b2aa0e7 100644 --- a/src/delta/operators/nonlocal.cpp +++ b/src/delta/operators/nonlocal.cpp @@ -17,17 +17,16 @@ static DeltaPlanFragment CompileNonLocalDeltaGuard(const DeltaOperatorInput &inp DeltaPlanFragment CompileAsofJoinDelta(const DeltaOperatorInput &input) { return CompileNonLocalDeltaGuard(input, DeltaOperatorStrategy::ASOF_AFFECTED_RECOMPUTE, "ASOF_JOIN", - "WINDOW_PARTITION, GROUP_RECOMPUTE, or CURRENT_DIFF_RECOMPUTE"); + "WINDOW_PARTITION, GROUP_RECOMPUTE, or FULL_REFRESH"); } DeltaPlanFragment CompilePositionalJoinDelta(const DeltaOperatorInput &input) { return CompileNonLocalDeltaGuard(input, DeltaOperatorStrategy::POSITIONAL_GLOBAL_RECOMPUTE, "POSITIONAL_JOIN", - "CURRENT_DIFF_RECOMPUTE"); + "FULL_REFRESH"); } DeltaPlanFragment CompileSampleDelta(const DeltaOperatorInput &input) { - return CompileNonLocalDeltaGuard(input, DeltaOperatorStrategy::SAMPLE_GLOBAL_RECOMPUTE, "SAMPLE", - "CURRENT_DIFF_RECOMPUTE"); + return CompileNonLocalDeltaGuard(input, DeltaOperatorStrategy::SAMPLE_GLOBAL_RECOMPUTE, "SAMPLE", "FULL_REFRESH"); } } // namespace duckdb diff --git a/src/include/core/ivm_view_classifier.hpp b/src/include/core/ivm_view_classifier.hpp index b6d12cf0..51fe814f 100644 --- a/src/include/core/ivm_view_classifier.hpp +++ b/src/include/core/ivm_view_classifier.hpp @@ -18,9 +18,8 @@ enum class DeltaStrategyReason { REPEATED_CTE_AGGREGATE_GROUP_FALLBACK, SEMI_ANTI_AGGREGATE_GROUP_FALLBACK, OUTER_JOIN_AGGREGATE_RECOMPUTE, - ASOF_CURRENT_DIFF_RECOMPUTE, - SAMPLE_CURRENT_DIFF_RECOMPUTE, - POSITIONAL_CURRENT_DIFF_RECOMPUTE + OUTER_JOIN_PRESERVED_TABLE_FUNCTION_RECOMPUTE, + INNER_DISTINCT_PROJECTION_RECOMPUTE }; enum class DeltaModelFeature { @@ -39,7 +38,6 @@ enum class DeltaModelFeature { ASOF_STATEFUL, SAMPLE_GLOBAL_RECOMPUTE, POSITIONAL_GLOBAL_RECOMPUTE, - CURRENT_DIFF_RECOMPUTE, FULL_ONLY }; @@ -167,6 +165,8 @@ struct DeltaViewModelInput { bool stored_query_has_top_k = false; bool has_hidden_minmax_having = false; bool has_computed_minmax_aggregate_projection = false; + bool has_computed_sum_aggregate_projection = false; + bool has_top_level_redundant_distinct = false; bool has_ducklake_source = false; }; @@ -225,7 +225,7 @@ const char *DeltaRuleKindName(DeltaRuleKind kind); const char *DeltaUnsupportedReasonName(DeltaUnsupportedReason reason); const char *DeltaUpdateSemanticsName(DeltaUpdateSemantics semantics); const char *DeltaAffectedDomainKindName(DeltaAffectedDomainKind kind); -bool IsDistinctAtTop(const PlanAnalysis &analysis, const vector &output_names); +bool IsDistinctAtTop(const CreateMVPlanFacts &facts, const vector &output_names); DeltaViewModel BuildDeltaViewModel(const DeltaViewModelInput &input); } // namespace duckdb diff --git a/src/include/core/openivm_constants.hpp b/src/include/core/openivm_constants.hpp index 05bb3e78..a152c7cd 100644 --- a/src/include/core/openivm_constants.hpp +++ b/src/include/core/openivm_constants.hpp @@ -37,6 +37,10 @@ constexpr const char *VAR_SQ_COL_PREFIX = "openivm_var_sq_"; // VARIANCE: no s constexpr const char *SUM_SQP_COL_PREFIX = "openivm_sum_sqp_"; // STDDEV_POP: sqrt + population denominator constexpr const char *VAR_SQP_COL_PREFIX = "openivm_var_sqp_"; // VAR_POP: no sqrt + population denominator constexpr const char *COUNT_COL_PREFIX = "openivm_count_"; +// COUNT(sum_argument) companions for user-visible SUM outputs. The suffix is +// the visible projection index, which associates the state with the exact +// bound output without inspecting SQL text or aliases. +constexpr const char *SUM_COUNT_COL_PREFIX = "openivm_nonnull_sum_count_"; // Hidden COUNT(*) injected into AGGREGATE_GROUP MVs that don't already have a // count aggregate. Tracks per-group cardinality so the cleanup can delete rows @@ -50,6 +54,19 @@ constexpr const char *RIGHT_MATCH_COUNT_COL = "openivm_right_match_count"; // Index suffix for GROUP BY unique index on MV data tables constexpr const char *INDEX_SUFFIX = "openivm_index"; +// Placeholders in the stored LEFT JOIN secondary-delta SQL, substituted at refresh time with a +// subquery yielding (__k, __m) = (join key, signed multiplicity) for that side's pending changes. +// They exist because the row source depends on the storage backend: a regular table reads +// openivm_delta_
filtered by timestamp, while a DuckLake table has no delta table at all and +// must read ducklake_table_insertions/deletions between two snapshot IDs -- and those IDs are only +// known at refresh, whereas this SQL is generated once at CREATE. +// Indexed because a chain of N tables has N-1 LEFT JOIN levels and EVERY level whose preserved side +// is itself a join needs its own secondary delta. Handling only the outermost level left deeper +// preserved-side counts undercounted for 4+ table chains. +constexpr const char *LJSEC_INNER_DELTA_PREFIX = "__OPENIVM_LJSEC_INNER_DELTA_"; +constexpr const char *LJSEC_PRES_DELTA_PREFIX = "__OPENIVM_LJSEC_PRES_DELTA_"; +constexpr const char *LJSEC_PLACEHOLDER_SUFFIX = "__"; + // Temporary table prefix for companion row snapshots constexpr const char *TEMP_TABLE_PREFIX = "openivm_old_"; @@ -67,6 +84,8 @@ constexpr const char *DISABLED_OPTIMIZERS = TEMPLATE_DATA_DEPENDENT_OPTIMIZERS; // DELIM joins to eliminate. Pure robustness guard — not flag-gated. constexpr const char *REFRESH_DISABLED_OPTIMIZERS = "deliminator"; +constexpr uint8_t LEGACY_CURRENT_DIFF_RECOMPUTE_TYPE = 10; + } // namespace openivm enum class RefreshType : uint8_t { @@ -78,11 +97,10 @@ enum class RefreshType : uint8_t { WINDOW_PARTITION, // window functions — partition-level recompute GROUP_RECOMPUTE, // inner-DISTINCT-under-AGG fallback: DELETE+INSERT only the GROUP BY keys touched by source deltas TOP_K, // Legacy enum value; current top-k support strips ORDER BY/LIMIT into the user-facing view - DISTINCT_INCREMENTAL, // inner-DISTINCT-under-AGG with aux state (openivm_distinct_aux_state=true): DBSP-correct - // distinct(R)=sgn(R[t]); per-tuple count table emits ±1 only on count transitions - SEMI_ANTI_RECOMPUTE, // SEMI/ANTI join aux state: per-left-tuple match counts, transition-scoped MV updates - CURRENT_DIFF_RECOMPUTE, // exact recompute inside incremental refresh; emits MV deltas from old/current diff - COUNT_DISTINCT_INCREMENTAL // COUNT(DISTINCT x) with per-(group,x) multiplicity aux state + DISTINCT_INCREMENTAL, // inner-DISTINCT-under-AGG with aux state (openivm_distinct_aux_state=true): DBSP-correct + // distinct(R)=sgn(R[t]); per-tuple count table emits ±1 only on count transitions + SEMI_ANTI_RECOMPUTE, // SEMI/ANTI join aux state: per-left-tuple match counts, transition-scoped MV updates + COUNT_DISTINCT_INCREMENTAL = 11 // COUNT(DISTINCT x) with per-(group,x) multiplicity aux state }; enum class GroupRecomputeAffectedMode : uint8_t { SOURCE_DELTA, SOURCE_DELTA_RELAX_AGGREGATE_FILTER, CURRENT_DIFF }; @@ -105,8 +123,6 @@ inline const char *RefreshTypeName(RefreshType type) { return "DISTINCT_INCREMENTAL"; case RefreshType::SEMI_ANTI_RECOMPUTE: return "SEMI_ANTI_RECOMPUTE"; - case RefreshType::CURRENT_DIFF_RECOMPUTE: - return "CURRENT_DIFF_RECOMPUTE"; case RefreshType::COUNT_DISTINCT_INCREMENTAL: return "COUNT_DISTINCT_INCREMENTAL"; case RefreshType::TOP_K: diff --git a/src/include/core/parser.hpp b/src/include/core/parser.hpp index 06e93537..bebef42a 100644 --- a/src/include/core/parser.hpp +++ b/src/include/core/parser.hpp @@ -13,18 +13,25 @@ class MaterializedViewParserExtension : public ParserExtension { explicit MaterializedViewParserExtension() { parse_function = ParseFunction; plan_function = PlanFunction; + parser_override = OverrideFunction; } static ParserExtensionParseResult ParseFunction(ParserExtensionInfo *info, const string &query); + static ParserOverrideResult OverrideFunction(ParserExtensionInfo *info, const string &query, + ParserOptions &options); static ParserExtensionPlanResult PlanFunction(ParserExtensionInfo *info, ClientContext &context, unique_ptr parse_data); }; +string MaterializedViewLifecycleQuery(ClientContext &context, const FunctionParameters ¶meters); +string MaterializedViewDropQuery(ClientContext &context, const FunctionParameters ¶meters); + struct MaterializedViewParseData : ParserExtensionParseData { unique_ptr statement; int64_t refresh_interval = -1; // seconds, -1 = not specified (manual only) bool is_replace = false; // CREATE OR REPLACE: drop old MV before creating string alter_sql; // non-empty for ALTER MATERIALIZED VIEW (executed directly in plan function) + string target_name; // parsed catalog-qualified CREATE/ALTER target unique_ptr Copy() const override { auto copy = make_uniq_base(statement->Copy()); @@ -32,6 +39,7 @@ struct MaterializedViewParseData : ParserExtensionParseData { data.refresh_interval = refresh_interval; data.is_replace = is_replace; data.alter_sql = alter_sql; + data.target_name = target_name; return copy; } diff --git a/src/include/core/parser_ddl.hpp b/src/include/core/parser_ddl.hpp index 37c8b856..97f6a077 100644 --- a/src/include/core/parser_ddl.hpp +++ b/src/include/core/parser_ddl.hpp @@ -2,16 +2,62 @@ #define OPENIVM_PARSER_DDL_HPP #include "duckdb.hpp" +#include "duckdb/main/client_context_state.hpp" #include "duckdb/parser/parser_extension.hpp" +#include + namespace duckdb { +struct DropInfo; + static constexpr const char *OPENIVM_DDL_CLEANUP_PREFIX = "openivm_cleanup:"; static constexpr const char *OPENIVM_DDL_PROFILE_PREFIX = "openivm_profile:"; static constexpr const char *OPENIVM_DDL_PROFILE_RECORD_PREFIX = "openivm_profile_record:"; static constexpr const char *OPENIVM_DDL_CREATE_DELTA_FROM_DATA_PREFIX = "openivm_create_delta_from_data:"; +static constexpr const char *OPENIVM_TRANSACTIONAL_DDL_FUNCTION = "openivm_transactional_ddl"; +static constexpr const char *OPENIVM_STAGED_DDL_FUNCTION = "openivm_staged_ddl"; + +enum class DDLExecutionMode : uint8_t { CALLER_TRANSACTION, STAGED_CROSS_CATALOG }; + +// Native lifecycle statements are rendered into a caller-transaction SQL +// program. Helper connections cannot observe that program's uncommitted +// metadata, so retain the metadata operations on the caller context and replay +// them into temporary shadow tables when a later lifecycle/refresh statement +// in the same transaction needs to compile. +class TransactionalMVMetadataState : public ClientContextState { +public: + static TransactionalMVMetadataState &Get(ClientContext &context); + static optional_ptr TryGet(ClientContext &context); + + void Register(ClientContext &context, const vector ¶meters, const string &view_name); + void RegisterSQL(const string &sql, const string &view_name); + void IncludeView(const string &view_name); + void Apply(Connection &connection) const; + + void TransactionCommit(MetaTransaction &transaction, ClientContext &context) override; + void TransactionRollback(MetaTransaction &transaction, ClientContext &context) override; + +private: + void Clear(); + + vector statements; + unordered_set view_names; +}; -void ConfigureDDLExecutorResult(ParserExtensionPlanResult &result); +void ConfigureDDLExecutorResult(ParserExtensionPlanResult &result, + DDLExecutionMode mode = DDLExecutionMode::STAGED_CROSS_CATALOG); +string RenderTransactionalDDL(ClientContext &context, const vector ¶meters); +void ExecuteStagedDDL(ClientContext &context, const vector ¶meters); +string BuildCreateDeltaFromDataOperation(const string &delta_table, const string &data_table, bool replace); +string BuildDropViewStatement(const DropInfo &drop_info); +string BuildDropTableStatement(const DropInfo &drop_info); +unique_ptr BindDropView(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names); +unique_ptr BindDropTable(ClientContext &context, TableFunctionBindInput &input, + vector &return_types, vector &names); +unique_ptr InitDropView(ClientContext &context, TableFunctionInitInput &input); +void ExecuteDropView(ClientContext &context, TableFunctionInput &input, DataChunk &output); } // namespace duckdb diff --git a/src/include/core/parser_plan_helpers.hpp b/src/include/core/parser_plan_helpers.hpp index 0309acb1..c8967262 100644 --- a/src/include/core/parser_plan_helpers.hpp +++ b/src/include/core/parser_plan_helpers.hpp @@ -85,6 +85,9 @@ struct CreateMVPlanFacts { bool has_bound_aggregate_filter = false; bool has_hidden_minmax_having_column = false; bool has_computed_minmax_aggregate_projection = false; + bool has_computed_sum_aggregate_projection = false; + bool has_top_level_redundant_distinct = false; + bool has_descendant_distinct = false; }; string BuildTopKSuffix(const vector &orders, idx_t limit_val, idx_t offset_val, @@ -94,9 +97,23 @@ string QualifyCreateSourceTable(const string &table_name, const string ¤t_ const string &default_db); string ExplainInitialLoadQuery(Connection &con, const string &label, const string &query); CreateMVPlanFacts BuildCreateMVPlanFacts(LogicalOperator *plan, const string ¤t_catalog); +bool ProducesAtMostOneRow(LogicalOperator &node); +bool IsRedundantDistinctOverGroupKeys(LogicalOperator &node); void AddJoinKeyColumn(const unique_ptr &expr, unordered_map> &join_key_cols); bool OuterJoinAggregateNeedsRecompute(const CreateMVPlanFacts &facts, idx_t group_index); -bool RelationExists(Connection &con, const string &qualified_name); +// delta_view_catalog_prefix must be the same prefix the view's delta table was created with +// (internal_catalog_prefix). Without it the generated INSERT targets an unqualified delta table, +// which fails for MVs whose state lives in another catalog -- e.g. DuckLake-backed views, where the +// delta table is dl.main.openivm_delta_. +// The returned SQL carries LJSEC_INNER_DELTA_PLACEHOLDER / LJSEC_PRES_DELTA_PLACEHOLDER where each +// side's pending changes belong; refresh substitutes them (see openivm_constants.hpp). The out_* +// params report the two sides' identities so refresh can pick the right row source per backend. +string BuildLeftJoinSecondaryDeltaSQL(ClientContext &context, const CreateMVPlanFacts &facts, + const vector &output_names, const string &view_name, + vector &preserved_cols, const string &delta_view_catalog_prefix, + vector &out_inner_tables, vector &out_inner_keys, + vector &out_pres_tables, vector &out_pres_keys); +bool OuterJoinPreservedSideHasTableFunction(const CreateMVPlanFacts &facts); vector DeriveGroupColumnNames(const CreateMVPlanFacts &facts, idx_t group_index, size_t group_count, const vector &output_names); vector DeriveScalarDelimKeyColumnNames(const CreateMVPlanFacts &facts, const vector &output_names); diff --git a/src/include/core/plan_rewrite_internal.hpp b/src/include/core/plan_rewrite_internal.hpp index 1e7e26a1..c354d3e2 100644 --- a/src/include/core/plan_rewrite_internal.hpp +++ b/src/include/core/plan_rewrite_internal.hpp @@ -11,6 +11,8 @@ AggregateFunction BindAggregateByName(ClientContext &context, const string &name LogicalOperator *FindProjectionAggregateInput(unique_ptr &plan, bool allow_having_filter); void RewriteDerivedAggregates(ClientContext &context, unique_ptr &plan, Optimizer &opt, bool is_top = true); +void InjectSumNonNullCounts(ClientContext &context, unique_ptr &plan); +void PropagateHiddenAggregateColumns(unique_ptr &plan); bool RewriteSafeSemiAntiDelimGets(ClientContext &context, unique_ptr &plan); } // namespace duckdb diff --git a/src/include/core/refresh_daemon.hpp b/src/include/core/refresh_daemon.hpp index 4cc916a1..2a3daf54 100644 --- a/src/include/core/refresh_daemon.hpp +++ b/src/include/core/refresh_daemon.hpp @@ -13,8 +13,8 @@ namespace duckdb { // Background thread that periodically refreshes materialized views with a REFRESH EVERY interval. // Holds a raw pointer to DatabaseInstance (valid for the lifetime of the extension). -// The daemon wakes every 30 seconds, checks which views are due, and refreshes them. -// Views already being refreshed (manual PRAGMA or cascade) are skipped via TryLockView. +// The daemon wakes every 30 seconds, checks which views are due, and refreshes them +// after acquiring the shared OpenIVM mutation gate. class RefreshDaemon { public: // Start the daemon thread. Safe to call multiple times (only starts once). @@ -23,6 +23,9 @@ class RefreshDaemon { // Stop the daemon and join the thread. Called on destruction or explicit shutdown. void Stop(); + // Request an immediate scheduling cycle (used by explicit daemon restart). + void Wake(); + ~RefreshDaemon(); // Get the effective interval for a view (may be larger than configured due to backoff). @@ -38,6 +41,7 @@ class RefreshDaemon { std::thread thread_; std::atomic shutdown_ {false}; std::atomic started_ {false}; + std::atomic wake_requested_ {false}; std::mutex cv_mutex_; std::condition_variable cv_; diff --git a/src/include/core/refresh_locks.hpp b/src/include/core/refresh_locks.hpp index 904045ca..b4adab34 100644 --- a/src/include/core/refresh_locks.hpp +++ b/src/include/core/refresh_locks.hpp @@ -2,105 +2,78 @@ #define REFRESH_LOCKS_HPP #include "duckdb.hpp" +#include "duckdb/main/client_context_state.hpp" +#include #include #include namespace duckdb { -// Provides per-view and per-delta-table mutexes for safe concurrent refresh. -// -// Per-view mutex: prevents two concurrent refreshes of the same MV (which would -// cause write-write conflicts on the MV table). -// -// Per-delta-table mutex: serializes the refresh's "read deltas + set last_update" -// critical section with the insert rule's delta row writes. This closes the window -// where concurrent DML deltas could be permanently skipped. -class RefreshLocks { +// Serializes OpenIVM mutations for one database. The gate is re-entrant for one +// logical owner because helper connections can execute on different worker threads. +class MutationGate { public: - // --- View-level locks (prevent concurrent refresh of same MV) --- - - // Blocking lock — used by PRAGMA refresh() (user explicitly wants to refresh, so wait). - static void LockView(const string &view_name); - - // Non-blocking try-lock — used by the refresh daemon (skip if busy). - // Returns true if the lock was acquired. - static bool TryLockView(const string &view_name); + void Lock(const void *owner); + void Unlock(const void *owner); - static void UnlockView(const string &view_name); - - // --- Delta-table-level locks (serialize delta reads/writes) --- - - // Blocking lock — held briefly by both refresh (read + timestamp update) - // and insert rule (delta row write). - static void LockDelta(const string &delta_table_name); +private: + mutex lock; + std::condition_variable condition; + const void *active_owner = nullptr; + idx_t depth = 0; +}; - static void UnlockDelta(const string &delta_table_name); +class RefreshLocks { +public: + static void LockMutation(DatabaseInstance &db, const void *owner); + static void UnlockMutation(DatabaseInstance &db, const void *owner); private: - static std::mutex &GetViewMutex(const string &view_name); - static std::mutex &GetDeltaMutex(const string &delta_table_name); + static MutationGate &GetMutationGate(DatabaseInstance &db); static std::mutex map_mutex_; - static std::unordered_map> view_mutexes_; - static std::unordered_map> delta_mutexes_; + static std::unordered_map> mutation_gates_; }; -// RAII guard for delta-table locks. Automatically unlocks on scope exit (including exceptions). -class DeltaLockGuard { - string name_; +class MutationLockGuard { + DatabaseInstance *db; + const void *owner; public: - explicit DeltaLockGuard(const string &delta_table_name) : name_(delta_table_name) { - RefreshLocks::LockDelta(name_); + explicit MutationLockGuard(ClientContext &owner_p) : db(&DatabaseInstance::GetDatabase(owner_p)), owner(&owner_p) { + RefreshLocks::LockMutation(*db, owner); } - ~DeltaLockGuard() { - RefreshLocks::UnlockDelta(name_); + MutationLockGuard(DatabaseInstance &db_p, const void *owner_p) : db(&db_p), owner(owner_p) { + RefreshLocks::LockMutation(*db, owner); } - DeltaLockGuard(const DeltaLockGuard &) = delete; - DeltaLockGuard &operator=(const DeltaLockGuard &) = delete; - DeltaLockGuard(DeltaLockGuard &&) = delete; - DeltaLockGuard &operator=(DeltaLockGuard &&) = delete; + ~MutationLockGuard() { + RefreshLocks::UnlockMutation(*db, owner); + } + MutationLockGuard(const MutationLockGuard &) = delete; + MutationLockGuard &operator=(const MutationLockGuard &) = delete; + MutationLockGuard(MutationLockGuard &&) = delete; + MutationLockGuard &operator=(MutationLockGuard &&) = delete; }; -// RAII guard for view locks. Automatically unlocks on scope exit (including exceptions). -class ViewLockGuard { - string name_; - +// Native lifecycle and refresh programs are returned as multiple DuckDB +// statements. Retain the mutation gate until the caller transaction ends. +class TransactionalMVLockState : public ClientContextState { public: - explicit ViewLockGuard(const string &view_name) : name_(view_name) { - RefreshLocks::LockView(name_); - } - ~ViewLockGuard() { - RefreshLocks::UnlockView(name_); - } - ViewLockGuard(const ViewLockGuard &) = delete; - ViewLockGuard &operator=(const ViewLockGuard &) = delete; - ViewLockGuard(ViewLockGuard &&) = delete; - ViewLockGuard &operator=(ViewLockGuard &&) = delete; -}; + static TransactionalMVLockState &Get(ClientContext &context); -// Non-blocking RAII guard for opportunistic view-lock checks. -class TryViewLockGuard { - string name_; - bool owns_lock_; + void AcquireMutationLock(); + void SetMutationOwner(const void *owner_token); -public: - explicit TryViewLockGuard(const string &view_name) - : name_(view_name), owns_lock_(RefreshLocks::TryLockView(name_)) { - } - ~TryViewLockGuard() { - if (owns_lock_) { - RefreshLocks::UnlockView(name_); - } - } - bool OwnsLock() const { - return owns_lock_; - } - TryViewLockGuard(const TryViewLockGuard &) = delete; - TryViewLockGuard &operator=(const TryViewLockGuard &) = delete; - TryViewLockGuard(TryViewLockGuard &&) = delete; - TryViewLockGuard &operator=(TryViewLockGuard &&) = delete; + void TransactionCommit(MetaTransaction &transaction, ClientContext &context) override; + void TransactionRollback(MetaTransaction &transaction, ClientContext &context) override; + +private: + void Release(); + + unique_ptr mutation_guard; + ClientContext *owner = nullptr; + const void *mutation_owner = nullptr; }; } // namespace duckdb diff --git a/src/include/core/refresh_metadata.hpp b/src/include/core/refresh_metadata.hpp index 2ceb6af7..faebe5ab 100644 --- a/src/include/core/refresh_metadata.hpp +++ b/src/include/core/refresh_metadata.hpp @@ -55,9 +55,23 @@ class RefreshMetadata { string schema_name; string table_name; }; + struct StoredViewLocation { + string catalog_name; + string schema_name; + }; + struct DeltaSource { + string table_name; + string catalog_type; + string catalog_name; + string schema_name; + }; SourceLocation GetSourceLocation(const string &view_name, const string &table_name, const string &fallback_catalog = "", const string &fallback_schema = ""); + StoredViewLocation GetStoredViewLocation(const string &view_name, const string &fallback_catalog = "", + const string &fallback_schema = ""); + vector GetDeltaSources(const string &view_name, const string &fallback_catalog = "", + const string &fallback_schema = ""); string ResolveDeltaQualifiedName(const string &view_name, const string &delta_table_name, const string &fallback_catalog = "", const string &fallback_schema = ""); @@ -80,14 +94,18 @@ class RefreshMetadata { // Get all downstream MV dependents in topological order (closest first). // For table→mv1→mv2→mv3, GetDownstreamViews("mv1") returns ["mv2", "mv3"]. vector GetDownstreamViews(const string &view_name); + vector GetDownstreamViewsStrict(const string &view_name); + bool HasDownstreamViews(const string &view_name); // Get refresh_interval in seconds for a view. Returns -1 if not set (manual only). int64_t GetRefreshInterval(const string &view_name); // Get all views with a non-null refresh_interval. - // Returns tuples of (view_name, interval_seconds, last_update_timestamp_string). + // Returns the stored relation identity plus its schedule and last refresh watermark. struct ScheduledView { string view_name; + string catalog_name; + string schema_name; int64_t interval_seconds; string last_update; }; @@ -99,7 +117,8 @@ class RefreshMetadata { // Build SQL to delete old delta rows that all dependent views have already consumed. // target: the (possibly schema-qualified) table to delete from. // metadata_key: the name used in openivm_delta_tables (unqualified delta name). - static string BuildDeltaCleanupSQL(const string &target, const string &metadata_key); + static string BuildDeltaCleanupSQL(const string &target, const string &metadata_key, + const string &delta_metadata_table = ""); // Get GROUP BY column names for a view. Returns empty vector if not stored. vector GetGroupColumns(const string &view_name); @@ -150,7 +169,7 @@ class RefreshMetadata { const string &catalog_name, const string &schema_name); static string BuildDuckLakeRefreshMetadataSQL(const string &view_name, const string &table_name, - const string &snapshot_expr); + const string &snapshot_expr, const string &delta_metadata_table = ""); void UpdateDuckLakeRefreshMetadata(const string &view_name, const string &table_name, int64_t snapshot_id); // --- Refresh history (learned cost model) --- @@ -235,9 +254,13 @@ class RefreshMetadata { string output_col; string source; string source_col; + // A target type for one CAST, or an encoded expression template for nested/TRY_CAST chains. + string source_cast; string lookup; string lookup_col; + string lookup_cast; string lookup_out; + string lookup_out_cast; }; bool GetWindowPartitionLineage(const string &view_name, vector &out); @@ -300,6 +323,24 @@ class RefreshMetadata { bool GetFilteredGroupCountAuxMeta(const string &view_name, FilteredGroupCountAuxMeta &out); static string FilteredGroupCountAuxMetaToJson(const FilteredGroupCountAuxMeta &meta); + // LEFT JOIN pipeline secondary-delta maintenance (Larson & Zhou). The secondary-delta SQL is + // generated once at CREATE time (from the plan) and stored; refresh substitutes the per-source + // delta timestamps and appends it between the primary-delta INSERT and the MERGE. Zero refresh-time + // plan walks. + struct LeftJoinSecondaryMeta { + string sql; // secondary-delta INSERT (run before the MERGE; carries LJSEC_* placeholders) + vector preserved_cols; + // Identities of the two sides whose pending changes the placeholders stand for. Refresh needs + // them to build the right row source per storage backend (delta table vs DuckLake snapshots). + vector inner_tables; // bare base-table names of the inner (null-supplying) sides + vector inner_keys; // join key columns on the inner sides + vector pres_tables; // bare base-table names of the preserved sides + vector pres_keys; // join key columns on the preserved sides + }; + + bool GetLeftJoinSecondaryMeta(const string &view_name, LeftJoinSecondaryMeta &out); + static string LeftJoinSecondaryMetaToJson(const LeftJoinSecondaryMeta &meta); + static vector ExpectedDistinctAuxColumns(const DistinctAuxMeta &meta); static vector ExpectedCountDistinctAuxColumns(const CountDistinctAuxMeta &meta); static vector ExpectedFilteredGroupCountAuxColumns(const FilteredGroupCountAuxMeta &meta); diff --git a/src/include/core/sql_utils.hpp b/src/include/core/sql_utils.hpp index e6bba854..39cd2a67 100644 --- a/src/include/core/sql_utils.hpp +++ b/src/include/core/sql_utils.hpp @@ -47,10 +47,18 @@ class SqlUtils { static string JoinQuotedColumns(const vector &columns); static string JoinQualifiedQuotedColumns(const vector &columns, const string &alias); static string BuildAllNullPredicate(const vector &columns); + static string BuildAnyNullPredicate(const vector &columns, const string &prefix = ""); static string BuildNullSafeMatch(const vector &columns, const string &lhs_alias, const string &rhs_alias); static string BuildNullSafeKeyPredicate(const vector &columns, const string &left_prefix, const string &right_prefix); - static string BuildFullRecomputeSQL(const string &data_table, const string &view_query_sql); + /// Cast specs retain the legacy single-target-type representation while encoding nested/TRY_CAST expressions. + static string BuildCastSpec(const string &target_type, bool try_cast); + static string ComposeCastSpecs(const string &outer_cast_spec, const string &inner_cast_spec); + static string ApplyCastSpec(const string &column_expression, const string &cast_spec); + // Pass unique_keys + temp_table when the data table carries a UNIQUE index, to get a form that + // never deletes and re-inserts the same key in one transaction (see the definition for why). + static string BuildFullRecomputeSQL(const string &data_table, const string &view_query_sql, + const vector &unique_keys = {}, const string &temp_table = ""); static string ReplaceAllOccurrences(string haystack, const string &needle, const string &replacement); static vector ReplaceEachPlainOccurrence(const string &haystack, const string &needle, const string &replacement); diff --git a/src/include/delta/operators/ducklake_join.hpp b/src/include/delta/operators/ducklake_join.hpp index 94c37672..660bbc14 100644 --- a/src/include/delta/operators/ducklake_join.hpp +++ b/src/include/delta/operators/ducklake_join.hpp @@ -9,6 +9,11 @@ namespace duckdb { struct JoinLeafInfo; +/// Collect DuckLake scans through join trees and transparent projection/filter +/// wrappers. Returns false when the plan contains a wrapper that requires the +/// legacy recursive compiler. +bool TryCollectDuckLakeJoinLeaves(LogicalOperator *node, vector &leaves, string &fallback_reason); + /// Build N join delta terms using DuckLake time-travel (AT VERSION). /// /// Instead of inclusion-exclusion (2^N - 1 terms), produces exactly N terms @@ -20,7 +25,7 @@ struct JoinLeafInfo; /// and is provably equivalent to inclusion-exclusion. vector> BuildDuckLakeJoinTerms(DeltaOperatorInput input, ClientContext &context, Binder &binder, const vector &leaves, - bool has_left_join); + bool has_left_join, bool flattened_leaves); } // namespace duckdb diff --git a/src/include/rules/schema_evolution.hpp b/src/include/rules/schema_evolution.hpp index 02efd52f..7499802c 100644 --- a/src/include/rules/schema_evolution.hpp +++ b/src/include/rules/schema_evolution.hpp @@ -4,9 +4,10 @@ namespace duckdb { -string FirstMVReferencingColumn(Connection &con, const string &delta_name, const string &table_name, - const string &col_name); -void RewriteDependentViewMetadataForRename(Connection &con, const string &delta_name, const string &table_name, +string FirstMVReferencingColumn(Connection &con, const string &delta_name, const string &source_catalog, + const string &source_schema, const string &table_name, const string &col_name); +void RewriteDependentViewMetadataForRename(Connection &con, const string &delta_name, const string &source_catalog, + const string &source_schema, const string &table_name, const string &old_name, const string &new_name); } // namespace duckdb diff --git a/src/include/rules/transactional_delta_capture.hpp b/src/include/rules/transactional_delta_capture.hpp new file mode 100644 index 00000000..70cdec82 --- /dev/null +++ b/src/include/rules/transactional_delta_capture.hpp @@ -0,0 +1,61 @@ +#ifndef TRANSACTIONAL_DELTA_CAPTURE_HPP +#define TRANSACTIONAL_DELTA_CAPTURE_HPP + +#include "duckdb/planner/operator/logical_extension_operator.hpp" + +namespace duckdb { + +class TableCatalogEntry; + +enum class DeltaCaptureMode : uint8_t { INSERT, DELETE, UPDATE }; + +// Streaming plan operator that writes the rows affected by a base-table DML statement +// to its delta table through the caller's transaction before passing the input onward. +class LogicalTransactionalDeltaCapture : public LogicalExtensionOperator { +public: + LogicalTransactionalDeltaCapture(TableCatalogEntry &base_table, TableCatalogEntry &delta_table, + DeltaCaptureMode mode, vector> update_expressions = {}, + vector update_columns = {}, optional_idx row_id_index = {}); + + TableCatalogEntry &base_table; + TableCatalogEntry &delta_table; + DeltaCaptureMode mode; + vector update_columns; + optional_idx row_id_index; + + PhysicalOperator &CreatePlan(ClientContext &context, PhysicalPlanGenerator &planner) override; + vector GetColumnBindings() override; + string GetName() const override; + string GetExtensionName() const override; + bool SupportSerialization() const override { + return false; + } + +protected: + void ResolveTypes() override; +}; + +// Transparent wrapper around a LogicalMergeInto. At physical-plan creation it decorates +// DuckDB's resolved action sinks, so only rows that actually INSERT/UPDATE/DELETE are captured. +class LogicalTransactionalMergeDeltaCapture : public LogicalExtensionOperator { +public: + LogicalTransactionalMergeDeltaCapture(TableCatalogEntry &base_table, TableCatalogEntry &delta_table); + + TableCatalogEntry &base_table; + TableCatalogEntry &delta_table; + + PhysicalOperator &CreatePlan(ClientContext &context, PhysicalPlanGenerator &planner) override; + vector GetColumnBindings() override; + string GetName() const override; + string GetExtensionName() const override; + bool SupportSerialization() const override { + return false; + } + +protected: + void ResolveTypes() override; +}; + +} // namespace duckdb + +#endif // TRANSACTIONAL_DELTA_CAPTURE_HPP diff --git a/src/include/upsert/refresh.hpp b/src/include/upsert/refresh.hpp index a619e0e9..38b56b40 100644 --- a/src/include/upsert/refresh.hpp +++ b/src/include/upsert/refresh.hpp @@ -8,9 +8,10 @@ namespace duckdb { -// Generates refresh SQL for each view (including cascaded views) and executes it -// under a per-view lock. This ensures concurrent refresh of the same view is serialized. +// Generates refresh SQL for each view (including cascaded views) and executes it. +// The caller must own the database-wide OpenIVM mutation gate. void UpsertDeltaQueriesLocked(ClientContext &context, const FunctionParameters ¶meters); +string TransactionalRefreshQuery(ClientContext &context, const FunctionParameters ¶meters); } // namespace duckdb diff --git a/src/include/upsert/refresh_compiler.hpp b/src/include/upsert/refresh_compiler.hpp index 9511508f..ee06f309 100644 --- a/src/include/upsert/refresh_compiler.hpp +++ b/src/include/upsert/refresh_compiler.hpp @@ -40,11 +40,14 @@ string CompileAggregateGroups(const string &view_name, optional_ptr &derived_output_expressions = {}, - bool derived_output_expressions_complete = false); + bool derived_output_expressions_complete = false, + const vector &preserved_side_cols = {}, bool *out_used_group_recompute = nullptr, + bool force_group_recompute = false); string CompileSimpleAggregates(const string &view_name, const vector &column_names, const string &view_query_sql = "", bool has_minmax = false, bool list_mode = false, const string &delta_ts_filter = "", const string &catalog_prefix = "", - bool insert_only = false, const vector &column_types = {}); + bool insert_only = false, const vector &column_types = {}, + bool *out_full_recompute = nullptr); string CompileProjectionsFilters(const string &view_name, const vector &column_names, const string &delta_ts_filter = "", const string &catalog_prefix = "", bool insert_only = false); @@ -52,7 +55,6 @@ string CompileWindowRecompute(const string &view_name, const string &view_query_ const string &catalog_prefix = "", const vector &partition_columns = {}, const vector &partition_delta_specs = {}, bool emit_cascade_delta = false, const string &affected_keys_sql = "", - const string &affected_key_cols = "", const string &affected_key_tuple = "", const vector &column_names = {}, bool running_window_incremental = false); string CompileFullRecompute(const string &view_name, const string &view_query_sql, const string &catalog_prefix = ""); diff --git a/src/include/upsert/refresh_cost_model.hpp b/src/include/upsert/refresh_cost_model.hpp index faba6fa1..2ed558bf 100644 --- a/src/include/upsert/refresh_cost_model.hpp +++ b/src/include/upsert/refresh_cost_model.hpp @@ -30,7 +30,7 @@ struct RefreshCostEstimate { // meaning of `incremental_compute` / `incremental_upsert`. For "incremental" // views the fields hold delta-driven IVM cost. For fixed strategy views they // hold that strategy's affected-domain cost. Known labels: "incremental", - // "group_recompute", "window_partition", "current_diff_recompute", + // "group_recompute", "window_partition", // "distinct_incremental", "semi_anti_recompute", and "full". string strategy_label; diff --git a/src/include/upsert/refresh_internal.hpp b/src/include/upsert/refresh_internal.hpp index 227ec985..9846d8bc 100644 --- a/src/include/upsert/refresh_internal.hpp +++ b/src/include/upsert/refresh_internal.hpp @@ -123,11 +123,18 @@ string BuildCompactDeltaViewSQL(const string &view_name, const string &delta_vie string BuildDeleteInsertRefreshSQL(const string &data_table, const string &view_query_sql, const string &recompute_alias, const string &delete_where, const string &insert_where, const string &statement_prefix = ""); +// When `upsert_keys` and `recompute_temp_table` are supplied, emits an INSERT OR REPLACE form that +// never deletes and re-inserts the same key inside one transaction. Required when the data table +// carries a UNIQUE index: DuckDB's on-disk unique index keeps deleted keys for constraint checking +// within the same transaction, so DELETE-then-INSERT of a surviving group raises a spurious +// "Duplicate key ... violates unique constraint". Only valid when such an index exists (INSERT OR +// REPLACE requires a UNIQUE/PK constraint); pass them empty to keep the plain DELETE+INSERT form. string BuildAffectedKeyRefreshSQL(const string &data_table, const string &view_query_sql, const string &affected_subquery, const string &target_alias, const string &recompute_alias, const string &affected_alias, const string &target_match, const string &recompute_match, - const string &affected_temp_table = ""); + const string &affected_temp_table = "", const vector &upsert_keys = {}, + const string &recompute_temp_table = ""); string BuildSignedMultisetDeltaInsertSQL(const string &delta_table, const string &old_source, const string &new_source, const string &statement_prefix = ""); bool IsSummableLogicalType(const LogicalType &type); @@ -140,7 +147,8 @@ string ResolveDuckLakeCatalogName(Connection &con, const string &view_catalog_na const string &attached_db_catalog_name); string BuildRecomputeQuery(RefreshMetadata &metadata, const string &view_name, const string &view_query_sql, bool cross_system, const string &attached_catalog = "", const string &attached_schema = "", - const string &catalog_prefix = "", string *out_post_meta = nullptr); + const string &catalog_prefix = "", const string &metadata_prefix = "", + string *out_post_meta = nullptr); string BuildFullOuterAffectedGroupRefresh(RefreshMetadata &metadata, const string &view_name, const vector &delta_table_names, const vector &group_cols, @@ -235,7 +243,7 @@ string GenerateRefreshSQL(ClientContext &context, const string &view_catalog_nam string *out_post_meta = nullptr, RefreshCompileProfile *compile_profile = nullptr, const DeltaActivityResult *precomputed_delta_activity = nullptr, RefreshCostEstimate *out_adaptive_estimate = nullptr, - const openivm::CompileFacts *facts = nullptr); + const openivm::CompileFacts *facts = nullptr, Connection *metadata_connection = nullptr); } // namespace duckdb diff --git a/src/openivm_extension.cpp b/src/openivm_extension.cpp index 18ab9ea4..795c272a 100644 --- a/src/openivm_extension.cpp +++ b/src/openivm_extension.cpp @@ -29,6 +29,7 @@ #include "duckdb/parser/tableref/subqueryref.hpp" #include "duckdb/planner/planner.hpp" #include "core/parser.hpp" +#include "core/parser_ddl.hpp" #include "rules/incremental_rewrite_rule.hpp" #include "rules/refresh_insert_rule.hpp" #include "core/openivm_debug.hpp" @@ -112,6 +113,9 @@ static duckdb::unique_ptr ComputeDeltaBind(ClientContext &context, input.named_parameters["view_schema_name"] = view_schema_name; Connection con(*context.db); + if (auto metadata_state = TransactionalMVMetadataState::TryGet(context)) { + metadata_state->Apply(con); + } string view_query = RefreshMetadata(con).GetViewQuery(view_name); if (view_query.empty()) { throw Exception(ExceptionType::CATALOG, @@ -157,6 +161,10 @@ static void LoadInternal(ExtensionLoader &loader) { // after the extension is loaded. Entry points also set the current ClientContext // explicitly because pre-existing local settings override global defaults. db_config.SetOption(PreserveInsertionOrderSetting::SettingIndex, Value::BOOLEAN(false)); + // CREATE/ALTER MATERIALIZED VIEW and transactional DROP VIEW are expanded into + // ordinary DuckDB statements by OpenIVM's parser override. FALLBACK leaves every + // statement OpenIVM does not recognize with DuckDB's native parser. + db_config.SetOption(AllowParserOverrideExtensionSetting::SettingIndex, Value("fallback")); db_config.AddExtensionOption("openivm_files_path", "path for compiled SQL reference files", LogicalType::VARCHAR); db_config.AddExtensionOption("openivm_refresh_mode", "refresh strategy: incremental, full, or auto", @@ -330,6 +338,8 @@ static void LoadInternal(ExtensionLoader &loader) { " ADD COLUMN IF NOT EXISTS count_distinct_aux_meta_json VARCHAR DEFAULT NULL"); con.Query("ALTER TABLE " + string(openivm::VIEWS_TABLE) + " ADD COLUMN IF NOT EXISTS semi_anti_aux_meta_json VARCHAR DEFAULT NULL"); + con.Query("ALTER TABLE " + string(openivm::VIEWS_TABLE) + + " ADD COLUMN IF NOT EXISTS leftjoin_secondary_meta_json VARCHAR DEFAULT NULL"); con.Query("ALTER TABLE " + string(openivm::VIEWS_TABLE) + " ADD COLUMN IF NOT EXISTS lineage_json VARCHAR DEFAULT NULL"); @@ -356,6 +366,19 @@ static void LoadInternal(ExtensionLoader &loader) { OptimizerExtension::Register(db_config, std::move(incremental_rewrite_rule)); OptimizerExtension::Register(db_config, std::move(refresh_insert_rule)); + loader.RegisterFunction(PragmaFunction::PragmaCall("openivm_materialized_view_lifecycle", + MaterializedViewLifecycleQuery, {LogicalType::VARCHAR})); + loader.RegisterFunction(PragmaFunction::PragmaCall("openivm_materialized_view_drop", MaterializedViewDropQuery, + {LogicalType::VARCHAR})); + loader.RegisterFunction(TableFunction( + "openivm_execute_drop_view", + {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::BOOLEAN}, + ExecuteDropView, BindDropView, InitDropView)); + loader.RegisterFunction(TableFunction( + "openivm_execute_drop_table", + {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::BOOLEAN, LogicalType::BOOLEAN}, + ExecuteDropView, BindDropTable, InitDropView)); + TableFunction compute_delta_function("ComputeDelta", {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}, ComputeDeltaFunction, ComputeDeltaBind, ComputeDeltaInit); @@ -387,13 +410,11 @@ static void LoadInternal(ExtensionLoader &loader) { con.Commit(); - // Use the locked pragma_function_t variant: generates SQL and executes it under a - // per-view mutex, preventing concurrent refresh from double-applying deltas. auto refresh_options = - PragmaFunction::PragmaCall("refresh_options", UpsertDeltaQueriesLocked, + PragmaFunction::PragmaCall("refresh_options", TransactionalRefreshQuery, {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}); loader.RegisterFunction(refresh_options); - auto refresh = PragmaFunction::PragmaCall("refresh", UpsertDeltaQueriesLocked, {LogicalType::VARCHAR}); + auto refresh = PragmaFunction::PragmaCall("refresh", TransactionalRefreshQuery, {LogicalType::VARCHAR}); loader.RegisterFunction(refresh); auto declare_rely_fk = PragmaFunction::PragmaCall( "openivm_declare_rely_fk", @@ -421,7 +442,7 @@ static void LoadInternal(ExtensionLoader &loader) { PragmaFunction::PragmaCall("refresh_history", RefreshCostHistoryQuery, {LogicalType::VARCHAR}); loader.RegisterFunction(refresh_history); auto refresh_cross_system = PragmaFunction::PragmaCall( - "refresh_cross_system", UpsertDeltaQueriesLocked, + "refresh_cross_system", TransactionalRefreshQuery, {LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR, LogicalType::VARCHAR}); loader.RegisterFunction(refresh_cross_system); @@ -489,17 +510,21 @@ static void LoadInternal(ExtensionLoader &loader) { loader.RegisterFunction(refresh_status); // PRAGMA refresh_start_daemon — (re)start the daemon on the caller's DB instance. - auto refresh_start_daemon = - PragmaFunction::PragmaCall("refresh_start_daemon", - [](ClientContext &context, const FunctionParameters &) -> string { - if (global_daemon) { - global_daemon->Stop(); - } - global_daemon = make_shared_ptr(); - global_daemon->Start(*context.db); - return "SELECT true AS started;"; - }, - {}); + auto refresh_start_daemon = PragmaFunction::PragmaCall( + "refresh_start_daemon", + [](ClientContext &context, const FunctionParameters &) -> string { + if (!context.transaction.IsAutoCommit()) { + throw TransactionException("The OpenIVM refresh daemon cannot be restarted inside a transaction"); + } + if (global_daemon) { + global_daemon->Stop(); + } + global_daemon = make_shared_ptr(); + global_daemon->Start(*context.db); + global_daemon->Wake(); + return "SELECT true AS started;"; + }, + {}); loader.RegisterFunction(refresh_start_daemon); // Start the refresh daemon unless disabled (e.g. shadow/compile-only DBs). diff --git a/src/rules/incremental_rewrite_rule.cpp b/src/rules/incremental_rewrite_rule.cpp index aea57447..55e6a5a8 100644 --- a/src/rules/incremental_rewrite_rule.cpp +++ b/src/rules/incremental_rewrite_rule.cpp @@ -2,6 +2,7 @@ #include "core/openivm_constants.hpp" #include "core/openivm_debug.hpp" +#include "core/parser_ddl.hpp" #include "core/parser_plan_helpers.hpp" #include "core/scoped_optimizer_settings.hpp" #include "core/sql_utils.hpp" @@ -85,6 +86,9 @@ void IncrementalRewriteRule::IncrementalRewriteRuleFunction(OptimizerExtensionIn auto view_schema = child_get->named_parameters["view_schema_name"].ToString(); Connection con(*input.context.db); + if (auto metadata_state = TransactionalMVMetadataState::TryGet(input.context)) { + metadata_state->Apply(con); + } auto v = con.Query("select sql_string from " + string(openivm::VIEWS_TABLE) + " where view_name = '" + SqlUtils::EscapeValue(view) + "';"); diff --git a/src/rules/refresh_insert_rule.cpp b/src/rules/refresh_insert_rule.cpp index 4f88a18f..800b90be 100644 --- a/src/rules/refresh_insert_rule.cpp +++ b/src/rules/refresh_insert_rule.cpp @@ -7,170 +7,353 @@ #include "core/refresh_locks.hpp" #include "core/sql_utils.hpp" #include "rules/column_hider.hpp" +#include "rules/transactional_delta_capture.hpp" -#include "lpts_pipeline.hpp" #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" -#include "duckdb/common/column_index.hpp" #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp" -#include "duckdb/function/table/read_csv.hpp" +#include "duckdb/common/enums/database_modification_type.hpp" +#include "duckdb/main/client_data.hpp" #include "duckdb/main/connection.hpp" +#include "duckdb/main/database_manager.hpp" #include "duckdb/optimizer/optimizer.hpp" #include "duckdb/parser/parsed_data/alter_table_info.hpp" #include "duckdb/parser/parsed_data/drop_info.hpp" -#include "duckdb/parser/parser.hpp" -#include "duckdb/parser/statement/logical_plan_statement.hpp" -#include "duckdb/planner/operator/logical_comparison_join.hpp" -#include "duckdb/parser/tableref/basetableref.hpp" -#include "duckdb/planner/expression.hpp" +#include "duckdb/parser/qualified_name.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" -#include "duckdb/planner/expression/bound_constant_expression.hpp" -#include "duckdb/planner/expression_iterator.hpp" -#include "duckdb/planner/operator/logical_aggregate.hpp" #include "duckdb/planner/operator/logical_delete.hpp" -#include "duckdb/planner/operator/logical_dummy_scan.hpp" -#include "duckdb/planner/operator/logical_expression_get.hpp" -#include "duckdb/planner/operator/logical_filter.hpp" -#include "duckdb/planner/operator/logical_get.hpp" #include "duckdb/planner/operator/logical_insert.hpp" +#include "duckdb/planner/operator/logical_merge_into.hpp" #include "duckdb/planner/operator/logical_projection.hpp" #include "duckdb/planner/operator/logical_simple.hpp" #include "duckdb/planner/operator/logical_update.hpp" -#include "duckdb/planner/planner.hpp" - -#include -#include +#include "duckdb/transaction/meta_transaction.hpp" namespace duckdb { -// PAC compatibility boundary: delta writes run through a fresh connection, so -// disable PAC checks when that extension is loaded in the caller session. -static void DisablePACIfLoaded(ClientContext &context, Connection &con) { - Value pac_val; - if (context.TryGetCurrentSetting("pac_check", pac_val)) { - con.Query("SET pac_check = false"); +class TransactionalHelperUndoState : public ClientContextState { +public: + static TransactionalHelperUndoState &Get(ClientContext &context) { + auto state = + context.registered_state->GetOrCreate("openivm_transactional_helper_undo"); + if (!state->mutation_guard) { + state->mutation_guard = make_uniq(context); + } + return *state; } -} -// Build the data column list from a delta table catalog entry, excluding metadata columns. -// Returns e.g. "id, name, val" (quoted) — the base table columns only. -static string BuildDeltaDataColumns(TableCatalogEntry &delta_entry) { - string cols; - for (auto &col : delta_entry.GetColumns().Logical()) { - if (col.GetName() == openivm::MULTIPLICITY_COL || col.GetName() == openivm::TIMESTAMP_COL) { - continue; + void AddRestoreSQL(string sql) { + restore_sql.push_back(std::move(sql)); + } + + void TransactionCommit(MetaTransaction &transaction, ClientContext &context) override { + Clear(); + } + + void TransactionRollback(MetaTransaction &transaction, ClientContext &context) override { + OPENIVM_DEBUG_PRINT("[TRANSACTIONAL DDL] restoring %zu metadata snapshots\n", restore_sql.size()); + try { + Connection con(*context.db); + auto schema_result = con.Query("SET schema='" + string(DEFAULT_SCHEMA) + "'"); + if (schema_result->HasError()) { + OPENIVM_DEBUG_PRINT("[TRANSACTIONAL DDL] metadata restore setup failed: %s\n", + schema_result->GetError().c_str()); + } else { + for (auto it = restore_sql.rbegin(); it != restore_sql.rend(); ++it) { + auto result = con.Query(*it); + if (result->HasError()) { + OPENIVM_DEBUG_PRINT("[TRANSACTIONAL DDL] metadata restore failed: %s\n", + result->GetError().c_str()); + } + } + } + } catch (std::exception &ex) { + // Transaction callbacks must always release the mutation gate. A + // helper-restore error is diagnostic here; the caller transaction has + // already rolled back and cannot report a second failure safely. + OPENIVM_DEBUG_PRINT("[TRANSACTIONAL DDL] metadata rollback callback failed: %s\n", ex.what()); } - if (!cols.empty()) { - cols += ", "; + Clear(); + } + +private: + void Clear() { + restore_sql.clear(); + mutation_guard.reset(); + } + + vector restore_sql; + unique_ptr mutation_guard; +}; + +static string BuildRestoreRowsSQL(MaterializedQueryResult &rows, const string &table_name) { + if (rows.RowCount() == 0) { + return ""; + } + string columns; + for (auto &name : rows.names) { + if (!columns.empty()) { + columns += ", "; + } + columns += SqlUtils::QuoteIdentifier(name); + } + string values; + for (idx_t row = 0; row < rows.RowCount(); row++) { + if (!values.empty()) { + values += ", "; + } + values += "("; + for (idx_t col = 0; col < rows.ColumnCount(); col++) { + if (col > 0) { + values += ", "; + } + values += rows.GetValue(col, row).ToSQLString(); } - cols += KeywordHelper::WriteOptionallyQuoted(col.GetName()); + values += ")"; } - return cols; + return "INSERT OR REPLACE INTO " + SqlUtils::QuoteIdentifier(table_name) + " (" + columns + ") VALUES " + values; } -// Build "INSERT INTO delta_t (col1, col2, ..., mul, ts)" prefix for delta writes. -static string BuildDeltaInsertPrefix(const string &full_delta_table_name, TableCatalogEntry &delta_entry) { - string col_list = BuildDeltaDataColumns(delta_entry); - return "INSERT INTO " + full_delta_table_name + " (" + col_list + ", " + string(openivm::MULTIPLICITY_COL) + ", " + - string(openivm::TIMESTAMP_COL) + ")"; +static void RegisterMetadataRestore(ClientContext &context, Connection &con, const string &table_name, + const string &predicate) { + auto rows = con.Query("SELECT * FROM " + SqlUtils::QuoteIdentifier(table_name) + " WHERE " + predicate); + if (rows->HasError()) { + throw CatalogException("OpenIVM could not snapshot helper metadata: %s", rows->GetError()); + } + auto restore = BuildRestoreRowsSQL(*rows, table_name); + if (!restore.empty()) { + TransactionalHelperUndoState::Get(context).AddRestoreSQL(std::move(restore)); + } } -// Build "SELECT col1, col2, ..., , now()::timestamp FROM " for delta writes. -static string BuildDeltaSelectFrom(TableCatalogEntry &delta_entry, const string &mul_val, const string &source) { - string cols = BuildDeltaDataColumns(delta_entry); - return "SELECT " + cols + ", " + mul_val + ", now()::timestamp FROM " + source; +static void ExecuteHelperMetadataSQL(Connection &con, const string &sql) { + auto result = con.Query(sql); + if (result->HasError()) { + throw CatalogException("OpenIVM metadata update failed: %s", result->GetError()); + } } -using BoundColumnNameMap = std::map, string>; +static void DropCatalogEntry(ClientContext &context, const string &catalog_name, const string &schema_name, + const string &entry_name, CatalogType type) { + DropInfo info; + info.type = type; + info.catalog = catalog_name; + info.schema = schema_name; + info.name = entry_name; + info.if_not_found = OnEntryNotFound::RETURN_NULL; + auto &catalog = Catalog::GetCatalog(context, catalog_name); + MetaTransaction::Get(context).ModifyDatabase(catalog.GetAttached(), DatabaseModificationType::DROP_CATALOG_ENTRY); + catalog.DropEntry(context, info); +} -static void CollectBoundColumnNames(LogicalOperator &op, BoundColumnNameMap &column_names) { - if (op.type == LogicalOperatorType::LOGICAL_GET) { - auto &get = op.Cast(); - auto bindings = get.GetColumnBindings(); - auto column_ids = get.GetColumnIds(); - for (auto &binding : bindings) { - if (binding.column_index >= column_ids.size()) { - continue; - } - const auto &column_name = get.GetColumnName(column_ids[binding.column_index]); - column_names[{binding.table_index, binding.column_index}] = - KeywordHelper::WriteOptionallyQuoted(column_name); - } - } - for (auto &child : op.children) { - CollectBoundColumnNames(*child, column_names); +static void DropQualifiedCatalogEntry(ClientContext &context, const string &qualified_name, + const string &fallback_catalog, const string &fallback_schema, CatalogType type) { + auto components = QualifiedName::ParseComponents(qualified_name); + if (components.size() == 1) { + DropCatalogEntry(context, fallback_catalog, fallback_schema, components[0], type); + } else if (components.size() == 2) { + DropCatalogEntry(context, fallback_catalog, components[0], components[1], type); + } else if (components.size() == 3) { + DropCatalogEntry(context, components[0], components[1], components[2], type); + } else { + throw InternalException("OpenIVM could not resolve internal relation '%s'", qualified_name); } } -static void QuoteBoundColumnRefs(Expression &expr, const BoundColumnNameMap &column_names) { - if (expr.GetExpressionClass() == ExpressionClass::BOUND_COLUMN_REF) { - auto &bcr = expr.Cast(); - auto entry = column_names.find({bcr.binding.table_index, bcr.binding.column_index}); - if (entry != column_names.end()) { - bcr.alias = entry->second; +static void AlterDeltaInCallerTransaction(ClientContext &context, AlterTableInfo &source_alter, + const string &catalog_name, const string &schema_name, + const string &delta_name) { + auto delta_alter = source_alter.Copy(); + delta_alter->catalog = catalog_name; + delta_alter->schema = schema_name; + delta_alter->name = delta_name; + auto &catalog = Catalog::GetCatalog(context, catalog_name); + MetaTransaction::Get(context).ModifyDatabase(catalog.GetAttached(), DatabaseModificationType::ALTER_TABLE); + catalog.Alter(context, *delta_alter); +} + +static pair ResolveDDLLocus(ClientContext &context, const string &catalog_name, + const string &schema_name) { + auto &default_entry = ClientData::Get(context).catalog_search_path->GetDefault(); + string default_catalog = + default_entry.catalog.empty() ? DatabaseManager::GetDefaultDatabase(context) : default_entry.catalog; + return { + catalog_name.empty() ? default_catalog : catalog_name, + schema_name.empty() ? (default_entry.schema.empty() ? DEFAULT_SCHEMA : default_entry.schema) : schema_name, + }; +} + +static bool SameRelationLocus(const string &left_catalog, const string &left_schema, const string &right_catalog, + const string &right_schema) { + return StringUtil::CIEquals(left_catalog, right_catalog) && StringUtil::CIEquals(left_schema, right_schema); +} + +static string MVInternalPrefix(ClientContext &context, const RefreshMetadata::StoredViewLocation &location, + const string &view_name) { + QueryErrorContext error_context; + auto entry = Catalog::GetEntry(context, location.catalog_name, location.schema_name, + EntryLookupInfo(CatalogType::VIEW_ENTRY, view_name, error_context), + OnEntryNotFound::RETURN_NULL); + if (entry) { + auto data_table = IncrementalTableNames::DataTableName(view_name); + auto &view = dynamic_cast(*entry); + auto data_ref = SqlUtils::FindTableReference(view.sql, data_table); + auto separator = data_ref.rfind('.'); + if (separator != string::npos) { + return data_ref.substr(0, separator + 1); } } - ExpressionIterator::EnumerateChildren(expr, [&](unique_ptr &child) { - if (child) { - QuoteBoundColumnRefs(*child, column_names); - } - }); + return SqlUtils::QualifiedPrefix(location.catalog_name, location.schema_name); } -static string QuotedExpressionString(const unique_ptr &expr, const BoundColumnNameMap &column_names) { - auto copy = expr->Copy(); - QuoteBoundColumnRefs(*copy, column_names); - return copy->ToString(); +static void DropTrackedMaterializedView(ClientContext &context, Connection &con, RefreshMetadata &metadata, + const string &view_name, bool drop_user_view) { + auto location = metadata.GetStoredViewLocation(view_name); + auto delta_sources = metadata.GetDeltaSources(view_name, location.catalog_name, location.schema_name); + auto internal_prefix = MVInternalPrefix(context, location, view_name); + auto escaped_view_name = SqlUtils::EscapeValue(view_name); + auto view_predicate = "view_name = '" + escaped_view_name + "'"; + RegisterMetadataRestore(context, con, openivm::VIEWS_TABLE, view_predicate); + RegisterMetadataRestore(context, con, openivm::DELTA_TABLES_TABLE, view_predicate); + auto dependency_predicate = "parent_view = '" + escaped_view_name + "' OR child_view = '" + escaped_view_name + "'"; + RegisterMetadataRestore(context, con, openivm::MV_DEPS_TABLE, dependency_predicate); + ExecuteHelperMetadataSQL(con, "DELETE FROM " + string(openivm::VIEWS_TABLE) + " WHERE view_name = '" + + escaped_view_name + "'"); + ExecuteHelperMetadataSQL(con, "DELETE FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name = '" + + escaped_view_name + "'"); + ExecuteHelperMetadataSQL(con, "DELETE FROM " + string(openivm::MV_DEPS_TABLE) + " WHERE " + dependency_predicate); + if (drop_user_view) { + DropCatalogEntry(context, location.catalog_name, location.schema_name, view_name, CatalogType::VIEW_ENTRY); + } + DropQualifiedCatalogEntry(context, + internal_prefix + KeywordHelper::WriteOptionallyQuoted(SqlUtils::DeltaName(view_name)), + location.catalog_name, location.schema_name, CatalogType::TABLE_ENTRY); + DropQualifiedCatalogEntry(context, + internal_prefix + + KeywordHelper::WriteOptionallyQuoted(IncrementalTableNames::DataTableName(view_name)), + location.catalog_name, location.schema_name, CatalogType::TABLE_ENTRY); + + for (auto &source : delta_sources) { + // DuckLake entries store the base table name — never drop it. + if (source.catalog_type == "ducklake") { + continue; + } + auto remaining = con.Query("SELECT count(*) FROM " + string(openivm::DELTA_TABLES_TABLE) + + " WHERE table_name = '" + SqlUtils::EscapeValue(source.table_name) + + "' AND COALESCE(source_catalog, '" + SqlUtils::EscapeValue(source.catalog_name) + + "') = '" + SqlUtils::EscapeValue(source.catalog_name) + + "' AND COALESCE(source_schema, '" + SqlUtils::EscapeValue(source.schema_name) + + "') = '" + SqlUtils::EscapeValue(source.schema_name) + "'"); + if (!remaining->HasError() && remaining->RowCount() > 0 && remaining->GetValue(0, 0).GetValue() == 0) { + DropCatalogEntry(context, source.catalog_name, source.schema_name, source.table_name, + CatalogType::TABLE_ENTRY); + } + } } -static string BuildDeltaInsertFromPlan(ClientContext &context, TableCatalogEntry &delta_entry, - const string &full_delta_table_name, unique_ptr &source_plan) { - string prefix = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry); - SqlDialect dialect = openivm::CompileFactsContextSlot::Get(context).target_dialect; - auto ast = LogicalPlanToAst(context, source_plan, dialect); - auto cte_list = AstToCteList(*ast, dialect); - string subquery_string = cte_list->ToQuery(false); - if (!subquery_string.empty() && subquery_string.back() == ';') { - subquery_string.pop_back(); - } - return prefix + " SELECT *, 1, now()::timestamp FROM (" + subquery_string + ")"; +static optional_ptr TryGetTrackedDeltaTable(ClientContext &context, TableCatalogEntry &table) { + const auto &table_name = table.name; + if (table_name.empty() || SqlUtils::IsDelta(table_name) || IncrementalTableNames::IsDataTable(table_name) || + table.catalog.GetCatalogType() == "ducklake") { + return nullptr; + } + auto delta_table = + Catalog::GetEntry(context, table.catalog.GetName(), table.schema.name, + SqlUtils::DeltaName(table_name), OnEntryNotFound::RETURN_NULL); + if (!delta_table) { + return nullptr; + } + return &delta_table->Cast(); } -static string BuildDeleteDeltaInsertFromPlan(ClientContext &context, TableCatalogEntry &delta_entry, - const string &full_delta_table_name, const string &full_table_name, - unique_ptr &source_plan) { - string prefix = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry); - string data_cols = BuildDeltaDataColumns(delta_entry); - SqlDialect dialect = openivm::CompileFactsContextSlot::Get(context).target_dialect; - auto ast = LogicalPlanToAst(context, source_plan, dialect); - auto cte_list = AstToCteList(*ast, dialect); - string subquery_string = cte_list->ToQuery(false); - if (!subquery_string.empty() && subquery_string.back() == ';') { - subquery_string.pop_back(); - } - // DuckDB DELETE children identify physical rows by rowid; read the base table - // columns back through that rowid set to materialize the negative delta tuple. - return prefix + " SELECT " + data_cols + ", -1, now()::timestamp FROM " + full_table_name + - " WHERE rowid IN (SELECT rowid FROM (" + subquery_string + ") openivm_deleted_rows)"; +static void ResolveInsertDefaults(OptimizerExtensionInput &input, LogicalInsert &insert) { + if (insert.column_index_map.empty()) { + return; + } + auto child_bindings = insert.children[0]->GetColumnBindings(); + vector> expressions; + for (auto &column : insert.table.GetColumns().Physical()) { + auto mapped_index = insert.column_index_map[column.Physical()]; + if (mapped_index == DConstants::INVALID_INDEX) { + expressions.push_back(insert.bound_defaults[column.StorageOid()]->Copy()); + } else { + if (mapped_index >= child_bindings.size()) { + throw InternalException("OpenIVM insert column mapping is out of range"); + } + expressions.push_back(make_uniq(column.Type(), child_bindings[mapped_index])); + } + } + auto projection = make_uniq(input.optimizer.binder.GenerateTableIndex(), std::move(expressions)); + projection->children.push_back(std::move(insert.children[0])); + insert.children[0] = std::move(projection); + insert.column_index_map = physical_index_vector_t(); + insert.expected_types = insert.table.GetTypes(); } -static bool IsRowIdColumn(const unique_ptr &expr) { - if (!expr || expr->type != ExpressionType::BOUND_COLUMN_REF) { - return false; +static idx_t FindExpressionBindingIndex(LogicalOperator &child, const Expression &expression) { + if (expression.type != ExpressionType::BOUND_COLUMN_REF) { + throw InternalException("OpenIVM expected a bound row-id column reference"); + } + auto &column_ref = expression.Cast(); + auto bindings = child.GetColumnBindings(); + for (idx_t index = 0; index < bindings.size(); index++) { + if (bindings[index] == column_ref.binding) { + return index; + } } - auto &col_ref = expr->Cast(); - return StringUtil::CIEquals(col_ref.GetName(), "rowid") || StringUtil::CIEquals(col_ref.alias, "rowid"); + throw InternalException("OpenIVM could not resolve the DML row-id binding"); } -static bool IsSemiJoinOnRowId(LogicalComparisonJoin &join) { - if (join.join_type != JoinType::SEMI) { - return false; +static void ResolveUpdateDefaults(OptimizerExtensionInput &input, LogicalUpdate &update) { + bool has_default = false; + for (auto &expression : update.expressions) { + has_default = has_default || expression->type == ExpressionType::VALUE_DEFAULT; + } + if (!has_default) { + return; + } + + auto child_bindings = update.children[0]->GetColumnBindings(); + auto child_types = update.children[0]->types; + if (child_bindings.empty() || child_bindings.size() != child_types.size()) { + throw InternalException("OpenIVM cannot normalize UPDATE defaults without a row-id input"); } - for (auto &condition : join.conditions) { - if (IsRowIdColumn(condition.left) || IsRowIdColumn(condition.right)) { - return true; + + auto projection_index = input.optimizer.binder.GenerateTableIndex(); + vector> projection_expressions; + projection_expressions.reserve(child_bindings.size() + update.expressions.size()); + for (idx_t index = 0; index + 1 < child_bindings.size(); index++) { + projection_expressions.push_back( + make_uniq(child_types[index], child_bindings[index])); + } + + vector default_indexes(update.expressions.size(), DConstants::INVALID_INDEX); + for (idx_t index = 0; index < update.expressions.size(); index++) { + if (update.expressions[index]->type != ExpressionType::VALUE_DEFAULT) { + continue; } + default_indexes[index] = projection_expressions.size(); + projection_expressions.push_back(update.bound_defaults[update.columns[index].index]->Copy()); } - return false; + + const auto row_id_output_index = projection_expressions.size(); + projection_expressions.push_back(make_uniq(child_types.back(), child_bindings.back())); + for (idx_t index = 0; index < update.expressions.size(); index++) { + auto return_type = update.expressions[index]->return_type; + idx_t output_index; + if (default_indexes[index] != DConstants::INVALID_INDEX) { + output_index = default_indexes[index]; + } else { + output_index = FindExpressionBindingIndex(*update.children[0], *update.expressions[index]); + D_ASSERT(output_index + 1 < child_bindings.size()); + } + update.expressions[index] = + make_uniq(return_type, ColumnBinding(projection_index, output_index)); + } + + auto projection = make_uniq(projection_index, std::move(projection_expressions)); + projection->children.push_back(std::move(update.children[0])); + update.children[0] = std::move(projection); + D_ASSERT(update.children[0]->GetColumnBindings().size() == row_id_output_index + 1); } RefreshInsertRule::RefreshInsertRule() { @@ -194,69 +377,53 @@ void RefreshInsertRule::RefreshInsertRuleFunction(OptimizerExtensionInput &input } auto table_name = drop_info->name; + auto target_locus = ResolveDDLLocus(input.context, drop_info->catalog, drop_info->schema); Connection con(*input.context.db); auto view_check = con.Query("SELECT 1 FROM " + string(openivm::VIEWS_TABLE) + " WHERE view_name = '" + SqlUtils::EscapeValue(table_name) + "'"); if (!view_check->HasError() && view_check->RowCount() > 0) { - // Acquire view lock to prevent cleanup during an in-flight refresh - ViewLockGuard view_guard(table_name); - OPENIVM_DEBUG_PRINT("[INSERT RULE] DROP TABLE '%s' — cleaning up IVM metadata\n", table_name.c_str()); RefreshMetadata metadata(con); - auto delta_tables = metadata.GetDeltaTables(table_name); - - con.Query("DELETE FROM " + string(openivm::VIEWS_TABLE) + " WHERE view_name = '" + - SqlUtils::EscapeValue(table_name) + "'"); - con.Query("DELETE FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name = '" + - SqlUtils::EscapeValue(table_name) + "'"); - con.Query("DROP TABLE IF EXISTS " + KeywordHelper::WriteOptionallyQuoted(SqlUtils::DeltaName(table_name))); - con.Query("DROP TABLE IF EXISTS " + - KeywordHelper::WriteOptionallyQuoted(IncrementalTableNames::DataTableName(table_name))); - - for (auto &dt : delta_tables) { - // DuckLake entries store the base table name — never drop it - if (metadata.IsDuckLakeTable(table_name, dt)) { - continue; - } - auto remaining = con.Query("SELECT count(*) FROM " + string(openivm::DELTA_TABLES_TABLE) + - " WHERE table_name = '" + SqlUtils::EscapeValue(dt) + "'"); - if (!remaining->HasError() && remaining->RowCount() > 0 && - remaining->GetValue(0, 0).GetValue() == 0) { - con.Query("DROP TABLE IF EXISTS " + KeywordHelper::WriteOptionallyQuoted(dt)); + auto location = metadata.GetStoredViewLocation(table_name); + if (SameRelationLocus(location.catalog_name, location.schema_name, target_locus.first, + target_locus.second)) { + TransactionalMVLockState::Get(input.context).AcquireMutationLock(); + OPENIVM_DEBUG_PRINT("[INSERT RULE] DROP TABLE '%s' — cleaning up IVM metadata\n", table_name.c_str()); + bool drop_user_view = drop_info->type == CatalogType::VIEW_ENTRY; + DropTrackedMaterializedView(input.context, con, metadata, table_name, drop_user_view); + if (drop_user_view) { + // The original logical DROP remains as an idempotent no-op. + drop_info->if_not_found = OnEntryNotFound::RETURN_NULL; } } } // Handle CASCADE: drop dependent MVs - auto dep_check = - con.Query("SELECT DISTINCT view_name FROM " + string(openivm::DELTA_TABLES_TABLE) + - " WHERE table_name = '" + SqlUtils::EscapeValue(SqlUtils::DeltaName(table_name)) + "'"); + auto dep_check = con.Query("SELECT DISTINCT view_name FROM " + string(openivm::DELTA_TABLES_TABLE) + + " WHERE table_name = '" + SqlUtils::EscapeValue(SqlUtils::DeltaName(table_name)) + + "' AND COALESCE(source_catalog, '" + SqlUtils::EscapeValue(target_locus.first) + + "') = '" + SqlUtils::EscapeValue(target_locus.first) + + "' AND COALESCE(source_schema, '" + SqlUtils::EscapeValue(target_locus.second) + + "') = '" + SqlUtils::EscapeValue(target_locus.second) + "'"); if (!dep_check->HasError() && dep_check->RowCount() > 0 && drop_info->cascade) { + TransactionalMVLockState::Get(input.context).AcquireMutationLock(); + RefreshMetadata cascade_metadata(con); + vector dependent_views; + unordered_set seen_dependents; for (size_t i = 0; i < dep_check->RowCount(); i++) { - auto dep_view = dep_check->GetValue(0, i).ToString(); - // Lock each dependent view before dropping - ViewLockGuard view_guard(dep_view); - RefreshMetadata dep_metadata(con); - auto dep_delta_tables = dep_metadata.GetDeltaTables(dep_view); - - con.Query("DELETE FROM " + string(openivm::VIEWS_TABLE) + " WHERE view_name = '" + - SqlUtils::EscapeValue(dep_view) + "'"); - con.Query("DELETE FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE view_name = '" + - SqlUtils::EscapeValue(dep_view) + "'"); - con.Query("DROP TABLE IF EXISTS " + - KeywordHelper::WriteOptionallyQuoted(SqlUtils::DeltaName(dep_view))); - con.Query("DROP TABLE IF EXISTS " + - KeywordHelper::WriteOptionallyQuoted(IncrementalTableNames::DataTableName(dep_view))); - con.Query("DROP VIEW IF EXISTS " + KeywordHelper::WriteOptionallyQuoted(dep_view)); - - for (auto &dt : dep_delta_tables) { - auto remaining = con.Query("SELECT count(*) FROM " + string(openivm::DELTA_TABLES_TABLE) + - " WHERE table_name = '" + SqlUtils::EscapeValue(dt) + "'"); - if (!remaining->HasError() && remaining->RowCount() > 0 && - remaining->GetValue(0, 0).GetValue() == 0) { - con.Query("DROP TABLE IF EXISTS " + KeywordHelper::WriteOptionallyQuoted(dt)); + auto direct_view = dep_check->GetValue(0, i).ToString(); + auto downstream = cascade_metadata.GetDownstreamViews(direct_view); + for (auto it = downstream.rbegin(); it != downstream.rend(); ++it) { + if (seen_dependents.insert(*it).second) { + dependent_views.push_back(*it); } } + if (seen_dependents.insert(direct_view).second) { + dependent_views.push_back(std::move(direct_view)); + } + } + for (auto &dep_view : dependent_views) { + DropTrackedMaterializedView(input.context, con, cascade_metadata, dep_view, true); } } @@ -276,15 +443,18 @@ void RefreshInsertRule::RefreshInsertRuleFunction(OptimizerExtensionInput &input string table_name = alter_info->name; string delta_name = SqlUtils::DeltaName(table_name); - string qdelta = KeywordHelper::WriteOptionallyQuoted(delta_name); + auto source_locus = ResolveDDLLocus(input.context, alter_info->catalog, alter_info->schema); Connection con(*input.context.db); // Check if a delta table exists for this base table (i.e., it's tracked by IVM) - auto delta_check = con.Query("SELECT 1 FROM information_schema.tables WHERE table_name = '" + + auto delta_check = con.Query("SELECT 1 FROM information_schema.tables WHERE table_catalog = '" + + SqlUtils::EscapeValue(source_locus.first) + "' AND table_schema = '" + + SqlUtils::EscapeValue(source_locus.second) + "' AND table_name = '" + SqlUtils::EscapeValue(delta_name) + "'"); if (delta_check->HasError() || delta_check->RowCount() == 0) { return; // not an IVM-tracked table } + TransactionalMVLockState::Get(input.context).AcquireMutationLock(); switch (alter_info->alter_table_type) { case AlterTableType::ADD_COLUMN: { @@ -294,9 +464,8 @@ void RefreshInsertRule::RefreshInsertRuleFunction(OptimizerExtensionInput &input } OPENIVM_DEBUG_PRINT("[INSERT RULE] ALTER TABLE ADD COLUMN '%s' — syncing delta table\n", add_info->new_column.Name().c_str()); - con.Query("ALTER TABLE " + qdelta + " ADD COLUMN IF NOT EXISTS " + - KeywordHelper::WriteOptionallyQuoted(add_info->new_column.Name()) + " " + - add_info->new_column.Type().ToString()); + AlterDeltaInCallerTransaction(input.context, *alter_info, source_locus.first, source_locus.second, + delta_name); break; } case AlterTableType::REMOVE_COLUMN: { @@ -305,15 +474,16 @@ void RefreshInsertRule::RefreshInsertRuleFunction(OptimizerExtensionInput &input break; } string col_name = remove_info->removed_column; - string referencing_mv = FirstMVReferencingColumn(con, delta_name, table_name, col_name); + string referencing_mv = FirstMVReferencingColumn(con, delta_name, source_locus.first, source_locus.second, + table_name, col_name); if (!referencing_mv.empty()) { throw CatalogException("Cannot drop column '" + col_name + "': it is referenced by materialized view '" + referencing_mv + "'. Drop the view first."); } OPENIVM_DEBUG_PRINT("[INSERT RULE] ALTER TABLE DROP COLUMN '%s' — syncing delta table\n", col_name.c_str()); - con.Query("ALTER TABLE " + qdelta + " DROP COLUMN IF EXISTS " + - KeywordHelper::WriteOptionallyQuoted(col_name)); + AlterDeltaInCallerTransaction(input.context, *alter_info, source_locus.first, source_locus.second, + delta_name); break; } case AlterTableType::RENAME_COLUMN: { @@ -323,11 +493,19 @@ void RefreshInsertRule::RefreshInsertRuleFunction(OptimizerExtensionInput &input } string old_name = rename_info->old_name; string new_name = rename_info->new_name; - RewriteDependentViewMetadataForRename(con, delta_name, table_name, old_name, new_name); + auto dependent_view_predicate = + "view_name IN (SELECT view_name FROM " + string(openivm::DELTA_TABLES_TABLE) + " WHERE table_name = '" + + SqlUtils::EscapeValue(delta_name) + "' AND COALESCE(source_catalog, '" + + SqlUtils::EscapeValue(source_locus.first) + "') = '" + SqlUtils::EscapeValue(source_locus.first) + + "' AND COALESCE(source_schema, '" + SqlUtils::EscapeValue(source_locus.second) + "') = '" + + SqlUtils::EscapeValue(source_locus.second) + "')"; + RegisterMetadataRestore(input.context, con, openivm::VIEWS_TABLE, dependent_view_predicate); + RewriteDependentViewMetadataForRename(con, delta_name, source_locus.first, source_locus.second, table_name, + old_name, new_name); OPENIVM_DEBUG_PRINT("[INSERT RULE] ALTER TABLE RENAME COLUMN '%s' → '%s' — syncing delta table\n", old_name.c_str(), new_name.c_str()); - con.Query("ALTER TABLE " + qdelta + " RENAME COLUMN " + KeywordHelper::WriteOptionallyQuoted(old_name) + - " TO " + KeywordHelper::WriteOptionallyQuoted(new_name)); + AlterDeltaInCallerTransaction(input.context, *alter_info, source_locus.first, source_locus.second, + delta_name); break; } default: @@ -340,379 +518,87 @@ void RefreshInsertRule::RefreshInsertRuleFunction(OptimizerExtensionInput &input return; } - auto root_name = root->GetName(); - if (root_name.rfind("INSERT", 0) != 0 && root_name.rfind("DELETE", 0) != 0 && root_name.rfind("UPDATE", 0) != 0) { - return; + auto dml_owner = &plan; + auto dml = dml_owner->get(); + while (dml->type != LogicalOperatorType::LOGICAL_INSERT && dml->type != LogicalOperatorType::LOGICAL_DELETE && + dml->type != LogicalOperatorType::LOGICAL_UPDATE && dml->type != LogicalOperatorType::LOGICAL_MERGE_INTO) { + if (dml->children.size() != 1) { + return; + } + dml_owner = &dml->children[0]; + dml = dml_owner->get(); } - switch (root->type) { + switch (dml->type) { case LogicalOperatorType::LOGICAL_INSERT: { - auto insert_node = dynamic_cast(root); - auto insert_table_name = insert_node->table.name; - OPENIVM_DEBUG_PRINT("[INSERT RULE] INSERT into '%s'\n", insert_table_name.c_str()); - - if (SqlUtils::IsDelta(insert_table_name) || insert_table_name.empty() || - IncrementalTableNames::IsDataTable(insert_table_name)) { - return; - } - // DuckLake tables have native change tracking — no delta writes needed - if (insert_node->table.catalog.GetCatalogType() == "ducklake") { - OPENIVM_DEBUG_PRINT("[INSERT RULE] Skipping delta for DuckLake table '%s'\n", insert_table_name.c_str()); + auto &insert = dml->Cast(); + const auto &table_name = insert.table.name; + auto delta_table = TryGetTrackedDeltaTable(input.context, insert.table); + if (!delta_table) { return; } - auto delta_table_catalog_entry = Catalog::GetEntry( - input.context, insert_node->table.catalog.GetName(), insert_node->table.schema.name, - SqlUtils::DeltaName(insert_table_name), OnEntryNotFound::RETURN_NULL); - - if (delta_table_catalog_entry) { - Connection con(*input.context.db); - DisablePACIfLoaded(input.context, con); - RefreshMetadata metadata(con); - if (metadata.IsBaseTable(insert_table_name)) { - string full_delta_table_name = SqlUtils::FullDeltaName( - insert_node->table.catalog.GetName(), insert_node->table.schema.name, insert_node->table.name); - if (insert_node->children[0]->type == LogicalOperatorType::LOGICAL_PROJECTION) { - auto &delta_entry_ins = delta_table_catalog_entry->Cast(); - string insert_query = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry_ins); - - auto projection = dynamic_cast(insert_node->children[0].get()); - if (projection->children[0]->type == LogicalOperatorType::LOGICAL_EXPRESSION_GET) { - insert_query += " VALUES "; - auto expression_get = dynamic_cast(projection->children[0].get()); - bool all_values_are_constants = true; - for (auto &expression : expression_get->expressions) { - for (auto &value : expression) { - if (value->type != ExpressionType::VALUE_CONSTANT) { - all_values_are_constants = false; - break; - } - } - if (!all_values_are_constants) { - break; - } - } - if (!all_values_are_constants) { - // DuckDB may bind VALUES literals through casts or other scalar expressions. Serialize the - // planned insert source instead of rejecting an otherwise valid base-table write. - insert_query = BuildDeltaInsertFromPlan(*con.context, delta_entry_ins, - full_delta_table_name, insert_node->children[0]); - } else { - for (auto &expression : expression_get->expressions) { - string values = "("; - for (auto &value : expression) { - auto constant = dynamic_cast(value.get()); - values += constant->value.ToSQLString() + ","; - } - values += "1, now()::timestamp),"; - insert_query += values; - } - insert_query.pop_back(); - } - } else { - auto &delta_entry = delta_table_catalog_entry->Cast(); - insert_query = BuildDeltaInsertFromPlan(*con.context, delta_entry, full_delta_table_name, - insert_node->children[0]); - } - OPENIVM_DEBUG_PRINT("[INSERT RULE] insert_query: %s\n", insert_query.c_str()); - { - DeltaLockGuard guard(SqlUtils::DeltaName(insert_table_name)); - auto r = con.Query(insert_query); - if (r->HasError()) { - throw Exception(ExceptionType::EXECUTOR, - "Cannot insert in delta table after insertion! " + r->GetError()); - } - } - - } else if (insert_node->children[0]->type == LogicalOperatorType::LOGICAL_GET) { - auto get = dynamic_cast(insert_node->children[0].get()); - auto *bind_data = dynamic_cast(get->bind_data.get()); - if (!bind_data) { - throw NotImplementedException( - "Only CSV file imports (read_csv) are supported for IVM delta tracking " - "via LOGICAL_GET. Other table functions are not yet supported."); - } - auto &delta_entry_csv = delta_table_catalog_entry->Cast(); - string prefix_csv = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry_csv); - auto files = bind_data->file_list->GetAllFiles(); - for (auto &file : files) { - auto query = prefix_csv + " SELECT *, 1, now()::timestamp FROM read_csv('" + file.path + "');"; - DeltaLockGuard guard(SqlUtils::DeltaName(insert_table_name)); - auto r = con.Query(query); - if (r->HasError()) { - throw Exception(ExceptionType::EXECUTOR, "Cannot insert in delta table! " + r->GetError()); - } - } - } else { - // Any other insert-source plan shape. DuckDB places a STREAMING_LIMIT/LOGICAL_LIMIT - // directly above the projection for larger INSERT ... SELECT (above a row threshold), - // so children[0] is neither LOGICAL_PROJECTION nor LOGICAL_GET. Serialize the whole - // source plan generically so the delta is still captured. Without this catch-all, such - // inserts updated the base table but silently skipped the delta, leaving the MV stale - // after the next refresh. - auto &delta_entry_other = delta_table_catalog_entry->Cast(); - string insert_query = BuildDeltaInsertFromPlan(*con.context, delta_entry_other, - full_delta_table_name, insert_node->children[0]); - OPENIVM_DEBUG_PRINT("[INSERT RULE] generic-plan delta insert_query: %s\n", insert_query.c_str()); - DeltaLockGuard guard(SqlUtils::DeltaName(insert_table_name)); - auto r = con.Query(insert_query); - if (r->HasError()) { - throw Exception(ExceptionType::EXECUTOR, - "Cannot insert in delta table after insertion! " + r->GetError()); - } - } - } - } - } break; - + ResolveInsertDefaults(input, insert); + auto capture = + make_uniq(insert.table, *delta_table, DeltaCaptureMode::INSERT); + capture->children.push_back(std::move(insert.children[0])); + insert.children[0] = std::move(capture); + OPENIVM_DEBUG_PRINT("[INSERT RULE] transactional INSERT delta capture for '%s'\n", table_name.c_str()); + break; + } case LogicalOperatorType::LOGICAL_DELETE: { - auto delete_node = dynamic_cast(root); - auto delete_table_name = delete_node->table.name; - OPENIVM_DEBUG_PRINT("[INSERT RULE] DELETE from '%s'\n", delete_table_name.c_str()); - if (SqlUtils::IsDelta(delete_table_name) || IncrementalTableNames::IsDataTable(delete_table_name)) { - return; - } - if (delete_node->table.catalog.GetCatalogType() == "ducklake") { - OPENIVM_DEBUG_PRINT("[INSERT RULE] Skipping delta for DuckLake table '%s'\n", delete_table_name.c_str()); + auto &delete_op = dml->Cast(); + const auto &table_name = delete_op.table.name; + auto delta_table = TryGetTrackedDeltaTable(input.context, delete_op.table); + if (!delta_table) { return; } - auto delta_table_catalog_entry = Catalog::GetEntry( - input.context, delete_node->table.catalog.GetName(), delete_node->table.schema.name, - SqlUtils::DeltaName(delete_table_name), OnEntryNotFound::RETURN_NULL); - - if (delta_table_catalog_entry) { - auto full_table_name = SqlUtils::FullName(delete_node->table.catalog.GetName(), - delete_node->table.schema.name, delete_node->table.name); - auto full_delta_table_name = SqlUtils::FullDeltaName( - delete_node->table.catalog.GetName(), delete_node->table.schema.name, delete_node->table.name); - Connection con(*input.context.db); - DisablePACIfLoaded(input.context, con); - RefreshMetadata metadata(con); - if (metadata.IsBaseTable(delete_table_name)) { - auto &delta_entry_del = delta_table_catalog_entry->Cast(); - string insert_string = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry_del) + " " + - BuildDeltaSelectFrom(delta_entry_del, "-1", full_table_name); - if (plan->children[0]->type == LogicalOperatorType::LOGICAL_FILTER) { - auto filter = dynamic_cast(plan->children[0].get()); - bool has_subquery = false; - for (auto &expr : filter->expressions) { - has_subquery = has_subquery || expr->HasSubquery(); - } - if (has_subquery) { - insert_string = BuildDeleteDeltaInsertFromPlan( - *con.context, delta_entry_del, full_delta_table_name, full_table_name, plan->children[0]); - } else { - BoundColumnNameMap column_names; - CollectBoundColumnNames(*filter->children[0], column_names); - insert_string += " where "; - for (idx_t i = 0; i < filter->expressions.size(); i++) { - if (i > 0) { - insert_string += " AND "; - } - insert_string += QuotedExpressionString(filter->expressions[i], column_names); - } - } - } else if (plan->children[0]->type == LogicalOperatorType::LOGICAL_GET) { - auto get = dynamic_cast(plan->children[0].get()); - if (!get->table_filters.filters.empty()) { - insert_string += " where "; - bool first_filter = true; - for (auto &entry : get->table_filters.filters) { - if (!first_filter) { - insert_string += " AND "; - } - first_filter = false; - auto col_name = get->GetColumnName(ColumnIndex(entry.first)); - col_name = KeywordHelper::WriteOptionallyQuoted(col_name); - insert_string += entry.second->ToString(col_name); - } - } - } else if (plan->children[0]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT) { - return; - } else if (plan->children[0]->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { - // DELETE FROM t WHERE rowid IN (subquery) compiles to a SEMI JOIN on - // rowid. LPTS cannot serialise the full SEMI JOIN because `rowid` is a - // virtual column whose binding is lost after the join. Serialise only - // the right child (the subquery returning rowids) and wrap it as - // WHERE rowid IN (...) against the base table. - auto *join = dynamic_cast(plan->children[0].get()); - if (join && IsSemiJoinOnRowId(*join) && !join->children.empty()) { - try { - string prefix_del = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry_del); - string data_cols = BuildDeltaDataColumns(delta_entry_del); - SqlDialect dialect = openivm::CompileFactsContextSlot::Get(*con.context).target_dialect; - auto ast = LogicalPlanToAst(*con.context, join->children[1], dialect); - auto cte_list = AstToCteList(*ast, dialect); - string rowid_sql = cte_list->ToQuery(false); - if (!rowid_sql.empty() && rowid_sql.back() == ';') { - rowid_sql.pop_back(); - } - insert_string = prefix_del + " SELECT " + data_cols + ", -1, now()::timestamp FROM " + - full_table_name + " WHERE rowid IN (" + rowid_sql + ")"; - } catch (...) { - throw NotImplementedException( - "DELETE with rowid IN (subquery) is not yet fully supported for IVM delta tracking"); - } - } else { - try { - insert_string = - BuildDeleteDeltaInsertFromPlan(*con.context, delta_entry_del, full_delta_table_name, - full_table_name, plan->children[0]); - } catch (...) { - throw NotImplementedException( - "DELETE with complex subqueries is not yet fully supported for IVM delta tracking"); - } - } - } else { - try { - string prefix_del = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry_del); - SqlDialect dialect = openivm::CompileFactsContextSlot::Get(*con.context).target_dialect; - auto ast = LogicalPlanToAst(*con.context, plan->children[0], dialect); - auto cte_list = AstToCteList(*ast, dialect); - string subquery_string = cte_list->ToQuery(false); - if (!subquery_string.empty() && subquery_string.back() == ';') { - subquery_string.pop_back(); - } - insert_string = prefix_del + " SELECT *, -1, now()::timestamp FROM (" + subquery_string + ")"; - } catch (...) { - throw NotImplementedException( - "DELETE with complex subqueries is not yet fully supported for IVM delta tracking"); - } - } - - { - DeltaLockGuard guard(SqlUtils::DeltaName(delete_table_name)); - auto r = con.Query(insert_string); - if (r->HasError()) { - throw Exception(ExceptionType::EXECUTOR, - "Cannot insert in delta table after deletion! " + r->GetError()); - } - } - } - } - } break; - + D_ASSERT(delete_op.expressions.size() == 1); + auto row_id_index = FindExpressionBindingIndex(*delete_op.children[0], *delete_op.expressions[0]); + auto capture = make_uniq( + delete_op.table, *delta_table, DeltaCaptureMode::DELETE, vector> {}, + vector {}, optional_idx(row_id_index)); + capture->children.push_back(std::move(delete_op.children[0])); + delete_op.children[0] = std::move(capture); + OPENIVM_DEBUG_PRINT("[INSERT RULE] transactional DELETE delta capture for '%s'\n", table_name.c_str()); + break; + } case LogicalOperatorType::LOGICAL_UPDATE: { - auto update_node = dynamic_cast(root); - auto update_table_name = update_node->table.name; - if (SqlUtils::IsDelta(update_table_name) || IncrementalTableNames::IsDataTable(update_table_name)) { + auto &update = dml->Cast(); + const auto &table_name = update.table.name; + auto delta_table = TryGetTrackedDeltaTable(input.context, update.table); + if (!delta_table) { return; } - if (update_node->table.catalog.GetCatalogType() == "ducklake") { - OPENIVM_DEBUG_PRINT("[INSERT RULE] Skipping delta for DuckLake table '%s'\n", update_table_name.c_str()); - return; + ResolveUpdateDefaults(input, update); + vector> update_expressions; + update_expressions.reserve(update.expressions.size()); + for (idx_t index = 0; index < update.expressions.size(); index++) { + update_expressions.push_back(update.expressions[index]->Copy()); } - auto delta_table_catalog_entry = Catalog::GetEntry( - input.context, update_node->table.catalog.GetName(), update_node->table.schema.name, - SqlUtils::DeltaName(update_table_name), OnEntryNotFound::RETURN_NULL); - - if (delta_table_catalog_entry) { - Connection con(*input.context.db); - DisablePACIfLoaded(input.context, con); - RefreshMetadata metadata(con); - if (!metadata.IsBaseTable(update_table_name)) { - break; - } - { - auto full_table_name = SqlUtils::FullName(update_node->table.catalog.GetName(), - update_node->table.schema.name, update_node->table.name); - auto full_delta_table_name = SqlUtils::FullDeltaName( - update_node->table.catalog.GetName(), update_node->table.schema.name, update_node->table.name); - auto *projection = dynamic_cast(update_node->children[0].get()); - if (!projection) { - OPENIVM_DEBUG_PRINT("[INSERT RULE] UPDATE skipped: no projection child (child type: %s)\n", - LogicalOperatorToString(update_node->children[0]->type).c_str()); - break; - } - - std::map update_values; - string where_string; - BoundColumnNameMap column_names; - CollectBoundColumnNames(*projection->children[0], column_names); - for (size_t i = 0; i < update_node->columns.size(); i++) { - auto column = update_node->columns[i].index; - update_values[to_string(column)] = QuotedExpressionString(projection->expressions[i], column_names); - } - - if (projection->children[0]->type == LogicalOperatorType::LOGICAL_FILTER) { - auto filter = dynamic_cast(projection->children[0].get()); - where_string += " where "; - for (idx_t i = 0; i < filter->expressions.size(); i++) { - if (i > 0) { - where_string += " AND "; - } - where_string += QuotedExpressionString(filter->expressions[i], column_names); - } - } else if (projection->children[0]->type == LogicalOperatorType::LOGICAL_GET) { - auto get = dynamic_cast(projection->children[0].get()); - if (!get->table_filters.filters.empty()) { - where_string += " where "; - bool first_filter = true; - for (auto &entry : get->table_filters.filters) { - if (!first_filter) { - where_string += " AND "; - } - first_filter = false; - auto col_name = get->GetColumnName(ColumnIndex(entry.first)); - col_name = KeywordHelper::WriteOptionallyQuoted(col_name); - where_string += entry.second->ToString(col_name); - } - } - } else if (projection->children[0]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT) { - return; - } else { - throw NotImplementedException("Only simple UPDATE statements are supported in IVM!"); - } - - auto &delta_entry_upd = delta_table_catalog_entry->Cast(); - string prefix_upd = BuildDeltaInsertPrefix(full_delta_table_name, delta_entry_upd); - string select_old = BuildDeltaSelectFrom(delta_entry_upd, "-1", full_table_name) + where_string; - // For select_new: use the update_values map to replace modified columns - string select_new = "SELECT "; - for (auto &col : delta_entry_upd.GetColumns().Logical()) { - if (col.GetName() == openivm::MULTIPLICITY_COL || col.GetName() == openivm::TIMESTAMP_COL) { - continue; - } - // Find the column's positional index in the base table - auto base_columns = update_node->table.GetColumns().GetColumnNames(); - for (size_t i = 0; i < base_columns.size(); i++) { - if (base_columns[i] == col.GetName()) { - if (update_values.find(to_string(i)) != update_values.end()) { - select_new += update_values[to_string(i)] + ", "; - } else { - select_new += KeywordHelper::WriteOptionallyQuoted(col.GetName()) + ", "; - } - break; - } - } - } - select_new += "1, now()::timestamp FROM " + full_table_name + where_string; - - { - DeltaLockGuard guard(SqlUtils::DeltaName(update_table_name)); - // ATOMIC UPDATE DELTA WRITES: - // The old-delete and new-insert rows MUST commit together. If they land - // in separate auto-commit transactions, a concurrent refresh can take a - // snapshot between them — seeing one but not the other. Since both rows - // target the same group, processing only one breaks consolidation (net - // count change should be 0, but ends up +1 or -1 → MV drift). - // - // Combining into a single multi-row INSERT via UNION ALL ensures both - // rows share one commit point and one `now()` value — either both visible - // in a given snapshot or neither. - string combined = prefix_upd + " " + "SELECT * FROM (" + select_old + - ") UNION ALL SELECT * FROM (" + select_new + ")"; - OPENIVM_DEBUG_PRINT("[INSERT RULE] combined UPDATE delta: %s\n", combined.c_str()); - auto r = con.Query(combined); - if (r->HasError()) { - throw Exception(ExceptionType::EXECUTOR, "Cannot insert UPDATE delta rows! " + r->GetError()); - } - } - } + auto row_id_index = update.children[0]->GetColumnBindings().size() - 1; + auto capture = make_uniq(update.table, *delta_table, DeltaCaptureMode::UPDATE, + std::move(update_expressions), update.columns, + optional_idx(row_id_index)); + capture->children.push_back(std::move(update.children[0])); + update.children[0] = std::move(capture); + OPENIVM_DEBUG_PRINT("[INSERT RULE] transactional UPDATE delta capture for '%s'\n", table_name.c_str()); + break; + } + case LogicalOperatorType::LOGICAL_MERGE_INTO: { + auto &merge = dml->Cast(); + const auto &table_name = merge.table.name; + auto delta_table = TryGetTrackedDeltaTable(input.context, merge.table); + if (!delta_table) { + return; } - } break; + auto capture = make_uniq(merge.table, *delta_table); + capture->children.push_back(std::move(*dml_owner)); + *dml_owner = std::move(capture); + OPENIVM_DEBUG_PRINT("[INSERT RULE] transactional MERGE action delta capture for '%s'\n", table_name.c_str()); + break; + } default: return; } } - } // namespace duckdb diff --git a/src/rules/schema_evolution.cpp b/src/rules/schema_evolution.cpp index 4d77224f..3e1fd304 100644 --- a/src/rules/schema_evolution.cpp +++ b/src/rules/schema_evolution.cpp @@ -2,7 +2,6 @@ #include "core/openivm_constants.hpp" #include "core/parser_plan_helpers.hpp" -#include "core/refresh_locks.hpp" #include "core/refresh_metadata.hpp" #include "core/sql_utils.hpp" #include "rules/column_hider.hpp" @@ -26,15 +25,29 @@ static bool NamesMatch(const string &left, const string &right) { return StringUtil::CIEquals(SqlUtils::LastIdentifierPart(left), SqlUtils::LastIdentifierPart(right)); } -static vector GetDependentViews(Connection &con, const string &delta_name) { - vector views; +struct DependentView { + string name; + string catalog_name; + string schema_name; +}; + +static vector GetDependentViews(Connection &con, const string &delta_name, const string &source_catalog, + const string &source_schema) { + vector views; auto result = con.Query("SELECT DISTINCT d.view_name FROM " + string(openivm::DELTA_TABLES_TABLE) + - " d WHERE d.table_name = '" + SqlUtils::EscapeValue(delta_name) + "' ORDER BY 1"); + " d WHERE d.table_name = '" + SqlUtils::EscapeValue(delta_name) + + "' AND COALESCE(d.source_catalog, '" + SqlUtils::EscapeValue(source_catalog) + "') = '" + + SqlUtils::EscapeValue(source_catalog) + "' AND COALESCE(d.source_schema, '" + + SqlUtils::EscapeValue(source_schema) + "') = '" + SqlUtils::EscapeValue(source_schema) + + "' ORDER BY 1"); if (result->HasError()) { return views; } + RefreshMetadata metadata(con); for (idx_t i = 0; i < result->RowCount(); i++) { - views.push_back(result->GetValue(0, i).ToString()); + auto name = result->GetValue(0, i).ToString(); + auto location = metadata.GetStoredViewLocation(name); + views.push_back({std::move(name), std::move(location.catalog_name), std::move(location.schema_name)}); } return views; } @@ -549,44 +562,45 @@ static bool AuxMetadataReferencesColumn(RefreshMetadata &metadata, const string return false; } -string FirstMVReferencingColumn(Connection &con, const string &delta_name, const string &table_name, - const string &col_name) { +string FirstMVReferencingColumn(Connection &con, const string &delta_name, const string &source_catalog, + const string &source_schema, const string &table_name, const string &col_name) { RefreshMetadata metadata(con); - for (auto &view_name : GetDependentViews(con, delta_name)) { - if (RewriteStoredViewQuery(con, metadata, view_name, table_name, col_name, col_name, /*persist=*/false) || - AuxMetadataReferencesColumn(metadata, view_name, table_name, col_name)) { - return view_name; + for (auto &view : GetDependentViews(con, delta_name, source_catalog, source_schema)) { + if (RewriteStoredViewQuery(con, metadata, view.name, table_name, col_name, col_name, /*persist=*/false) || + AuxMetadataReferencesColumn(metadata, view.name, table_name, col_name)) { + return view.name; } } return ""; } -void RewriteDependentViewMetadataForRename(Connection &con, const string &delta_name, const string &table_name, +void RewriteDependentViewMetadataForRename(Connection &con, const string &delta_name, const string &source_catalog, + const string &source_schema, const string &table_name, const string &old_name, const string &new_name) { RefreshMetadata metadata(con); - for (auto &view_name : GetDependentViews(con, delta_name)) { - ViewLockGuard view_guard(view_name); - bool tx_open = false; - try { - con.BeginTransaction(); - tx_open = true; - RewriteStoredViewQuery(con, metadata, view_name, table_name, old_name, new_name, /*persist=*/true); - RewriteDistinctAuxMeta(con, metadata, view_name, table_name, old_name, new_name); - RewriteFilteredGroupCountMeta(con, metadata, view_name, table_name, old_name, new_name); - RewriteSemiAntiAuxMeta(con, metadata, view_name, table_name, old_name, new_name); - RewriteLineageMeta(con, metadata, view_name, table_name, old_name, new_name); - RewriteWindowGroupColumnSources(con, metadata, view_name, old_name, new_name); - con.Commit(); - tx_open = false; - } catch (std::exception &) { - if (tx_open) { - try { - con.Rollback(); - } catch (std::exception &) { - } + auto views = GetDependentViews(con, delta_name, source_catalog, source_schema); + bool tx_open = false; + try { + con.BeginTransaction(); + tx_open = true; + for (auto &view : views) { + RewriteStoredViewQuery(con, metadata, view.name, table_name, old_name, new_name, /*persist=*/true); + RewriteDistinctAuxMeta(con, metadata, view.name, table_name, old_name, new_name); + RewriteFilteredGroupCountMeta(con, metadata, view.name, table_name, old_name, new_name); + RewriteSemiAntiAuxMeta(con, metadata, view.name, table_name, old_name, new_name); + RewriteLineageMeta(con, metadata, view.name, table_name, old_name, new_name); + RewriteWindowGroupColumnSources(con, metadata, view.name, old_name, new_name); + } + con.Commit(); + tx_open = false; + } catch (std::exception &) { + if (tx_open) { + try { + con.Rollback(); + } catch (std::exception &) { } - throw; } + throw; } } diff --git a/src/rules/transactional_delta_capture.cpp b/src/rules/transactional_delta_capture.cpp new file mode 100644 index 00000000..0d508d09 --- /dev/null +++ b/src/rules/transactional_delta_capture.cpp @@ -0,0 +1,985 @@ +#include "rules/transactional_delta_capture.hpp" + +#include "core/openivm_constants.hpp" +#include "core/openivm_debug.hpp" +#include "core/refresh_locks.hpp" + +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/execution/operator/persistent/physical_delete.hpp" +#include "duckdb/execution/operator/persistent/physical_insert.hpp" +#include "duckdb/execution/operator/persistent/physical_merge_into.hpp" +#include "duckdb/execution/operator/persistent/physical_update.hpp" +#include "duckdb/execution/physical_operator.hpp" +#include "duckdb/planner/binder.hpp" +#include "duckdb/planner/expression_binder/check_binder.hpp" +#include "duckdb/planner/expression/bound_reference_expression.hpp" +#include "duckdb/storage/data_table.hpp" +#include "duckdb/storage/optimistic_data_writer.hpp" +#include "duckdb/storage/table/append_state.hpp" +#include "duckdb/storage/table/scan_state.hpp" +#include "duckdb/transaction/duck_transaction.hpp" +#include "duckdb/transaction/local_storage.hpp" +#include "duckdb/transaction/transaction_context.hpp" + +#include + +namespace duckdb { + +namespace { + +static void EnterDeltaWritePhase(ClientContext &context) { + TransactionalMVLockState::Get(context).AcquireMutationLock(); +} + +class TransactionalDeltaAppendState { +public: + TransactionalDeltaAppendState(ClientContext &context, const vector> &generated_expressions, + const vector &delta_types) + : generated_executor(context, generated_expressions) { + vector generated_types; + generated_types.reserve(generated_expressions.size()); + for (auto &expression : generated_expressions) { + generated_types.push_back(expression->return_type); + } + generated_values.Initialize(Allocator::Get(context), generated_types); + delta_rows.Initialize(Allocator::Get(context), delta_types); + } + + ExpressionExecutor generated_executor; + DataChunk generated_values; + DataChunk delta_rows; + TableAppendState local_append_state; + PhysicalIndex collection_index = PhysicalIndex(DConstants::INVALID_INDEX); + unique_ptr optimistic_writer; + bool entered_write_phase = false; + bool append_finalized = false; +}; + +class TransactionalDeltaAppender { +public: + TransactionalDeltaAppender(TableCatalogEntry &base_table_p, TableCatalogEntry &delta_table_p, + vector> generated_expressions_p) + : delta_table(delta_table_p), delta_types(delta_table.GetTypes()), + generated_expressions(std::move(generated_expressions_p)) { + BuildDeltaColumnMap(base_table_p); + } + + void EnterWritePhase(ClientContext &context, TransactionalDeltaAppendState &state) const { + if (state.entered_write_phase) { + return; + } + EnterDeltaWritePhase(context); + state.entered_write_phase = true; + } + + void Append(ClientContext &context, DataChunk &base_rows, int32_t multiplicity, + TransactionalDeltaAppendState &state) const { + if (base_rows.size() == 0) { + return; + } + auto &delta_rows = state.delta_rows; + delta_rows.Reset(); + if (!generated_expressions.empty()) { + state.generated_values.Reset(); + state.generated_executor.Execute(base_rows, state.generated_values); + } + for (idx_t delta_index = 0; delta_index < delta_column_map.size(); delta_index++) { + if (delta_index == multiplicity_index.GetIndex()) { + delta_rows.data[delta_index].Reference(Value::INTEGER(multiplicity)); + } else if (delta_index == timestamp_index.GetIndex()) { + delta_rows.data[delta_index].Reference(Value::TIMESTAMP(Timestamp::GetCurrentTimestamp())); + } else { + auto generated_index = generated_column_map[delta_index]; + if (generated_index != DConstants::INVALID_INDEX) { + delta_rows.data[delta_index].Reference(state.generated_values.data[generated_index]); + } else { + auto base_index = delta_column_map[delta_index]; + D_ASSERT(base_index != DConstants::INVALID_INDEX); + delta_rows.data[delta_index].Reference(base_rows.data[base_index]); + } + } + } + delta_rows.SetCardinality(base_rows); + auto &storage = delta_table.GetStorage(); + if (!state.collection_index.IsValid()) { + lock_guard guard(append_lock); + state.optimistic_writer = make_uniq(context, storage); + auto optimistic_collection = state.optimistic_writer->CreateCollection(storage, delta_types); + auto &collection = *optimistic_collection->collection; + collection.InitializeEmpty(); + collection.InitializeAppend(state.local_append_state); + state.collection_index = storage.CreateOptimisticCollection(context, std::move(optimistic_collection)); + } + auto &optimistic_collection = storage.GetOptimisticCollection(context, state.collection_index); + auto &collection = *optimistic_collection.collection; + if (collection.Append(delta_rows, state.local_append_state)) { + state.optimistic_writer->WriteNewRowGroup(optimistic_collection); + } + } + + void Finalize(ClientContext &context, TransactionalDeltaAppendState &state) const { + if (state.append_finalized || !state.collection_index.IsValid()) { + return; + } + state.append_finalized = true; + auto &storage = delta_table.GetStorage(); + auto &optimistic_collection = storage.GetOptimisticCollection(context, state.collection_index); + auto &collection = *optimistic_collection.collection; + TransactionData transaction_data(0, 0); + collection.FinalizeAppend(transaction_data, state.local_append_state); + + lock_guard guard(append_lock); + if (collection.GetTotalRows() < storage.GetRowGroupSize()) { + vector> no_constraints; + LocalAppendState append_state; + storage.InitializeLocalAppend(append_state, delta_table, context, no_constraints); + auto &transaction = DuckTransaction::Get(context, delta_table.catalog); + for (auto &chunk : collection.Chunks(transaction)) { + storage.LocalAppend(append_state, context, chunk, false); + } + storage.FinalizeLocalAppend(append_state); + return; + } + state.optimistic_writer->WriteUnflushedRowGroups(optimistic_collection); + state.optimistic_writer->FinalFlush(); + storage.LocalMerge(context, optimistic_collection); + storage.GetOptimisticWriter(context).Merge(*state.optimistic_writer); + } + + const vector> &GeneratedExpressions() const { + return generated_expressions; + } + + const vector &DeltaTypes() const { + return delta_types; + } + +private: + void BuildDeltaColumnMap(TableCatalogEntry &base_table) { + case_insensitive_map_t base_column_index; + idx_t base_index = 0; + for (auto &column : base_table.GetColumns().Physical()) { + base_column_index[column.Name()] = base_index++; + } + + idx_t generated_index = 0; + for (auto &column : delta_table.GetColumns().Physical()) { + generated_column_map.push_back(DConstants::INVALID_INDEX); + if (column.Name() == openivm::MULTIPLICITY_COL) { + delta_column_map.push_back(DConstants::INVALID_INDEX); + multiplicity_index = column.Physical().index; + continue; + } + if (column.Name() == openivm::TIMESTAMP_COL) { + delta_column_map.push_back(DConstants::INVALID_INDEX); + timestamp_index = column.Physical().index; + continue; + } + auto entry = base_column_index.find(column.Name()); + if (entry == base_column_index.end()) { + if (generated_index >= generated_expressions.size()) { + throw InternalException("OpenIVM could not bind generated delta column '%s'", column.Name()); + } + delta_column_map.push_back(DConstants::INVALID_INDEX); + generated_column_map.back() = generated_index++; + continue; + } + delta_column_map.push_back(entry->second); + } + if (generated_index != generated_expressions.size()) { + throw InternalException("OpenIVM generated delta column count does not match bound expressions"); + } + if (!multiplicity_index.IsValid() || !timestamp_index.IsValid()) { + throw InternalException("OpenIVM delta table is missing multiplicity or timestamp metadata"); + } + } + + TableCatalogEntry &delta_table; + vector delta_types; + vector> generated_expressions; + vector delta_column_map; + vector generated_column_map; + optional_idx multiplicity_index; + optional_idx timestamp_index; + mutable mutex append_lock; +}; + +class TransactionalBaseRowFetcher { +public: + explicit TransactionalBaseRowFetcher(TableCatalogEntry &base_table_p) + : base_table(base_table_p), base_types(base_table.GetTypes()) { + column_ids.reserve(base_types.size()); + for (idx_t column = 0; column < base_types.size(); column++) { + column_ids.emplace_back(column); + } + } + + void Fetch(ClientContext &context, Vector &row_ids, idx_t count, DataChunk &rows, + ColumnFetchState &fetch_state) const { + rows.Reset(); + auto &transaction = DuckTransaction::Get(context, base_table.catalog); + base_table.GetStorage().Fetch(transaction, rows, column_ids, row_ids, count, fetch_state); + } + + bool CanFetch(ClientContext &context, row_t row_id) const { + auto &transaction = DuckTransaction::Get(context, base_table.catalog); + auto &storage = base_table.GetStorage(); + if (row_id < MAX_ROW_ID) { + return storage.CanFetch(transaction, row_id); + } + return transaction.GetLocalStorage().CanFetch(storage, row_id); + } + + const vector &Types() const { + return base_types; + } + +private: + TableCatalogEntry &base_table; + vector base_types; + vector column_ids; +}; + +static constexpr idx_t ROW_ID_DEDUP_SHARDS = 16; + +struct RowIdDedupShard { + mutex lock; + unordered_set captured_row_ids; +}; + +static idx_t SelectUnseenRows(DataChunk &input, idx_t row_id_index, array &shards, + array, ROW_ID_DEDUP_SHARDS> &candidates, SelectionVector &selection, + DataChunk &selected_input) { + auto &row_ids = input.data[row_id_index]; + row_ids.Flatten(input.size()); + auto row_id_data = FlatVector::GetData(row_ids); + for (auto &candidate_rows : candidates) { + candidate_rows.clear(); + } + for (idx_t row = 0; row < input.size(); row++) { + auto shard = static_cast(std::hash {}(row_id_data[row])) % ROW_ID_DEDUP_SHARDS; + candidates[shard].push_back(row); + } + idx_t selected_count = 0; + for (idx_t shard_index = 0; shard_index < ROW_ID_DEDUP_SHARDS; shard_index++) { + auto &shard = shards[shard_index]; + lock_guard guard(shard.lock); + for (auto row : candidates[shard_index]) { + if (shard.captured_row_ids.insert(row_id_data[row]).second) { + selection.set_index(selected_count++, row); + } + } + } + if (selected_count == 0) { + return 0; + } + selected_input.Reference(input); + if (selected_count != input.size()) { + selected_input.Slice(selection, selected_count); + } + selected_input.data[row_id_index].Flatten(selected_count); + return selected_count; +} + +class TransactionalDeltaCaptureGlobalState : public GlobalOperatorState { +public: + array shards; +}; + +class TransactionalDeltaCaptureLocalState : public OperatorState { +public: + TransactionalDeltaCaptureLocalState(ExecutionContext &context, + const vector> &update_expressions, + const vector> &generated_expressions, + const vector &delta_types, const vector &base_types, + const vector &input_types, bool capture_existing_rows) + : append_state(context.client, generated_expressions, delta_types), + update_executor(context.client, update_expressions), selection(STANDARD_VECTOR_SIZE) { + vector update_types; + update_types.reserve(update_expressions.size()); + for (auto &expression : update_expressions) { + update_types.push_back(expression->return_type); + } + update_values.Initialize(Allocator::Get(context.client), update_types); + if (!capture_existing_rows) { + return; + } + old_rows.Initialize(Allocator::Get(context.client), base_types); + new_rows.Initialize(Allocator::Get(context.client), base_types); + selected_input.InitializeEmpty(input_types); + } + + TransactionalDeltaAppendState append_state; + ExpressionExecutor update_executor; + DataChunk update_values; + DataChunk selected_input; + DataChunk old_rows; + DataChunk new_rows; + SelectionVector selection; + ColumnFetchState fetch_state; + array, ROW_ID_DEDUP_SHARDS> dedup_candidates; +}; + +class PhysicalTransactionalDeltaCapture : public PhysicalOperator { +public: + PhysicalTransactionalDeltaCapture(PhysicalPlan &physical_plan, PhysicalOperator &child, + TableCatalogEntry &base_table_p, TableCatalogEntry &delta_table_p, + DeltaCaptureMode mode_p, vector> update_expressions_p, + vector> generated_expressions_p, + vector update_columns_p, optional_idx row_id_index_p, + idx_t estimated_cardinality) + : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, child.GetTypes(), estimated_cardinality), + mode(mode_p), update_expressions(std::move(update_expressions_p)), + update_columns(std::move(update_columns_p)), row_id_index(row_id_index_p), + appender(make_shared_ptr(base_table_p, delta_table_p, + std::move(generated_expressions_p))), + row_fetcher(base_table_p) { + children.push_back(child); + } + + unique_ptr GetGlobalOperatorState(ClientContext &context) const override { + return make_uniq(); + } + + unique_ptr GetOperatorState(ExecutionContext &context) const override { + return make_uniq( + context, update_expressions, appender->GeneratedExpressions(), appender->DeltaTypes(), row_fetcher.Types(), + children[0].get().types, mode != DeltaCaptureMode::INSERT); + } + + OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk, + GlobalOperatorState &gstate_p, OperatorState &state_p) const override { + auto &gstate = gstate_p.Cast(); + auto &state = state_p.Cast(); + appender->EnterWritePhase(context.client, state.append_state); + + if (mode == DeltaCaptureMode::INSERT) { + appender->Append(context.client, input, 1, state.append_state); + } else { + CaptureDeleteOrUpdate(context, input, gstate, state); + } + chunk.Reference(input); + return OperatorResultType::NEED_MORE_INPUT; + } + + OperatorFinalizeResultType FinalExecute(ExecutionContext &context, DataChunk &chunk, GlobalOperatorState &gstate_p, + OperatorState &state_p) const override { + auto &state = state_p.Cast(); + appender->Finalize(context.client, state.append_state); + return OperatorFinalizeResultType::FINISHED; + } + + bool RequiresFinalExecute() const override { + return true; + } + + bool ParallelOperator() const override { + return true; + } + + string GetName() const override { + return "OPENIVM_TRANSACTIONAL_DELTA_CAPTURE"; + } + +private: + void CaptureDeleteOrUpdate(ExecutionContext &context, DataChunk &input, + TransactionalDeltaCaptureGlobalState &gstate, + TransactionalDeltaCaptureLocalState &state) const { + D_ASSERT(row_id_index.IsValid()); + state.selected_input.Reset(); + idx_t selected_count; + // Only row-id deduplication is shared. Sharding keeps disjoint chunks from + // serializing behind one global mutex; preimage fetch and expression work + // remain parallel. + selected_count = SelectUnseenRows(input, row_id_index.GetIndex(), gstate.shards, state.dedup_candidates, + state.selection, state.selected_input); + if (selected_count == 0) { + return; + } + + row_fetcher.Fetch(context.client, state.selected_input.data[row_id_index.GetIndex()], selected_count, + state.old_rows, state.fetch_state); + appender->Append(context.client, state.old_rows, -1, state.append_state); + + if (mode != DeltaCaptureMode::UPDATE) { + return; + } + state.update_values.Reset(); + state.update_executor.Execute(state.selected_input, state.update_values); + state.new_rows.Reset(); + for (idx_t column = 0; column < row_fetcher.Types().size(); column++) { + state.new_rows.data[column].Reference(state.old_rows.data[column]); + } + for (idx_t update_index = 0; update_index < update_columns.size(); update_index++) { + state.new_rows.data[update_columns[update_index].index].Reference(state.update_values.data[update_index]); + } + state.new_rows.SetCardinality(state.old_rows); + appender->Append(context.client, state.new_rows, 1, state.append_state); + } + + DeltaCaptureMode mode; + vector> update_expressions; + vector update_columns; + optional_idx row_id_index; + shared_ptr appender; + TransactionalBaseRowFetcher row_fetcher; +}; + +class MergeDeltaCaptureExecutionState { +public: + mutex lock; + std::condition_variable preimage_ready; + unordered_set capturing_row_ids; + unordered_set captured_row_ids; + idx_t finalized_actions = 0; +}; + +class MergeDeltaCaptureCoordinator { +public: + MergeDeltaCaptureCoordinator(TableCatalogEntry &base_table, shared_ptr appender_p, + const vector &action_input_types_p, idx_t action_count_p) + : row_fetcher(base_table), appender(std::move(appender_p)), action_input_types(action_input_types_p), + action_count(action_count_p) { + } + + shared_ptr GetExecutionState(ClientContext &context) { + // Action sinks are separate physical operators, but their preimages form one statement-level delta. + const auto query_id = context.transaction.GetActiveQuery(); + lock_guard guard(lock); + for (auto entry = states.begin(); entry != states.end();) { + if (entry->second.expired()) { + entry = states.erase(entry); + } else { + entry++; + } + } + auto existing = states.find(query_id); + if (existing != states.end()) { + if (auto state = existing->second.lock()) { + return state; + } + } + auto state = make_shared_ptr(); + states[query_id] = state; + return state; + } + + void CapturePreimages(ClientContext &context, MergeDeltaCaptureExecutionState &state, Vector &input_row_ids, + idx_t count, vector &reserved_row_ids, Vector &fetch_row_ids, DataChunk &rows, + ColumnFetchState &fetch_state, TransactionalDeltaAppendState &append_state) const { + UnifiedVectorFormat row_id_data; + input_row_ids.ToUnifiedFormat(count, row_id_data); + auto row_ids = UnifiedVectorFormat::GetData(row_id_data); + reserved_row_ids.clear(); + { + // Reserve before fetching so actions touching the same row wait, while disjoint rows remain parallel. + lock_guard guard(state.lock); + for (idx_t row = 0; row < count; row++) { + auto index = row_id_data.sel->get_index(row); + D_ASSERT(row_id_data.validity.RowIsValid(index)); + auto row_id = row_ids[index]; + if (state.captured_row_ids.find(row_id) != state.captured_row_ids.end() || + !state.capturing_row_ids.insert(row_id).second) { + continue; + } + reserved_row_ids.push_back(row_id); + } + } + + if (!reserved_row_ids.empty()) { + auto fetch_data = FlatVector::GetData(fetch_row_ids); + for (idx_t index = 0; index < reserved_row_ids.size(); index++) { + fetch_data[index] = reserved_row_ids[index]; + } + try { + row_fetcher.Fetch(context, fetch_row_ids, reserved_row_ids.size(), rows, fetch_state); + D_ASSERT(rows.size() == reserved_row_ids.size()); + appender->EnterWritePhase(context, append_state); + appender->Append(context, rows, -1, append_state); + lock_guard guard(state.lock); + for (auto row_id : reserved_row_ids) { + state.capturing_row_ids.erase(row_id); + state.captured_row_ids.insert(row_id); + } + state.preimage_ready.notify_all(); + } catch (std::exception &) { + lock_guard guard(state.lock); + for (auto row_id : reserved_row_ids) { + state.capturing_row_ids.erase(row_id); + } + state.preimage_ready.notify_all(); + throw; + } + } + + unique_lock guard(state.lock); + state.preimage_ready.wait(guard, [&]() { + for (idx_t row = 0; row < count; row++) { + auto index = row_id_data.sel->get_index(row); + if (state.capturing_row_ids.find(row_ids[index]) != state.capturing_row_ids.end()) { + return false; + } + } + return true; + }); + } + + void FinalizeAction(ClientContext &context, MergeDeltaCaptureExecutionState &state) const { + { + lock_guard guard(state.lock); + state.finalized_actions++; + if (state.finalized_actions != action_count) { + return; + } + D_ASSERT(state.capturing_row_ids.empty()); + } + OPENIVM_DEBUG_PRINT("[INSERT RULE] finalizing MERGE deltas for %zu target rows\n", + state.captured_row_ids.size()); + TransactionalDeltaAppendState append_state(context, appender->GeneratedExpressions(), appender->DeltaTypes()); + appender->EnterWritePhase(context, append_state); + Vector row_ids(LogicalType::ROW_TYPE, STANDARD_VECTOR_SIZE); + DataChunk rows; + rows.Initialize(Allocator::Get(context), row_fetcher.Types()); + ColumnFetchState fetch_state; + auto row_id_data = FlatVector::GetData(row_ids); + idx_t count = 0; + idx_t invisible_count = 0; + for (auto row_id : state.captured_row_ids) { + if (!row_fetcher.CanFetch(context, row_id)) { + invisible_count++; + continue; + } + row_id_data[count++] = row_id; + if (count != STANDARD_VECTOR_SIZE) { + continue; + } + row_fetcher.Fetch(context, row_ids, count, rows, fetch_state); + appender->Append(context, rows, 1, append_state); + count = 0; + } + if (count > 0) { + row_fetcher.Fetch(context, row_ids, count, rows, fetch_state); + appender->Append(context, rows, 1, append_state); + } + appender->Finalize(context, append_state); + OPENIVM_DEBUG_PRINT("[INSERT RULE] skipped %zu invisible MERGE target postimages\n", invisible_count); + } + + const TransactionalBaseRowFetcher &RowFetcher() const { + return row_fetcher; + } + + const shared_ptr &Appender() const { + return appender; + } + + const vector &ActionInputTypes() const { + return action_input_types; + } + +private: + TransactionalBaseRowFetcher row_fetcher; + shared_ptr appender; + vector action_input_types; + idx_t action_count; + mutex lock; + unordered_map> states; +}; + +class MergeActionDeltaCaptureGlobalState : public GlobalSinkState { +public: + MergeActionDeltaCaptureGlobalState(unique_ptr child_state_p, + shared_ptr execution_state_p) + : child_state(std::move(child_state_p)), execution_state(std::move(execution_state_p)) { + } + + unique_ptr child_state; + shared_ptr execution_state; + array delete_insert_shards; +}; + +class MergeActionDeltaCaptureLocalState : public LocalSinkState { +public: + MergeActionDeltaCaptureLocalState(ExecutionContext &context, PhysicalOperator &action_op, + const TransactionalDeltaAppender &appender, const vector &base_types, + const vector &action_input_types, + const vector &delegated_types, bool capture_delete_insert_update, + bool has_update_defaults) + : child_state(action_op.GetLocalSinkState(context)), + append_state(context.client, appender.GeneratedExpressions(), appender.DeltaTypes()), + selection(STANDARD_VECTOR_SIZE), fetch_row_ids(LogicalType::ROW_TYPE, STANDARD_VECTOR_SIZE) { + if (action_op.type == PhysicalOperatorType::INSERT) { + return; + } + preimage_rows.Initialize(Allocator::Get(context.client), base_types); + if (!capture_delete_insert_update) { + return; + } + selected_input.InitializeEmpty(action_input_types); + auto &update = action_op.Cast(); + vector update_types; + update_types.reserve(update.expressions.size()); + for (auto &expression : update.expressions) { + update_types.push_back(expression->return_type); + } + update_values.Initialize(Allocator::Get(context.client), update_types); + pending_new_rows.Initialize(Allocator::Get(context.client), base_types); + if (has_update_defaults) { + default_executor = make_uniq(context.client, update.bound_defaults); + delegated_input.Initialize(Allocator::Get(context.client), delegated_types); + } + } + + unique_ptr child_state; + TransactionalDeltaAppendState append_state; + unique_ptr default_executor; + DataChunk delegated_input; + DataChunk update_values; + DataChunk selected_input; + DataChunk preimage_rows; + DataChunk pending_new_rows; + SelectionVector selection; + Vector fetch_row_ids; + ColumnFetchState fetch_state; + vector reserved_row_ids; + array, ROW_ID_DEDUP_SHARDS> dedup_candidates; + bool prepared_for_input = false; +}; + +class PhysicalMergeActionDeltaCapture : public PhysicalOperator { +public: + PhysicalMergeActionDeltaCapture(PhysicalPlan &physical_plan, PhysicalOperator &action_op_p, + shared_ptr coordinator_p, idx_t estimated_cardinality) + : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, action_op_p.GetTypes(), + estimated_cardinality), + action_op(action_op_p), coordinator(std::move(coordinator_p)), + capture_delete_insert_update(action_op.type == PhysicalOperatorType::UPDATE && + action_op.Cast().update_is_del_and_insert) { + if (action_op.type != PhysicalOperatorType::INSERT && action_op.type != PhysicalOperatorType::UPDATE && + action_op.type != PhysicalOperatorType::DELETE_OPERATOR) { + throw InternalException("OpenIVM cannot capture unsupported MERGE action operator %s", action_op.GetName()); + } + if (capture_delete_insert_update) { + NormalizeUpdateDefaults(coordinator->ActionInputTypes()); + } + } + + unique_ptr GetGlobalSinkState(ClientContext &context) const override { + return make_uniq(action_op.GetGlobalSinkState(context), + coordinator->GetExecutionState(context)); + } + + unique_ptr GetLocalSinkState(ExecutionContext &context) const override { + auto &capture_input_types = delegated_types.empty() ? coordinator->ActionInputTypes() : delegated_types; + return make_uniq( + context, action_op, *coordinator->Appender(), coordinator->RowFetcher().Types(), capture_input_types, + delegated_types, capture_delete_insert_update, !default_update_indexes.empty()); + } + + SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override { + auto &gstate = input.global_state.Cast(); + auto &lstate = input.local_state.Cast(); + auto &action_input = PrepareActionInput(chunk, lstate); + if (action_op.type != PhysicalOperatorType::INSERT) { + const auto row_id_index = action_op.type == PhysicalOperatorType::DELETE_OPERATOR + ? action_op.Cast().row_id_index + : action_input.ColumnCount() - 1; + coordinator->CapturePreimages(context.client, *gstate.execution_state, action_input.data[row_id_index], + action_input.size(), lstate.reserved_row_ids, lstate.fetch_row_ids, + lstate.preimage_rows, lstate.fetch_state, lstate.append_state); + if (capture_delete_insert_update && !lstate.prepared_for_input) { + PrepareDeleteInsertRows(context, action_input, row_id_index, gstate, lstate); + lstate.prepared_for_input = true; + } + } + + OperatorSinkInput child_input {*gstate.child_state, *lstate.child_state, input.interrupt_state}; + auto result = action_op.Sink(context, action_input, child_input); + if (result == SinkResultType::BLOCKED) { + return result; + } + coordinator->Appender()->EnterWritePhase(context.client, lstate.append_state); + if (action_op.type == PhysicalOperatorType::INSERT) { + coordinator->Appender()->Append(context.client, action_input, 1, lstate.append_state); + } else if (capture_delete_insert_update) { + coordinator->Appender()->Append(context.client, lstate.pending_new_rows, 1, lstate.append_state); + lstate.prepared_for_input = false; + } + return result; + } + + SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override { + auto &gstate = input.global_state.Cast(); + auto &lstate = input.local_state.Cast(); + OperatorSinkCombineInput child_input {*gstate.child_state, *lstate.child_state, input.interrupt_state}; + auto result = action_op.Combine(context, child_input); + if (result == SinkCombineResultType::FINISHED) { + coordinator->Appender()->Finalize(context.client, lstate.append_state); + } + return result; + } + + SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const override { + auto &gstate = input.global_state.Cast(); + OperatorSinkFinalizeInput child_input {*gstate.child_state, input.interrupt_state}; + auto result = action_op.Finalize(pipeline, event, context, child_input); + if (result != SinkFinalizeType::BLOCKED) { + coordinator->FinalizeAction(context, *gstate.execution_state); + } + return result; + } + + unique_ptr GetGlobalSourceState(ClientContext &context) const override { + auto &gstate = sink_state->Cast(); + action_op.sink_state = std::move(gstate.child_state); + return action_op.GetGlobalSourceState(context); + } + + unique_ptr GetLocalSourceState(ExecutionContext &context, + GlobalSourceState &gstate) const override { + return action_op.GetLocalSourceState(context, gstate); + } + + SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const override { + return action_op.GetData(context, chunk, input); + } + + string GetName() const override { + return "OPENIVM_MERGE_ACTION_DELTA_CAPTURE"; + } + +private: + void NormalizeUpdateDefaults(const vector &input_types) { + auto &update = action_op.Cast(); + for (idx_t index = 0; index < update.expressions.size(); index++) { + if (update.expressions[index]->GetExpressionType() == ExpressionType::VALUE_DEFAULT) { + default_update_indexes.push_back(index); + } + } + if (default_update_indexes.empty()) { + return; + } + if (input_types.empty()) { + throw InternalException("OpenIVM MERGE UPDATE DEFAULT capture is missing its row-id input"); + } + delegated_types.reserve(input_types.size() + default_update_indexes.size()); + for (idx_t index = 0; index + 1 < input_types.size(); index++) { + delegated_types.push_back(input_types[index]); + } + for (auto update_index : default_update_indexes) { + auto &expression = update.expressions[update_index]; + const auto reference_index = delegated_types.size(); + delegated_types.push_back(expression->return_type); + expression = make_uniq(expression->return_type, reference_index); + } + delegated_types.push_back(input_types.back()); + } + + DataChunk &PrepareActionInput(DataChunk &input, MergeActionDeltaCaptureLocalState &state) const { + if (default_update_indexes.empty()) { + return input; + } + if (state.prepared_for_input) { + return state.delegated_input; + } + D_ASSERT(state.default_executor); + state.default_executor->SetChunk(input); + state.delegated_input.Reset(); + for (idx_t index = 0; index + 1 < input.ColumnCount(); index++) { + state.delegated_input.data[index].Reference(input.data[index]); + } + auto &update = action_op.Cast(); + idx_t delegated_index = input.ColumnCount() - 1; + for (auto update_index : default_update_indexes) { + state.default_executor->ExecuteExpression(update.columns[update_index].index, + state.delegated_input.data[delegated_index++]); + } + state.delegated_input.data[delegated_index].Reference(input.data.back()); + state.delegated_input.SetCardinality(input); + return state.delegated_input; + } + + void PrepareDeleteInsertRows(ExecutionContext &context, DataChunk &input, idx_t row_id_index, + MergeActionDeltaCaptureGlobalState &gstate, + MergeActionDeltaCaptureLocalState &lstate) const { + lstate.selected_input.Reset(); + auto selected_count = SelectUnseenRows(input, row_id_index, gstate.delete_insert_shards, + lstate.dedup_candidates, lstate.selection, lstate.selected_input); + lstate.pending_new_rows.Reset(); + if (selected_count == 0) { + return; + } + coordinator->RowFetcher().Fetch(context.client, lstate.selected_input.data[row_id_index], selected_count, + lstate.preimage_rows, lstate.fetch_state); + D_ASSERT(lstate.preimage_rows.size() == selected_count); + auto &update = action_op.Cast(); + lstate.update_values.Reset(); + lstate.update_values.SetCardinality(lstate.selected_input); + for (idx_t index = 0; index < update.expressions.size(); index++) { + auto &expression = *update.expressions[index]; + D_ASSERT(expression.GetExpressionType() == ExpressionType::BOUND_REF); + auto &reference = expression.Cast(); + lstate.update_values.data[index].Reference(lstate.selected_input.data[reference.index]); + } + for (idx_t column = 0; column < coordinator->RowFetcher().Types().size(); column++) { + lstate.pending_new_rows.data[column].Reference(lstate.preimage_rows.data[column]); + } + for (idx_t index = 0; index < update.columns.size(); index++) { + lstate.pending_new_rows.data[update.columns[index].index].Reference(lstate.update_values.data[index]); + } + lstate.pending_new_rows.SetCardinality(lstate.preimage_rows); + } + + PhysicalOperator &action_op; + shared_ptr coordinator; + bool capture_delete_insert_update; + vector default_update_indexes; + vector delegated_types; +}; + +static vector> BindGeneratedExpressions(ClientContext &context, TableCatalogEntry &base_table, + TableCatalogEntry &delta_table) { + case_insensitive_set_t physical_columns; + for (auto &column : base_table.GetColumns().Physical()) { + physical_columns.insert(column.Name()); + } + vector> generated_expressions; + auto binder = Binder::CreateBinder(context); + physical_index_set_t bound_columns; + CheckBinder generated_binder(*binder, context, base_table.name, base_table.GetColumns(), bound_columns); + for (auto &column : delta_table.GetColumns().Physical()) { + if (column.Name() == openivm::MULTIPLICITY_COL || column.Name() == openivm::TIMESTAMP_COL || + physical_columns.find(column.Name()) != physical_columns.end()) { + continue; + } + auto &base_column = base_table.GetColumns().GetColumn(column.Name()); + if (!base_column.Generated()) { + throw InternalException("OpenIVM delta column '%s' is absent from the base table storage", column.Name()); + } + generated_binder.target_type = base_column.Type(); + auto generated_expression = base_column.GeneratedExpression().Copy(); + generated_expressions.push_back(generated_binder.Bind(generated_expression)); + } + return generated_expressions; +} + +} // namespace + +LogicalTransactionalDeltaCapture::LogicalTransactionalDeltaCapture(TableCatalogEntry &base_table_p, + TableCatalogEntry &delta_table_p, + DeltaCaptureMode mode_p, + vector> update_expressions, + vector update_columns_p, + optional_idx row_id_index_p) + : LogicalExtensionOperator(std::move(update_expressions)), base_table(base_table_p), delta_table(delta_table_p), + mode(mode_p), update_columns(std::move(update_columns_p)), row_id_index(row_id_index_p) { + if (mode == DeltaCaptureMode::INSERT) { + if (!expressions.empty() || !update_columns.empty() || row_id_index.IsValid()) { + throw InternalException("OpenIVM INSERT delta capture received UPDATE/DELETE state"); + } + } else if (!row_id_index.IsValid()) { + throw InternalException("OpenIVM DELETE/UPDATE delta capture is missing its row-id column"); + } else if (mode == DeltaCaptureMode::DELETE && (!expressions.empty() || !update_columns.empty())) { + throw InternalException("OpenIVM DELETE delta capture received UPDATE state"); + } else if (mode == DeltaCaptureMode::UPDATE && expressions.size() != update_columns.size()) { + throw InternalException("OpenIVM UPDATE delta capture expression/column counts do not match"); + } +} + +PhysicalOperator &LogicalTransactionalDeltaCapture::CreatePlan(ClientContext &context, PhysicalPlanGenerator &planner) { + D_ASSERT(children.size() == 1); + auto &child = planner.CreatePlan(*children[0]); + planner.dependencies.AddDependency(base_table); + planner.dependencies.AddDependency(delta_table); + + auto generated_expressions = BindGeneratedExpressions(context, base_table, delta_table); + return planner.Make(child, base_table, delta_table, mode, std::move(expressions), + std::move(generated_expressions), std::move(update_columns), + row_id_index, estimated_cardinality); +} + +vector LogicalTransactionalDeltaCapture::GetColumnBindings() { + D_ASSERT(children.size() == 1); + return children[0]->GetColumnBindings(); +} + +string LogicalTransactionalDeltaCapture::GetName() const { + return "OPENIVM_TRANSACTIONAL_DELTA_CAPTURE"; +} + +string LogicalTransactionalDeltaCapture::GetExtensionName() const { + return "openivm_transactional_delta_capture"; +} + +void LogicalTransactionalDeltaCapture::ResolveTypes() { + D_ASSERT(children.size() == 1); + types = children[0]->types; +} + +LogicalTransactionalMergeDeltaCapture::LogicalTransactionalMergeDeltaCapture(TableCatalogEntry &base_table_p, + TableCatalogEntry &delta_table_p) + : base_table(base_table_p), delta_table(delta_table_p) { +} + +PhysicalOperator &LogicalTransactionalMergeDeltaCapture::CreatePlan(ClientContext &context, + PhysicalPlanGenerator &planner) { + D_ASSERT(children.size() == 1); + auto &child = planner.CreatePlan(*children[0]); + if (child.type != PhysicalOperatorType::MERGE_INTO) { + throw InternalException("OpenIVM MERGE delta capture expected a physical MERGE operator"); + } + planner.dependencies.AddDependency(base_table); + planner.dependencies.AddDependency(delta_table); + auto generated_expressions = BindGeneratedExpressions(context, base_table, delta_table); + auto appender = + make_shared_ptr(base_table, delta_table, std::move(generated_expressions)); + auto &merge = child.Cast(); + idx_t action_count = 0; + bool requires_serial_execution = false; + for (auto &action : merge.actions) { + if (!action->op) { + continue; + } + action_count++; + if (action->op->type == PhysicalOperatorType::UPDATE && + action->op->Cast().update_is_del_and_insert) { + requires_serial_execution = true; + } + } + auto coordinator = make_shared_ptr(base_table, appender, + merge.children[0].get().types, action_count); + for (auto &action : merge.actions) { + if (!action->op) { + continue; + } + action->op = planner.Make(*action->op, coordinator, estimated_cardinality); + } + if (requires_serial_execution) { + // DuckDB and capture must choose the same first match for delete-and-insert updates. + merge.parallel = false; + } + return child; +} + +vector LogicalTransactionalMergeDeltaCapture::GetColumnBindings() { + D_ASSERT(children.size() == 1); + return children[0]->GetColumnBindings(); +} + +string LogicalTransactionalMergeDeltaCapture::GetName() const { + return "OPENIVM_TRANSACTIONAL_MERGE_DELTA_CAPTURE"; +} + +string LogicalTransactionalMergeDeltaCapture::GetExtensionName() const { + return "openivm_transactional_merge_delta_capture"; +} + +void LogicalTransactionalMergeDeltaCapture::ResolveTypes() { + D_ASSERT(children.size() == 1); + types = children[0]->types; +} + +} // namespace duckdb diff --git a/src/upsert/refresh.cpp b/src/upsert/refresh.cpp index e85c0b92..72910c8b 100644 --- a/src/upsert/refresh.cpp +++ b/src/upsert/refresh.cpp @@ -2,6 +2,7 @@ #include "core/openivm_constants.hpp" #include "core/openivm_debug.hpp" +#include "core/parser_ddl.hpp" #include "core/refresh_metadata.hpp" #include "core/refresh_locks.hpp" #include "core/sql_utils.hpp" @@ -109,31 +110,25 @@ static bool TrySkipEmptyRefresh(ClientContext &context, RefreshMetadata &metadat const string &view_name, const string &attached_db_catalog_name, const string &attached_db_schema_name, DeltaActivityResult *active_activity); -// Generate and execute refresh SQL for a single view under its per-view lock. +static void UseMetadataSchema(Connection &con) { + auto result = con.Query("SET schema='" + string(DEFAULT_SCHEMA) + "'"); + if (result->HasError()) { + throw CatalogException("OpenIVM could not select its metadata schema: %s", result->GetError()); + } +} + +// Generate and execute refresh SQL for a single view while the caller owns the mutation gate. // When openivm_adaptive_refresh is on, also computes a cost estimate before execution // and records execution history for the learned cost model. -static bool RefreshViewLocked(ClientContext &context, const string &view_catalog_name, const string &view_schema_name, - const string &vn, bool cross_system, const string &attached_db_catalog_name, - const string &attached_db_schema_name, bool skip_empty_refresh) { +static void RefreshViewSerialized(ClientContext &context, const string &view_catalog_name, + const string &view_schema_name, const string &vn, bool cross_system, + const string &attached_db_catalog_name, const string &attached_db_schema_name, + bool skip_empty_refresh) { RefreshProfiler profiler(context, vn); - auto lock_start = std::chrono::steady_clock::now(); - ViewLockGuard view_guard(vn); - // Acquire delta-table locks in sorted order to serialize parallel refreshes that - // share base tables (e.g. mv_A and mv_B both reading STOCK → both write to - // `delta_STOCK` inside their transactions → "Conflict on tuple deletion!" when - // the second tx tries to delete rows the first already processed). Sorting - // guarantees the same acquisition order across all views, so no deadlock is - // possible between concurrent refreshes. - vector> delta_guards; + profiler.AddMeasuredStep("acquire_locks", 0, "database mutation gate pre-acquired"); Connection probe_con(*context.db.get()); + UseMetadataSchema(probe_con); RefreshMetadata probe_meta(probe_con); - auto delta_table_names = probe_meta.GetDeltaTables(vn); - std::sort(delta_table_names.begin(), delta_table_names.end()); - delta_table_names.erase(std::unique(delta_table_names.begin(), delta_table_names.end()), delta_table_names.end()); - for (auto &dt : delta_table_names) { - delta_guards.push_back(make_uniq(dt)); - } - profiler.AddStep("acquire_locks", lock_start, to_string(delta_table_names.size()) + " delta locks"); DeltaActivityResult delta_activity; DeltaActivityResult *precomputed_delta_activity = nullptr; if (skip_empty_refresh) { @@ -141,7 +136,7 @@ static bool RefreshViewLocked(ClientContext &context, const string &view_catalog attached_db_catalog_name, attached_db_schema_name, &delta_activity)) { profiler.AddTotal(); profiler.Flush(*context.db.get()); - return true; + return; } if (!delta_activity.active_delta_table_names.empty() || delta_activity.requires_full_refresh) { precomputed_delta_activity = &delta_activity; @@ -153,6 +148,7 @@ static bool RefreshViewLocked(ClientContext &context, const string &view_catalog // failure modes (e.g. rebinding errors thrown by Query itself, not reported as // HasError()). Rollback-then-throw keeps the WAL clean and leaves the DB valid. Connection exec_con(*context.db.get()); + UseMetadataSchema(exec_con); bool tx_open = false; try { bool adaptive_refresh = SqlUtils::GetBoolSetting(context, "openivm_adaptive_refresh", false); @@ -175,9 +171,8 @@ static bool RefreshViewLocked(ClientContext &context, const string &view_catalog "sql_bytes=" + to_string(sql.size()) + ", meta_pre_bytes=" + to_string(meta_pre_sql.size()) + ", meta_post_bytes=" + to_string(meta_post_sql.size())); - // IVM-generated SQL can nest deeply for multi-table joins + CTEs (N-term telescoping - // over 7+ tables produces hundreds of chained projections). Lift the default 1000 - // expression-depth limit so the binder doesn't reject legitimate generated plans. + // IVM-generated SQL can nest deeply for multi-table joins + CTEs. Keep the + // established guard while rejecting unexpectedly recursive generated plans. exec_con.Query("SET max_expression_depth = 10000"); // The generated refresh SQL already contains an explicit decorrelated plan. DuckDB's // deliminator can recurse through very deep stress-query CTE/subquery expansions and @@ -190,6 +185,7 @@ static bool RefreshViewLocked(ClientContext &context, const string &view_catalog // Refresh SQL uses fully qualified internal data/delta names. DuckLake-targeted // MVs write those objects in DuckLake; native MVs keep them in the physical DB. OPENIVM_DEBUG_PRINT("[UPSERT] Executing refresh SQL:\n%s\n", sql.c_str()); + OPENIVM_DEBUG_PRINT("[UPSERT] Generated refresh SQL size: %zu bytes\n", sql.size()); // Wrap the entire refresh in a transaction so that a failed refresh leaves the MV // and delta tables in a clean state (atomically rolled back). The refresh_in_progress @@ -333,7 +329,7 @@ static bool RefreshViewLocked(ClientContext &context, const string &view_catalog } profiler.AddTotal(); profiler.Flush(*context.db.get()); - return false; + return; } catch (...) { // Ensure the transaction is rolled back before we propagate the exception. // This covers the case where Query() itself threw (vs returning HasError) — @@ -438,6 +434,11 @@ void UpsertDeltaQueriesLocked(ClientContext &context, const FunctionParameters & bool cross_system = false; Connection con(*context.db.get()); + // Hooks run through this helper connection while the caller owns the + // database-wide mutation gate. Give tracked DML in the hook the same logical + // owner so delta capture re-enters the gate instead of waiting on its caller. + TransactionalMVLockState::Get(*con.context).SetMutationOwner(&context); + UseMetadataSchema(con); if (parameters.values.size() == 3) { view_catalog_name = StringValue::Get(parameters.values[0]); @@ -463,7 +464,7 @@ void UpsertDeltaQueriesLocked(ClientContext &context, const FunctionParameters & // cross_system detection: the view's catalog differs from the fresh connection's physical // default. Metadata tables (openivm_views etc.) live in the physical default; data/view // tables live in view_catalog_name. DuckDB forbids cross-catalog writes in one transaction, - // so RefreshViewLocked must split the refresh SQL into data ops and metadata ops. + // so RefreshViewSerialized must split the refresh SQL into data ops and metadata ops. if (!view_catalog_name.empty()) { Connection probe(*context.db.get()); string probe_default; @@ -485,17 +486,14 @@ void UpsertDeltaQueriesLocked(ClientContext &context, const FunctionParameters & RefreshMetadata metadata(con); - // Each view is generated + executed under its own per-view lock. - // This ensures cascaded views are also protected from concurrent refresh. - // Upstream cascade: refresh ancestors first (this may populate our delta tables). if (cascade_mode == "upstream" || cascade_mode == "both") { auto upstream = metadata.GetUpstreamViews(view_name); for (auto &dep : upstream) { auto dep_location = ResolveViewLocation(con, dep, view_catalog_name, view_schema_name); - RefreshViewLocked(context, dep_location.catalog_name, dep_location.schema_name, dep, - dep_location.cross_system, attached_db_catalog_name, attached_db_schema_name, - /*skip_empty_refresh=*/true); + RefreshViewSerialized(context, dep_location.catalog_name, dep_location.schema_name, dep, + dep_location.cross_system, attached_db_catalog_name, attached_db_schema_name, + /*skip_empty_refresh=*/true); } } @@ -514,30 +512,30 @@ void UpsertDeltaQueriesLocked(ClientContext &context, const FunctionParameters & // Hook-bearing refreshes keep the old pre-hook empty skip semantics. Hook-free refreshes // compute the same delta activity under the view lock and reuse it during SQL generation. - if (has_refresh_hook && TrySkipEmptyRefresh(context, metadata, con, view_catalog_name, view_schema_name, view_name, - attached_db_catalog_name, attached_db_schema_name, nullptr)) { - return; - } - - if (!hook_sql.empty() && hook_mode == "before") { - auto hr = con.Query(hook_sql); - if (hr->HasError()) { - Printer::Print("Warning: before-hook for '" + view_name + "' failed: " + hr->GetError()); + bool skip_current_node = + has_refresh_hook && TrySkipEmptyRefresh(context, metadata, con, view_catalog_name, view_schema_name, view_name, + attached_db_catalog_name, attached_db_schema_name, nullptr); + if (!skip_current_node) { + if (!hook_sql.empty() && hook_mode == "before") { + auto hr = con.Query(hook_sql); + if (hr->HasError()) { + Printer::Print("Warning: before-hook for '" + view_name + "' failed: " + hr->GetError()); + } } - } - if (hook_mode != "replace") { - if (RefreshViewLocked(context, view_catalog_name, view_schema_name, view_name, cross_system, - attached_db_catalog_name, attached_db_schema_name, !has_refresh_hook)) { - return; + if (hook_mode != "replace") { + RefreshViewSerialized(context, view_catalog_name, view_schema_name, view_name, cross_system, + attached_db_catalog_name, attached_db_schema_name, !has_refresh_hook); } - } - if (!hook_sql.empty() && (hook_mode == "after" || hook_mode == "replace")) { - auto hr = con.Query(hook_sql); - if (hr->HasError()) { - Printer::Print("Warning: " + hook_mode + "-hook for '" + view_name + "' failed: " + hr->GetError()); + if (!hook_sql.empty() && (hook_mode == "after" || hook_mode == "replace")) { + auto hr = con.Query(hook_sql); + if (hr->HasError()) { + Printer::Print("Warning: " + hook_mode + "-hook for '" + view_name + "' failed: " + hr->GetError()); + } } + } else { + OPENIVM_DEBUG_PRINT("[UPSERT] Skipped refresh node '%s'; continuing cascade traversal\n", view_name.c_str()); } // Downstream cascade: refresh dependents after @@ -545,11 +543,173 @@ void UpsertDeltaQueriesLocked(ClientContext &context, const FunctionParameters & auto downstream = metadata.GetDownstreamViews(view_name); for (auto &dep : downstream) { auto dep_location = ResolveViewLocation(con, dep, view_catalog_name, view_schema_name); - RefreshViewLocked(context, dep_location.catalog_name, dep_location.schema_name, dep, - dep_location.cross_system, attached_db_catalog_name, attached_db_schema_name, - /*skip_empty_refresh=*/true); + RefreshViewSerialized(context, dep_location.catalog_name, dep_location.schema_name, dep, + dep_location.cross_system, attached_db_catalog_name, attached_db_schema_name, + /*skip_empty_refresh=*/true); + } + } +} + +static string BuildTransactionalRefreshViewSQL(ClientContext &context, Connection &metadata_con, + const string &view_catalog_name, const string &view_schema_name, + const string &view_name, const string &attached_db_catalog_name, + const string &attached_db_schema_name) { + RefreshMetadata metadata(metadata_con); + auto delta_tables = metadata.GetDeltaTables(view_name); + DeltaActivityResult conservative_activity; + conservative_activity.has_join = metadata.HasJoin(view_name) || delta_tables.size() > 1; + conservative_activity.tables_with_changes = delta_tables.size(); + conservative_activity.any_has_deletes = true; + conservative_activity.all_ducklake = false; + conservative_activity.active_delta_table_names = delta_tables; + + // The planning connection cannot see transaction-local delta rows. Compile all + // registered native sources conservatively; the returned SQL executes through + // the caller context and therefore sees exactly the caller's transaction. + return GenerateRefreshSQL(context, view_catalog_name, view_schema_name, view_name, false, attached_db_catalog_name, + attached_db_schema_name, nullptr, nullptr, nullptr, &conservative_activity, nullptr, + nullptr, &metadata_con); +} + +string TransactionalRefreshQuery(ClientContext &context, const FunctionParameters ¶meters) { + if (context.transaction.IsAutoCommit()) { + MutationLockGuard mutation_guard(context); + // TODO: Replace query-pragma expansion with a native refresh operator/table + // function that owns compilation, execution, and lock lifetime in one caller + // transaction. Query pragmas are expanded through a preprocessing transaction; + // its transaction-end callback can release ClientContextState locks before the + // returned multi-statement program has fully completed. That boundary permits a + // concurrent parent/child refresh to enter early and produce an MVCC update + // conflict. Until refresh has a native execution boundary, retain the established + // locked helper executor for autocommit calls. Explicit caller transactions use + // the program below so their DML, MV changes, metadata, and rollback remain atomic. + UpsertDeltaQueriesLocked(context, parameters); + return "SELECT true AS Success"; + } + string view_catalog_name; + string view_schema_name; + string attached_db_catalog_name; + string attached_db_schema_name; + string view_name; + bool cross_system = false; + if (parameters.values.size() != 1 && parameters.values.size() != 3 && parameters.values.size() != 5) { + throw InvalidInputException("OpenIVM refresh received an unsupported argument list"); + } + view_name = StringValue::Get(parameters.values.back()); + if (parameters.values.size() >= 3) { + view_catalog_name = StringValue::Get(parameters.values[0]); + view_schema_name = StringValue::Get(parameters.values[1]); + } else { + auto &default_entry = ClientData::Get(context).catalog_search_path->GetDefault(); + view_catalog_name = default_entry.catalog; + view_schema_name = default_entry.schema.empty() ? DEFAULT_SCHEMA : default_entry.schema; + } + Connection metadata_con(*context.db); + UseMetadataSchema(metadata_con); + if (auto metadata_state = TransactionalMVMetadataState::TryGet(context)) { + metadata_state->IncludeView(view_name); + metadata_state->Apply(metadata_con); + } + + if (parameters.values.size() == 3) { + view_catalog_name = StringValue::Get(parameters.values[0]); + view_schema_name = StringValue::Get(parameters.values[1]); + view_name = StringValue::Get(parameters.values[2]); + } else if (parameters.values.size() == 5) { + view_catalog_name = StringValue::Get(parameters.values[0]); + view_schema_name = StringValue::Get(parameters.values[1]); + attached_db_catalog_name = StringValue::Get(parameters.values[2]); + attached_db_schema_name = StringValue::Get(parameters.values[3]); + view_name = StringValue::Get(parameters.values[4]); + cross_system = true; + } else if (parameters.values.size() == 1) { + view_name = StringValue::Get(parameters.values[0]); + auto resolved = ResolveViewCatalogFromContext(context, metadata_con, view_name); + view_catalog_name = resolved.view_catalog_name; + view_schema_name = resolved.view_schema_name; + cross_system = resolved.cross_system; + } + if (RefreshMetadata(metadata_con).GetViewQuery(view_name).empty()) { + throw CatalogException("Materialized view '%s' does not exist", view_name); + } + + if (!view_catalog_name.empty()) { + auto default_result = metadata_con.Query("SELECT current_database()"); + if (!default_result->HasError() && default_result->RowCount() > 0 && !default_result->GetValue(0, 0).IsNull() && + default_result->GetValue(0, 0).ToString() != view_catalog_name) { + cross_system = true; + } + } + // Retain only the database-wide mutation gate before choosing the execution + // path. Cross-system refresh delegates to a helper that acquires its own view + // lock; retaining that non-recursive view lock here would self-deadlock. + TransactionalMVLockState::Get(context).AcquireMutationLock(); + if (cross_system) { + // DuckDB cannot commit writes to the native metadata catalog and an + // attached external catalog in one transaction. Keep the staged path for + // that boundary; native catalogs use the caller-transaction program below. + UpsertDeltaQueriesLocked(context, parameters); + return "SELECT true AS Success"; + } + RefreshMetadata metadata(metadata_con); + string cascade_mode = "downstream"; + Value cascade_value; + if (context.TryGetCurrentSetting("openivm_cascade_refresh", cascade_value) && !cascade_value.IsNull()) { + cascade_mode = StringUtil::Lower(cascade_value.ToString()); + } + + vector refresh_order; + if (cascade_mode == "upstream" || cascade_mode == "both") { + auto upstream = metadata.GetUpstreamViews(view_name); + refresh_order.insert(refresh_order.end(), upstream.begin(), upstream.end()); + } + refresh_order.push_back(view_name); + if (cascade_mode == "downstream" || cascade_mode == "both") { + auto downstream = metadata.GetDownstreamViews(view_name); + refresh_order.insert(refresh_order.end(), downstream.begin(), downstream.end()); + } + + string program; + unordered_set seen; + vector ordered_nodes; + for (auto &node : refresh_order) { + if (!seen.insert(node).second) { + continue; + } + auto location = ResolveViewLocation(metadata_con, node, view_catalog_name, view_schema_name); + if (location.cross_system) { + throw NotImplementedException( + "Transactional native refresh cannot include cross-catalog dependent view '%s'", node); + } + ordered_nodes.push_back(node); + } + + for (auto &node : ordered_nodes) { + auto location = ResolveViewLocation(metadata_con, node, view_catalog_name, view_schema_name); + string hook_sql; + string hook_mode; + auto hooks = metadata_con.Query("SELECT hook_sql, mode FROM openivm_refresh_hooks" + " WHERE view_name = '" + + SqlUtils::EscapeValue(node) + "'"); + if (!hooks->HasError() && hooks->RowCount() > 0) { + hook_sql = hooks->GetValue(0, 0).ToString(); + hook_mode = StringUtil::Lower(hooks->GetValue(1, 0).ToString()); + } + if (!hook_sql.empty() && hook_mode == "before") { + program += hook_sql + ";\n"; + } + if (hook_mode != "replace") { + program += + BuildTransactionalRefreshViewSQL(context, metadata_con, location.catalog_name, location.schema_name, + node, attached_db_catalog_name, attached_db_schema_name); + program += "\n"; + } + if (!hook_sql.empty() && (hook_mode == "after" || hook_mode == "replace")) { + program += hook_sql + ";\n"; } } + program += "SELECT true AS Success"; + return program; } } // namespace duckdb diff --git a/src/upsert/refresh_compiler.cpp b/src/upsert/refresh_compiler.cpp index 9263da60..66ecb0db 100644 --- a/src/upsert/refresh_compiler.cpp +++ b/src/upsert/refresh_compiler.cpp @@ -7,6 +7,7 @@ #include "upsert/refresh_internal.hpp" #include +#include namespace duckdb { @@ -329,6 +330,10 @@ static string BuildUpdatedAggregateColumn(const string &col) { return "COALESCE(v." + col + " + d." + col + ", v." + col + ", d." + col + ")"; } +static string BuildNullableSum(const string &sum_expr, const string &count_expr) { + return "CASE WHEN " + count_expr + " = 0 THEN NULL ELSE " + sum_expr + " END"; +} + static string BuildNullSafeExtremumUpdate(const string &col, const string &fn) { return "CASE WHEN v." + col + " IS NULL THEN d." + col + " WHEN d." + col + " IS NULL THEN v." + col + " ELSE " + fn + "(v." + col + ", d." + col + ") END"; @@ -496,6 +501,32 @@ static DerivedAggDecomposition DetectDerivedAggColumns(const vector &col return result; } +static unordered_map DetectSumNullCountColumns(const vector &columns) { + unordered_map result; + const string prefix = openivm::SUM_COUNT_COL_PREFIX; + for (auto &column : columns) { + if (column.size() <= prefix.size() || column.compare(0, prefix.size(), prefix) != 0) { + continue; + } + idx_t projection_index = 0; + for (idx_t i = prefix.size(); i < column.size(); i++) { + if (!std::isdigit(static_cast(column[i]))) { + throw InternalException("Invalid SUM count state column '%s'", column); + } + idx_t digit = column[i] - '0'; + if (projection_index > (std::numeric_limits::max() - digit) / 10) { + throw InternalException("Invalid SUM count state column '%s'", column); + } + projection_index = projection_index * 10 + digit; + } + if (projection_index >= columns.size()) { + throw InternalException("Invalid SUM count state column '%s'", column); + } + result[SqlUtils::QuoteIdentifier(columns[projection_index])] = SqlUtils::QuoteIdentifier(column); + } + return result; +} + string CompileAggregateGroups(const string &view_name, optional_ptr index_delta_view_catalog_entry, vector column_names, const string &view_query_sql, bool has_minmax, bool list_mode, const string &delta_ts_filter, const vector &group_column_names, @@ -505,7 +536,14 @@ string CompileAggregateGroups(const string &view_name, optional_ptr &derived_output_expressions, - bool derived_output_expressions_complete) { + bool derived_output_expressions_complete, const vector &preserved_side_cols, + bool *out_used_group_recompute, bool force_group_recompute) { + if (out_used_group_recompute) { + *out_used_group_recompute = false; + } + auto is_preserved_side = [&](const string &c) { + return std::find(preserved_side_cols.begin(), preserved_side_cols.end(), c) != preserved_side_cols.end(); + }; string data_table = catalog_prefix + SqlUtils::QuoteIdentifier(IncrementalTableNames::DataTableName(view_name)); string delta_view = catalog_prefix + SqlUtils::QuoteIdentifier(SqlUtils::DeltaName(view_name)); @@ -558,15 +596,19 @@ string CompileAggregateGroups(const string &view_name, optional_ptr upsert_keys; + string recompute_temp; + if (index_delta_view_catalog_entry) { + upsert_keys = keys; + recompute_temp = SqlUtils::QuoteIdentifier("openivm_recompute_" + view_name); + } return BuildAffectedKeyRefreshSQL(data_table, view_query_sql, " " + affected, "openivm_tgt", - "openivm_recompute", "openivm_aff", match_delete, match_insert); + "openivm_recompute", "openivm_aff", match_delete, match_insert, + /*affected_temp_table=*/"", upsert_keys, recompute_temp); } // CTE: consolidate deltas per group @@ -879,7 +947,7 @@ string CompileAggregateGroups(const string &view_name, optional_ptrsecond)); + inserted_column_expressions[raw_column] = BuildNullableSum("d." + column, "d." + sum_count->second); + } else if (insert_only && agg_type == "min") { updated_column_expressions[raw_column] = BuildNullSafeExtremumUpdate(column, "LEAST"); } else if (insert_only && agg_type == "max") { updated_column_expressions[raw_column] = BuildNullSafeExtremumUpdate(column, "GREATEST"); @@ -939,7 +1012,7 @@ string CompileAggregateGroups(const string &view_name, optional_ptr
. test/sql/left_join_pipeline_secondary_delta.test covers only +# regular tables and cannot catch either. + +require openivm + +require parquet + +statement ok +INSTALL ducklake; + +statement ok +LOAD ducklake; + +statement ok +ATTACH '__TEST_DIR__/dl_lj_pipeline_secondary.db' AS dl (TYPE ducklake); + +statement ok +SET openivm_cascade_refresh = 'off'; + +statement ok +CREATE TABLE dl.cust (cid INT, nation VARCHAR); + +statement ok +CREATE TABLE dl.ord (oid INT, cid INT); + +statement ok +CREATE TABLE dl.line (lid INT, oid INT, amt INT); + +statement ok +INSERT INTO dl.cust VALUES (1,'A'),(2,'A'),(3,'B'); + +statement ok +INSERT INTO dl.ord VALUES (10,1),(11,1),(12,2); + +# o10 has 2 lines, o11 has 1 line, o12 has none (already NULL-padded) +statement ok +INSERT INTO dl.line VALUES (100,10,5),(101,10,7),(102,11,9); + +# n_ord counts the INTERMEDIATE preserved-side column -- the aggregate the secondary delta protects +statement ok +CREATE MATERIALIZED VIEW dl.dlmv AS + SELECT c.nation, COUNT(o.oid) AS n_ord, COUNT(l.lid) AS n_line, SUM(l.amt) AS rev + FROM dl.cust c LEFT JOIN dl.ord o ON o.cid = c.cid LEFT JOIN dl.line l ON l.oid = o.oid + GROUP BY c.nation; + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM dl.cust c LEFT JOIN dl.ord o ON o.cid=c.cid LEFT JOIN dl.line l ON l.oid=o.oid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, rev FROM dl.dlmv +); +---- +0 + +# Batched mixed DML before ONE refresh: +# DELETE line 102 -> o11 loses its LAST line => NULL-padded row must reappear; n_ord must NOT drop +# INSERT line 103 -> o12 gains its FIRST line => NULL-padded row removed +# DELETE line 100 -> o10 still has 101 => no transition +statement ok +DELETE FROM dl.line WHERE lid = 102; + +statement ok +INSERT INTO dl.line VALUES (103, 12, 4); + +statement ok +DELETE FROM dl.line WHERE lid = 100; + +statement ok +PRAGMA refresh('dlmv'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM dl.cust c LEFT JOIN dl.ord o ON o.cid=c.cid LEFT JOIN dl.line l ON l.oid=o.oid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, rev FROM dl.dlmv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, rev FROM dl.dlmv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM dl.cust c LEFT JOIN dl.ord o ON o.cid=c.cid LEFT JOIN dl.line l ON l.oid=o.oid GROUP BY c.nation +); +---- +0 + +# every order still counted: o10,o11,o12 for A plus a NULL-padded row for B's order-less customer +query III +SELECT nation, n_ord, n_line FROM dl.dlmv ORDER BY nation; +---- +A 3 2 +B 0 0 + +# Second batch: preserved side (orders) changes too, so BOTH sides' snapshot ranges are exercised +statement ok +INSERT INTO dl.ord VALUES (13,3); + +statement ok +INSERT INTO dl.line VALUES (104,11,3),(105,13,8); + +statement ok +DELETE FROM dl.line WHERE lid = 101; + +statement ok +DELETE FROM dl.ord WHERE oid = 12; + +statement ok +PRAGMA refresh('dlmv'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM dl.cust c LEFT JOIN dl.ord o ON o.cid=c.cid LEFT JOIN dl.line l ON l.oid=o.oid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, rev FROM dl.dlmv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, rev FROM dl.dlmv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM dl.cust c LEFT JOIN dl.ord o ON o.cid=c.cid LEFT JOIN dl.line l ON l.oid=o.oid GROUP BY c.nation +); +---- +0 + +# Third batch: an order loses its last line AND is deleted in the same batch (the downward-transition +# case), while another order gains its first line (the upward case that must NOT be suppressed) +statement ok +DELETE FROM dl.line WHERE lid = 105; + +statement ok +DELETE FROM dl.ord WHERE oid = 13; + +statement ok +INSERT INTO dl.ord VALUES (14,2); + +statement ok +INSERT INTO dl.line VALUES (106,14,6); + +statement ok +PRAGMA refresh('dlmv'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM dl.cust c LEFT JOIN dl.ord o ON o.cid=c.cid LEFT JOIN dl.line l ON l.oid=o.oid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, rev FROM dl.dlmv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, rev FROM dl.dlmv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM dl.cust c LEFT JOIN dl.ord o ON o.cid=c.cid LEFT JOIN dl.line l ON l.oid=o.oid GROUP BY c.nation +); +---- +0 diff --git a/test/sql/ducklake_projection.test b/test/sql/ducklake_projection.test index b6395199..a5f22e92 100644 --- a/test/sql/ducklake_projection.test +++ b/test/sql/ducklake_projection.test @@ -15,6 +15,26 @@ LOAD ducklake; statement ok ATTACH '__TEST_DIR__/ducklake_projection.db' AS dl (TYPE ducklake); +# Snapshot metadata is keyed by an unqualified source name. Reject two +# physically distinct DuckLake sources with the same name instead of silently +# sharing a watermark. +statement ok +ATTACH '__TEST_DIR__/ducklake_projection_second.db' AS dl_second (TYPE ducklake); + +statement ok +CREATE TABLE dl.same_name (id INTEGER); + +statement ok +CREATE TABLE dl_second.same_name (id INTEGER); + +statement error +CREATE MATERIALIZED VIEW colliding_ducklake_sources AS +SELECT l.id AS left_id, r.id AS right_id +FROM dl.same_name l +JOIN dl_second.same_name r ON l.id = r.id; +---- +cannot reference different source tables with the same unqualified name + # ========================================== # 1. Simple SELECT projection # ========================================== diff --git a/test/sql/group_recompute_persistent_unique_index.test b/test/sql/group_recompute_persistent_unique_index.test new file mode 100644 index 00000000..74eb2592 --- /dev/null +++ b/test/sql/group_recompute_persistent_unique_index.test @@ -0,0 +1,318 @@ +# name: test/sql/group_recompute_persistent_unique_index.test +# description: AGGREGATE_GROUP views carry a UNIQUE index on the data table. On a PERSISTENT database +# group: [sql] + +# DuckDB's on-disk unique index keeps deleted keys for constraint checking within a +# transaction, so the group-recompute path's DELETE-then-INSERT of a surviving group +# raised a spurious "Duplicate key ... violates unique constraint" and the refresh +# failed outright. Must use `load` + `restart`: on an in-memory database the index is +# built in-session and the bug does not fire, which is why the rest of the suite (all +# in-memory) never caught it. + +load __TEST_DIR__/openivm_group_recompute_unique_index.db + +require openivm + +statement ok +SET openivm_cascade_refresh = 'off'; + +statement ok +CREATE TABLE cust (cid INT, nation VARCHAR); + +statement ok +CREATE TABLE ord (oid INT, cid INT, amt INT); + +statement ok +INSERT INTO cust VALUES (1,'A'),(2,'A'),(3,'B'),(4,'C'); + +statement ok +INSERT INTO ord VALUES (10,1,5),(11,1,7),(12,2,9),(13,3,4); + +# AGGREGATE_GROUP over a LEFT JOIN -> data table gets the UNIQUE index on the group key. +statement ok +CREATE MATERIALIZED VIEW mv AS + SELECT c.nation, COUNT(o.oid) AS n_ord, SUM(o.amt) AS total + FROM cust c LEFT JOIN ord o ON o.cid = c.cid + GROUP BY c.nation; + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), SUM(o.amt) FROM cust c LEFT JOIN ord o ON o.cid=c.cid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, total FROM mv +); +---- +0 + +# NULL group keys do not conflict in DuckDB UNIQUE indexes. Both the forced-full +# and affected-group upsert forms must explicitly remove the old NULL-key row +# before inserting its replacement. +statement ok +CREATE TABLE nullable_full_src (k INT, val INT); + +statement ok +INSERT INTO nullable_full_src VALUES (NULL, 10), (1, 20); + +statement ok +CREATE MATERIALIZED VIEW nullable_full_mv AS + SELECT k, SUM(val) AS total FROM nullable_full_src GROUP BY k; + +statement ok +SET openivm_refresh_mode = 'full'; + +statement ok +INSERT INTO nullable_full_src VALUES (NULL, 5), (1, 2); + +statement ok +PRAGMA refresh('nullable_full_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM nullable_full_mv + EXCEPT ALL + SELECT k, SUM(val) FROM nullable_full_src GROUP BY k +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT k, SUM(val) FROM nullable_full_src GROUP BY k + EXCEPT ALL + SELECT * FROM nullable_full_mv +); +---- +0 + +statement ok +SET openivm_refresh_mode = 'incremental'; + +statement ok +CREATE TABLE nullable_gr_left (id INT, k INT); + +statement ok +CREATE TABLE nullable_gr_right (id INT, val INT); + +statement ok +INSERT INTO nullable_gr_left VALUES (1, NULL), (2, 1); + +statement ok +INSERT INTO nullable_gr_right VALUES (1, 10), (2, 20); + +statement ok +CREATE MATERIALIZED VIEW nullable_gr_mv AS + SELECT l.k, COUNT(r.id) AS matches, SUM(r.val) AS total + FROM nullable_gr_left l LEFT JOIN nullable_gr_right r ON l.id = r.id + GROUP BY l.k; + +statement ok +SET openivm_left_join_merge = false; + +statement ok +INSERT INTO nullable_gr_right VALUES (1, 11), (2, 21); + +statement ok +UPDATE nullable_gr_right SET val = 12 WHERE id = 1 AND val = 11; + +statement ok +DELETE FROM nullable_gr_right WHERE id = 2 AND val = 20; + +statement ok +PRAGMA refresh('nullable_gr_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM nullable_gr_mv + EXCEPT ALL + SELECT l.k, COUNT(r.id), SUM(r.val) + FROM nullable_gr_left l LEFT JOIN nullable_gr_right r ON l.id = r.id + GROUP BY l.k +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.k, COUNT(r.id), SUM(r.val) + FROM nullable_gr_left l LEFT JOIN nullable_gr_right r ON l.id = r.id + GROUP BY l.k + EXCEPT ALL + SELECT * FROM nullable_gr_mv +); +---- +0 + +statement ok +SET openivm_left_join_merge = true; + +# Reopen the database so the unique index is loaded from disk rather than built in this session. +# Without this restart the bug does not reproduce. +restart + +statement ok +SET openivm_cascade_refresh = 'off'; + +# openivm_left_join_merge=false routes CompileAggregateGroups to its group-recompute branch, which +# recomputes affected groups: nation 'A' survives the recompute, so it is deleted and re-inserted. +statement ok +SET openivm_left_join_merge = false; + +statement ok +INSERT INTO ord VALUES (14,1,3); + +statement ok +DELETE FROM ord WHERE oid = 12; + +statement ok +PRAGMA refresh('mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), SUM(o.amt) FROM cust c LEFT JOIN ord o ON o.cid=c.cid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, total FROM mv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, total FROM mv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), SUM(o.amt) FROM cust c LEFT JOIN ord o ON o.cid=c.cid GROUP BY c.nation +); +---- +0 + +# A group that disappears entirely must still be removed (the recompute yields no row for it, so the +# upsert form has to delete it rather than leave the stale row behind). +statement ok +DELETE FROM cust WHERE nation = 'C'; + +statement ok +PRAGMA refresh('mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), SUM(o.amt) FROM cust c LEFT JOIN ord o ON o.cid=c.cid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, total FROM mv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, total FROM mv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), SUM(o.amt) FROM cust c LEFT JOIN ord o ON o.cid=c.cid GROUP BY c.nation +); +---- +0 + +query I +SELECT COUNT(*) FROM mv WHERE nation = 'C'; +---- +0 + +# refresh_mode='full' reaches the same recompute branch on a persistent DB. +statement ok +SET openivm_left_join_merge = true; + +statement ok +SET openivm_refresh_mode = 'full'; + +statement ok +INSERT INTO ord VALUES (15,3,11); + +statement ok +PRAGMA refresh('mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), SUM(o.amt) FROM cust c LEFT JOIN ord o ON o.cid=c.cid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, total FROM mv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, total FROM mv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), SUM(o.amt) FROM cust c LEFT JOIN ord o ON o.cid=c.cid GROUP BY c.nation +); +---- +0 + +# The interrupted-refresh recovery route goes through BuildFullRecomputeSQL +# (DELETE FROM data; INSERT INTO data ), which re-inserts every key deleted in the same +# transaction and so hits the same on-disk unique-index behaviour. Distinct code path from the +# group-recompute branch above -- it needs its own coverage. + +# Reopen again: the on-disk index must be freshly loaded for this to reproduce. Once earlier +# statements in a session have modified the index, the bug stops firing. +restart + +statement ok +SET openivm_cascade_refresh = 'off'; + +statement ok +SET openivm_refresh_mode = 'incremental'; + +statement ok +INSERT INTO ord VALUES (16,1,6); + +# Simulate a crash mid-refresh: set the flag without refreshing. +statement ok +UPDATE openivm_views SET refresh_in_progress = true WHERE view_name = 'mv'; + +statement ok +PRAGMA refresh('mv'); + +query I +SELECT refresh_in_progress FROM openivm_views WHERE view_name = 'mv'; +---- +false + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), SUM(o.amt) FROM cust c LEFT JOIN ord o ON o.cid=c.cid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, total FROM mv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, total FROM mv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), SUM(o.amt) FROM cust c LEFT JOIN ord o ON o.cid=c.cid GROUP BY c.nation +); +---- +0 + +# A group that vanishes must also be dropped by the recovery recompute. +restart + +statement ok +SET openivm_cascade_refresh = 'off'; + +statement ok +DELETE FROM cust WHERE nation = 'B'; + +statement ok +UPDATE openivm_views SET refresh_in_progress = true WHERE view_name = 'mv'; + +statement ok +PRAGMA refresh('mv'); + +query I +SELECT COUNT(*) FROM mv WHERE nation = 'B'; +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), SUM(o.amt) FROM cust c LEFT JOIN ord o ON o.cid=c.cid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, total FROM mv +); +---- +0 diff --git a/test/sql/insert_rule.test b/test/sql/insert_rule.test index 6e992aae..305adfeb 100644 --- a/test/sql/insert_rule.test +++ b/test/sql/insert_rule.test @@ -1112,8 +1112,8 @@ SELECT count(*) FROM ( # DuckDB places a STREAMING_LIMIT directly above the projection for large LIMIT values, # so the INSERT's child is neither LOGICAL_PROJECTION nor LOGICAL_GET. The delta-capture # rule previously handled only those two shapes and silently skipped delta population for -# any other shape, leaving the MV stale after refresh. The catch-all branch serializes the -# source plan so the delta is captured for these shapes too. +# any other shape, leaving the MV stale after refresh. Transactional capture now wraps the +# DML input as a streaming physical operator, independent of the source-plan shape. # ========================================== statement ok @@ -1157,3 +1157,728 @@ query I SELECT count(*) FROM (SELECT * FROM bulk_base EXCEPT ALL SELECT * FROM mv_bulk); ---- 0 + +# ========================================== +# Regression: delta capture shares the base DML transaction. +# A rollback or failed constraint must roll back the corresponding delta rows too. +# ========================================== + +statement ok +CREATE TABLE tx_atomic_src ( + id INTEGER PRIMARY KEY, + grp INTEGER, + v INTEGER NOT NULL DEFAULT 7 +); + +statement ok +INSERT INTO tx_atomic_src VALUES (1, 1, 10), (2, 1, 20), (3, 2, 30); + +statement ok +CREATE MATERIALIZED VIEW tx_atomic_mv AS +SELECT grp, SUM(v) AS total, COUNT(*) AS n +FROM tx_atomic_src +GROUP BY grp; + +# Batch conflicting DML before one rollback. The id=5 row is inserted, updated, and +# deleted in the same transaction; none of the eight logical delta rows may survive. +statement ok +BEGIN TRANSACTION; + +statement ok +INSERT INTO tx_atomic_src (id, grp) VALUES (4, 2); + +statement ok +UPDATE tx_atomic_src SET v = 15 WHERE id = 1; + +statement ok +DELETE FROM tx_atomic_src WHERE id = 2; + +statement ok +INSERT INTO tx_atomic_src VALUES (5, 3, 50); + +statement ok +UPDATE tx_atomic_src SET v = 55 WHERE id = 5; + +statement ok +DELETE FROM tx_atomic_src WHERE id = 5; + +statement ok +ROLLBACK; + +query I +SELECT count(*) FROM openivm_delta_tx_atomic_src; +---- +0 + +statement ok +PRAGMA refresh('tx_atomic_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_atomic_mv + EXCEPT ALL + SELECT grp, SUM(v), COUNT(*) FROM tx_atomic_src GROUP BY grp +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT grp, SUM(v), COUNT(*) FROM tx_atomic_src GROUP BY grp + EXCEPT ALL + SELECT * FROM tx_atomic_mv +); +---- +0 + +# The same batch must remain incremental when committed. +statement ok +BEGIN TRANSACTION; + +statement ok +INSERT INTO tx_atomic_src (id, grp) VALUES (4, 2); + +statement ok +UPDATE tx_atomic_src SET v = 15 WHERE id = 1; + +statement ok +DELETE FROM tx_atomic_src WHERE id = 2; + +statement ok +INSERT INTO tx_atomic_src VALUES (5, 3, 50); + +statement ok +UPDATE tx_atomic_src SET v = 55 WHERE id = 5; + +statement ok +DELETE FROM tx_atomic_src WHERE id = 5; + +statement ok +COMMIT; + +query I +SELECT count(*) FROM openivm_delta_tx_atomic_src; +---- +8 + +statement ok +PRAGMA refresh('tx_atomic_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_atomic_mv + EXCEPT ALL + SELECT grp, SUM(v), COUNT(*) FROM tx_atomic_src GROUP BY grp +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT grp, SUM(v), COUNT(*) FROM tx_atomic_src GROUP BY grp + EXCEPT ALL + SELECT * FROM tx_atomic_mv +); +---- +0 + +# Constraint failures happen after the capture operator has seen input, so these prove +# that both writes are part of the same transaction and roll back together. +statement error +INSERT INTO tx_atomic_src VALUES (1, 9, 999); +---- +Duplicate key + +statement error +UPDATE tx_atomic_src SET v = NULL WHERE id = 1; +---- +NOT NULL + +query I +SELECT count(*) FROM openivm_delta_tx_atomic_src; +---- +8 + +statement ok +PRAGMA refresh('tx_atomic_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_atomic_mv + EXCEPT ALL + SELECT grp, SUM(v), COUNT(*) FROM tx_atomic_src GROUP BY grp +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT grp, SUM(v), COUNT(*) FROM tx_atomic_src GROUP BY grp + EXCEPT ALL + SELECT * FROM tx_atomic_mv +); +---- +0 + +# Generated columns are not physically stored in the base table, so capture must evaluate +# them for both old and new rows. RETURNING adds a projection above the DML operator; it must +# not hide the affected DML plan from the capture rule. +statement ok +CREATE TABLE tx_generated_src ( + a INTEGER, + doubled INTEGER GENERATED ALWAYS AS (a * 2) +); + +statement ok +INSERT INTO tx_generated_src (a) VALUES (1), (2); + +statement ok +CREATE MATERIALIZED VIEW tx_generated_mv AS SELECT a, doubled FROM tx_generated_src; + +query II +INSERT INTO tx_generated_src (a) VALUES (3) RETURNING a, doubled; +---- +3 6 + +query II +UPDATE tx_generated_src SET a = a + 10 WHERE a = 1 RETURNING a, doubled; +---- +11 22 + +query II +DELETE FROM tx_generated_src WHERE a = 2 RETURNING a, doubled; +---- +2 4 + +query I +SELECT count(*) FROM openivm_delta_tx_generated_src; +---- +4 + +statement ok +PRAGMA refresh('tx_generated_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_generated_mv + EXCEPT ALL + SELECT * FROM tx_generated_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_generated_src + EXCEPT ALL + SELECT * FROM tx_generated_mv +); +---- +0 + +# UPDATE DEFAULT must be evaluated once and shared by base-table update and delta capture. +# A sequence default makes duplicate evaluation observable. +statement ok +CREATE SEQUENCE tx_default_seq START 100; + +statement ok +CREATE TABLE tx_default_src ( + id INTEGER, + v BIGINT DEFAULT nextval('tx_default_seq') +); + +statement ok +INSERT INTO tx_default_src VALUES (1, 0); + +statement ok +CREATE MATERIALIZED VIEW tx_default_mv AS SELECT id, v FROM tx_default_src; + +statement ok +UPDATE tx_default_src SET v = DEFAULT WHERE id = 1; + +query I +SELECT v FROM tx_default_src; +---- +100 + +query I +SELECT v FROM openivm_delta_tx_default_src WHERE openivm_multiplicity = 1; +---- +100 + +statement ok +PRAGMA refresh('tx_default_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_default_mv + EXCEPT ALL + SELECT * FROM tx_default_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM tx_default_src + EXCEPT ALL + SELECT * FROM tx_default_mv +); +---- +0 + +# ========================================== +# Actual outcomes for ON CONFLICT / OR REPLACE +# ========================================== + +statement ok +CREATE TABLE conflict_src ( + id INTEGER PRIMARY KEY, + v INTEGER NOT NULL DEFAULT 7, + label VARCHAR +); + +statement ok +INSERT INTO conflict_src VALUES (1, 10, 'one'), (2, 20, 'two'); + +statement ok +CREATE MATERIALIZED VIEW conflict_mv AS SELECT id, v, label FROM conflict_src; + +# The rejected id=1 input is not an affected row and must not become a delta. +query III +INSERT INTO conflict_src VALUES (1, 999, 'ignored'), (3, 30, 'three') +ON CONFLICT DO NOTHING +RETURNING id, v, label; +---- +3 30 three + +query III +SELECT id, v, openivm_multiplicity +FROM openivm_delta_conflict_src +ORDER BY id, openivm_multiplicity; +---- +3 30 1 + +statement ok +PRAGMA refresh('conflict_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_mv + EXCEPT ALL + SELECT * FROM conflict_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_src + EXCEPT ALL + SELECT * FROM conflict_mv +); +---- +0 + +statement ok +DELETE FROM openivm_delta_conflict_src; + +# Action predicates are resolved by DuckDB before capture: id=2 performs DO NOTHING, +# id=1 updates, and id=4 inserts. +query III +INSERT INTO conflict_src VALUES (1, 5, 'one-updated'), (2, -1, 'ignored'), (4, 40, 'four') +ON CONFLICT(id) DO UPDATE +SET v = conflict_src.v + excluded.v, label = excluded.label +WHERE excluded.v > 0 +RETURNING id, v, label; +---- +1 15 one-updated +4 40 four + +query III +SELECT id, v, openivm_multiplicity +FROM openivm_delta_conflict_src +ORDER BY id, openivm_multiplicity, v; +---- +1 10 -1 +1 15 1 +4 40 1 + +statement ok +PRAGMA refresh('conflict_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_mv + EXCEPT ALL + SELECT * FROM conflict_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_src + EXCEPT ALL + SELECT * FROM conflict_mv +); +---- +0 + +statement ok +DELETE FROM openivm_delta_conflict_src; + +# OR REPLACE is an update for the conflicting row and an insert for the fresh row. +query III +INSERT OR REPLACE INTO conflict_src VALUES (1, 50, 'replaced'), (5, 50, 'five') +RETURNING id, v, label; +---- +1 50 replaced +5 50 five + +query III +SELECT id, v, openivm_multiplicity +FROM openivm_delta_conflict_src +ORDER BY id, openivm_multiplicity, v; +---- +1 15 -1 +1 50 1 +5 50 1 + +statement ok +PRAGMA refresh('conflict_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_mv + EXCEPT ALL + SELECT * FROM conflict_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_src + EXCEPT ALL + SELECT * FROM conflict_mv +); +---- +0 + +# Capture runs before DuckDB's action sink. A failed conflict update must roll back +# both its provisional deltas and the base-table statement. +statement ok +DELETE FROM openivm_delta_conflict_src; + +statement error +INSERT INTO conflict_src VALUES (1, 1, 'bad') +ON CONFLICT(id) DO UPDATE SET v = NULL; +---- +NOT NULL + +query I +SELECT count(*) FROM openivm_delta_conflict_src; +---- +0 + +# Batch conflict update/insert, replacement, UPDATE, and DELETE before one refresh. +# id=6 is inserted, updated, and deleted in the same batch. +statement ok +INSERT INTO conflict_src VALUES (1, 1, 'again'), (6, 60, 'six') +ON CONFLICT(id) DO UPDATE SET v = conflict_src.v + excluded.v, label = excluded.label; + +statement ok +INSERT OR REPLACE INTO conflict_src VALUES (4, 44, 'four-replaced'); + +statement ok +UPDATE conflict_src SET v = 66 WHERE id = 6; + +statement ok +DELETE FROM conflict_src WHERE id = 6; + +query I +SELECT count(*) FROM openivm_delta_conflict_src; +---- +8 + +statement ok +PRAGMA refresh('conflict_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_mv + EXCEPT ALL + SELECT * FROM conflict_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_src + EXCEPT ALL + SELECT * FROM conflict_mv +); +---- +0 + +# A volatile UPDATE DEFAULT is evaluated once and shared by capture and the action sink. +statement ok +CREATE SEQUENCE conflict_default_seq START 100; + +statement ok +CREATE TABLE conflict_default_src ( + id INTEGER PRIMARY KEY, + v BIGINT DEFAULT nextval('conflict_default_seq') +); + +statement ok +INSERT INTO conflict_default_src VALUES (1, 0); + +statement ok +CREATE MATERIALIZED VIEW conflict_default_mv AS SELECT id, v FROM conflict_default_src; + +query II +INSERT INTO conflict_default_src(id) VALUES (1) +ON CONFLICT(id) DO UPDATE SET v = DEFAULT +RETURNING id, v; +---- +1 101 + +query III +SELECT id, v, openivm_multiplicity +FROM openivm_delta_conflict_default_src +ORDER BY openivm_multiplicity; +---- +1 0 -1 +1 101 1 + +statement ok +PRAGMA refresh('conflict_default_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_default_mv + EXCEPT ALL + SELECT * FROM conflict_default_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_default_src + EXCEPT ALL + SELECT * FROM conflict_default_mv +); +---- +0 + +# Generated delta columns are evaluated from both the old and resolved new row. +statement ok +CREATE TABLE conflict_generated_src ( + id INTEGER PRIMARY KEY, + a INTEGER, + doubled INTEGER GENERATED ALWAYS AS (a * 2) +); + +statement ok +INSERT INTO conflict_generated_src(id, a) VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW conflict_generated_mv AS +SELECT id, a, doubled FROM conflict_generated_src; + +statement ok +INSERT INTO conflict_generated_src(id, a) VALUES (1, 5), (2, 20) +ON CONFLICT(id) DO UPDATE SET a = conflict_generated_src.a + excluded.a; + +query IIII +SELECT id, a, doubled, openivm_multiplicity +FROM openivm_delta_conflict_generated_src +ORDER BY id, openivm_multiplicity; +---- +1 10 20 -1 +1 15 30 1 +2 20 40 1 + +statement ok +PRAGMA refresh('conflict_generated_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_generated_mv + EXCEPT ALL + SELECT * FROM conflict_generated_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM conflict_generated_src + EXCEPT ALL + SELECT * FROM conflict_generated_mv +); +---- +0 + +# Native MERGE uses the same resolved action boundary. Exercise DELETE, UPDATE, +# and INSERT actions together, including action-specific RETURNING rows. +statement ok +CREATE TABLE merge_src (id INTEGER PRIMARY KEY, v INTEGER); + +statement ok +INSERT INTO merge_src VALUES (1, 10), (2, 20); + +statement ok +CREATE MATERIALIZED VIEW merge_mv AS SELECT id, v FROM merge_src; + +query TII +MERGE INTO merge_src +USING (VALUES (1, 5), (2, -1), (3, 30)) changes(id, v) +ON merge_src.id = changes.id +WHEN MATCHED AND changes.v < 0 THEN DELETE +WHEN MATCHED THEN UPDATE SET v = merge_src.v + changes.v +WHEN NOT MATCHED THEN INSERT VALUES (changes.id, changes.v) +RETURNING merge_action, id, v; +---- +DELETE 2 20 +UPDATE 1 15 +INSERT 3 30 + +query III +SELECT id, v, openivm_multiplicity +FROM openivm_delta_merge_src +ORDER BY id, openivm_multiplicity; +---- +1 10 -1 +1 15 1 +2 20 -1 +3 30 1 + +statement ok +PRAGMA refresh('merge_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM merge_mv + EXCEPT ALL + SELECT * FROM merge_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM merge_src + EXCEPT ALL + SELECT * FROM merge_mv +); +---- +0 + +# Duplicate source matches can make DuckDB choose a different update row than the +# input's first occurrence when it sorts and deduplicates target row IDs. Capture +# the target's actual final value rather than predicting which source row wins. +statement ok +CREATE TABLE merge_duplicate_src (id INTEGER PRIMARY KEY, v INTEGER); + +statement ok +INSERT INTO merge_duplicate_src SELECT i, 0 FROM range(2500) r(i); + +statement ok +CREATE MATERIALIZED VIEW merge_duplicate_mv AS SELECT id, v FROM merge_duplicate_src; + +statement ok +CREATE TABLE merge_duplicate_changes AS +SELECT i AS id, x AS v +FROM range(2500) r(i), (VALUES (1), (2)) values_table(x); + +statement ok +MERGE INTO merge_duplicate_src +USING merge_duplicate_changes +ON merge_duplicate_src.id = merge_duplicate_changes.id +WHEN MATCHED THEN UPDATE SET v = merge_duplicate_changes.v; + +statement ok +PRAGMA refresh('merge_duplicate_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM merge_duplicate_mv + EXCEPT ALL + SELECT * FROM merge_duplicate_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM merge_duplicate_src + EXCEPT ALL + SELECT * FROM merge_duplicate_mv +); +---- +0 + +# A MERGE can update a committed row while deleting a row inserted earlier in +# the same transaction. The deleted local row ID must not be refetched as a +# positive postimage when the MERGE actions finalize. +statement ok +CREATE TABLE merge_mixed_visibility_src (id INTEGER PRIMARY KEY, v INTEGER); + +statement ok +INSERT INTO merge_mixed_visibility_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW merge_mixed_visibility_mv AS +SELECT id, v FROM merge_mixed_visibility_src; + +statement ok +BEGIN TRANSACTION; + +statement ok +INSERT INTO merge_mixed_visibility_src VALUES (2, 20); + +statement ok +MERGE INTO merge_mixed_visibility_src +USING (VALUES (1, 5), (2, -1)) changes(id, v) +ON merge_mixed_visibility_src.id = changes.id +WHEN MATCHED AND changes.v < 0 THEN DELETE +WHEN MATCHED THEN UPDATE SET v = merge_mixed_visibility_src.v + changes.v; + +statement ok +COMMIT; + +query III +SELECT id, v, openivm_multiplicity +FROM openivm_delta_merge_mixed_visibility_src +ORDER BY id, v, openivm_multiplicity; +---- +1 10 -1 +1 15 1 +2 20 -1 +2 20 1 + +statement ok +PRAGMA refresh('merge_mixed_visibility_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM merge_mixed_visibility_mv + EXCEPT ALL + SELECT * FROM merge_mixed_visibility_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM merge_mixed_visibility_src + EXCEPT ALL + SELECT * FROM merge_mixed_visibility_mv +); +---- +0 diff --git a/test/sql/lateral.test b/test/sql/lateral.test index f0d9ea64..4cd2cf25 100644 --- a/test/sql/lateral.test +++ b/test/sql/lateral.test @@ -610,6 +610,87 @@ SELECT COUNT(*) FROM ( ---- 0 +# DISTINCT inside the left input of a LATERAL aggregate is not a top-level +# DISTINCT. It remains incrementally maintainable via affected-group recompute. +statement ok +CREATE TABLE lat_distinct_customer (id INT, warehouse_id INT, state VARCHAR); + +statement ok +INSERT INTO lat_distinct_customer VALUES + (1, 1, 'CA'), (2, 1, 'CA'), (3, 1, 'NY'), (4, 2, 'CA'); + +statement ok +CREATE MATERIALIZED VIEW mv_lat_distinct_count AS + SELECT c.warehouse_id, c.state, agg.customer_count + FROM ( + SELECT DISTINCT warehouse_id, state + FROM lat_distinct_customer + ) c + JOIN LATERAL ( + SELECT COUNT(*) AS customer_count + FROM lat_distinct_customer + WHERE warehouse_id = c.warehouse_id AND state = c.state + ) agg ON TRUE; + +query I +SELECT type FROM openivm_views WHERE view_name = 'mv_lat_distinct_count'; +---- +6 + +statement ok +UPDATE lat_distinct_customer SET state = 'TX' WHERE id = 2; + +statement ok +DELETE FROM lat_distinct_customer WHERE id = 3; + +statement ok +INSERT INTO lat_distinct_customer VALUES (5, 2, 'WA'), (6, 3, 'CA'); + +statement ok +UPDATE lat_distinct_customer SET warehouse_id = 3 WHERE id = 6; + +statement ok +DELETE FROM lat_distinct_customer WHERE id = 5; + +statement ok +PRAGMA refresh('mv_lat_distinct_count'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM mv_lat_distinct_count + EXCEPT ALL + SELECT c.warehouse_id, c.state, agg.customer_count + FROM ( + SELECT DISTINCT warehouse_id, state + FROM lat_distinct_customer + ) c + JOIN LATERAL ( + SELECT COUNT(*) AS customer_count + FROM lat_distinct_customer + WHERE warehouse_id = c.warehouse_id AND state = c.state + ) agg ON TRUE +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT c.warehouse_id, c.state, agg.customer_count + FROM ( + SELECT DISTINCT warehouse_id, state + FROM lat_distinct_customer + ) c + JOIN LATERAL ( + SELECT COUNT(*) AS customer_count + FROM lat_distinct_customer + WHERE warehouse_id = c.warehouse_id AND state = c.state + ) agg ON TRUE + EXCEPT ALL + SELECT * FROM mv_lat_distinct_count +); +---- +0 + statement ok INSERT INTO lat_history VALUES (1, 1, 7); diff --git a/test/sql/left_join.test b/test/sql/left_join.test index 9c38de70..a94e1572 100644 --- a/test/sql/left_join.test +++ b/test/sql/left_join.test @@ -1020,6 +1020,135 @@ SELECT COUNT(*) FROM ( ---- 0 +# A table function on the preserved side has no source delta of its own. +# LEFT JOIN MERGE cannot infer which zero-match generated groups must remain, +# so these aggregates use current-vs-stored affected-group recomputation. +statement ok +CREATE TABLE tf_customer (id INT, warehouse_id INT, balance INT); + +statement ok +INSERT INTO tf_customer VALUES (1, 1, 10), (2, 1, 20), (3, 2, 30); + +statement ok +CREATE MATERIALIZED VIEW mv_tf_customer_count AS + SELECT generated_id AS warehouse_id, COUNT(c.id) AS customer_count + FROM generate_series(1, 5) generated(generated_id) + LEFT JOIN tf_customer c ON c.warehouse_id = generated.generated_id + GROUP BY generated_id; + +query I +SELECT type FROM openivm_views WHERE view_name = 'mv_tf_customer_count'; +---- +6 + +# Conflicting insert/delete/update operations are deliberately batched before +# one refresh. In particular, groups 3 and 5 must retain their generated +# zero-count rows. +statement ok +UPDATE tf_customer SET balance = 11 WHERE id = 1; + +statement ok +UPDATE tf_customer SET warehouse_id = 2 WHERE id = 2; + +statement ok +DELETE FROM tf_customer WHERE id = 3; + +statement ok +INSERT INTO tf_customer VALUES (4, 1, 40), (5, 4, 50); + +statement ok +UPDATE tf_customer SET balance = 41 WHERE id = 4; + +statement ok +DELETE FROM tf_customer WHERE id = 5; + +statement ok +PRAGMA refresh('mv_tf_customer_count'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM mv_tf_customer_count + EXCEPT ALL + SELECT generated_id, COUNT(c.id) + FROM generate_series(1, 5) generated(generated_id) + LEFT JOIN tf_customer c ON c.warehouse_id = generated.generated_id + GROUP BY generated_id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT generated_id, COUNT(c.id) + FROM generate_series(1, 5) generated(generated_id) + LEFT JOIN tf_customer c ON c.warehouse_id = generated.generated_id + GROUP BY generated_id + EXCEPT ALL + SELECT * FROM mv_tf_customer_count +); +---- +0 + +statement ok +CREATE TABLE tf_stock (id INT, quantity INT); + +statement ok +INSERT INTO tf_stock VALUES (1, 3), (2, 12), (3, 27); + +statement ok +CREATE MATERIALIZED VIEW mv_tf_stock_thresholds AS + SELECT threshold, COUNT(s.id) AS low_stock_items + FROM range(5, 30, 5) generated(threshold) + LEFT JOIN tf_stock s ON s.quantity < generated.threshold + GROUP BY threshold; + +query I +SELECT type FROM openivm_views WHERE view_name = 'mv_tf_stock_thresholds'; +---- +6 + +statement ok +UPDATE tf_stock SET quantity = 18 WHERE id = 1; + +statement ok +DELETE FROM tf_stock WHERE id = 2; + +statement ok +INSERT INTO tf_stock VALUES (4, 7), (5, 22); + +statement ok +UPDATE tf_stock SET quantity = 2 WHERE id = 5; + +statement ok +DELETE FROM tf_stock WHERE id = 4; + +statement ok +PRAGMA refresh('mv_tf_stock_thresholds'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM mv_tf_stock_thresholds + EXCEPT ALL + SELECT threshold, COUNT(s.id) + FROM range(5, 30, 5) generated(threshold) + LEFT JOIN tf_stock s ON s.quantity < generated.threshold + GROUP BY threshold +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT threshold, COUNT(s.id) + FROM range(5, 30, 5) generated(threshold) + LEFT JOIN tf_stock s ON s.quantity < generated.threshold + GROUP BY threshold + EXCEPT ALL + SELECT * FROM mv_tf_stock_thresholds +); +---- +0 + # Delete from a deeper-right table: the matched MV row should # revert to NULL on that side. statement ok diff --git a/test/sql/left_join_deep_chain_secondary.test b/test/sql/left_join_deep_chain_secondary.test new file mode 100644 index 00000000..a02dcd31 --- /dev/null +++ b/test/sql/left_join_deep_chain_secondary.test @@ -0,0 +1,251 @@ +# name: test/sql/left_join_deep_chain_secondary.test +# description: LEFT JOIN chains DEEPER than 3 tables need a secondary delta at EVERY level whose +# group: [sql] + +# preserved side is itself a join, not just the outermost one. With cust ⟕ ord ⟕ line ⟕ supp, +# deleting an order's last line stranded that order in COUNT(o.oid) because only the +# outermost (⟕ supp) level was corrected and the ⟕ line level was not. 3-table chains were +# already covered by left_join_pipeline_secondary_delta.test and passed, so this shape needs +# its own coverage at 4 and 5 tables. + +require openivm + +statement ok +SET openivm_files_path = '__TEST_DIR__'; + +statement ok +SET openivm_cascade_refresh = 'off'; + +statement ok +CREATE TABLE cust (cid INT, nation VARCHAR); + +statement ok +CREATE TABLE ord (oid INT, cid INT); + +statement ok +CREATE TABLE line (lid INT, oid INT, sid INT, amt INT); + +statement ok +CREATE TABLE supp (sid INT, sname VARCHAR); + +statement ok +CREATE TABLE part (pid INT, sid INT, pname VARCHAR); + +statement ok +INSERT INTO cust VALUES (1,'A'),(2,'B'); + +statement ok +INSERT INTO ord VALUES (10,1),(11,2); + +statement ok +INSERT INTO line VALUES (100,10,500,5),(101,11,501,7); + +statement ok +INSERT INTO supp VALUES (500,'S500'),(501,'S501'); + +statement ok +INSERT INTO part VALUES (900,500,'P900'); + +statement ok +CREATE MATERIALIZED VIEW mv4 AS + SELECT c.nation, COUNT(o.oid) AS n_ord, COUNT(l.lid) AS n_line, COUNT(s.sid) AS n_supp + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid LEFT JOIN supp s ON s.sid=l.sid + GROUP BY c.nation; + +statement ok +CREATE MATERIALIZED VIEW mv5 AS + SELECT c.nation, COUNT(o.oid) AS n_ord, COUNT(l.lid) AS n_line, COUNT(s.sid) AS n_supp, COUNT(p.pid) AS n_part + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid LEFT JOIN supp s ON s.sid=l.sid + LEFT JOIN part p ON p.sid=s.sid + GROUP BY c.nation; + +# order 11 loses its LAST line: its supplier and part vanish too, but the ORDER itself must still count +statement ok +DELETE FROM line WHERE lid = 101; + +statement ok +PRAGMA refresh('mv4'); + +statement ok +PRAGMA refresh('mv5'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), COUNT(s.sid) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid LEFT JOIN supp s ON s.sid=l.sid + GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, n_supp FROM mv4 +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, n_supp FROM mv4 + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), COUNT(s.sid) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid LEFT JOIN supp s ON s.sid=l.sid + GROUP BY c.nation +); +---- +0 + +# the specific regression: nation B's order survives in n_ord even though its line/supp/part are gone +query III +SELECT nation, n_ord, n_line FROM mv4 ORDER BY nation; +---- +A 1 1 +B 1 0 + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), COUNT(s.sid), COUNT(p.pid) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid LEFT JOIN supp s ON s.sid=l.sid + LEFT JOIN part p ON p.sid=s.sid + GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, n_supp, n_part FROM mv5 +); +---- +0 + +query I +SELECT COUNT(*) FROM ( +SELECT nation, n_ord, n_line, n_supp, n_part FROM mv5 + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), COUNT(s.sid), COUNT(p.pid) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid LEFT JOIN supp s ON s.sid=l.sid + LEFT JOIN part p ON p.sid=s.sid + GROUP BY c.nation +); +---- +0 + +# Repeated guards for the same leaf/key pair share one transition-count plan. +# In this 3-leaf chain the serialized plan has three transition definitions +# (one in the outer join compilation and two across its nested compilation), +# with five current_count references each. Cloning per inclusion/exclusion term +# produced more copies. +statement ok +CREATE TABLE transition_share_a (id INTEGER); + +statement ok +CREATE TABLE transition_share_b (id INTEGER, aid INTEGER); + +statement ok +CREATE TABLE transition_share_c (id INTEGER, bid INTEGER); + +statement ok +INSERT INTO transition_share_a VALUES (1), (2); + +statement ok +INSERT INTO transition_share_b VALUES (10, 1), (20, 2); + +statement ok +INSERT INTO transition_share_c VALUES (100, 10), (200, 20); + +statement ok +CREATE MATERIALIZED VIEW transition_share_mv AS +SELECT a.id AS aid, b.id AS bid, c.id AS cid +FROM transition_share_a a +LEFT JOIN transition_share_b b ON b.aid = a.id +LEFT JOIN transition_share_c c ON c.bid = b.id; + +statement ok +INSERT INTO transition_share_a VALUES (3); + +statement ok +INSERT INTO transition_share_b VALUES (30, 3); + +statement ok +INSERT INTO transition_share_c VALUES (300, 30); + +statement ok +PRAGMA refresh('transition_share_mv'); + +query I +SELECT len(regexp_extract_all(content, 'current_count')) +FROM read_text('__TEST_DIR__/openivm_upsert_queries_transition_share_mv.sql'); +---- +15 + +query I +SELECT count(*) FROM ( + SELECT * FROM transition_share_mv + EXCEPT ALL + SELECT a.id, b.id, c.id + FROM transition_share_a a + LEFT JOIN transition_share_b b ON b.aid = a.id + LEFT JOIN transition_share_c c ON c.bid = b.id +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT a.id, b.id, c.id + FROM transition_share_a a + LEFT JOIN transition_share_b b ON b.aid = a.id + LEFT JOIN transition_share_c c ON c.bid = b.id + EXCEPT ALL + SELECT * FROM transition_share_mv +); +---- +0 + +# Second batch: a middle-level match reappears (order 11 gains a line again) plus an unrelated insert, +# exercising the upward transition at a non-outermost level. +statement ok +INSERT INTO line VALUES (102, 11, 501, 3); + +statement ok +INSERT INTO ord VALUES (12, 1); + +statement ok +PRAGMA refresh('mv4'); + +statement ok +PRAGMA refresh('mv5'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), COUNT(s.sid) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid LEFT JOIN supp s ON s.sid=l.sid + GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, n_supp FROM mv4 +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, n_supp FROM mv4 + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), COUNT(s.sid) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid LEFT JOIN supp s ON s.sid=l.sid + GROUP BY c.nation +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), COUNT(s.sid), COUNT(p.pid) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid LEFT JOIN supp s ON s.sid=l.sid + LEFT JOIN part p ON p.sid=s.sid + GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, n_supp, n_part FROM mv5 +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, n_supp, n_part FROM mv5 + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), COUNT(s.sid), COUNT(p.pid) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid LEFT JOIN supp s ON s.sid=l.sid + LEFT JOIN part p ON p.sid=s.sid + GROUP BY c.nation +); +---- +0 diff --git a/test/sql/left_join_emptied_group.test b/test/sql/left_join_emptied_group.test new file mode 100644 index 00000000..667e1cf3 --- /dev/null +++ b/test/sql/left_join_emptied_group.test @@ -0,0 +1,182 @@ +# name: test/sql/left_join_emptied_group.test +# description: LEFT JOIN aggregate MVs must DROP a group once its last preserved-side row is deleted, +# group: [sql] + +# while KEEPING groups that merely have no match (NULL-padded). Previously the empty-group +# cleanup was skipped for any view carrying openivm_match_count, so emptied groups lingered +# forever as zeroed rows (e.g. `B | 0 | 0`) where a recompute drops them. The two cases are +# only distinguishable via openivm_count_star, which counts LEFT JOIN OUTPUT rows: it is 1 +# for a NULL-padded group and 0 for an emptied one. That in turn requires the secondary +# delta to emit the null-padded reappearance at EVERY level, including a single LEFT JOIN. + +require openivm + +statement ok +SET openivm_cascade_refresh = 'off'; + +statement ok +CREATE TABLE cu (id INT, name VARCHAR); + +statement ok +CREATE TABLE od (cust_id INT, amount INT); + +statement ok +INSERT INTO cu VALUES (1,'Alice'),(2,'Bob'),(3,'Carol'),(4,'Dave'); + +# Alice has 2 orders, Bob 1, Carol none (already NULL-padded), Dave none +statement ok +INSERT INTO od VALUES (1,100),(1,200),(2,50); + +statement ok +CREATE MATERIALIZED VIEW mv_lj AS + SELECT c.name, SUM(o.amount) AS total, COUNT(o.amount) AS cnt + FROM cu c LEFT JOIN od o ON c.id = o.cust_id + GROUP BY c.name; + +query I +SELECT COUNT(*) FROM ( + SELECT c.name, SUM(o.amount), COUNT(o.amount) FROM cu c LEFT JOIN od o ON c.id=o.cust_id GROUP BY c.name + EXCEPT ALL SELECT name, total, cnt FROM mv_lj +); +---- +0 + +# Batch: Alice loses ALL her orders (group must SURVIVE as NULL/0 -- she still has a customer row), +# and Dave's customer row is deleted (group must DISAPPEAR). +statement ok +DELETE FROM od WHERE cust_id = 1; + +statement ok +DELETE FROM cu WHERE id = 4; + +statement ok +PRAGMA refresh('mv_lj'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.name, SUM(o.amount), COUNT(o.amount) FROM cu c LEFT JOIN od o ON c.id=o.cust_id GROUP BY c.name + EXCEPT ALL SELECT name, total, cnt FROM mv_lj +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT name, total, cnt FROM mv_lj + EXCEPT ALL + SELECT c.name, SUM(o.amount), COUNT(o.amount) FROM cu c LEFT JOIN od o ON c.id=o.cust_id GROUP BY c.name +); +---- +0 + +# Alice unmatched but present; Carol still NULL-padded; Dave gone entirely +query III +SELECT name, total, cnt FROM mv_lj ORDER BY name; +---- +Alice NULL 0 +Bob 50 1 +Carol NULL 0 + +# count_star must reflect OUTPUT rows: 1 for a surviving unmatched group, not 0 +query I +SELECT openivm_count_star FROM openivm_data_mv_lj WHERE name = 'Alice'; +---- +1 + +# Deleting the last customer of an already-unmatched group must drop it +statement ok +DELETE FROM cu WHERE id = 3; + +statement ok +PRAGMA refresh('mv_lj'); + +query I +SELECT COUNT(*) FROM mv_lj WHERE name = 'Carol'; +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT c.name, SUM(o.amount), COUNT(o.amount) FROM cu c LEFT JOIN od o ON c.id=o.cust_id GROUP BY c.name + EXCEPT ALL SELECT name, total, cnt FROM mv_lj +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT name, total, cnt FROM mv_lj + EXCEPT ALL + SELECT c.name, SUM(o.amount), COUNT(o.amount) FROM cu c LEFT JOIN od o ON c.id=o.cust_id GROUP BY c.name +); +---- +0 + +# A group can come back: re-add Alice's order, and re-add Carol entirely +statement ok +INSERT INTO od VALUES (1,999); + +statement ok +INSERT INTO cu VALUES (3,'Carol'); + +statement ok +PRAGMA refresh('mv_lj'); + +query III +SELECT name, total, cnt FROM mv_lj ORDER BY name; +---- +Alice 999 1 +Bob 50 1 +Carol NULL 0 + +query I +SELECT COUNT(*) FROM ( + SELECT name, total, cnt FROM mv_lj + EXCEPT ALL + SELECT c.name, SUM(o.amount), COUNT(o.amount) FROM cu c LEFT JOIN od o ON c.id=o.cust_id GROUP BY c.name +); +---- +0 + +# ---- same lifecycle in a 3-table LEFT JOIN pipeline ---- +statement ok +CREATE TABLE li (oid INT, cust_id INT, qty INT); + +statement ok +INSERT INTO li VALUES (1,1,3),(2,2,4); + +statement ok +CREATE MATERIALIZED VIEW mv_pipe AS + SELECT c.name, COUNT(o.amount) AS n_ord, COUNT(l.oid) AS n_li + FROM cu c LEFT JOIN od o ON c.id = o.cust_id LEFT JOIN li l ON l.cust_id = c.id + GROUP BY c.name; + +statement ok +DELETE FROM cu WHERE id = 2; + +statement ok +PRAGMA refresh('mv_pipe'); + +query I +SELECT COUNT(*) FROM mv_pipe WHERE name = 'Bob'; +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT c.name, COUNT(o.amount), COUNT(l.oid) + FROM cu c LEFT JOIN od o ON c.id=o.cust_id LEFT JOIN li l ON l.cust_id=c.id GROUP BY c.name + EXCEPT ALL SELECT name, n_ord, n_li FROM mv_pipe +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT name, n_ord, n_li FROM mv_pipe + EXCEPT ALL + SELECT c.name, COUNT(o.amount), COUNT(l.oid) + FROM cu c LEFT JOIN od o ON c.id=o.cust_id LEFT JOIN li l ON l.cust_id=c.id GROUP BY c.name +); +---- +0 diff --git a/test/sql/left_join_group_by_join_key.test b/test/sql/left_join_group_by_join_key.test new file mode 100644 index 00000000..2b16dcb8 --- /dev/null +++ b/test/sql/left_join_group_by_join_key.test @@ -0,0 +1,156 @@ +# name: test/sql/left_join_group_by_join_key.test +# description: GROUP BY a column that is ALSO the deepest LEFT JOIN key. The secondary-delta generator +# group: [sql] + +# aliases the preserved-side join key "__k" and each group column "__g", but a single +# output column can only carry one alias -- so when the key IS a group column the group +# alias won and the emitted SQL still referenced X."__k", failing to bind with +# 'Values list "X" does not have a column named "__k"'. Refresh failed outright for every +# such view, which is a common star-schema shape (GROUP BY the dimension key you join on). + +require openivm + +statement ok +SET openivm_cascade_refresh = 'off'; + +statement ok +CREATE TABLE fact (fid INT, k INT); + +statement ok +CREATE TABLE dm1 (k1 INT, v1 INT); + +statement ok +CREATE TABLE dm2 (k2 INT, v2 INT); + +statement ok +INSERT INTO fact VALUES (1,100),(2,100),(3,200); + +statement ok +INSERT INTO dm1 VALUES (100,1),(200,1); + +statement ok +INSERT INTO dm2 VALUES (100,2),(200,2); + +# f.k is both the GROUP BY column and the join key used by both LEFT JOINs +statement ok +CREATE MATERIALIZED VIEW mv_gk AS + SELECT f.k, COUNT(f.fid) AS n, COUNT(x1.k1) AS c1, COUNT(x2.k2) AS c2 + FROM fact f LEFT JOIN dm1 x1 ON x1.k1 = f.k LEFT JOIN dm2 x2 ON x2.k2 = f.k + GROUP BY f.k; + +query I +SELECT COUNT(*) FROM ( + SELECT f.k, COUNT(f.fid), COUNT(x1.k1), COUNT(x2.k2) + FROM fact f LEFT JOIN dm1 x1 ON x1.k1=f.k LEFT JOIN dm2 x2 ON x2.k2=f.k GROUP BY f.k + EXCEPT ALL SELECT k, n, c1, c2 FROM mv_gk +); +---- +0 + +# Batched mixed DML. Every group keeps at least one fact row, so this isolates the alias fix from the +# separate emptied-group lifecycle behaviour. +statement ok +INSERT INTO fact VALUES (4,200),(5,100); + +statement ok +DELETE FROM fact WHERE fid = 2; + +statement ok +PRAGMA refresh('mv_gk'); + +query I +SELECT COUNT(*) FROM ( + SELECT f.k, COUNT(f.fid), COUNT(x1.k1), COUNT(x2.k2) + FROM fact f LEFT JOIN dm1 x1 ON x1.k1=f.k LEFT JOIN dm2 x2 ON x2.k2=f.k GROUP BY f.k + EXCEPT ALL SELECT k, n, c1, c2 FROM mv_gk +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT k, n, c1, c2 FROM mv_gk + EXCEPT ALL + SELECT f.k, COUNT(f.fid), COUNT(x1.k1), COUNT(x2.k2) + FROM fact f LEFT JOIN dm1 x1 ON x1.k1=f.k LEFT JOIN dm2 x2 ON x2.k2=f.k GROUP BY f.k +); +---- +0 + +query IIII +SELECT k, n, c1, c2 FROM mv_gk ORDER BY k; +---- +100 2 2 2 +200 2 2 2 + +# A dimension-side change at the deepest level, which is what drives the secondary delta +statement ok +INSERT INTO dm2 VALUES (100,22); + +statement ok +PRAGMA refresh('mv_gk'); + +query I +SELECT COUNT(*) FROM ( + SELECT f.k, COUNT(f.fid), COUNT(x1.k1), COUNT(x2.k2) + FROM fact f LEFT JOIN dm1 x1 ON x1.k1=f.k LEFT JOIN dm2 x2 ON x2.k2=f.k GROUP BY f.k + EXCEPT ALL SELECT k, n, c1, c2 FROM mv_gk +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT k, n, c1, c2 FROM mv_gk + EXCEPT ALL + SELECT f.k, COUNT(f.fid), COUNT(x1.k1), COUNT(x2.k2) + FROM fact f LEFT JOIN dm1 x1 ON x1.k1=f.k LEFT JOIN dm2 x2 ON x2.k2=f.k GROUP BY f.k +); +---- +0 + +# Deeper star (5 dimensions) grouped by the same join key +statement ok +CREATE TABLE dm3 (k3 INT, v3 INT); + +statement ok +CREATE TABLE dm4 (k4 INT, v4 INT); + +statement ok +INSERT INTO dm3 VALUES (100,3),(200,3); + +statement ok +INSERT INTO dm4 VALUES (100,4),(200,4); + +statement ok +CREATE MATERIALIZED VIEW mv_gk4 AS + SELECT f.k, COUNT(f.fid) AS n, COUNT(x1.k1) AS c1, COUNT(x3.k3) AS c3, COUNT(x4.k4) AS c4 + FROM fact f LEFT JOIN dm1 x1 ON x1.k1=f.k LEFT JOIN dm3 x3 ON x3.k3=f.k LEFT JOIN dm4 x4 ON x4.k4=f.k + GROUP BY f.k; + +statement ok +INSERT INTO fact VALUES (6,100); + +statement ok +PRAGMA refresh('mv_gk4'); + +query I +SELECT COUNT(*) FROM ( + SELECT f.k, COUNT(f.fid), COUNT(x1.k1), COUNT(x3.k3), COUNT(x4.k4) + FROM fact f LEFT JOIN dm1 x1 ON x1.k1=f.k LEFT JOIN dm3 x3 ON x3.k3=f.k LEFT JOIN dm4 x4 ON x4.k4=f.k + GROUP BY f.k + EXCEPT ALL SELECT k, n, c1, c3, c4 FROM mv_gk4 +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT k, n, c1, c3, c4 FROM mv_gk4 + EXCEPT ALL + SELECT f.k, COUNT(f.fid), COUNT(x1.k1), COUNT(x3.k3), COUNT(x4.k4) + FROM fact f LEFT JOIN dm1 x1 ON x1.k1=f.k LEFT JOIN dm3 x3 ON x3.k3=f.k LEFT JOIN dm4 x4 ON x4.k4=f.k + GROUP BY f.k +); +---- +0 diff --git a/test/sql/left_join_pipeline_secondary_delta.test b/test/sql/left_join_pipeline_secondary_delta.test new file mode 100644 index 00000000..e895e20c --- /dev/null +++ b/test/sql/left_join_pipeline_secondary_delta.test @@ -0,0 +1,866 @@ +# name: test/sql/left_join_pipeline_secondary_delta.test +# description: LEFT JOIN pipeline aggregate over a preserved/intermediate-side column must stay correct under +# group: [sql] + +# inner-table deletes (Larson & Zhou secondary deltas). Deleting an order's last line must re-insert +# the NULL-padded row so COUNT(o.oid) still counts that order. Regression for the secondary-delta bug. + +require openivm + +statement ok +SET openivm_cascade_refresh = 'off'; + +# ============================================================ +# Setup: cust (preserved) LEFT JOIN ord (intermediate) LEFT JOIN line (inner) +# n_ord = COUNT(o.oid) <- aggregate over the INTERMEDIATE preserved-side column (the buggy case) +# n_line = COUNT(l.lid), rev = SUM(l.amt) <- inner-side aggregates +# ============================================================ + +statement ok +CREATE TABLE cust (cid INT, nation VARCHAR); + +statement ok +CREATE TABLE ord (oid INT, cid INT); + +statement ok +CREATE TABLE line (lid INT, oid INT, amt INT); + +statement ok +INSERT INTO cust VALUES (1,'A'),(2,'A'),(3,'B'); + +# c1 -> o10,o11 ; c2 -> o12 ; c3 -> (no orders) +statement ok +INSERT INTO ord VALUES (10,1),(11,1),(12,2); + +# o10 has 2 lines, o11 has 1 line, o12 has NO lines (already NULL-padded) +statement ok +INSERT INTO line VALUES (100,10,5),(101,10,7),(102,11,9); + +statement ok +CREATE MATERIALIZED VIEW mv AS + SELECT c.nation, COUNT(o.oid) AS n_ord, COUNT(l.lid) AS n_line, SUM(l.amt) AS rev + FROM cust c + LEFT JOIN ord o ON o.cid = c.cid + LEFT JOIN line l ON l.oid = o.oid + GROUP BY c.nation; + +# initial cross-check (both directions) +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid) AS n_ord, COUNT(l.lid) AS n_line, SUM(l.amt) AS rev + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, rev FROM mv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, rev FROM mv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation +); +---- +0 + +# ============================================================ +# Unsupported LEFT JOIN secondary-delta shapes must use affected-group +# recomputation. The arithmetic correction is valid only for one direct +# equality predicate between distinct source tables. +# ============================================================ + +# SUM over the preserved side cannot be corrected by the secondary-delta +# generator. When the unmatched row becomes matched, the primary delta must +# not be merged without its missing null-padded-row retraction. +statement ok +CREATE TABLE ljfb_left (id INT, val INT); + +statement ok +CREATE TABLE ljfb_right (id INT, payload INT); + +statement ok +INSERT INTO ljfb_left VALUES (1, 10), (2, 20); + +statement ok +CREATE MATERIALIZED VIEW ljfb_preserved_sum AS + SELECT l.id, SUM(l.val) AS total + FROM ljfb_left l LEFT JOIN ljfb_right r ON l.id = r.id + GROUP BY l.id; + +query I +SELECT type FROM openivm_views WHERE view_name = 'ljfb_preserved_sum'; +---- +6 + +statement ok +INSERT INTO ljfb_right VALUES (1, 100), (2, 200); + +statement ok +UPDATE ljfb_right SET payload = 101 WHERE id = 1; + +statement ok +DELETE FROM ljfb_right WHERE id = 2; + +statement ok +PRAGMA refresh('ljfb_preserved_sum'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM ljfb_preserved_sum + EXCEPT ALL + SELECT l.id, SUM(l.val) + FROM ljfb_left l LEFT JOIN ljfb_right r ON l.id = r.id + GROUP BY l.id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.id, SUM(l.val) + FROM ljfb_left l LEFT JOIN ljfb_right r ON l.id = r.id + GROUP BY l.id + EXCEPT ALL + SELECT * FROM ljfb_preserved_sum +); +---- +0 + +# A self-referencing LEFT JOIN cannot use independent preserved/inner delta +# accounting because both occurrences share one physical delta table. +statement ok +CREATE TABLE ljfb_self (id INT, parent_id INT, val INT); + +statement ok +INSERT INTO ljfb_self VALUES (1, NULL, 10), (2, 1, 20), (3, 99, 30); + +statement ok +CREATE MATERIALIZED VIEW ljfb_self_mv AS + SELECT child.id, COUNT(parent.id) AS parents, SUM(parent.val) AS parent_total + FROM ljfb_self child LEFT JOIN ljfb_self parent ON child.parent_id = parent.id + GROUP BY child.id; + +query I +SELECT type FROM openivm_views WHERE view_name = 'ljfb_self_mv'; +---- +6 + +statement ok +INSERT INTO ljfb_self VALUES (99, NULL, 90), (4, 1, 40); + +statement ok +UPDATE ljfb_self SET parent_id = 99 WHERE id = 2; + +statement ok +DELETE FROM ljfb_self WHERE id = 1; + +statement ok +PRAGMA refresh('ljfb_self_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM ljfb_self_mv + EXCEPT ALL + SELECT child.id, COUNT(parent.id), SUM(parent.val) + FROM ljfb_self child LEFT JOIN ljfb_self parent ON child.parent_id = parent.id + GROUP BY child.id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT child.id, COUNT(parent.id), SUM(parent.val) + FROM ljfb_self child LEFT JOIN ljfb_self parent ON child.parent_id = parent.id + GROUP BY child.id + EXCEPT ALL + SELECT * FROM ljfb_self_mv +); +---- +0 + +# A compound equality predicate cannot be reduced to conditions[0]: another +# row sharing only the first key must not hide an unmatched/matched transition. +statement ok +CREATE TABLE ljfb_comp_left (a INT, b INT); + +statement ok +CREATE TABLE ljfb_comp_right (a INT, b INT, val INT); + +statement ok +INSERT INTO ljfb_comp_left VALUES (1, 1), (1, 2), (2, 1); + +statement ok +INSERT INTO ljfb_comp_right VALUES (1, 2, 20); + +statement ok +CREATE MATERIALIZED VIEW ljfb_comp_mv AS + SELECT l.a, l.b, COUNT(r.val) AS matches, SUM(r.val) AS total + FROM ljfb_comp_left l LEFT JOIN ljfb_comp_right r ON l.a = r.a AND l.b = r.b + GROUP BY l.a, l.b; + +query I +SELECT type FROM openivm_views WHERE view_name = 'ljfb_comp_mv'; +---- +6 + +statement ok +INSERT INTO ljfb_comp_right VALUES (1, 1, 10), (2, 1, 30); + +statement ok +UPDATE ljfb_comp_right SET val = 11 WHERE a = 1 AND b = 1; + +statement ok +DELETE FROM ljfb_comp_right WHERE a = 2 AND b = 1; + +statement ok +PRAGMA refresh('ljfb_comp_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM ljfb_comp_mv + EXCEPT ALL + SELECT l.a, l.b, COUNT(r.val), SUM(r.val) + FROM ljfb_comp_left l LEFT JOIN ljfb_comp_right r ON l.a = r.a AND l.b = r.b + GROUP BY l.a, l.b +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.a, l.b, COUNT(r.val), SUM(r.val) + FROM ljfb_comp_left l LEFT JOIN ljfb_comp_right r ON l.a = r.a AND l.b = r.b + GROUP BY l.a, l.b + EXCEPT ALL + SELECT * FROM ljfb_comp_mv +); +---- +0 + +# Non-equality predicates need the complete predicate to find groups that lose +# their final match, so they also use affected-group recomputation. +statement ok +CREATE TABLE ljfb_range_left (id INT, threshold INT); + +statement ok +CREATE TABLE ljfb_range_right (id INT, val INT); + +statement ok +INSERT INTO ljfb_range_left VALUES (1, 10), (2, 20); + +statement ok +INSERT INTO ljfb_range_right VALUES (10, 5), (20, 15); + +statement ok +CREATE MATERIALIZED VIEW ljfb_range_mv AS + SELECT l.id, COUNT(r.id) AS matches, SUM(r.val) AS total + FROM ljfb_range_left l LEFT JOIN ljfb_range_right r ON r.val < l.threshold + GROUP BY l.id; + +query I +SELECT type FROM openivm_views WHERE view_name = 'ljfb_range_mv'; +---- +6 + +statement ok +INSERT INTO ljfb_range_right VALUES (30, 8), (40, 25); + +statement ok +UPDATE ljfb_range_right SET val = 12 WHERE id = 10; + +statement ok +DELETE FROM ljfb_range_right WHERE id = 20; + +statement ok +PRAGMA refresh('ljfb_range_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM ljfb_range_mv + EXCEPT ALL + SELECT l.id, COUNT(r.id), SUM(r.val) + FROM ljfb_range_left l LEFT JOIN ljfb_range_right r ON r.val < l.threshold + GROUP BY l.id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.id, COUNT(r.id), SUM(r.val) + FROM ljfb_range_left l LEFT JOIN ljfb_range_right r ON r.val < l.threshold + GROUP BY l.id + EXCEPT ALL + SELECT * FROM ljfb_range_mv +); +---- +0 + +# A right-only ON predicate is represented as a filter on the inner subtree, +# not necessarily as a second LogicalComparisonJoin condition. Base-table +# key counts would include inactive rows and miss the last-match transition. +statement ok +CREATE TABLE ljfb_filter_left (id INT); + +statement ok +CREATE TABLE ljfb_filter_right (id INT, active BOOLEAN, val INT); + +statement ok +INSERT INTO ljfb_filter_left VALUES (1), (2); + +statement ok +INSERT INTO ljfb_filter_right VALUES (1, true, 10), (1, false, 99), (2, true, 20); + +statement ok +CREATE MATERIALIZED VIEW ljfb_filter_mv AS + SELECT l.id, COUNT(r.val) AS matches, SUM(r.val) AS total + FROM ljfb_filter_left l LEFT JOIN ljfb_filter_right r ON l.id = r.id AND r.active = true + GROUP BY l.id; + +query I +SELECT type FROM openivm_views WHERE view_name = 'ljfb_filter_mv'; +---- +6 + +statement ok +INSERT INTO ljfb_filter_right VALUES (2, false, 200), (1, true, 11); + +statement ok +UPDATE ljfb_filter_right SET val = 12 WHERE id = 1 AND active; + +statement ok +DELETE FROM ljfb_filter_right WHERE id = 2 AND active; + +statement ok +PRAGMA refresh('ljfb_filter_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM ljfb_filter_mv + EXCEPT ALL + SELECT l.id, COUNT(r.val), SUM(r.val) + FROM ljfb_filter_left l LEFT JOIN ljfb_filter_right r ON l.id = r.id AND r.active = true + GROUP BY l.id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.id, COUNT(r.val), SUM(r.val) + FROM ljfb_filter_left l LEFT JOIN ljfb_filter_right r ON l.id = r.id AND r.active = true + GROUP BY l.id + EXCEPT ALL + SELECT * FROM ljfb_filter_mv +); +---- +0 + +# A filter in the preserved subtree changes which keys contribute outer rows. +# Raw preserved-table counts cannot model false->true and true->false transitions, +# so this shape must use affected-group recomputation. +statement ok +CREATE TABLE ljfb_pres_filter_left (id INT, active BOOLEAN); + +statement ok +CREATE TABLE ljfb_pres_filter_right (id INT, val INT); + +statement ok +INSERT INTO ljfb_pres_filter_left VALUES (1, false), (2, true), (3, true); + +statement ok +INSERT INTO ljfb_pres_filter_right VALUES (2, 20); + +statement ok +CREATE MATERIALIZED VIEW ljfb_pres_filter_mv AS + SELECT l.id, COUNT(r.val) AS matches + FROM (SELECT * FROM ljfb_pres_filter_left WHERE active) l + LEFT JOIN ljfb_pres_filter_right r ON l.id = r.id + GROUP BY l.id; + +query I +SELECT type FROM openivm_views WHERE view_name = 'ljfb_pres_filter_mv'; +---- +6 + +statement ok +UPDATE ljfb_pres_filter_left SET active = true WHERE id = 1; + +statement ok +UPDATE ljfb_pres_filter_left SET active = false WHERE id = 2; + +statement ok +INSERT INTO ljfb_pres_filter_right VALUES (1, 10), (3, 30); + +statement ok +DELETE FROM ljfb_pres_filter_right WHERE id = 3; + +statement ok +PRAGMA refresh('ljfb_pres_filter_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM ljfb_pres_filter_mv + EXCEPT ALL + SELECT l.id, COUNT(r.val) + FROM (SELECT * FROM ljfb_pres_filter_left WHERE active) l + LEFT JOIN ljfb_pres_filter_right r ON l.id = r.id + GROUP BY l.id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.id, COUNT(r.val) + FROM (SELECT * FROM ljfb_pres_filter_left WHERE active) l + LEFT JOIN ljfb_pres_filter_right r ON l.id = r.id + GROUP BY l.id + EXCEPT ALL + SELECT * FROM ljfb_pres_filter_mv +); +---- +0 + +# COUNT over a nullable preserved-side column contributes zero for a +# NULL-padded row when the preserved value itself is NULL. The scalar +# secondary correction is safe only when the source column is proven NOT NULL. +statement ok +CREATE TABLE ljfb_nullable_count_left (id INT, val INT); + +statement ok +CREATE TABLE ljfb_nullable_count_right (id INT, payload INT); + +statement ok +INSERT INTO ljfb_nullable_count_left VALUES (1, NULL), (2, 20); + +statement ok +INSERT INTO ljfb_nullable_count_right VALUES (1, 100), (2, 200); + +statement ok +CREATE MATERIALIZED VIEW ljfb_nullable_count_mv AS + SELECT l.id, COUNT(l.val) AS preserved_values, COUNT(r.payload) AS matches + FROM ljfb_nullable_count_left l + LEFT JOIN ljfb_nullable_count_right r ON l.id = r.id + GROUP BY l.id; + +query I +SELECT type FROM openivm_views WHERE view_name = 'ljfb_nullable_count_mv'; +---- +6 + +statement ok +DELETE FROM ljfb_nullable_count_right WHERE id = 1; + +statement ok +UPDATE ljfb_nullable_count_right SET payload = 201 WHERE id = 2; + +statement ok +INSERT INTO ljfb_nullable_count_right VALUES (2, 202); + +statement ok +PRAGMA refresh('ljfb_nullable_count_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM ljfb_nullable_count_mv + EXCEPT ALL + SELECT l.id, COUNT(l.val), COUNT(r.payload) + FROM ljfb_nullable_count_left l + LEFT JOIN ljfb_nullable_count_right r ON l.id = r.id + GROUP BY l.id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.id, COUNT(l.val), COUNT(r.payload) + FROM ljfb_nullable_count_left l + LEFT JOIN ljfb_nullable_count_right r ON l.id = r.id + GROUP BY l.id + EXCEPT ALL + SELECT * FROM ljfb_nullable_count_mv +); +---- +0 + +# Secondary source identities are structured arrays: a comma in a valid quoted +# table identifier must not split one join level into multiple levels. +statement ok +CREATE TABLE lj_comma_left (id INT); + +statement ok +CREATE TABLE "lj,r" (id INT, val INT); + +statement ok +INSERT INTO lj_comma_left VALUES (1), (2); + +statement ok +INSERT INTO "lj,r" VALUES (1, 10), (2, 20); + +statement ok +CREATE MATERIALIZED VIEW lj_comma_mv AS + SELECT l.id, COUNT(r.val) AS matches, SUM(r.val) AS total + FROM lj_comma_left l LEFT JOIN "lj,r" r ON l.id = r.id + GROUP BY l.id; + +query I +SELECT type FROM openivm_views WHERE view_name = 'lj_comma_mv'; +---- +0 + +query I +SELECT contains(leftjoin_secondary_meta_json, '"inner_tables":["lj,r"]')::INT +FROM openivm_views WHERE view_name = 'lj_comma_mv'; +---- +1 + +statement ok +DELETE FROM "lj,r" WHERE id = 1; + +statement ok +INSERT INTO "lj,r" VALUES (2, 21); + +statement ok +UPDATE "lj,r" SET val = 22 WHERE id = 2 AND val = 21; + +statement ok +PRAGMA refresh('lj_comma_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM lj_comma_mv + EXCEPT ALL + SELECT l.id, COUNT(r.val), SUM(r.val) + FROM lj_comma_left l LEFT JOIN "lj,r" r ON l.id = r.id + GROUP BY l.id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.id, COUNT(r.val), SUM(r.val) + FROM lj_comma_left l LEFT JOIN "lj,r" r ON l.id = r.id + GROUP BY l.id + EXCEPT ALL + SELECT * FROM lj_comma_mv +); +---- +0 + +# Existing MVs can still carry the legacy comma-delimited identity fields. +# If such metadata is ambiguous, refresh must use affected-group recompute +# instead of silently dropping the required secondary correction. +statement ok +UPDATE openivm_views +SET leftjoin_secondary_meta_json = + replace(leftjoin_secondary_meta_json, '"inner_tables":["lj,r"]', '"inner_table":"lj,r"') +WHERE view_name = 'lj_comma_mv'; + +statement ok +UPDATE openivm_views +SET leftjoin_secondary_meta_json = + replace(leftjoin_secondary_meta_json, '"inner_keys":["id"]', '"inner_key":"id"') +WHERE view_name = 'lj_comma_mv'; + +statement ok +UPDATE openivm_views +SET leftjoin_secondary_meta_json = + replace(leftjoin_secondary_meta_json, '"pres_tables":["lj_comma_left"]', '"pres_table":"lj_comma_left"') +WHERE view_name = 'lj_comma_mv'; + +statement ok +UPDATE openivm_views +SET leftjoin_secondary_meta_json = + replace(leftjoin_secondary_meta_json, '"pres_keys":["id"]', '"pres_key":"id"') +WHERE view_name = 'lj_comma_mv'; + +statement ok +DELETE FROM "lj,r" WHERE id = 2; + +statement ok +PRAGMA refresh('lj_comma_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT * FROM lj_comma_mv + EXCEPT ALL + SELECT l.id, COUNT(r.val), SUM(r.val) + FROM lj_comma_left l LEFT JOIN "lj,r" r ON l.id = r.id + GROUP BY l.id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT l.id, COUNT(r.val), SUM(r.val) + FROM lj_comma_left l LEFT JOIN "lj,r" r ON l.id = r.id + GROUP BY l.id + EXCEPT ALL + SELECT * FROM lj_comma_mv +); +---- +0 + +# ============================================================ +# Batched mixed DML before ONE refresh (exercises secondary deltas in BOTH directions): +# - DELETE line 102 -> o11 loses its LAST line => NULL-padded (A,o11,NULL) reappears; COUNT(o.oid) must NOT drop +# - INSERT line 103 -> o12 gains its FIRST line => NULL-padded (A,o12,NULL) removed, real row added +# - DELETE line 100 -> o10 still has l101 => no secondary delta, just n_line/rev drop +# - INSERT ord 13 -> c3 (B) gains an order with no lines => (B,o13,NULL) +# - INSERT cust 4 -> new B customer, no orders => (B,NULL,NULL) +# ============================================================ + +statement ok +DELETE FROM line WHERE lid = 102; + +statement ok +INSERT INTO line VALUES (103, 12, 4); + +statement ok +DELETE FROM line WHERE lid = 100; + +statement ok +INSERT INTO ord VALUES (13, 3); + +statement ok +INSERT INTO cust VALUES (4, 'B'); + +statement ok +PRAGMA refresh('mv'); + +# cross-check both directions after the batched refresh +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid) AS n_ord, COUNT(l.lid) AS n_line, SUM(l.amt) AS rev + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, rev FROM mv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, rev FROM mv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation +); +---- +0 + +# ============================================================ +# Second batch: drive an order's line count back up and another down, plus a delete of a whole order +# ============================================================ + +statement ok +INSERT INTO line VALUES (104, 11, 3), (105, 13, 8); + + +statement ok +DELETE FROM line WHERE lid = 101; + +statement ok +DELETE FROM ord WHERE oid = 12; + +statement ok +PRAGMA refresh('mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid) AS n_ord, COUNT(l.lid) AS n_line, SUM(l.amt) AS rev + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, rev FROM mv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, rev FROM mv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation +); +---- +0 + +# ============================================================ +# Third batch: preserved-side row that is BRAND NEW in the same batch as its first inner row. +# ord 20 is inserted AND immediately gets its first line, all before one refresh. No NULL-padded +# (B,o20,NULL) row was ever materialized, so the secondary-delta correction must NOT fire for it +# (firing would subtract a dangling row that never existed). Also covers the same-batch churn case: +# ord 21 is inserted with a line and then both are deleted again (net effect: never existed). +# ============================================================ + +statement ok +INSERT INTO ord VALUES (20, 3); + +statement ok +INSERT INTO line VALUES (200, 20, 11); + +statement ok +INSERT INTO ord VALUES (21, 3); + +statement ok +INSERT INTO line VALUES (201, 21, 6); + +statement ok +DELETE FROM line WHERE lid = 201; + +statement ok +DELETE FROM ord WHERE oid = 21; + +statement ok +PRAGMA refresh('mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid) AS n_ord, COUNT(l.lid) AS n_line, SUM(l.amt) AS rev + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, rev FROM mv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, rev FROM mv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation +); +---- +0 + +# ============================================================ +# Fourth batch: an order's LAST line and the order ITSELF deleted together in one batch (the +# downward-transition double-count case). ord 20 loses line 200 and is deleted at the same time, so +# the kept-outer-join term must not emit a phantom NULL-padded (B,o20,NULL) row on top of the real +# removal supplied by the higher-order term. Simultaneously ord 22 (pre-existing after this batch's +# insert of its own line) exercises the upward direction, which must NOT be excluded. +# ============================================================ + +statement ok +INSERT INTO ord VALUES (22, 1); + +statement ok +PRAGMA refresh('mv'); + +statement ok +DELETE FROM line WHERE lid = 200; + +statement ok +DELETE FROM ord WHERE oid = 20; + +statement ok +INSERT INTO line VALUES (202, 22, 4); + +statement ok +PRAGMA refresh('mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid) AS n_ord, COUNT(l.lid) AS n_line, SUM(l.amt) AS rev + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, rev FROM mv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, rev FROM mv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation +); +---- +0 + +# ============================================================ +# Fifth batch: the same pipeline refreshed with openivm_left_join_merge=false, which routes +# CompileAggregateGroups to group-recompute instead of the delta-arithmetic MERGE. Plain coverage +# that a pipeline view with secondary-delta metadata still refreshes correctly on the recompute +# path, and that switching back to the MERGE path afterwards is also correct (the secondary is a +# per-refresh decision, not a permanent one). The existing left_join.test coverage of this flag uses +# a single LEFT JOIN, which never generates secondary metadata, so this shape was untested. +# +# NOTE: this does NOT reproduce the duplicate-key refresh failure seen on the group-recompute path +# at TPC-H scale -- that failure is PRE-EXISTING (it reproduces on commit 7697928, before any of the +# secondary-delta work) and needs 2+ MVs over shared base tables or a prior MERGE-path refresh. +# Tracked separately; do not treat this section as covering it. +# ============================================================ + +statement ok +SET openivm_left_join_merge = false; + +statement ok +INSERT INTO ord VALUES (30, 2); + +statement ok +INSERT INTO line VALUES (300, 30, 12); + +statement ok +DELETE FROM line WHERE lid = 104; + +statement ok +PRAGMA refresh('mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid) AS n_ord, COUNT(l.lid) AS n_line, SUM(l.amt) AS rev + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, rev FROM mv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, rev FROM mv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation +); +---- +0 + +statement ok +SET openivm_left_join_merge = true; + +# Back on the MERGE path, a further batch must still be correct (the guard must not have +# disabled the secondary permanently -- it is a per-refresh decision). + +statement ok +DELETE FROM line WHERE lid = 300; + +statement ok +INSERT INTO line VALUES (301, 22, 5); + +statement ok +PRAGMA refresh('mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT c.nation, COUNT(o.oid) AS n_ord, COUNT(l.lid) AS n_line, SUM(l.amt) AS rev + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation + EXCEPT ALL SELECT nation, n_ord, n_line, rev FROM mv +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT nation, n_ord, n_line, rev FROM mv + EXCEPT ALL + SELECT c.nation, COUNT(o.oid), COUNT(l.lid), SUM(l.amt) + FROM cust c LEFT JOIN ord o ON o.cid=c.cid LEFT JOIN line l ON l.oid=o.oid GROUP BY c.nation +); +---- +0 diff --git a/test/sql/nonlocal_operator_recompute.test b/test/sql/nonlocal_operator_recompute.test index 94dcda4b..212c846c 100644 --- a/test/sql/nonlocal_operator_recompute.test +++ b/test/sql/nonlocal_operator_recompute.test @@ -9,7 +9,7 @@ require openivm # ========================================== -# SAMPLE is exact current/new diff recompute, not a local delta +# SAMPLE falls back to full refresh; it is not a local delta # ========================================== statement ok @@ -36,7 +36,7 @@ CREATE MATERIALIZED VIEW mv_oprec_sample AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_sample'; ---- -10 +3 statement ok INSERT INTO oprec_sample VALUES (7, 1, 70), (8, 3, 80); @@ -95,8 +95,61 @@ SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_sample_volatile'; ---- 3 +# Persisted legacy CURRENT_DIFF_RECOMPUTE metadata (type 10) maps to the +# current full-refresh strategy during refresh. + +statement ok +CREATE TABLE oprec_legacy_type10 (id INT, grp INT, v INT); + +statement ok +INSERT INTO oprec_legacy_type10 VALUES + (1, 1, 10), + (2, 1, 20), + (3, 2, 30), + (4, 2, 40); + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_legacy_type10 AS + SELECT id, grp, v + FROM oprec_legacy_type10 USING SAMPLE reservoir(3 ROWS) REPEATABLE (31); + +statement ok +UPDATE openivm_views SET type = 10 WHERE view_name = 'mv_oprec_legacy_type10'; + +statement ok +INSERT INTO oprec_legacy_type10 VALUES (5, 3, 50); + +statement ok +DELETE FROM oprec_legacy_type10 WHERE id = 2; + +statement ok +UPDATE oprec_legacy_type10 SET v = 15 WHERE id = 1; + +statement ok +PRAGMA refresh('mv_oprec_legacy_type10'); + +query I +SELECT count(*) FROM ( + SELECT id, grp, v + FROM oprec_legacy_type10 USING SAMPLE reservoir(3 ROWS) REPEATABLE (31) + EXCEPT ALL + SELECT id, grp, v FROM mv_oprec_legacy_type10 +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, grp, v FROM mv_oprec_legacy_type10 + EXCEPT ALL + SELECT id, grp, v + FROM oprec_legacy_type10 USING SAMPLE reservoir(3 ROWS) REPEATABLE (31) +); +---- +0 + # ========================================== -# POSITIONAL JOIN is exact current/new diff recompute +# POSITIONAL JOIN falls back to full refresh # ========================================== statement ok @@ -129,7 +182,7 @@ CREATE MATERIALIZED VIEW mv_oprec_positional AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_positional'; ---- -10 +3 statement ok INSERT INTO oprec_pos_l VALUES (4, 'd'); @@ -183,6 +236,212 @@ SELECT count(*) FROM ( ---- 0 +# ========================================== +# FULL_REFRESH non-local projections still emit cascade deltas +# ========================================== + +statement ok +SET openivm_cascade_refresh = 'downstream'; + +statement ok +CREATE TABLE oprec_sample_cascade (id INT, grp INT, v INT); + +statement ok +INSERT INTO oprec_sample_cascade VALUES + (1, 1, 10), + (2, 1, 20), + (3, 1, 30), + (4, 2, 40), + (5, 2, 50), + (6, 3, 60); + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_sample_cascade_parent AS + SELECT id, grp, v + FROM oprec_sample_cascade USING SAMPLE reservoir(4 ROWS) REPEATABLE (29); + +query I +SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_sample_cascade_parent'; +---- +3 + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_sample_cascade_child AS + SELECT grp, COUNT(*) AS cnt, SUM(v) AS total_v + FROM mv_oprec_sample_cascade_parent + GROUP BY grp; + +statement ok +INSERT INTO oprec_sample_cascade VALUES (7, 2, 70), (8, 4, 80); + +statement ok +DELETE FROM oprec_sample_cascade WHERE id = 2; + +statement ok +UPDATE oprec_sample_cascade SET v = 15 WHERE id = 1; + +statement ok +PRAGMA refresh('mv_oprec_sample_cascade_parent'); + +query I +SELECT count(*) FROM ( + SELECT id, grp, v + FROM oprec_sample_cascade USING SAMPLE reservoir(4 ROWS) REPEATABLE (29) + EXCEPT ALL + SELECT id, grp, v FROM mv_oprec_sample_cascade_parent +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, grp, v FROM mv_oprec_sample_cascade_parent + EXCEPT ALL + SELECT id, grp, v + FROM oprec_sample_cascade USING SAMPLE reservoir(4 ROWS) REPEATABLE (29) +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT grp, COUNT(*) AS cnt, SUM(v) AS total_v + FROM ( + SELECT id, grp, v + FROM oprec_sample_cascade USING SAMPLE reservoir(4 ROWS) REPEATABLE (29) + ) s + GROUP BY grp + EXCEPT ALL + SELECT grp, cnt, total_v FROM mv_oprec_sample_cascade_child +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT grp, cnt, total_v FROM mv_oprec_sample_cascade_child + EXCEPT ALL + SELECT grp, COUNT(*) AS cnt, SUM(v) AS total_v + FROM ( + SELECT id, grp, v + FROM oprec_sample_cascade USING SAMPLE reservoir(4 ROWS) REPEATABLE (29) + ) s + GROUP BY grp +); +---- +0 + +statement ok +CREATE TABLE oprec_pos_cascade_l (id INT, lval INT); + +statement ok +CREATE TABLE oprec_pos_cascade_r (rid INT, rval INT); + +statement ok +INSERT INTO oprec_pos_cascade_l VALUES (1, 10), (2, 20), (3, 30); + +statement ok +INSERT INTO oprec_pos_cascade_r VALUES (10, 100), (20, 200); + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_pos_cascade_parent AS + SELECT + l.id AS id, + l.lval AS lval, + r.rid AS rid, + r.rval AS rval, + coalesce(l.id, 0) + coalesce(r.rid, 0) AS id_score, + coalesce(l.lval, 0) + coalesce(r.rval, 0) AS val_score + FROM oprec_pos_cascade_l l + POSITIONAL JOIN oprec_pos_cascade_r r; + +query I +SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_pos_cascade_parent'; +---- +3 + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_pos_cascade_child AS + SELECT COUNT(*) AS cnt, SUM(id_score) AS total_ids, SUM(val_score) AS total_vals + FROM mv_oprec_pos_cascade_parent; + +statement ok +INSERT INTO oprec_pos_cascade_l VALUES (4, 40); + +statement ok +DELETE FROM oprec_pos_cascade_r WHERE rid = 20; + +statement ok +UPDATE oprec_pos_cascade_l SET lval = 15 WHERE id = 1; + +statement ok +PRAGMA refresh('mv_oprec_pos_cascade_parent'); + +query I +SELECT count(*) FROM ( + SELECT + l.id AS id, + l.lval AS lval, + r.rid AS rid, + r.rval AS rval, + coalesce(l.id, 0) + coalesce(r.rid, 0) AS id_score, + coalesce(l.lval, 0) + coalesce(r.rval, 0) AS val_score + FROM oprec_pos_cascade_l l + POSITIONAL JOIN oprec_pos_cascade_r r + EXCEPT ALL + SELECT id, lval, rid, rval, id_score, val_score FROM mv_oprec_pos_cascade_parent +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, lval, rid, rval, id_score, val_score FROM mv_oprec_pos_cascade_parent + EXCEPT ALL + SELECT + l.id AS id, + l.lval AS lval, + r.rid AS rid, + r.rval AS rval, + coalesce(l.id, 0) + coalesce(r.rid, 0) AS id_score, + coalesce(l.lval, 0) + coalesce(r.rval, 0) AS val_score + FROM oprec_pos_cascade_l l + POSITIONAL JOIN oprec_pos_cascade_r r +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT + COUNT(*) AS cnt, + SUM(coalesce(l.id, 0) + coalesce(r.rid, 0)) AS total_ids, + SUM(coalesce(l.lval, 0) + coalesce(r.rval, 0)) AS total_vals + FROM oprec_pos_cascade_l l + POSITIONAL JOIN oprec_pos_cascade_r r + EXCEPT ALL + SELECT cnt, total_ids, total_vals FROM mv_oprec_pos_cascade_child +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT cnt, total_ids, total_vals FROM mv_oprec_pos_cascade_child + EXCEPT ALL + SELECT + COUNT(*) AS cnt, + SUM(coalesce(l.id, 0) + coalesce(r.rid, 0)) AS total_ids, + SUM(coalesce(l.lval, 0) + coalesce(r.rval, 0)) AS total_vals + FROM oprec_pos_cascade_l l + POSITIONAL JOIN oprec_pos_cascade_r r +); +---- +0 + +statement ok +SET openivm_cascade_refresh = 'off'; + # ========================================== # Hidden non-local operators inside materialized CTEs are still classified conservatively # ========================================== @@ -214,7 +473,7 @@ CREATE MATERIALIZED VIEW mv_oprec_cte_sample AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_cte_sample'; ---- -10 +3 statement ok INSERT INTO oprec_cte_sample VALUES (7, 2, 70), (8, 4, 80); @@ -286,7 +545,7 @@ CREATE MATERIALIZED VIEW mv_oprec_cte_positional AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_cte_positional'; ---- -10 +3 statement ok INSERT INTO oprec_cte_pos_l VALUES (4, 40); diff --git a/test/sql/nonlocal_operator_recompute_asof.test b/test/sql/nonlocal_operator_recompute_asof.test index 91addd50..957f47a3 100644 --- a/test/sql/nonlocal_operator_recompute_asof.test +++ b/test/sql/nonlocal_operator_recompute_asof.test @@ -5,7 +5,7 @@ require openivm # ========================================== -# ASOF projections use exact current/new diff recompute +# ASOF projections fall back to full refresh # ========================================== statement ok @@ -42,7 +42,7 @@ CREATE MATERIALIZED VIEW mv_oprec_asof_projection AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_asof_projection'; ---- -10 +3 statement ok INSERT INTO oprec_asof_prices VALUES ('A', TIMESTAMP '2024-01-01 10:19:00', 190); @@ -95,6 +95,151 @@ SELECT count(*) FROM ( ---- 0 +# FULL_REFRESH ASOF projections still emit signed deltas for downstream cascade. + +statement ok +SET openivm_cascade_refresh = 'downstream'; + +statement ok +CREATE TABLE oprec_asof_cascade_trades (symbol VARCHAR, ts TIMESTAMP, qty INT); + +statement ok +CREATE TABLE oprec_asof_cascade_prices (symbol VARCHAR, ts TIMESTAMP, price INT); + +statement ok +INSERT INTO oprec_asof_cascade_trades VALUES + ('A', TIMESTAMP '2024-01-02 10:00:00', 1), + ('A', TIMESTAMP '2024-01-02 10:20:00', 2), + ('B', TIMESTAMP '2024-01-02 10:15:00', 3); + +statement ok +INSERT INTO oprec_asof_cascade_prices VALUES + ('A', TIMESTAMP '2024-01-02 09:00:00', 100), + ('A', TIMESTAMP '2024-01-02 10:18:00', 180), + ('B', TIMESTAMP '2024-01-02 10:12:00', 120); + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_asof_cascade_parent AS + SELECT + t.symbol AS symbol, + t.ts AS trade_ts, + t.qty AS qty, + p.ts AS price_ts, + p.price AS price + FROM oprec_asof_cascade_trades t + ASOF JOIN oprec_asof_cascade_prices p + ON t.symbol = p.symbol + AND t.ts >= p.ts; + +query I +SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_asof_cascade_parent'; +---- +3 + +statement ok +CREATE MATERIALIZED VIEW mv_oprec_asof_cascade_child AS + SELECT symbol, COUNT(*) AS cnt, SUM(qty * price) AS total_value + FROM mv_oprec_asof_cascade_parent + GROUP BY symbol; + +statement ok +INSERT INTO oprec_asof_cascade_prices VALUES ('A', TIMESTAMP '2024-01-02 10:19:00', 190); + +statement ok +INSERT INTO oprec_asof_cascade_trades VALUES ('A', TIMESTAMP '2024-01-02 10:21:00', 4); + +statement ok +DELETE FROM oprec_asof_cascade_prices WHERE symbol = 'B' AND ts = TIMESTAMP '2024-01-02 10:12:00'; + +statement ok +UPDATE oprec_asof_cascade_trades SET qty = 6 WHERE symbol = 'A' AND ts = TIMESTAMP '2024-01-02 10:00:00'; + +statement ok +PRAGMA refresh('mv_oprec_asof_cascade_parent'); + +query I +SELECT count(*) FROM ( + SELECT + t.symbol AS symbol, + t.ts AS trade_ts, + t.qty AS qty, + p.ts AS price_ts, + p.price AS price + FROM oprec_asof_cascade_trades t + ASOF JOIN oprec_asof_cascade_prices p + ON t.symbol = p.symbol + AND t.ts >= p.ts + EXCEPT ALL + SELECT symbol, trade_ts, qty, price_ts, price FROM mv_oprec_asof_cascade_parent +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT symbol, trade_ts, qty, price_ts, price FROM mv_oprec_asof_cascade_parent + EXCEPT ALL + SELECT + t.symbol AS symbol, + t.ts AS trade_ts, + t.qty AS qty, + p.ts AS price_ts, + p.price AS price + FROM oprec_asof_cascade_trades t + ASOF JOIN oprec_asof_cascade_prices p + ON t.symbol = p.symbol + AND t.ts >= p.ts +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT symbol, COUNT(*) AS cnt, SUM(qty * price) AS total_value + FROM ( + SELECT + t.symbol AS symbol, + t.ts AS trade_ts, + t.qty AS qty, + p.ts AS price_ts, + p.price AS price + FROM oprec_asof_cascade_trades t + ASOF JOIN oprec_asof_cascade_prices p + ON t.symbol = p.symbol + AND t.ts >= p.ts + ) q + GROUP BY symbol + EXCEPT ALL + SELECT symbol, cnt, total_value FROM mv_oprec_asof_cascade_child +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT symbol, cnt, total_value FROM mv_oprec_asof_cascade_child + EXCEPT ALL + SELECT symbol, COUNT(*) AS cnt, SUM(qty * price) AS total_value + FROM ( + SELECT + t.symbol AS symbol, + t.ts AS trade_ts, + t.qty AS qty, + p.ts AS price_ts, + p.price AS price + FROM oprec_asof_cascade_trades t + ASOF JOIN oprec_asof_cascade_prices p + ON t.symbol = p.symbol + AND t.ts >= p.ts + ) q + GROUP BY symbol +); +---- +0 + +statement ok +SET openivm_cascade_refresh = 'off'; + # Grouped ASOF outputs use group recompute with current/new affected keys. statement ok @@ -285,8 +430,9 @@ SELECT count(*) FROM ( ---- 0 -# ASOF windows whose partition key comes from the RHS match must use current/new diff: -# a newer RHS row can move existing left rows out of the old matched partition. +# ASOF windows whose partition key comes from the RHS match fall back to full +# refresh: a newer RHS row can move existing left rows out of the old matched +# partition, so partition-local recompute is not safe. statement ok CREATE TABLE oprec_asof_rhs_win_trades (symbol VARCHAR, ts TIMESTAMP, qty INT); @@ -329,7 +475,7 @@ CREATE MATERIALIZED VIEW mv_oprec_asof_rhs_window AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_oprec_asof_rhs_window'; ---- -10 +3 statement ok INSERT INTO oprec_asof_rhs_win_prices VALUES ('A', TIMESTAMP '2024-02-02 10:15:00', 150); diff --git a/test/sql/pipeline.test b/test/sql/pipeline.test index 7d298c0c..efb7c8df 100644 --- a/test/sql/pipeline.test +++ b/test/sql/pipeline.test @@ -752,6 +752,209 @@ SELECT COUNT(*) FROM ( ---- 0 +# ============================================================ +# Empty current node must not stop downstream cascade traversal +# ============================================================ + +statement ok +SET openivm_cascade_refresh = 'downstream'; + +statement ok +CREATE TABLE cascade_skip_root_src (id INT, amount INT); + +statement ok +CREATE TABLE cascade_skip_side_src (id INT, bonus INT); + +statement ok +INSERT INTO cascade_skip_root_src VALUES (1, 10), (2, 20); + +statement ok +INSERT INTO cascade_skip_side_src VALUES (1, 1), (2, 2); + +statement ok +CREATE MATERIALIZED VIEW cascade_skip_parent AS + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id; + +statement ok +CREATE MATERIALIZED VIEW cascade_skip_child AS + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM cascade_skip_parent p + JOIN cascade_skip_side_src s USING (id) + GROUP BY p.id; + +# Leave a parent delta pending for the child, then invoke a downstream cascade +# when the parent's own source delta is empty. +statement ok +SET openivm_cascade_refresh = 'off'; + +statement ok +INSERT INTO cascade_skip_root_src VALUES (3, 30), (4, 40); + +statement ok +UPDATE cascade_skip_root_src SET amount = amount + 5 WHERE id IN (1, 3); + +statement ok +DELETE FROM cascade_skip_root_src WHERE id IN (2, 4); + +statement ok +PRAGMA refresh('cascade_skip_parent'); + +query I +SELECT COUNT(*) FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + EXCEPT ALL + SELECT id, total FROM cascade_skip_parent +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, total FROM cascade_skip_parent + EXCEPT ALL + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id +); +---- +0 + +statement ok +SET openivm_cascade_refresh = 'downstream'; + +statement ok +PRAGMA refresh('cascade_skip_parent'); + +query I +SELECT COUNT(*) FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + EXCEPT ALL + SELECT id, total FROM cascade_skip_parent +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, total FROM cascade_skip_parent + EXCEPT ALL + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + ) p + JOIN cascade_skip_side_src s USING (id) + GROUP BY p.id + EXCEPT ALL + SELECT id, score, cnt FROM cascade_skip_child +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, score, cnt FROM cascade_skip_child + EXCEPT ALL + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + ) p + JOIN cascade_skip_side_src s USING (id) + GROUP BY p.id +); +---- +0 + +# The child can also have independent source changes while the parent has none. +# Batch conflicting DML before one refresh: inserted rows are updated, and one +# inserted row is deleted before the cascade. +statement ok +INSERT INTO cascade_skip_side_src VALUES (3, 3), (4, 4); + +statement ok +UPDATE cascade_skip_side_src SET bonus = bonus + 10 WHERE id IN (1, 3); + +statement ok +DELETE FROM cascade_skip_side_src WHERE id IN (2, 4); + +statement ok +PRAGMA refresh('cascade_skip_parent'); + +query I +SELECT COUNT(*) FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + EXCEPT ALL + SELECT id, total FROM cascade_skip_parent +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, total FROM cascade_skip_parent + EXCEPT ALL + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + ) p + JOIN cascade_skip_side_src s USING (id) + GROUP BY p.id + EXCEPT ALL + SELECT id, score, cnt FROM cascade_skip_child +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, score, cnt FROM cascade_skip_child + EXCEPT ALL + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM ( + SELECT id, SUM(amount) AS total + FROM cascade_skip_root_src + GROUP BY id + ) p + JOIN cascade_skip_side_src s USING (id) + GROUP BY p.id +); +---- +0 + +statement ok +SET openivm_cascade_refresh = 'upstream'; + # Insert more data statement ok INSERT INTO pipe_up VALUES ('a', 5), ('b', 20); diff --git a/test/sql/refresh_hooks.test b/test/sql/refresh_hooks.test index 60a5ae17..07b51ac1 100644 --- a/test/sql/refresh_hooks.test +++ b/test/sql/refresh_hooks.test @@ -134,3 +134,265 @@ query I SELECT COUNT(*) FROM hook_log; ---- 0 + +# ============================================================ +# Regression: A hook can mutate a tracked source without deadlocking +# ============================================================ + +statement ok +CREATE TABLE tracked_hook_src (id INTEGER, amount INTEGER); + +statement ok +CREATE MATERIALIZED VIEW tracked_hook_mv AS +SELECT id, amount FROM tracked_hook_src; + +statement ok +INSERT INTO openivm_refresh_hooks VALUES ( + 'tracked_hook_mv', + 'INSERT INTO tracked_hook_src VALUES (4, 40)', + 'before' +); + +# Keep one batch pending so the before-hook runs, and mix conflicting mutations +# before the single refresh to exercise consolidated delta capture. +statement ok +INSERT INTO tracked_hook_src VALUES (1, 10), (2, 20), (3, 30); + +statement ok +UPDATE tracked_hook_src SET amount = amount + 5 WHERE id IN (1, 2); + +statement ok +DELETE FROM tracked_hook_src WHERE id IN (2, 3); + +statement ok +PRAGMA refresh('tracked_hook_mv'); + +query I +SELECT COUNT(*) FROM ( + SELECT id, amount FROM tracked_hook_mv + EXCEPT ALL + SELECT id, amount FROM tracked_hook_src +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, amount FROM tracked_hook_src + EXCEPT ALL + SELECT id, amount FROM tracked_hook_mv +); +---- +0 + +# ============================================================ +# Test 5: A scheduled refresh executes its hook exactly once +# ============================================================ + +statement ok +BEGIN TRANSACTION; + +statement error +PRAGMA refresh_start_daemon; +---- +cannot be restarted inside a transaction + +statement ok +ROLLBACK; + +statement ok +CREATE TABLE daemon_hook_src (id INTEGER); + +statement ok +CREATE MATERIALIZED VIEW daemon_hook_mv AS +SELECT count(*) AS cnt FROM daemon_hook_src; + +statement ok +INSERT INTO openivm_refresh_hooks VALUES ( + 'daemon_hook_mv', + 'INSERT INTO hook_log VALUES(''daemon-hook ran'')', + 'after' +); + +statement ok +INSERT INTO daemon_hook_src VALUES (1), (2); + +statement ok +UPDATE openivm_views SET refresh_interval = 60 +WHERE view_name = 'daemon_hook_mv'; + +statement ok +UPDATE openivm_delta_tables +SET last_update = now()::TIMESTAMP - INTERVAL '2 minutes' +WHERE view_name = 'daemon_hook_mv'; + +statement ok +PRAGMA refresh_start_daemon; + +statement ok +SELECT sleep_ms(1500); + +query I +SELECT count(*) FROM hook_log WHERE msg = 'daemon-hook ran'; +---- +1 + +query I +SELECT count(*) FROM ( + SELECT * FROM daemon_hook_mv + EXCEPT ALL + SELECT count(*) AS cnt FROM daemon_hook_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT count(*) AS cnt FROM daemon_hook_src + EXCEPT ALL + SELECT * FROM daemon_hook_mv +); +---- +0 + +statement ok +UPDATE openivm_views SET refresh_interval = NULL WHERE view_name = 'daemon_hook_mv'; + +statement ok +DELETE FROM hook_log; + +# Dropping an MV removes its hook configuration, so a later MV with the same +# name cannot inherit executable metadata from the dropped object. +statement ok +CREATE TABLE dropped_hook_src (id INTEGER); + +statement ok +CREATE MATERIALIZED VIEW dropped_hook_mv AS +SELECT count(*) AS cnt FROM dropped_hook_src; + +statement ok +INSERT INTO openivm_refresh_hooks VALUES ( + 'dropped_hook_mv', + 'INSERT INTO hook_log VALUES(''stale hook ran'')', + 'after' +); + +statement ok +DROP VIEW dropped_hook_mv; + +query I +SELECT count(*) FROM openivm_refresh_hooks WHERE view_name = 'dropped_hook_mv'; +---- +0 + +# ============================================================ +# Test 6: An empty hook-bearing node does not stop its cascade +# ============================================================ + +statement ok +SET openivm_cascade_refresh = 'downstream'; + +statement ok +CREATE TABLE hook_cascade_root_src (id INT, amount INT); + +statement ok +CREATE TABLE hook_cascade_side_src (id INT, bonus INT); + +statement ok +INSERT INTO hook_cascade_root_src VALUES (1, 10), (2, 20), (3, 30); + +statement ok +INSERT INTO hook_cascade_side_src VALUES (1, 1), (2, 2); + +statement ok +CREATE MATERIALIZED VIEW hook_cascade_parent AS + SELECT id, SUM(amount) AS total + FROM hook_cascade_root_src + GROUP BY id; + +statement ok +CREATE MATERIALIZED VIEW hook_cascade_child AS + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM hook_cascade_parent p + JOIN hook_cascade_side_src s USING (id) + GROUP BY p.id; + +statement ok +INSERT INTO openivm_refresh_hooks VALUES ( + 'hook_cascade_parent', + 'INSERT INTO hook_log VALUES(''empty-parent hook ran'')', + 'before' +); + +# Only the child's independent source changes. The parent's pre-hook empty +# semantics still skip its hook, but traversal must refresh the child. +statement ok +INSERT INTO hook_cascade_side_src VALUES (3, 3), (4, 4); + +statement ok +UPDATE hook_cascade_side_src SET bonus = bonus + 10 WHERE id IN (1, 3); + +statement ok +DELETE FROM hook_cascade_side_src WHERE id IN (2, 4); + +statement ok +PRAGMA refresh('hook_cascade_parent'); + +query I +SELECT COUNT(*) FROM ( + SELECT id, SUM(amount) AS total + FROM hook_cascade_root_src + GROUP BY id + EXCEPT ALL + SELECT id, total FROM hook_cascade_parent +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, total FROM hook_cascade_parent + EXCEPT ALL + SELECT id, SUM(amount) AS total + FROM hook_cascade_root_src + GROUP BY id +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM ( + SELECT id, SUM(amount) AS total + FROM hook_cascade_root_src + GROUP BY id + ) p + JOIN hook_cascade_side_src s USING (id) + GROUP BY p.id + EXCEPT ALL + SELECT id, score, cnt FROM hook_cascade_child +); +---- +0 + +query I +SELECT COUNT(*) FROM ( + SELECT id, score, cnt FROM hook_cascade_child + EXCEPT ALL + SELECT p.id, SUM(p.total + s.bonus) AS score, COUNT(*) AS cnt + FROM ( + SELECT id, SUM(amount) AS total + FROM hook_cascade_root_src + GROUP BY id + ) p + JOIN hook_cascade_side_src s USING (id) + GROUP BY p.id +); +---- +0 + +query I +SELECT COUNT(*) FROM hook_log; +---- +0 diff --git a/test/sql/scalar_subquery_fold.test b/test/sql/scalar_subquery_fold.test index 7674797d..600891bf 100644 --- a/test/sql/scalar_subquery_fold.test +++ b/test/sql/scalar_subquery_fold.test @@ -63,7 +63,8 @@ SELECT count(*) FROM ( # ========================================== # Stacked constant scalar subqueries inside CASE and an aggregate expression -# over a GROUP BY: still folds and incrementalizes (AGGREGATE_GROUP = 0). +# over a GROUP BY: still folds and incrementally recomputes affected groups +# because SUM(amt) + constant is a computed aggregate output. # ========================================== statement ok @@ -79,7 +80,7 @@ CREATE MATERIALIZED VIEW mv_ssf_agg AS query I SELECT type FROM openivm_views WHERE view_name = 'mv_ssf_agg'; ---- -0 +6 statement ok INSERT INTO ssf_t VALUES (8, 'GC', 80, 2), (9, NULL, 90, 1); diff --git a/test/sql/schema_evolution.test b/test/sql/schema_evolution.test index a863d285..95600b7b 100644 --- a/test/sql/schema_evolution.test +++ b/test/sql/schema_evolution.test @@ -720,3 +720,278 @@ SELECT count(*) FROM ( ); ---- 0 + +# ========================================== +# Test 12: same-named sources are isolated by catalog and schema +# ========================================== + +statement ok +CREATE SCHEMA se_identity_a; + +statement ok +CREATE SCHEMA se_identity_b; + +statement ok +CREATE TABLE se_identity_a.shared_t (id INT, v INT); + +statement ok +CREATE TABLE se_identity_b.shared_t (id INT, v INT); + +statement ok +INSERT INTO se_identity_a.shared_t VALUES (1, 10), (2, 20); + +statement ok +INSERT INTO se_identity_b.shared_t VALUES (1, 100), (2, 200); + +statement ok +CREATE MATERIALIZED VIEW se_identity_a_mv AS +SELECT id, v FROM se_identity_a.shared_t; + +statement ok +CREATE MATERIALIZED VIEW se_identity_b_mv AS +SELECT id, v FROM se_identity_b.shared_t; + +statement ok +ALTER TABLE se_identity_a.shared_t RENAME COLUMN v TO amount; + +statement ok +INSERT INTO se_identity_a.shared_t VALUES (3, 30), (4, 40); + +statement ok +UPDATE se_identity_a.shared_t SET amount = 11 WHERE id = 1; + +statement ok +DELETE FROM se_identity_a.shared_t WHERE id = 2; + +statement ok +INSERT INTO se_identity_b.shared_t VALUES (3, 300), (4, 400); + +statement ok +UPDATE se_identity_b.shared_t SET v = 101 WHERE id = 1; + +statement ok +DELETE FROM se_identity_b.shared_t WHERE id = 2; + +statement ok +PRAGMA refresh('se_identity_a_mv'); + +statement ok +PRAGMA refresh('se_identity_b_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM se_identity_a_mv + EXCEPT ALL + SELECT id, amount FROM se_identity_a.shared_t +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, amount FROM se_identity_a.shared_t + EXCEPT ALL + SELECT * FROM se_identity_a_mv +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM se_identity_b_mv + EXCEPT ALL + SELECT id, v FROM se_identity_b.shared_t +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, v FROM se_identity_b.shared_t + EXCEPT ALL + SELECT * FROM se_identity_b_mv +); +---- +0 + +# ========================================== +# Test 13: CASCADE only removes exact-schema dependents +# ========================================== + +statement ok +CREATE TABLE se_identity_a.drop_shared (id INT); + +statement ok +CREATE TABLE se_identity_b.drop_shared (id INT); + +statement ok +INSERT INTO se_identity_a.drop_shared VALUES (1); + +statement ok +INSERT INTO se_identity_b.drop_shared VALUES (10); + +statement ok +CREATE MATERIALIZED VIEW se_identity_drop_a_mv AS +SELECT * FROM se_identity_a.drop_shared; + +statement ok +CREATE MATERIALIZED VIEW se_identity_drop_b_mv AS +SELECT * FROM se_identity_b.drop_shared; + +statement ok +DROP TABLE se_identity_a.drop_shared CASCADE; + +query II +SELECT + (SELECT count(*) FROM openivm_views WHERE view_name = 'se_identity_drop_a_mv'), + (SELECT count(*) FROM openivm_views WHERE view_name = 'se_identity_drop_b_mv'); +---- +0 1 + +statement ok +INSERT INTO se_identity_b.drop_shared VALUES (20), (30); + +statement ok +DELETE FROM se_identity_b.drop_shared WHERE id = 10; + +statement ok +PRAGMA refresh('se_identity_drop_b_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM se_identity_drop_b_mv + EXCEPT ALL + SELECT * FROM se_identity_b.drop_shared +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM se_identity_b.drop_shared + EXCEPT ALL + SELECT * FROM se_identity_drop_b_mv +); +---- +0 + +# ========================================== +# Test 14: tracked schema changes roll back atomically +# ========================================== + +statement ok +CREATE TABLE se_rollback_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO se_rollback_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW se_rollback_mv AS +SELECT id, v FROM se_rollback_src; + +statement ok +BEGIN TRANSACTION; + +statement ok +ALTER TABLE se_rollback_src RENAME COLUMN v TO amount; + +statement ok +ROLLBACK; + +query I +SELECT count(*) FROM duckdb_columns() +WHERE table_name = 'se_rollback_src' AND column_name = 'v'; +---- +1 + +query I +SELECT count(*) FROM duckdb_columns() +WHERE table_name = 'openivm_delta_se_rollback_src' AND column_name = 'v'; +---- +1 + +statement ok +INSERT INTO se_rollback_src VALUES (2, 20), (3, 30); + +statement ok +UPDATE se_rollback_src SET v = 11 WHERE id = 1; + +statement ok +DELETE FROM se_rollback_src WHERE id = 3; + +statement ok +PRAGMA refresh('se_rollback_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM se_rollback_mv + EXCEPT ALL + SELECT * FROM se_rollback_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM se_rollback_src + EXCEPT ALL + SELECT * FROM se_rollback_mv +); +---- +0 + +# ========================================== +# Test 15: failed rename restores exact metadata +# ========================================== + +statement ok +CREATE TABLE se_failed_rename_src (id INTEGER, old_col INTEGER, new_col INTEGER); + +statement ok +INSERT INTO se_failed_rename_src VALUES (1, 10, 100), (2, 20, 200); + +statement ok +CREATE MATERIALIZED VIEW se_failed_rename_mv AS +SELECT id, old_col AS old_value, new_col AS new_value +FROM se_failed_rename_src; + +statement error +ALTER TABLE se_failed_rename_src RENAME COLUMN old_col TO new_col; +---- +already exists + +# Batch conflicting changes before one refresh. A lossy new_col -> old_col +# rollback rewrite makes the two projected values equal and fails bag equality. +statement ok +INSERT INTO se_failed_rename_src VALUES (3, 30, 300), (4, 40, 400); + +statement ok +UPDATE se_failed_rename_src +SET old_col = old_col + 1, new_col = new_col + 2 +WHERE id IN (1, 3); + +statement ok +DELETE FROM se_failed_rename_src WHERE id IN (2, 4); + +statement ok +PRAGMA refresh('se_failed_rename_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM se_failed_rename_mv + EXCEPT ALL + SELECT id, old_col AS old_value, new_col AS new_value + FROM se_failed_rename_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, old_col AS old_value, new_col AS new_value + FROM se_failed_rename_src + EXCEPT ALL + SELECT * FROM se_failed_rename_mv +); +---- +0 diff --git a/test/sql/transactional_lifecycle.test b/test/sql/transactional_lifecycle.test new file mode 100644 index 00000000..9b46ad29 --- /dev/null +++ b/test/sql/transactional_lifecycle.test @@ -0,0 +1,1439 @@ +# name: test/sql/transactional_lifecycle.test +# description: Materialized-view lifecycle operations participate in the caller transaction +# group: [sql] + +require openivm + +require parquet + +statement ok +INSTALL ducklake; + +statement ok +LOAD ducklake; + +load __TEST_DIR__/openivm_transactional_lifecycle.db + +# CREATE rollback removes every object created for the MV. +statement ok +CREATE TABLE lifecycle_create_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_create_src VALUES (1, 10); + +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE MATERIALIZED VIEW lifecycle_create_mv AS +SELECT id, v FROM lifecycle_create_src; + +statement ok +ROLLBACK; + +query IIII +SELECT + (SELECT count(*) FROM duckdb_views() WHERE view_name = 'lifecycle_create_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_data_lifecycle_create_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_delta_lifecycle_create_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_views'); +---- +0 0 0 0 + +# REPLACE rollback restores the old definition, data, and refresh metadata. +statement ok +CREATE TABLE lifecycle_replace_src (id INTEGER, old_v INTEGER, new_v INTEGER); + +statement ok +INSERT INTO lifecycle_replace_src VALUES (1, 10, 100); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_replace_mv REFRESH EVERY '5 minutes' AS +SELECT id, old_v AS v FROM lifecycle_replace_src; + +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE OR REPLACE MATERIALIZED VIEW lifecycle_replace_mv REFRESH EVERY '10 minutes' AS +SELECT id, new_v AS v FROM lifecycle_replace_src; + +statement ok +ROLLBACK; + +query II +SELECT id, v FROM lifecycle_replace_mv; +---- +1 10 + +query I +SELECT refresh_interval FROM openivm_views WHERE view_name = 'lifecycle_replace_mv'; +---- +300 + +# A replacement that fails during initial materialization also preserves the old MV. +statement ok +CREATE TABLE lifecycle_failed_replace_src (id INTEGER, txt VARCHAR); + +statement ok +INSERT INTO lifecycle_failed_replace_src VALUES (1, 'not-an-integer'); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_failed_replace_mv AS +SELECT id, txt FROM lifecycle_failed_replace_src; + +statement error +CREATE OR REPLACE MATERIALIZED VIEW lifecycle_failed_replace_mv AS +SELECT id, CAST(txt AS INTEGER) AS txt FROM lifecycle_failed_replace_src; +---- +Could not convert + +query II +SELECT id, txt FROM lifecycle_failed_replace_mv; +---- +1 not-an-integer + +# ALTER rollback leaves the previous schedule unchanged. +statement ok +BEGIN TRANSACTION; + +statement ok +ALTER MATERIALIZED VIEW lifecycle_replace_mv SET REFRESH EVERY '30 minutes'; + +statement ok +ROLLBACK; + +query I +SELECT refresh_interval FROM openivm_views WHERE view_name = 'lifecycle_replace_mv'; +---- +300 + +# DROP rollback restores the user view, backing state, metadata, and refreshability. +statement ok +CREATE TABLE lifecycle_drop_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_drop_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_drop_mv AS +SELECT id, v FROM lifecycle_drop_src; + +statement ok +BEGIN TRANSACTION; + +statement ok +DROP VIEW lifecycle_drop_mv; + +statement ok +ROLLBACK; + +query III +SELECT + (SELECT count(*) FROM duckdb_views() WHERE view_name = 'lifecycle_drop_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_data_lifecycle_drop_mv'), + (SELECT count(*) FROM openivm_views WHERE view_name = 'lifecycle_drop_mv'); +---- +1 1 1 + +statement ok +INSERT INTO lifecycle_drop_src VALUES (2, 20); + +statement ok +PRAGMA refresh('lifecycle_drop_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_drop_mv + EXCEPT ALL + SELECT * FROM lifecycle_drop_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_drop_src + EXCEPT ALL + SELECT * FROM lifecycle_drop_mv +); +---- +0 + +# A source dropped with CASCADE in the same transaction that created its MV +# removes the caller-visible objects and uncommitted metadata atomically. +statement ok +CREATE TABLE lifecycle_same_tx_cascade_src (id INTEGER, v INTEGER); + +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE MATERIALIZED VIEW lifecycle_same_tx_cascade_mv AS +SELECT id, v FROM lifecycle_same_tx_cascade_src; + +statement ok +INSERT INTO openivm_refresh_hooks VALUES ( + 'lifecycle_same_tx_cascade_mv', + 'SELECT true', + 'after' +); + +statement ok +DROP TABLE lifecycle_same_tx_cascade_src CASCADE; + +statement ok +COMMIT; + +query IIIIIII +SELECT + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'lifecycle_same_tx_cascade_src'), + (SELECT count(*) FROM duckdb_views() WHERE view_name = 'lifecycle_same_tx_cascade_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_data_lifecycle_same_tx_cascade_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_delta_lifecycle_same_tx_cascade_mv'), + (SELECT count(*) FROM openivm_views WHERE view_name = 'lifecycle_same_tx_cascade_mv'), + (SELECT count(*) FROM openivm_delta_tables WHERE view_name = 'lifecycle_same_tx_cascade_mv'), + (SELECT count(*) FROM openivm_refresh_hooks WHERE view_name = 'lifecycle_same_tx_cascade_mv'); +---- +0 0 0 0 0 0 0 + +# Rolling the same sequence back restores the source and removes everything +# created for the rolled-back MV. +statement ok +CREATE TABLE lifecycle_same_tx_cascade_rollback_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_same_tx_cascade_rollback_src VALUES (1, 10); + +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE MATERIALIZED VIEW lifecycle_same_tx_cascade_rollback_mv AS +SELECT id, v FROM lifecycle_same_tx_cascade_rollback_src; + +statement ok +DROP TABLE lifecycle_same_tx_cascade_rollback_src CASCADE; + +statement ok +ROLLBACK; + +query IIII +SELECT + (SELECT count(*) FROM lifecycle_same_tx_cascade_rollback_src), + (SELECT count(*) FROM duckdb_views() WHERE view_name = 'lifecycle_same_tx_cascade_rollback_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_data_lifecycle_same_tx_cascade_rollback_mv'), + (SELECT count(*) FROM openivm_views WHERE view_name = 'lifecycle_same_tx_cascade_rollback_mv'); +---- +1 0 0 0 + +# Autocommit cascade uses one locked helper transaction and removes hook +# configuration together with the tracked MV. +statement ok +CREATE TABLE lifecycle_autocommit_cascade_src (id INTEGER); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_autocommit_cascade_mv AS +SELECT id FROM lifecycle_autocommit_cascade_src; + +statement ok +INSERT INTO openivm_refresh_hooks VALUES ( + 'lifecycle_autocommit_cascade_mv', + 'SELECT true', + 'after' +); + +statement ok +DROP TABLE lifecycle_autocommit_cascade_src CASCADE; + +query III +SELECT + (SELECT count(*) FROM duckdb_views() WHERE view_name = 'lifecycle_autocommit_cascade_mv'), + (SELECT count(*) FROM openivm_views WHERE view_name = 'lifecycle_autocommit_cascade_mv'), + (SELECT count(*) FROM openivm_refresh_hooks WHERE view_name = 'lifecycle_autocommit_cascade_mv'); +---- +0 0 0 + +# Metadata replay is based on the statement target, not literals in a valid +# backing-table query. +statement ok +CREATE TABLE lifecycle_metadata_literal_src (id INTEGER); + +statement ok +INSERT INTO lifecycle_metadata_literal_src VALUES (1); + +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE MATERIALIZED VIEW lifecycle_metadata_literal_mv AS +SELECT id, 'openivm_views' AS marker FROM lifecycle_metadata_literal_src; + +statement ok +INSERT INTO lifecycle_metadata_literal_src VALUES (2); + +statement ok +PRAGMA refresh('lifecycle_metadata_literal_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_metadata_literal_mv + EXCEPT ALL + SELECT id, 'openivm_views' AS marker FROM lifecycle_metadata_literal_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, 'openivm_views' AS marker FROM lifecycle_metadata_literal_src + EXCEPT ALL + SELECT * FROM lifecycle_metadata_literal_mv +); +---- +0 + +statement ok +COMMIT; + +# Native source deltas live beside the source, including non-default schemas. +statement ok +CREATE SCHEMA lifecycle_alt; + +statement ok +CREATE TABLE lifecycle_alt.drop_src (id INTEGER); + +statement ok +INSERT INTO lifecycle_alt.drop_src VALUES (1), (2); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_alt_drop_mv AS +SELECT id FROM lifecycle_alt.drop_src; + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_alt_drop_mv + EXCEPT ALL + SELECT * FROM lifecycle_alt.drop_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_alt.drop_src + EXCEPT ALL + SELECT * FROM lifecycle_alt_drop_mv +); +---- +0 + +query I +SELECT count(*) FROM information_schema.tables +WHERE table_schema = 'lifecycle_alt' AND table_name = 'openivm_delta_drop_src'; +---- +1 + +statement ok +DROP VIEW lifecycle_alt_drop_mv; + +query I +SELECT count(*) FROM information_schema.tables +WHERE table_schema = 'lifecycle_alt' AND table_name = 'openivm_delta_drop_src'; +---- +0 + +# Two-part native names identify a schema, not a catalog. The generated view, +# data table, lifecycle lock, and refresh lookup must all use that same locus. +statement ok +CREATE TABLE lifecycle_alt.qualified_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_alt.qualified_src VALUES (1, 10), (2, 20); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_alt.qualified_mv AS +SELECT id, v FROM lifecycle_alt.qualified_src; + +statement ok +INSERT INTO lifecycle_alt.qualified_src VALUES (3, 30), (4, 40); + +statement ok +UPDATE lifecycle_alt.qualified_src SET v = 11 WHERE id = 1; + +statement ok +DELETE FROM lifecycle_alt.qualified_src WHERE id = 2; + +statement ok +PRAGMA refresh_options('openivm_transactional_lifecycle', 'lifecycle_alt', 'qualified_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_alt.qualified_mv + EXCEPT ALL + SELECT * FROM lifecycle_alt.qualified_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_alt.qualified_src + EXCEPT ALL + SELECT * FROM lifecycle_alt.qualified_mv +); +---- +0 + +statement ok +ALTER MATERIALIZED VIEW "lifecycle_alt"."qualified_mv" SET REFRESH EVERY '5 minutes'; + +query I +SELECT count(*) FROM openivm_views +WHERE view_name = 'qualified_mv' + AND view_catalog = 'openivm_transactional_lifecycle' + AND view_schema = 'lifecycle_alt' + AND refresh_interval = 300; +---- +1 + +# A failed refresh in an explicit transaction must not change the caller's +# catalog search path while compiling against metadata in main. +statement ok +CREATE TABLE lifecycle_alt.refresh_failure_src (id INTEGER, txt VARCHAR); + +statement ok +INSERT INTO lifecycle_alt.refresh_failure_src VALUES (1, '10'); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_alt.refresh_failure_mv AS +SELECT id, CAST(txt AS INTEGER) AS v FROM lifecycle_alt.refresh_failure_src; + +statement ok +USE openivm_transactional_lifecycle.lifecycle_alt; + +statement ok +BEGIN TRANSACTION; + +statement ok +INSERT INTO refresh_failure_src VALUES (2, 'not-an-integer'); + +statement error +PRAGMA refresh_options('openivm_transactional_lifecycle', 'lifecycle_alt', 'refresh_failure_mv'); +---- +Could not convert + +statement ok +ROLLBACK; + +query T +SELECT current_schema(); +---- +lifecycle_alt + +statement ok +INSERT INTO refresh_failure_src VALUES (2, '20'), (3, '30'); + +statement ok +DELETE FROM refresh_failure_src WHERE id = 3; + +statement ok +PRAGMA refresh_options('openivm_transactional_lifecycle', 'lifecycle_alt', 'refresh_failure_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM refresh_failure_mv + EXCEPT ALL + SELECT id, CAST(txt AS INTEGER) FROM refresh_failure_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, CAST(txt AS INTEGER) FROM refresh_failure_src + EXCEPT ALL + SELECT * FROM refresh_failure_mv +); +---- +0 + +statement ok +USE openivm_transactional_lifecycle.main; + +# Quoted qualified targets may contain characters that are not legal in bare +# identifiers; ALTER must preserve the component boundaries. +statement ok +CREATE SCHEMA "lifecycle-alt-quoted"; + +statement ok +CREATE TABLE "lifecycle-alt-quoted"."qualified-src" (id INTEGER, v INTEGER); + +statement ok +INSERT INTO "lifecycle-alt-quoted"."qualified-src" VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW "lifecycle-alt-quoted"."qualified-mv" AS +SELECT id, v FROM "lifecycle-alt-quoted"."qualified-src"; + +statement ok +ALTER MATERIALIZED VIEW "lifecycle-alt-quoted"."qualified-mv" SET REFRESH EVERY '7 minutes'; + +query I +SELECT count(*) FROM openivm_views +WHERE view_name = 'qualified-mv' + AND view_schema = 'lifecycle-alt-quoted' + AND refresh_interval = 420; +---- +1 + +query I +SELECT count(*) FROM ( + SELECT * FROM "lifecycle-alt-quoted"."qualified-mv" + EXCEPT ALL + SELECT * FROM "lifecycle-alt-quoted"."qualified-src" +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM "lifecycle-alt-quoted"."qualified-src" + EXCEPT ALL + SELECT * FROM "lifecycle-alt-quoted"."qualified-mv" +); +---- +0 + +# Qualified ALTER must resolve the complete target instead of updating another +# schema's same-bare metadata row. +statement error +ALTER MATERIALIZED VIEW "lifecycle_alt"."qualified-mv" SET REFRESH EVERY '9 minutes'; +---- +does not exist in OpenIVM metadata + +query I +SELECT refresh_interval FROM openivm_views +WHERE view_name = 'qualified-mv' AND view_schema = 'lifecycle-alt-quoted'; +---- +420 + +statement ok +INSERT INTO "lifecycle-alt-quoted"."qualified-src" VALUES (2, 20), (3, 30); + +statement ok +UPDATE "lifecycle-alt-quoted"."qualified-src" SET v = 11 WHERE id = 1; + +statement ok +DELETE FROM "lifecycle-alt-quoted"."qualified-src" WHERE id = 3; + +statement ok +PRAGMA refresh_options('openivm_transactional_lifecycle', 'lifecycle-alt-quoted', 'qualified-mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM "lifecycle-alt-quoted"."qualified-mv" + EXCEPT ALL + SELECT * FROM "lifecycle-alt-quoted"."qualified-src" +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM "lifecycle-alt-quoted"."qualified-src" + EXCEPT ALL + SELECT * FROM "lifecycle-alt-quoted"."qualified-mv" +); +---- +0 + +# A schema-qualified DROP of an ordinary same-named view must not clean up the +# tracked MV in another schema. Metadata currently uses a bare view name, so +# cleanup must first prove the exact catalog entry is OpenIVM's generated view. +statement ok +CREATE TABLE lifecycle_drop_collision_src (id INTEGER); + +statement ok +INSERT INTO lifecycle_drop_collision_src VALUES (1); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_drop_collision_mv AS +SELECT * FROM lifecycle_drop_collision_src; + +statement ok +CREATE SCHEMA lifecycle_drop_collision_schema; + +statement ok +CREATE VIEW lifecycle_drop_collision_schema.lifecycle_drop_collision_mv AS +SELECT 99 AS id; + +statement ok +DROP VIEW lifecycle_drop_collision_schema.lifecycle_drop_collision_mv; + +query III +SELECT + (SELECT count(*) FROM duckdb_views() WHERE schema_name = 'main' + AND view_name = 'lifecycle_drop_collision_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_data_lifecycle_drop_collision_mv'), + (SELECT count(*) FROM openivm_views WHERE view_name = 'lifecycle_drop_collision_mv'); +---- +1 1 1 + +statement ok +INSERT INTO lifecycle_drop_collision_src VALUES (2); + +statement ok +PRAGMA refresh('lifecycle_drop_collision_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_drop_collision_mv + EXCEPT ALL + SELECT * FROM lifecycle_drop_collision_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_drop_collision_src + EXCEPT ALL + SELECT * FROM lifecycle_drop_collision_mv +); +---- +0 + +# Refresh rollback restores the MV and leaves its source delta available for retry. +statement ok +CREATE TABLE lifecycle_refresh_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_refresh_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_refresh_mv AS +SELECT id, v FROM lifecycle_refresh_src; + +statement ok +INSERT INTO lifecycle_refresh_src VALUES (2, 20); + +statement ok +BEGIN TRANSACTION; + +statement ok +PRAGMA refresh('lifecycle_refresh_mv'); + +query II +SELECT id, v FROM lifecycle_refresh_mv ORDER BY id; +---- +1 10 +2 20 + +statement ok +ROLLBACK; + +query II +SELECT id, v FROM lifecycle_refresh_mv ORDER BY id; +---- +1 10 + +query I +SELECT count(*) FROM openivm_delta_lifecycle_refresh_src; +---- +1 + +statement ok +PRAGMA refresh('lifecycle_refresh_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_refresh_mv + EXCEPT ALL + SELECT * FROM lifecycle_refresh_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_refresh_src + EXCEPT ALL + SELECT * FROM lifecycle_refresh_mv +); +---- +0 + +# Transaction-local DML is visible to refresh and rolls back together. +statement ok +CREATE TABLE lifecycle_local_refresh_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_local_refresh_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_local_refresh_mv AS +SELECT id, v FROM lifecycle_local_refresh_src; + +statement ok +BEGIN TRANSACTION; + +statement ok +INSERT INTO lifecycle_local_refresh_src VALUES (2, 20); + +statement ok +PRAGMA refresh('lifecycle_local_refresh_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_local_refresh_mv + EXCEPT ALL + SELECT * FROM lifecycle_local_refresh_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_local_refresh_src + EXCEPT ALL + SELECT * FROM lifecycle_local_refresh_mv +); +---- +0 + +statement ok +ROLLBACK; + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_local_refresh_mv + EXCEPT ALL + SELECT * FROM lifecycle_local_refresh_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_local_refresh_src + EXCEPT ALL + SELECT * FROM lifecycle_local_refresh_mv +); +---- +0 + +# A source table, MV metadata, backing tables, and refresh created in one +# explicit transaction must all be visible through the caller's snapshot. +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE TABLE lifecycle_same_tx_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_same_tx_src VALUES (1, 10), (2, 20); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_same_tx_mv AS +SELECT id, v FROM lifecycle_same_tx_src; + +statement ok +INSERT INTO lifecycle_same_tx_src VALUES (3, 30), (4, 40); + +statement ok +UPDATE lifecycle_same_tx_src SET v = 21 WHERE id = 2; + +statement ok +DELETE FROM lifecycle_same_tx_src WHERE id = 4; + +statement ok +PRAGMA refresh('lifecycle_same_tx_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_same_tx_mv + EXCEPT ALL + SELECT * FROM lifecycle_same_tx_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_same_tx_src + EXCEPT ALL + SELECT * FROM lifecycle_same_tx_mv +); +---- +0 + +statement ok +COMMIT; + +# A same-transaction replacement must compile refresh from the replacement +# definition and source metadata, not the committed definition visible to a +# helper connection. +statement ok +CREATE TABLE lifecycle_same_tx_replace_src (id INTEGER, old_v INTEGER, new_v INTEGER); + +statement ok +INSERT INTO lifecycle_same_tx_replace_src VALUES (1, 10, 100), (2, 20, 200); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_same_tx_replace_mv AS +SELECT id, old_v AS v FROM lifecycle_same_tx_replace_src; + +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE OR REPLACE MATERIALIZED VIEW lifecycle_same_tx_replace_mv AS +SELECT id, new_v AS v FROM lifecycle_same_tx_replace_src; + +statement ok +INSERT INTO lifecycle_same_tx_replace_src VALUES (3, 30, 300), (4, 40, 400); + +statement ok +UPDATE lifecycle_same_tx_replace_src SET new_v = 201 WHERE id = 2; + +statement ok +DELETE FROM lifecycle_same_tx_replace_src WHERE id = 4; + +statement ok +PRAGMA refresh('lifecycle_same_tx_replace_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_same_tx_replace_mv + EXCEPT ALL + SELECT id, new_v AS v FROM lifecycle_same_tx_replace_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, new_v AS v FROM lifecycle_same_tx_replace_src + EXCEPT ALL + SELECT * FROM lifecycle_same_tx_replace_mv +); +---- +0 + +statement ok +COMMIT; + +# Replaying one transaction-local lifecycle change must retain the complete +# downstream dependency closure and every child's source metadata. +statement ok +CREATE TABLE lifecycle_cascade_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_cascade_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_cascade_parent AS +SELECT id, v FROM lifecycle_cascade_src; + +statement ok +CREATE MATERIALIZED VIEW lifecycle_cascade_child AS +SELECT id, v FROM lifecycle_cascade_parent; + +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE OR REPLACE MATERIALIZED VIEW lifecycle_cascade_parent AS +SELECT id, v FROM lifecycle_cascade_src; + +statement ok +INSERT INTO lifecycle_cascade_src VALUES (2, 20), (3, 30); + +statement ok +UPDATE lifecycle_cascade_src SET v = 11 WHERE id = 1; + +statement ok +DELETE FROM lifecycle_cascade_src WHERE id = 3; + +statement ok +PRAGMA refresh('lifecycle_cascade_parent'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_cascade_child + EXCEPT ALL + SELECT id, v FROM lifecycle_cascade_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, v FROM lifecycle_cascade_src + EXCEPT ALL + SELECT * FROM lifecycle_cascade_child +); +---- +0 + +statement ok +COMMIT; + +# A lifecycle change to one view must not hide an unrelated refresh target from +# the helper metadata shadow. +statement ok +BEGIN TRANSACTION; + +statement ok +ALTER MATERIALIZED VIEW lifecycle_cascade_parent SET REFRESH MANUAL; + +statement ok +INSERT INTO lifecycle_refresh_src VALUES (3, 30); + +statement ok +PRAGMA refresh('lifecycle_refresh_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_refresh_mv + EXCEPT ALL + SELECT * FROM lifecycle_refresh_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_refresh_src + EXCEPT ALL + SELECT * FROM lifecycle_refresh_mv +); +---- +0 + +statement ok +COMMIT; + +# CREATE followed by DROP in one transaction must not orphan generated state +# or metadata that a helper connection cannot yet observe. +statement ok +CREATE TABLE lifecycle_same_tx_drop_src (id INTEGER); + +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE MATERIALIZED VIEW lifecycle_same_tx_drop_mv AS +SELECT * FROM lifecycle_same_tx_drop_src; + +statement ok +DROP VIEW lifecycle_same_tx_drop_mv; + +statement ok +COMMIT; + +query IIIII +SELECT + (SELECT count(*) FROM duckdb_views() WHERE view_name = 'lifecycle_same_tx_drop_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_data_lifecycle_same_tx_drop_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_delta_lifecycle_same_tx_drop_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_delta_lifecycle_same_tx_drop_src'), + (SELECT count(*) FROM openivm_views WHERE view_name = 'lifecycle_same_tx_drop_mv'); +---- +0 0 0 0 0 + +# DROP ownership is the exact catalog/schema identity recorded at CREATE time. +# A same-named ordinary view that deliberately references the backing table must +# not be able to destroy the tracked MV. +statement ok +CREATE TABLE lifecycle_exact_drop_src (id INTEGER); + +statement ok +INSERT INTO lifecycle_exact_drop_src VALUES (1), (2); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_exact_drop_mv AS +SELECT * FROM lifecycle_exact_drop_src; + +statement ok +CREATE SCHEMA lifecycle_spoof; + +statement ok +CREATE VIEW lifecycle_spoof.lifecycle_exact_drop_mv AS +SELECT * FROM main.openivm_data_lifecycle_exact_drop_mv; + +statement ok +DROP VIEW lifecycle_spoof.lifecycle_exact_drop_mv; + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_exact_drop_mv + EXCEPT ALL + SELECT * FROM lifecycle_exact_drop_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_exact_drop_src + EXCEPT ALL + SELECT * FROM lifecycle_exact_drop_mv +); +---- +0 + +query III +SELECT + (SELECT count(*) FROM openivm_views WHERE view_name = 'lifecycle_exact_drop_mv'), + (SELECT count(*) FROM duckdb_tables() WHERE table_name = 'openivm_data_lifecycle_exact_drop_mv'), + (SELECT count(*) FROM duckdb_views() + WHERE schema_name = 'lifecycle_spoof' AND view_name = 'lifecycle_exact_drop_mv'); +---- +1 1 0 + +# A transaction-local DROP must be visible to a subsequent CREATE of the same +# MV name, including refresh compilation against the replacement metadata. +statement ok +CREATE TABLE lifecycle_drop_recreate_src (id INTEGER, old_v INTEGER, new_v INTEGER); + +statement ok +INSERT INTO lifecycle_drop_recreate_src VALUES (1, 10, 100); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_drop_recreate_mv AS +SELECT id, old_v AS v FROM lifecycle_drop_recreate_src; + +statement ok +BEGIN TRANSACTION; + +statement ok +DROP VIEW lifecycle_drop_recreate_mv; + +statement ok +CREATE MATERIALIZED VIEW lifecycle_drop_recreate_mv AS +SELECT id, new_v AS v FROM lifecycle_drop_recreate_src; + +statement ok +INSERT INTO lifecycle_drop_recreate_src VALUES (2, 20, 200); + +statement ok +UPDATE lifecycle_drop_recreate_src SET new_v = 101 WHERE id = 1; + +statement ok +PRAGMA refresh('lifecycle_drop_recreate_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_drop_recreate_mv + EXCEPT ALL + SELECT id, new_v AS v FROM lifecycle_drop_recreate_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, new_v AS v FROM lifecycle_drop_recreate_src + EXCEPT ALL + SELECT * FROM lifecycle_drop_recreate_mv +); +---- +0 + +statement ok +COMMIT; + +# Stabilizing generated metadata timestamps must not rewrite now() text inside +# the stored view SQL. +statement ok +CREATE TABLE lifecycle_now_literal_src (id INTEGER); + +statement ok +INSERT INTO lifecycle_now_literal_src VALUES (1); + +statement ok +BEGIN TRANSACTION; + +statement ok +CREATE MATERIALIZED VIEW lifecycle_now_literal_mv AS +SELECT id, 'now()' AS marker FROM lifecycle_now_literal_src; + +statement ok +INSERT INTO lifecycle_now_literal_src VALUES (2); + +statement ok +PRAGMA refresh('lifecycle_now_literal_mv'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_now_literal_mv + EXCEPT ALL + SELECT id, 'now()' AS marker FROM lifecycle_now_literal_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, 'now()' AS marker FROM lifecycle_now_literal_src + EXCEPT ALL + SELECT * FROM lifecycle_now_literal_mv +); +---- +0 + +statement ok +COMMIT; + +# A refresh keeps its OpenIVM locks until the surrounding transaction ends. +# The second connection starts while the first transaction is sleeping after +# refresh and must wait for rollback before it can consume the restored delta. +statement ok +CREATE TABLE lifecycle_lock_src (id INTEGER, v INTEGER); + +statement ok +INSERT INTO lifecycle_lock_src VALUES (1, 10); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_lock_mv AS +SELECT id, v FROM lifecycle_lock_src; + +statement ok +INSERT INTO lifecycle_lock_src VALUES (2, 20); + +statement ok +CREATE TABLE lifecycle_lock_timing (phase VARCHAR, observed_at TIMESTAMP); + +concurrentloop threadid 0 2 + +onlyif threadid=0 +statement ok +BEGIN TRANSACTION; + +onlyif threadid=0 +statement ok +PRAGMA refresh('lifecycle_lock_mv'); + +onlyif threadid=0 +statement ok +SELECT sleep_ms(1200); + +onlyif threadid=0 +statement ok +ROLLBACK; + +onlyif threadid=1 +statement ok +SELECT sleep_ms(200); + +onlyif threadid=1 +statement ok +INSERT INTO lifecycle_lock_timing VALUES ('before', now()::TIMESTAMP); + +onlyif threadid=1 +statement ok +PRAGMA refresh('lifecycle_lock_mv'); + +onlyif threadid=1 +statement ok +INSERT INTO lifecycle_lock_timing VALUES ('after', now()::TIMESTAMP); + +endloop + +query I +SELECT epoch_ms(max(observed_at)) - epoch_ms(min(observed_at)) >= 700 +FROM lifecycle_lock_timing; +---- +true + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_lock_mv + EXCEPT ALL + SELECT * FROM lifecycle_lock_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_lock_src + EXCEPT ALL + SELECT * FROM lifecycle_lock_mv +); +---- +0 + +# DROP TABLE CASCADE companion cleanup must roll back with the base-table drop +# and must remove the complete downstream MV closure when committed. +statement ok +CREATE TABLE lifecycle_drop_chain_src (id INTEGER); + +statement ok +INSERT INTO lifecycle_drop_chain_src VALUES (1); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_drop_chain_parent AS +SELECT * FROM lifecycle_drop_chain_src; + +statement ok +CREATE MATERIALIZED VIEW lifecycle_drop_chain_child AS +SELECT * FROM lifecycle_drop_chain_parent; + +statement ok +BEGIN TRANSACTION; + +statement ok +DROP TABLE lifecycle_drop_chain_src CASCADE; + +statement ok +ROLLBACK; + +query I +SELECT count(*) FROM openivm_views +WHERE view_name IN ('lifecycle_drop_chain_parent', 'lifecycle_drop_chain_child'); +---- +2 + +statement ok +INSERT INTO lifecycle_drop_chain_src VALUES (2), (3); + +statement ok +DELETE FROM lifecycle_drop_chain_src WHERE id = 3; + +statement ok +PRAGMA refresh('lifecycle_drop_chain_parent'); + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_drop_chain_parent + EXCEPT ALL + SELECT * FROM lifecycle_drop_chain_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_drop_chain_src + EXCEPT ALL + SELECT * FROM lifecycle_drop_chain_parent +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_drop_chain_child + EXCEPT ALL + SELECT * FROM lifecycle_drop_chain_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_drop_chain_src + EXCEPT ALL + SELECT * FROM lifecycle_drop_chain_child +); +---- +0 + +statement ok +DROP TABLE lifecycle_drop_chain_src CASCADE; + +query I +SELECT count(*) FROM openivm_views +WHERE view_name IN ('lifecycle_drop_chain_parent', 'lifecycle_drop_chain_child'); +---- +0 + +query I +SELECT count(*) FROM duckdb_views() +WHERE view_name IN ('lifecycle_drop_chain_parent', 'lifecycle_drop_chain_child'); +---- +0 + +# DuckLake cannot share the native metadata transaction. Its replacement data +# is therefore built under an unpublished staging name and published only after +# materialization succeeds. +statement ok +ATTACH '__TEST_DIR__/openivm_transactional_lifecycle.ducklake' AS lifecycle_dl (TYPE ducklake); + +# Cascading from a native source removes dependent MV state, but DuckLake +# sources are external base tables and must never be treated as native delta +# tables eligible for cleanup. +statement ok +CREATE TABLE lifecycle_mixed_native (id INTEGER); + +statement ok +INSERT INTO lifecycle_mixed_native VALUES (1), (2); + +statement ok +CREATE TABLE lifecycle_dl.lifecycle_mixed_dimension (id INTEGER, label VARCHAR); + +statement ok +INSERT INTO lifecycle_dl.lifecycle_mixed_dimension VALUES (1, 'one'), (3, 'three'); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_mixed_mv AS +SELECT n.id, d.label +FROM lifecycle_mixed_native n +LEFT JOIN lifecycle_dl.lifecycle_mixed_dimension d ON n.id = d.id; + +query I +SELECT count(*) FROM ( + SELECT * FROM lifecycle_mixed_mv + EXCEPT ALL + SELECT n.id, d.label + FROM lifecycle_mixed_native n + LEFT JOIN lifecycle_dl.lifecycle_mixed_dimension d ON n.id = d.id +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT n.id, d.label + FROM lifecycle_mixed_native n + LEFT JOIN lifecycle_dl.lifecycle_mixed_dimension d ON n.id = d.id + EXCEPT ALL + SELECT * FROM lifecycle_mixed_mv +); +---- +0 + +statement ok +DROP TABLE lifecycle_mixed_native CASCADE; + +query I +SELECT count(*) FROM lifecycle_dl.lifecycle_mixed_dimension; +---- +2 + +query I +SELECT count(*) FROM openivm_views WHERE view_name = 'lifecycle_mixed_mv'; +---- +0 + +# Staged DuckLake lifecycle DDL must share the caller's logical mutation +# ownership after tracked native DML in the same explicit transaction. +statement ok +CREATE TABLE lifecycle_staged_lock_src (id INTEGER); + +statement ok +INSERT INTO lifecycle_staged_lock_src VALUES (1); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_staged_lock_native_mv AS +SELECT * FROM lifecycle_staged_lock_src; + +statement ok +CREATE TABLE lifecycle_dl.lifecycle_staged_lock_dl_src (id INTEGER); + +statement ok +INSERT INTO lifecycle_dl.lifecycle_staged_lock_dl_src VALUES (10); + +statement ok +BEGIN TRANSACTION; + +statement ok +INSERT INTO lifecycle_staged_lock_src VALUES (2); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_dl.lifecycle_staged_lock_dl_mv AS +SELECT * FROM lifecycle_dl.lifecycle_staged_lock_dl_src; + +statement ok +COMMIT; + +statement ok +CREATE TABLE lifecycle_dl.replace_src (id INTEGER, old_v INTEGER, txt VARCHAR); + +statement ok +INSERT INTO lifecycle_dl.replace_src VALUES (1, 10, 'bad'); + +statement ok +CREATE MATERIALIZED VIEW lifecycle_dl.replace_mv AS +SELECT id, old_v AS v FROM lifecycle_dl.replace_src; + +statement error +CREATE OR REPLACE MATERIALIZED VIEW lifecycle_dl.replace_mv AS +SELECT id, CAST(txt AS INTEGER) AS v FROM lifecycle_dl.replace_src; +---- +Could not convert + +query II +SELECT id, v FROM lifecycle_dl.replace_mv; +---- +1 10 + +query I +SELECT count(*) FROM duckdb_tables() +WHERE database_name = 'lifecycle_dl' + AND table_name = 'openivm_stage_replace_mv'; +---- +0 + +statement ok +CREATE OR REPLACE MATERIALIZED VIEW lifecycle_dl.replace_mv AS +SELECT id, length(txt) AS v FROM lifecycle_dl.replace_src; + +statement ok +INSERT INTO lifecycle_dl.replace_src VALUES (2, 20, 'four'); + +statement ok +PRAGMA refresh('replace_mv'); + +query I +SELECT count(*) FROM ( + SELECT id, v FROM lifecycle_dl.replace_mv + EXCEPT ALL + SELECT id, length(txt) AS v FROM lifecycle_dl.replace_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, length(txt) AS v FROM lifecycle_dl.replace_src + EXCEPT ALL + SELECT id, v FROM lifecycle_dl.replace_mv +); +---- +0 + +# Explicit cross-system refresh delegates before retaining the caller's +# non-recursive view lock. +statement ok +INSERT INTO lifecycle_dl.replace_src VALUES (3, 30, 'three'); + +statement ok +BEGIN TRANSACTION; + +statement ok +PRAGMA refresh('replace_mv'); + +statement ok +COMMIT; + +query I +SELECT count(*) FROM ( + SELECT id, v FROM lifecycle_dl.replace_mv + EXCEPT ALL + SELECT id, length(txt) AS v FROM lifecycle_dl.replace_src +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT id, length(txt) AS v FROM lifecycle_dl.replace_src + EXCEPT ALL + SELECT id, v FROM lifecycle_dl.replace_mv +); +---- +0 diff --git a/test/sql/window.test b/test/sql/window.test index e3313746..8af55c78 100644 --- a/test/sql/window.test +++ b/test/sql/window.test @@ -545,3 +545,110 @@ SELECT count(*) FROM ( ); ---- 0 + +# ========================================== +# Regression: NULL partition keys use window equality semantics +# ========================================== + +statement ok +CREATE TABLE w_null_part (id INT, grp VARCHAR, val INT); + +statement ok +INSERT INTO w_null_part VALUES + (1, NULL, 10), + (2, NULL, 20), + (3, 'a', 5); + +statement ok +CREATE MATERIALIZED VIEW mv_null_part AS +SELECT id, grp, val, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY val, id) AS rn +FROM w_null_part; + +# Batch conflicting changes in the NULL partition before one refresh. +statement ok +INSERT INTO w_null_part VALUES + (4, NULL, 15), + (5, NULL, 30); + +statement ok +UPDATE w_null_part SET val = 25 WHERE id = 1; + +statement ok +DELETE FROM w_null_part WHERE id IN (2, 5); + +statement ok +PRAGMA refresh('mv_null_part'); + +query I +SELECT count(*) FROM ( + SELECT id, grp, val, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY val, id) AS rn + FROM w_null_part + EXCEPT ALL + SELECT * FROM mv_null_part +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM mv_null_part + EXCEPT ALL + SELECT id, grp, val, ROW_NUMBER() OVER (PARTITION BY grp ORDER BY val, id) AS rn + FROM w_null_part +); +---- +0 + +# Composite partition keys must also match mixed-NULL tuples. +statement ok +CREATE TABLE w_null_comp (id INT, dept VARCHAR, team VARCHAR, val INT); + +statement ok +INSERT INTO w_null_comp VALUES + (1, 'eng', NULL, 10), + (2, 'eng', NULL, 20), + (3, NULL, 'ops', 30), + (4, 'hr', 'rec', 40); + +statement ok +CREATE MATERIALIZED VIEW mv_null_comp AS +SELECT id, dept, team, val, + ROW_NUMBER() OVER (PARTITION BY dept, team ORDER BY val, id) AS rn +FROM w_null_comp; + +statement ok +INSERT INTO w_null_comp VALUES + (5, 'eng', NULL, 15), + (6, NULL, 'ops', 35), + (7, 'eng', NULL, 50); + +statement ok +UPDATE w_null_comp SET val = 25 WHERE id = 1; + +statement ok +DELETE FROM w_null_comp WHERE id IN (2, 7); + +statement ok +PRAGMA refresh('mv_null_comp'); + +query I +SELECT count(*) FROM ( + SELECT id, dept, team, val, + ROW_NUMBER() OVER (PARTITION BY dept, team ORDER BY val, id) AS rn + FROM w_null_comp + EXCEPT ALL + SELECT * FROM mv_null_comp +); +---- +0 + +query I +SELECT count(*) FROM ( + SELECT * FROM mv_null_comp + EXCEPT ALL + SELECT id, dept, team, val, + ROW_NUMBER() OVER (PARTITION BY dept, team ORDER BY val, id) AS rn + FROM w_null_comp +); +---- +0 diff --git a/test/sql/window_running_aggregate_incremental.test b/test/sql/window_running_aggregate_incremental.test index 0c4a26c4..70b26501 100644 --- a/test/sql/window_running_aggregate_incremental.test +++ b/test/sql/window_running_aggregate_incremental.test @@ -18,7 +18,9 @@ INSERT INTO wrai_sales VALUES (1, 'a', 1, 10), (2, 'a', 2, 5), (3, 'b', 1, 7), - (4, 'c', 1, 3); + (4, 'c', 1, 3), + (11, NULL, 1, 4), + (12, NULL, 2, 6); statement ok CREATE MATERIALIZED VIEW wrai_sum_mv AS @@ -34,7 +36,8 @@ statement ok INSERT INTO wrai_sales VALUES (5, 'a', 3, 8), (6, 'b', 2, 9), - (7, 'c', 2, 4); + (7, 'c', 2, 4), + (13, NULL, 3, 2); query I SELECT CASE WHEN string_agg(sql, ' ' ORDER BY stmt_order) LIKE '%openivm_run_fast_wrai_sum_mv%' @@ -94,7 +97,8 @@ statement ok INSERT INTO wrai_sales VALUES (8, 'a', 0, 11), (9, 'b', 3, 2), - (10, 'd', 1, 6); + (10, 'd', 1, 6), + (14, NULL, 0, 13); statement ok PRAGMA refresh('wrai_sum_mv');