Refactor

CMakeLists.txt

@@ -57,6 +57,7 @@ set(EXTENSION_SOURCES
     src/upsert/refresh_index_regen.cpp
     src/rules/incremental_rewrite_rule.cpp
     src/rules/refresh_insert_rule.cpp
+    src/rules/transactional_delta_capture.cpp
     src/rules/schema_evolution.cpp
     src/rules/column_hider.cpp
     src/core/refresh_locks.cpp

benchmark/queries/ducklake_0251.sql

@@ -0,0 +1,5 @@
+-- {"operators": "AGGREGATE,CASE", "complexity": "medium", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": true, "tables": "ORDER_LINE", "ducklake": true}
+SELECT OL_W_ID, OL_D_ID, OL_O_ID, OL_NUMBER,
+       SUM(CASE WHEN OL_DELIVERY_D IS NULL THEN OL_AMOUNT ELSE NULL END) AS pending_amount
+FROM dl.ORDER_LINE
+GROUP BY OL_W_ID, OL_D_ID, OL_O_ID, OL_NUMBER;

benchmark/queries/tpcc/ducklake_0309.sql

@@ -1140,5 +1140,5 @@ SELECT
 	min_balance,
 	max_balance
 FROM topk_projected
-ORDER BY total_amount DESC, warehouse_id ASC, district_id ASC, customer_id ASC
+ORDER BY ROUND(total_amount, 10) DESC, warehouse_id ASC, district_id ASC, customer_id ASC
 LIMIT 25 OFFSET 2

benchmark/queries/tpcc/ducklake_0316.sql

@@ -0,0 +1,2 @@
+-- {"operators": "DUCKLAKE,OUTER_JOIN,WINDOW,CAST", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": true, "has_case": false, "tables": "CUSTOMER,OORDER", "ducklake": true, "openivm_verified": true}
+SELECT c.C_W_ID AS warehouse_id, c.C_D_ID AS district_id, c.C_ID AS customer_id, o.O_ID AS order_id, ROW_NUMBER() OVER (PARTITION BY c.C_W_ID ORDER BY o.O_ID DESC NULLS LAST, c.C_D_ID, c.C_ID) AS rn FROM dl.CUSTOMER c LEFT JOIN dl.OORDER o ON c.C_W_ID = o.O_W_ID AND c.C_D_ID = o.O_D_ID AND CAST(CAST(c.C_ID AS BIGINT) AS VARCHAR) = CAST(o.O_C_ID AS VARCHAR);

benchmark/queries/tpcc/ducklake_0317.sql

@@ -0,0 +1,2 @@
+-- {"operators": "DUCKLAKE,OUTER_JOIN,WINDOW,CAST", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": true, "has_case": false, "tables": "CUSTOMER,OORDER", "ducklake": true, "openivm_verified": true}
+SELECT c.C_W_ID AS warehouse_id, c.C_D_ID AS district_id, c.C_ID AS customer_id, o.O_ID AS order_id, ROW_NUMBER() OVER (PARTITION BY c.C_W_ID ORDER BY o.O_ID DESC NULLS LAST, c.C_D_ID, c.C_ID) AS rn FROM dl.CUSTOMER c LEFT JOIN dl.OORDER o ON c.C_W_ID = o.O_W_ID AND c.C_D_ID = o.O_D_ID AND CAST(TRY_CAST(CAST(c.C_ID AS VARCHAR) AS BIGINT) AS VARCHAR) = CAST(o.O_C_ID AS VARCHAR);

benchmark/queries/tpcc/ducklake_0318.sql

@@ -0,0 +1,12 @@
+-- {"operators": "DUCKLAKE,WINDOW", "complexity": "low", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE", "ducklake": true, "openivm_verified": true}
+SELECT
+    OL_W_ID AS warehouse_id,
+    OL_D_ID AS district_id,
+    OL_O_ID AS order_id,
+    OL_NUMBER AS line_number,
+    OL_DELIVERY_D AS delivery_at,
+    ROW_NUMBER() OVER (
+        PARTITION BY OL_DELIVERY_D
+        ORDER BY OL_W_ID, OL_D_ID, OL_O_ID, OL_NUMBER
+    ) AS delivery_row_number
+FROM dl.ORDER_LINE;

benchmark/queries/tpcc/ducklake_0319.sql

@@ -0,0 +1,9 @@
+-- {"operators": "DISTINCT,SUBQUERY", "complexity": "medium", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "CUSTOMER", "ducklake": true, "openivm_verified": true}
+SELECT payment_count % 2 AS parity
+FROM (
+    SELECT distinct_payment_count AS payment_count
+    FROM (
+        SELECT DISTINCT C_PAYMENT_CNT AS distinct_payment_count
+        FROM dl.CUSTOMER
+    )
+);

benchmark/queries/tpcc/query_2094.sql

@@ -1,2 +1,2 @@
--- {"operators": "ASOF_JOIN,FILTER", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"}
+-- {"operators": "ASOF_JOIN,FILTER", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"}
 SELECT ol.OL_W_ID, ol.OL_D_ID, ol.OL_O_ID, o.O_ENTRY_D FROM ORDER_LINE ol ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D WHERE ol.OL_AMOUNT > 0;

benchmark/queries/tpcc/query_2095.sql

@@ -1,2 +1,2 @@
--- {"operators": "ASOF_JOIN,LEFT_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"}
+-- {"operators": "ASOF_JOIN,LEFT_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"}
 SELECT ol.OL_W_ID, ol.OL_I_ID, o.O_C_ID FROM ORDER_LINE ol ASOF LEFT JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D;

benchmark/queries/tpcc/query_2097.sql

@@ -1,2 +1,2 @@
--- {"operators": "ASOF_JOIN,INNER_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER,CUSTOMER"}
+-- {"operators": "ASOF_JOIN,INNER_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER,CUSTOMER", "non_incr_reason": "op:ASOF_JOIN"}
 SELECT c.C_W_ID, c.C_D_ID, c.C_ID, ol.OL_AMOUNT FROM ORDER_LINE ol ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D JOIN CUSTOMER c ON c.C_W_ID = o.O_W_ID AND c.C_D_ID = o.O_D_ID AND c.C_ID = o.O_C_ID;

benchmark/queries/tpcc/query_2099.sql

@@ -1,2 +1,2 @@
--- {"operators": "ASOF_JOIN,CTE", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"}
+-- {"operators": "ASOF_JOIN,CTE", "complexity": "high", "is_incremental": false, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"}
 WITH delivered AS (SELECT * FROM ORDER_LINE WHERE OL_DELIVERY_D IS NOT NULL) SELECT d.OL_W_ID, d.OL_O_ID, o.O_C_ID FROM delivered d ASOF JOIN OORDER o ON d.OL_W_ID = o.O_W_ID AND d.OL_D_ID = o.O_D_ID AND d.OL_DELIVERY_D >= o.O_ENTRY_D;

benchmark/queries/tpcc/query_2101.sql

@@ -1,2 +1,2 @@
--- {"operators": "ASOF_JOIN,FULL_OUTER_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER,STOCK"}
+-- {"operators": "ASOF_JOIN,FULL_OUTER_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER,STOCK", "non_incr_reason": "op:ASOF_JOIN"}
 SELECT COALESCE(s.S_W_ID, ol.OL_W_ID) AS w_id, COALESCE(s.S_I_ID, ol.OL_I_ID) AS item_id FROM ORDER_LINE ol ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D FULL OUTER JOIN STOCK s ON s.S_W_ID = ol.OL_SUPPLY_W_ID AND s.S_I_ID = ol.OL_I_ID;

benchmark/queries/tpcc/query_2102.sql

@@ -1,2 +1,2 @@
--- {"operators": "ASOF_JOIN,SUBQUERY", "complexity": "high", "is_incremental": true, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"}
+-- {"operators": "ASOF_JOIN,SUBQUERY", "complexity": "high", "is_incremental": false, "has_nulls": true, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"}
 SELECT * FROM (SELECT ol.OL_W_ID, ol.OL_D_ID, o.O_ENTRY_D FROM ORDER_LINE ol ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D) q WHERE q.O_ENTRY_D IS NOT NULL;

benchmark/queries/tpcc/query_2103.sql

@@ -1,2 +1,2 @@
--- {"operators": "ASOF_JOIN,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER"}
+-- {"operators": "ASOF_JOIN,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE,OORDER", "non_incr_reason": "op:ASOF_JOIN"}
 SELECT ol.OL_W_ID, u.label, o.O_ENTRY_D FROM ORDER_LINE ol CROSS JOIN UNNEST(['delivery', 'entry']) AS u(label) ASOF JOIN OORDER o ON ol.OL_W_ID = o.O_W_ID AND ol.OL_D_ID = o.O_D_ID AND ol.OL_DELIVERY_D >= o.O_ENTRY_D;

benchmark/queries/tpcc/query_2114.sql

@@ -1,2 +1,2 @@
--- {"operators": "SAMPLE", "complexity": "medium", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK"}
+-- {"operators": "SAMPLE", "complexity": "medium", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "non_incr_reason": "op:SAMPLE"}
 SELECT S_W_ID, S_I_ID, S_QUANTITY FROM STOCK USING SAMPLE reservoir(10 ROWS) REPEATABLE (42);

benchmark/queries/tpcc/query_2115.sql

@@ -1,2 +1,2 @@
--- {"operators": "SAMPLE,AGGREGATE", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE"}
+-- {"operators": "SAMPLE,AGGREGATE", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ORDER_LINE", "non_incr_reason": "op:SAMPLE"}
 SELECT OL_W_ID, SUM(OL_AMOUNT) AS sampled_amount FROM (SELECT * FROM ORDER_LINE USING SAMPLE reservoir(25 ROWS) REPEATABLE (7)) sampled_order_line GROUP BY OL_W_ID;

benchmark/queries/tpcc/query_2116.sql

@@ -1,2 +1,2 @@
--- {"operators": "SAMPLE,WINDOW", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "openivm_verified": true}
+-- {"operators": "SAMPLE,WINDOW", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "openivm_verified": true, "non_incr_reason": "op:SAMPLE"}
 SELECT S_W_ID, S_I_ID, S_QUANTITY, ROW_NUMBER() OVER (PARTITION BY S_W_ID ORDER BY S_QUANTITY DESC) AS sampled_rank FROM STOCK USING SAMPLE reservoir(20 ROWS) REPEATABLE (11);

benchmark/queries/tpcc/query_2117.sql

@@ -1,2 +1,2 @@
--- {"operators": "SAMPLE,INNER_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ITEM,STOCK"}
+-- {"operators": "SAMPLE,INNER_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ITEM,STOCK", "non_incr_reason": "op:SAMPLE"}
 SELECT i.I_ID, i.I_NAME, s.S_W_ID, s.S_QUANTITY FROM (SELECT * FROM STOCK USING SAMPLE reservoir(20 ROWS) REPEATABLE (13)) s JOIN ITEM i ON s.S_I_ID = i.I_ID;

benchmark/queries/tpcc/query_2118.sql

@@ -1,2 +1,2 @@
--- {"operators": "SAMPLE,LEFT_JOIN,AGGREGATE", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ITEM,STOCK"}
+-- {"operators": "SAMPLE,LEFT_JOIN,AGGREGATE", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "ITEM,STOCK", "non_incr_reason": "op:SAMPLE"}
 SELECT i.I_IM_ID, COUNT(s.S_I_ID) AS sampled_stock_rows FROM ITEM i LEFT JOIN (SELECT * FROM STOCK USING SAMPLE reservoir(30 ROWS) REPEATABLE (17)) s ON i.I_ID = s.S_I_ID GROUP BY i.I_IM_ID;

benchmark/queries/tpcc/query_2119.sql

@@ -1,2 +1,2 @@
--- {"operators": "SAMPLE,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK"}
+-- {"operators": "SAMPLE,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "non_incr_reason": "op:SAMPLE"}
 SELECT s.S_W_ID, u.threshold FROM (SELECT * FROM STOCK USING SAMPLE reservoir(15 ROWS) REPEATABLE (19)) s CROSS JOIN UNNEST([10, 50, 90]) AS u(threshold) WHERE s.S_QUANTITY >= u.threshold;

benchmark/queries/tpcc/query_2121.sql

@@ -1,2 +1,2 @@
--- {"operators": "SAMPLE,PIVOT,AGGREGATE", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK"}
+-- {"operators": "SAMPLE,PIVOT,AGGREGATE", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK", "non_incr_reason": "op:SAMPLE"}
 SELECT * FROM (PIVOT (SELECT * FROM STOCK USING SAMPLE reservoir(20 ROWS) REPEATABLE (29)) ON S_W_ID IN (1, 2, 3) USING SUM(S_QUANTITY) GROUP BY S_I_ID) p;

benchmark/queries/tpcc/query_2122.sql

@@ -1,2 +1,2 @@
--- {"operators": "POSITIONAL_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM"}
+-- {"operators": "POSITIONAL_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM", "non_incr_reason": "join:POSITIONAL"}
 SELECT s.S_W_ID, s.S_I_ID, i.I_ID, i.I_NAME FROM STOCK s POSITIONAL JOIN ITEM i;

benchmark/queries/tpcc/query_2123.sql

@@ -1,2 +1,2 @@
--- {"operators": "POSITIONAL_JOIN,AGGREGATE", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM"}
+-- {"operators": "POSITIONAL_JOIN,AGGREGATE", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM", "non_incr_reason": "join:POSITIONAL"}
 SELECT s.S_W_ID, COUNT(i.I_ID) AS paired_items, SUM(s.S_QUANTITY) AS paired_qty FROM STOCK s POSITIONAL JOIN ITEM i GROUP BY s.S_W_ID;

benchmark/queries/tpcc/query_2124.sql

@@ -1,2 +1,2 @@
--- {"operators": "POSITIONAL_JOIN,WINDOW", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "CUSTOMER,OORDER", "openivm_verified": true}
+-- {"operators": "POSITIONAL_JOIN,WINDOW", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "CUSTOMER,OORDER", "openivm_verified": true, "non_incr_reason": "join:POSITIONAL"}
 SELECT c.C_W_ID, c.C_ID, o.O_ID, ROW_NUMBER() OVER (ORDER BY c.C_W_ID, c.C_ID) AS pos_rank FROM CUSTOMER c POSITIONAL JOIN OORDER o;

benchmark/queries/tpcc/query_2125.sql

@@ -1,2 +1,2 @@
--- {"operators": "POSITIONAL_JOIN,LEFT_JOIN", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM,ORDER_LINE"}
+-- {"operators": "POSITIONAL_JOIN,LEFT_JOIN", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM,ORDER_LINE", "non_incr_reason": "join:POSITIONAL"}
 SELECT s.S_W_ID, s.S_I_ID, i.I_NAME, ol.OL_AMOUNT FROM STOCK s POSITIONAL JOIN ITEM i LEFT JOIN ORDER_LINE ol ON ol.OL_I_ID = i.I_ID AND ol.OL_SUPPLY_W_ID = s.S_W_ID;

benchmark/queries/tpcc/query_2126.sql

@@ -1,2 +1,2 @@
--- {"operators": "POSITIONAL_JOIN,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": true, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM"}
+-- {"operators": "POSITIONAL_JOIN,UNNEST,TABLE_FUNCTION", "complexity": "high", "is_incremental": false, "has_nulls": false, "has_cast": false, "has_case": false, "tables": "STOCK,ITEM", "non_incr_reason": "join:POSITIONAL"}
 SELECT s.S_W_ID, i.I_ID, u.label FROM STOCK s POSITIONAL JOIN ITEM i CROSS JOIN UNNEST(['left_pos', 'right_pos']) AS u(label);