diff --git a/antalya/docs/design/alter-table-export-part-partition.md b/antalya/docs/design/alter-table-export-part-partition.md index d513e4b80e41..600bcd440995 100644 --- a/antalya/docs/design/alter-table-export-part-partition.md +++ b/antalya/docs/design/alter-table-export-part-partition.md @@ -171,10 +171,11 @@ The design should address the following topics in the near future. - `tests/queries/0_stateless/03572_export_merge_tree_part_special_columns.sh` - `tests/queries/0_stateless/03572_export_replicated_merge_tree_part_to_object_storage.sh` - `tests/queries/0_stateless/03572_export_replicated_merge_tree_part_to_object_storage_simple.sql` -- `tests/queries/0_stateless/03604_export_merge_tree_partition.sh` +- `tests/queries/0_stateless/05027_export_partition_merge_tree.sh` +- `tests/queries/0_stateless/05028_export_partition_replicated_merge_tree.sh` - `tests/queries/0_stateless/03608_export_merge_tree_part_filename_pattern.sh` - `tests/integration/test_export_merge_tree_part_to_object_storage/test.py` -- `tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py` +- `tests/integration/test_export_partition_to_object_storage/test.py` --- @@ -619,8 +620,9 @@ path — a REST / Glue fixture. The commit-file test above only exercises plain object-storage atomicity. Do not add `no-parallel` to any new test unless explicitly required by shared S3 bucket paths; -`03604` currently has the tag and should be re-examined to see whether unique per-run paths -remove the need. +the replicated wrappers (`05028`, `05030`) carry the tag inherited from the test they replaced, +and should be re-examined: the shared bodies now derive every table and S3 filename from +`$CLICKHOUSE_DATABASE`, so the per-run paths are already unique. ### Integration tests — `tests/integration` @@ -628,19 +630,25 @@ remove the need. - `test_export_merge_tree_part_to_object_storage/` — part export in a multi-node setup. PR 1618 makes minor adjustments. -- `test_export_replicated_mt_partition_to_object_storage/` — partition export across - replicas, including `wait_for_export_status`, retry counting, and replica failure +- `test_export_partition_to_object_storage/` — partition export to a plain object-storage + destination, including `wait_for_export_status`, retry counting, and replica failure scenarios. PR 1618 removes the `s3_retries.xml` config and reshapes several test cases - against the new shared helpers. + against the new shared helpers. The scenarios that do not depend on cross-replica + coordination request the `source_engine` fixture and run once per MergeTree flavour; the + rest stay replicated-only. **New in PR 1618:** - `test_export_merge_tree_part_to_iceberg/` — per-part export to an Iceberg destination, covering golden path, sidecar emission, manifest shape, and error paths. -- `test_export_replicated_mt_partition_to_iceberg/` — distributed partition export to - Iceberg across replicas, including `test_export_task_timeout_kills_stuck_pending_task` - (uses the `export_partition_commit_always_throw` failpoint to exhaust the commit path, - then asserts the timeout transitions the task to `KILLED`). +- `test_export_partition_to_iceberg/` — partition export to Iceberg, including + `test_export_task_timeout_kills_stuck_pending_task` (uses the + `export_partition_commit_always_throw` failpoint to exhaust the commit path, then asserts + the timeout transitions the task to `KILLED`). Parametrized over the source engine on the + same terms as the object-storage suite. +- `test_export_mt_partition_to_object_storage/` — the plain-`MergeTree` behavior the unified + suites cannot express: a Keeper-free cluster (proving no ensemble is needed) and + restart-resume from the on-disk task descriptor. - `test_storage_iceberg_with_spark/test_export_partition_iceberg.py` — catalog-less Iceberg round-trip; Spark reads ClickHouse-written data and verifies schema, partition layout, and snapshot atomicity. @@ -653,6 +661,28 @@ remove the need. - `tests/integration/helpers/iceberg_export_stats.py` — sidecar decoders and stats assertion helpers. +**Known divergence between the engines:** + +- Dispatch-time destination failures do not fail a `Replicated*MergeTree` task. When the + destination is dropped or recreated with an incompatible schema after the export was + scheduled but before the part task is dispatched, the plain scheduler fails the task, while + the replicated path leaves it `PENDING` until + `export_merge_tree_partition_task_timeout_seconds` (a day by default). For the dropped + destination the cause is that `UNKNOWN_TABLE` is missing from + `ExportPartitionUtils::isNonRetryableExportError`, so the failure counts as retryable. The + schema-mismatch case is not explained by that, since `INCOMPATIBLE_COLUMNS` is in the + non-retryable set — the replicated dispatch appears not to reach the classification. + `test_export_partition_to_object_storage/test_failures.py::test_dispatch_fails_when_destination_dropped` + and `::test_dispatch_fails_when_destination_schema_incompatible` cover the plain path only; + parametrize them over `source_engine` once the replicated path fails the task too. +- The commit failpoint `export_partition_commit_always_throw` only exists in + `ExportPartitionUtils::commit`, the ZooKeeper-coordinated commit routine. A plain `MergeTree` + commits through `MergeTreePartitionExportScheduler::tryCommit`, so its commit-failure handling + (retry, `FAILED` transition, timeout kill) has no failpoint coverage. +- An `IN PARTITION`-scoped mutation blocks the export of every other partition on a plain + `MergeTree`, because its mutations snapshot is not partition-scoped. See "Pending mutations" + under Plain (non-replicated) MergeTree in `docs/en/antalya/partition_export.md`. + **Remaining gaps to add:** - Initiating-replica dies mid-commit (post-data-file-write, pre-catalog-CAS) — asserts a @@ -668,7 +698,7 @@ remove the need. `EXPORT PARTITION` or the new Keeper manifest fields. Invocation: -`python -m ci.praktika run "integration" --test test_export_merge_tree_part_to_object_storage,test_export_replicated_mt_partition_to_object_storage,test_export_merge_tree_part_to_iceberg,test_export_replicated_mt_partition_to_iceberg,test_storage_iceberg_with_spark`. +`python -m ci.praktika run "integration" --test test_export_merge_tree_part_to_object_storage test_export_partition_to_object_storage test_export_merge_tree_part_to_iceberg test_export_partition_to_iceberg test_export_mt_partition_to_object_storage test_storage_iceberg_with_spark`. ### Failpoints diff --git a/ci/jobs/scripts/integration_tests_configs.py b/ci/jobs/scripts/integration_tests_configs.py index 673a86f0bd72..586df87d9a2a 100644 --- a/ci/jobs/scripts/integration_tests_configs.py +++ b/ci/jobs/scripts/integration_tests_configs.py @@ -58,7 +58,19 @@ class TC: True, "pins azurite to fixed host port 10000 (Spark emulator mode); concurrent --dist=each workers collide on bind", ), - TC("test_export_replicated_mt_partition_to_object_storage/", True, "ZooKeeper can't handle too many parallel requests"), + # The rest of test_export_partition_to_object_storage/ (validation, lifecycle) and all of + # test_export_partition_to_iceberg/ are parallel-safe: each xdist worker gets its own cluster, + # and those modules only create tables and assert on synchronous rejects. + TC( + "test_export_partition_to_object_storage/test_failures.py", + True, + "paces itself against retry back-off and scheduler ticks while object storage is cut off; host load makes the timings flaky", + ), + TC( + "test_export_partition_to_object_storage/test_replication.py", + True, + "ZooKeeper can't handle too many parallel requests", + ), ] IMAGES_ENV = { diff --git a/docs/en/antalya/partition_export.md b/docs/en/antalya/partition_export.md index 8522a7cbb036..1ef7ba27e81f 100644 --- a/docs/en/antalya/partition_export.md +++ b/docs/en/antalya/partition_export.md @@ -2,15 +2,20 @@ ## Overview -The `ALTER TABLE EXPORT PARTITION` command exports entire partitions from Replicated*MergeTree tables to object storage (S3, Azure Blob Storage, etc.) or data lakes like Apache Iceberg tables (with and without catalogs), typically in Parquet format. This feature coordinates export part operations across all replicas using ZooKeeper. +The `ALTER TABLE EXPORT PARTITION` command exports entire partitions from `MergeTree`-family tables to object storage (S3, Azure Blob Storage, etc.) or data lakes like Apache Iceberg tables (with and without catalogs), typically in Parquet format. -The set of parts that are exported is based on the list of parts the replica that received the export command sees. The other replicas will assist in the export process if they have those parts locally. Otherwise they will ignore it. +- On `Replicated*MergeTree` tables the export is coordinated across all replicas using ZooKeeper. +- On plain (non-replicated) `MergeTree` tables the export runs entirely on the single node that received the command. No ZooKeeper / `clickhouse-keeper` ensemble is required. See [Plain (non-replicated) MergeTree](#plain-non-replicated-mergetree) below. -The partition export tasks can be observed through `system.replicated_partition_exports`. The table is served from each replica's in-memory mirror, so queries do not contact ZooKeeper and are cheap to run. The mirror is refreshed on the manifest-updater poll cycle and on every status change, so a freshly written exception or terminal state may take up to one poll interval to appear. Individual part export progress can be observed as usual through `system.exports`. +The set of parts that are exported is based on the list of parts the replica that received the export command sees. On `Replicated*MergeTree`, the other replicas will assist in the export process if they have those parts locally. Otherwise they will ignore it. + +The partition export tasks of both engines can be observed through `system.partition_exports`. The table is served from an in-memory mirror, so queries do not contact ZooKeeper or disk and are cheap to run. For a `Replicated*MergeTree` source the mirror is refreshed on the manifest-updater poll cycle and on every status change, so a freshly written exception or terminal state may take up to one poll interval to appear; for a plain `MergeTree` source it is updated synchronously with every state change. Individual part export progress can be observed as usual through `system.exports`. + +`system.replicated_partition_exports` is kept as an alias of `system.partition_exports` for backwards compatibility. It returns exactly the same rows, including exports of plain `MergeTree` tables. Filter on `source_table` (or join against `system.tables`) if you need only one engine. The same partition can not be exported to the same destination more than once. There are two ways to override this behavior: either by setting the `export_merge_tree_partition_force_export` setting or waiting for the task to expire. -The export task can be killed by issuing the kill command: `KILL EXPORT PARTITION `. +The export task can be killed by issuing the kill command: `KILL EXPORT PARTITION `. The task is persistent - it should be resumed after crashes, failures and etc. @@ -35,6 +40,19 @@ The source partition must not be split in the destination. This is validated at Each MergeTree part will become a separate file with the following name convention: `//_.`. To ensure atomicity, a commit file containing the relative paths of all exported parts is also shipped. A data file should only be considered part of the dataset if a commit file references it. The commit file will be named using the following convention: `/commit__`. +## Plain (non-replicated) MergeTree {#plain-non-replicated-mergetree} + +The command, its settings, the partition-key compatibility rules and the destination file layout are the same for both engines. Only the coordination differs: + +- There is no ZooKeeper coordination and no `clickhouse-keeper` ensemble is needed. The node that received the command persists the task descriptor on the source table's disk and a local background scheduler drives it to completion. +- The task is still persistent: a node that is restarted (or killed) mid-export resumes the export from the on-disk descriptor. +- There are no other replicas to assist, so a single node exports every part of the partition. `source_replica` is empty in `system.partition_exports` and `last_exception_per_replica` holds at most one entry, whose `replica` is empty. +- The commit-path columns (`committed_metadata_file`, `committed_manifest_list`, `committed_manifest_file`, `committed_marker_file`) are always empty, because a plain `MergeTree` does not persist the commit paths. See [Columns that depend on the source engine](#columns-that-depend-on-the-source-engine). + +### Pending mutations {#plain-merge-tree-pending-mutations} + +The pending-mutation gate is more conservative than on a `Replicated*MergeTree`. A plain `MergeTree` does not scope its mutation snapshot by partition, so a mutation restricted with `IN PARTITION` still marks the parts of every other partition as having pending mutations, and exporting an unaffected partition is refused with `PENDING_MUTATIONS_NOT_ALLOWED`. The gate fails closed - it never exports data that a pending mutation would have changed - so the effect is that you may have to wait for an unrelated mutation to finish, or set `export_merge_tree_part_throw_on_pending_mutations` to `false`. + ## Syntax ```sql @@ -66,7 +84,7 @@ TO TABLE [destination_database.]destination_table - **Type**: `Bool` - **Default**: `false` -- **Description**: Enable export replicated merge tree partition feature. It is experimental and not yet ready for production use. +- **Description**: Enable the `EXPORT PARTITION` feature for both `Replicated*MergeTree` and plain `MergeTree` tables. It is experimental and not yet ready for production use. ### Query Settings @@ -183,19 +201,19 @@ WHERE partition_id = '2020' AND destination_table = 's3_table' ``` -The `WHERE` clause filters exports from the `system.replicated_partition_exports` table. You can use any columns from that table in the filter. +The `WHERE` clause filters exports from the `system.partition_exports` table, which covers both `Replicated*MergeTree` and plain `MergeTree` sources, so a single `KILL EXPORT PARTITION` targets either engine. You can use any column of that table in the filter (for example `partition_id`, `source_table`, `destination_table`). ## Monitoring ### Active and Completed Exports -Monitor partition exports using the `system.replicated_partition_exports` table: +Monitor partition exports using the `system.partition_exports` table: ```sql -arthur :) select * from system.replicated_partition_exports Format Vertical; +arthur :) select * from system.partition_exports Format Vertical; SELECT * -FROM system.replicated_partition_exports +FROM system.partition_exports FORMAT Vertical Query id: 9efc271a-a501-44d1-834f-bc4d20156164 @@ -259,28 +277,38 @@ Status values include: ### Exception columns -- `last_exception_per_replica` is an `Array(Tuple(replica String, message String, part String, time DateTime, count UInt64))`. Each tuple is the most recent exception observed by a single replica plus a best-effort within-replica `count`. Replicas that have never reported an exception are omitted. +- `last_exception_per_replica` is an `Array(Tuple(replica String, message String, part String, time DateTime, count UInt64))`. Each tuple is the most recent exception observed by a single replica plus a best-effort within-replica `count`. Replicas that have never reported an exception are omitted. A plain `MergeTree` export runs on a single node, so it contributes at most one tuple and its `replica` is empty. - `exception_count` is the sum of every `count` in `last_exception_per_replica`. Each replica owns its own counter, so cross-replica updates do not race; the sum is exact w.r.t. the snapshot returned. Within a single replica concurrent failing writers may under-count by one. ### Per-part destination file paths -- `destination_file_paths` is a `Map(String, Array(String))` keyed by source part name. Each value is the list of file paths written to the destination object storage when that part was exported (a single part can produce multiple files depending on `max_bytes` / `max_rows`). If a refresh cannot read a processed entry from ZooKeeper, the affected key holds the sentinel `` instead of silently under-counting. +- `destination_file_paths` is a `Map(String, Array(String))` keyed by source part name. Each value is the list of file paths written to the destination object storage when that part was exported (a single part can produce multiple files depending on `max_bytes` / `max_rows`). On a `Replicated*MergeTree` source, if a refresh cannot read a processed entry from ZooKeeper, the affected key holds the sentinel `` instead of silently under-counting. ### Commit info columns -These columns surface paths produced by the destination storage during commit, so it is possible to inspect what was written without consulting the destination directly: +These columns surface paths produced by the destination storage during commit, so it is possible to inspect what was written without consulting the destination directly. They are populated for `Replicated*MergeTree` sources only; a plain `MergeTree` does not persist the commit paths, so they stay empty there even after a successful commit. - `committed_metadata_file` — for Iceberg destinations: path of the new `vN.metadata.json` written by the commit. Empty for non-Iceberg destinations and before the commit lands. If the commit was already finished by a previous run (detected via the transaction id stored in the snapshot summary), this column carries a human-readable sentinel string instead of a path because the original committer's paths are not recoverable from inside the impl. - `committed_manifest_list` — for Iceberg destinations: path of the manifest list file (`snap-*.avro`) referenced by the new snapshot. Empty under the same conditions as `committed_metadata_file`. - `committed_manifest_file` — for Iceberg destinations: path of the manifest file referenced by `committed_manifest_list`. Empty under the same conditions as `committed_metadata_file`. - `committed_marker_file` — for plain object storage destinations: path of the per-transaction commit marker file written by the destination. Empty for Iceberg destinations and for tasks that have not committed yet. +### Columns that depend on the source engine {#columns-that-depend-on-the-source-engine} + +Rows for plain `MergeTree` sources share the schema with replicated ones, and leave the columns that only make sense with cross-replica coordination empty: + +- `source_replica` — empty, since there is a single node. +- `last_exception_per_replica` — at most one tuple, with an empty `replica`. +- `committed_metadata_file`, `committed_manifest_list`, `committed_manifest_file`, `committed_marker_file` — always empty. + +`local_backoff_per_part` is local to the node answering the query for both engines. + To pick the latest exception across replicas: ```sql SELECT arraySort(x -> -x.time, last_exception_per_replica)[1] AS latest_exception -FROM system.replicated_partition_exports +FROM system.partition_exports WHERE source_table = 'rmt_table' AND destination_table = 's3_table'; ``` diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 76c96586705c..6824036e4199 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -267,7 +267,8 @@ BlockIO InterpreterKillQueryQuery::execute() Block exports_block = getSelectResult( "source_database, source_table, transaction_id, destination_database, destination_table, partition_id", - "system.replicated_partition_exports"); + "system.partition_exports"); + if (exports_block.empty()) return res_io; diff --git a/src/Storages/ExportReplicatedMergeTreePartitionTaskEntry.h b/src/Storages/ExportReplicatedMergeTreePartitionTaskEntry.h index 36c0ef303fbf..bacdcc92ae54 100644 --- a/src/Storages/ExportReplicatedMergeTreePartitionTaskEntry.h +++ b/src/Storages/ExportReplicatedMergeTreePartitionTaskEntry.h @@ -3,8 +3,8 @@ #include #include #include +#include #include -#include "Core/QualifiedTableName.h" #include #include #include @@ -60,8 +60,8 @@ struct ExportReplicatedMergeTreePartitionTaskEntry std::string getCompositeKey() const { - const auto qualified_table_name = QualifiedTableName {manifest.destination_database, manifest.destination_table}; - return manifest.partition_id + "_" + qualified_table_name.getFullName(); + return ExportPartitionUtils::compositeKey( + manifest.partition_id, manifest.destination_database, manifest.destination_table); } std::string getTransactionId() const diff --git a/src/Storages/MergeTree/ExportPartitionKey.h b/src/Storages/MergeTree/ExportPartitionKey.h new file mode 100644 index 000000000000..27466f279963 --- /dev/null +++ b/src/Storages/MergeTree/ExportPartitionKey.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +namespace DB +{ + +namespace ExportPartitionUtils +{ + +/// Identifies a partition export task by its (partition, destination) triple. Both MergeTree +/// flavours key their task registry on it, and both use it to name the node holding the task +/// descriptor: a ZooKeeper child of the table's `exports` path for a `Replicated*MergeTree`, a +/// file under the table's export directory for a plain `MergeTree`. So it has to be injective +/// and safe to use as a path component. +/// +/// `escapeForFileName` percent-encodes every character that is not alphanumeric or `_`, so an +/// escaped component can never contain the `.` used here as a separator. Note that `_` survives +/// escaping and therefore cannot be used as a separator: without the escaping, destinations whose +/// qualified names flatten to the same string (`` `db.x`.`y` `` and `` `db`.`x.y` ``) would share +/// a key, and the second export of a partition would be rejected as a duplicate of the first. +inline String compositeKey( + const String & partition_id, const String & destination_database, const String & destination_table) +{ + return escapeForFileName(partition_id) + "." + + escapeForFileName(destination_database) + "." + + escapeForFileName(destination_table); +} + +} + +} diff --git a/src/Storages/MergeTree/ExportPartitionManifestUpdatingTask.cpp b/src/Storages/MergeTree/ExportPartitionManifestUpdatingTask.cpp index 077f06349c5d..8a82a729d4fe 100644 --- a/src/Storages/MergeTree/ExportPartitionManifestUpdatingTask.cpp +++ b/src/Storages/MergeTree/ExportPartitionManifestUpdatingTask.cpp @@ -214,6 +214,19 @@ namespace return false; } + size_t calculatePartsToDo( + size_t number_of_parts, + const std::map> & destination_file_paths_per_part) + { + /// A failed listing publishes the marker as a key and says nothing about progress, while a + /// marker as a value only means that leaf's paths were unreadable - the leaf exists, so + /// that part is done and still counts. + if (destination_file_paths_per_part.contains(String(zk_sync_failed_marker))) + return number_of_parts; + + return number_of_parts - destination_file_paths_per_part.size(); + } + bool skipReadingDestinationFilePaths( ExportReplicatedMergeTreePartitionTaskEntry::Status status, const std::map> & cached_paths, @@ -389,7 +402,7 @@ ExportPartitionManifestUpdatingTask::ExportPartitionManifestUpdatingTask(Storage { } -std::vector ExportPartitionManifestUpdatingTask::getPartitionExportsInfo() const +std::vector ExportPartitionManifestUpdatingTask::getPartitionExportsInfo() const { const auto model = storage.export_partition_manifests.get(); @@ -398,14 +411,14 @@ std::vector ExportPartitionManifestUpdatingTask:: const auto backoff = storage.export_merge_tree_partition_task_scheduler->getLocalBackoffSnapshot(); - std::vector infos; + std::vector infos; infos.reserve(model->size()); for (const auto & entry : model->get()) { const auto & manifest = entry.manifest; - ReplicatedPartitionExportInfo info; + PartitionExportInfo info; info.destination_database = manifest.destination_database; info.destination_table = manifest.destination_table; @@ -415,7 +428,7 @@ std::vector ExportPartitionManifestUpdatingTask:: info.create_time = manifest.create_time; info.source_replica = manifest.source_replica; info.parts_count = manifest.number_of_parts; - info.parts_to_do = manifest.parts.size(); + info.parts_to_do = calculatePartsToDo(manifest.number_of_parts, entry.destination_file_paths_per_part); info.parts = manifest.parts; info.status = magic_enum::enum_name(entry.status); @@ -424,7 +437,7 @@ std::vector ExportPartitionManifestUpdatingTask:: for (const auto & [_, ex] : entry.last_exception_per_replica) { total_exception_count += ex.count; - info.last_exception_per_replica.push_back(ex); + info.last_exception_per_replica.push_back({ex.replica, ex.message, ex.part, ex.time, ex.count}); } info.exception_count = total_exception_count; diff --git a/src/Storages/MergeTree/ExportPartitionManifestUpdatingTask.h b/src/Storages/MergeTree/ExportPartitionManifestUpdatingTask.h index 129629f6ed9b..2c26fd6afb3e 100644 --- a/src/Storages/MergeTree/ExportPartitionManifestUpdatingTask.h +++ b/src/Storages/MergeTree/ExportPartitionManifestUpdatingTask.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include namespace DB { @@ -25,7 +25,7 @@ class ExportPartitionManifestUpdatingTask /// Returns a snapshot of every replicated partition export task tracked by this /// replica's in-memory mirror. No ZooKeeper traffic; safe to call from query threads. - std::vector getPartitionExportsInfo() const; + std::vector getPartitionExportsInfo() const; private: StorageReplicatedMergeTree & storage; diff --git a/src/Storages/MergeTree/ExportPartitionTaskScheduler.cpp b/src/Storages/MergeTree/ExportPartitionTaskScheduler.cpp index de271bc01ec7..f491c565fad1 100644 --- a/src/Storages/MergeTree/ExportPartitionTaskScheduler.cpp +++ b/src/Storages/MergeTree/ExportPartitionTaskScheduler.cpp @@ -12,6 +12,7 @@ #include "Storages/MergeTree/MergeTreePartExportManifest.h" #include "Formats/FormatFactory.h" #include +#include #include namespace ProfileEvents @@ -41,34 +42,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -namespace -{ - /// Capped exponential back-off, matching the standard ClickHouse convention - /// (see ZooKeeperRetriesControl): delay = min(initial << (retry_count - 1), max). - /// `retry_count` is the number of failures so far (>= 1 when a retry is pending). - /// The shift is guarded against overflow by saturating to `max_backoff_seconds`. - size_t computeRetryBackoffSeconds(size_t retry_count, size_t initial_backoff_seconds, size_t max_backoff_seconds) - { - const size_t initial = std::min(initial_backoff_seconds, max_backoff_seconds); - - if (retry_count <= 1 || initial == 0) - return initial; - - const size_t shift = retry_count - 1; - - /// If shifting would overflow size_t, the result is certainly clamped to the cap. - static constexpr size_t bits = sizeof(size_t) * 8; - if (shift >= bits) - return max_backoff_seconds; - - const size_t headroom = std::numeric_limits::max() >> shift; - if (initial > headroom) - return max_backoff_seconds; - - return std::min(initial << shift, max_backoff_seconds); - } -} - ExportPartitionTaskScheduler::ExportPartitionTaskScheduler(StorageReplicatedMergeTree & storage_) : storage(storage_) { @@ -318,7 +291,7 @@ time_t ExportPartitionTaskScheduler::registerLocalBackoff( auto & backoff = parts.try_emplace(part_name).first->second; ++backoff.attempts; - const auto backoff_seconds = computeRetryBackoffSeconds( + const auto backoff_seconds = ExportPartitionUtils::computeRetryBackoffSeconds( backoff.attempts, manifest.retry_initial_backoff_seconds, manifest.retry_max_backoff_seconds); const auto now = time(nullptr); /// Clamp so a huge configured back-off cannot overflow time_t (now is a normal wall-clock value). diff --git a/src/Storages/MergeTree/ExportPartitionUtils.cpp b/src/Storages/MergeTree/ExportPartitionUtils.cpp index be172a927aa8..57997c4ee0dc 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.cpp +++ b/src/Storages/MergeTree/ExportPartitionUtils.cpp @@ -7,7 +7,12 @@ #include "Storages/ExportReplicatedMergeTreePartitionManifest.h" #include "Storages/ExportReplicatedMergeTreePartitionTaskEntry.h" #include +#include +#include +#include +#include #include +#include #include #include #include @@ -37,7 +42,9 @@ #if USE_AVRO #include +#include #include +#include #endif namespace ProfileEvents @@ -95,6 +102,7 @@ namespace Setting { extern const SettingsBool export_merge_tree_part_allow_lossy_cast; #if USE_AVRO + extern const SettingsBool allow_insert_into_iceberg; extern const SettingsTimezone iceberg_partition_timezone; #endif extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; @@ -155,6 +163,27 @@ namespace ExportPartitionUtils return non_retryable_codes.contains(code); } + size_t computeRetryBackoffSeconds(size_t retry_count, size_t initial_backoff_seconds, size_t max_backoff_seconds) + { + const size_t initial = std::min(initial_backoff_seconds, max_backoff_seconds); + + if (retry_count <= 1 || initial == 0) + return initial; + + const size_t shift = retry_count - 1; + + /// If shifting would overflow size_t, the result is certainly clamped to the cap. + static constexpr size_t bits = sizeof(size_t) * 8; + if (shift >= bits) + return max_backoff_seconds; + + const size_t headroom = std::numeric_limits::max() >> shift; + if (initial > headroom) + return max_backoff_seconds; + + return std::min(initial << shift, max_backoff_seconds); + } + Block getPartitionSourceBlockForIcebergCommit( MergeTreeData & storage, const String & partition_id, const std::vector & exported_part_names) { @@ -203,7 +232,8 @@ namespace ExportPartitionUtils return block; } - ContextPtr getContextCopyWithTaskSettings(const ContextPtr & context, const ExportReplicatedMergeTreePartitionManifest & manifest) + template + ContextPtr getContextCopyWithTaskSettings(const ContextPtr & context, const ManifestT & manifest) { auto context_copy = Context::createCopy(context); context_copy->makeQueryContextForExportPart(); @@ -265,6 +295,93 @@ namespace ExportPartitionUtils return context_copy; } + template ContextPtr getContextCopyWithTaskSettings( + const ContextPtr &, const ExportReplicatedMergeTreePartitionManifest &); + template ContextPtr getContextCopyWithTaskSettings( + const ContextPtr &, const MergeTreePartitionExportTask &); + + std::string extractDestinationIcebergMetadataJson( + const StorageMetadataPtr & source_metadata, + const StorageMetadataPtr & destination_metadata, + const StoragePtr & dest_storage, + const MergeTreeData::DataPartsVector & parts, + const String & partition_id, + const ContextPtr & context) + { + if (!dest_storage->isDataLake()) + { + verifyPlainPartitionCompatibility(source_metadata, destination_metadata, parts, partition_id, context); + return {}; + } + +#if USE_AVRO + auto * object_storage = dynamic_cast(dest_storage.get()); + auto * object_storage_cluster = dynamic_cast(dest_storage.get()); + + /// in theory this should never happen, but just in case + if (!object_storage && !object_storage_cluster) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Destination storage {} is not a StorageObjectStorage", dest_storage->getName()); + + IcebergMetadata * iceberg_metadata = nullptr; + if (object_storage) + iceberg_metadata = dynamic_cast(object_storage->getExternalMetadata(context)); + else if (object_storage_cluster) + iceberg_metadata = dynamic_cast(object_storage_cluster->getExternalMetadata(context)); + if (!iceberg_metadata) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Destination storage {} is a data lake but not an iceberg table", dest_storage->getName()); + + if (!context->getSettingsRef()[Setting::allow_insert_into_iceberg]) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "Iceberg writes are experimental. " + "To allow its usage, enable the setting `allow_insert_into_iceberg` on the initiator (query, session or profile) - replicas inherit it from the scheduled task."); + + const auto metadata_object = iceberg_metadata->getMetadataJSON(context); + + verifyIcebergPartitionCompatibility( + metadata_object, source_metadata, destination_metadata, parts, partition_id, context); + + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + oss.exceptions(std::ios::failbit); + metadata_object->stringify(oss); + return oss.str(); +#else + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Data lake export requires Avro support"); +#endif + } + + IStorage::ExportPartitionCommitInfo commitExportedPaths( + const String & transaction_id, + const String & partition_id, + const String & iceberg_metadata_json, + bool write_full_path_in_iceberg_metadata, + const std::optional & iceberg_partition_timezone, + const std::vector & exported_paths, + const std::vector & exported_part_names, + const StoragePtr & destination_storage, + MergeTreeData & source_storage, + const ContextPtr & context_in) + { + auto context = Context::createCopy(context_in); + context->setSetting("write_full_path_in_iceberg_metadata", write_full_path_in_iceberg_metadata); + + if (iceberg_partition_timezone) + context->setSetting("iceberg_partition_timezone", *iceberg_partition_timezone); + + IStorage::IcebergCommitExportPartitionArguments iceberg_args; + + if (!iceberg_metadata_json.empty()) + { + iceberg_args.metadata_json_string = iceberg_metadata_json; + const auto source_metadata = source_storage.getInMemoryMetadataPtr(context, false); + if (source_metadata->hasPartitionKey()) + iceberg_args.partition_source_block = + getPartitionSourceBlockForIcebergCommit(source_storage, partition_id, exported_part_names); + } + + return destination_storage->commitExportPartitionTransaction( + transaction_id, partition_id, exported_paths, iceberg_args, context); + } + /// Collect all the exported paths from the processed parts /// If multiRead is supported by the keeper implementation, it is done in a single request /// Otherwise, multiple async requests are sent @@ -331,12 +448,6 @@ namespace ExportPartitionUtils MergeTreeData & source_storage, const String & replica_name) { - auto context = Context::createCopy(context_in); - context->setSetting("write_full_path_in_iceberg_metadata", manifest.write_full_path_in_iceberg_metadata); - - if (manifest.iceberg_partition_timezone) - context->setSetting("iceberg_partition_timezone", *manifest.iceberg_partition_timezone); - /// Failpoint used by integration tests to force persistent commit failure and exercise /// the commit-attempts budget / FAILED state transition. fiu_do_on(FailPoints::export_partition_commit_always_throw, @@ -383,19 +494,17 @@ namespace ExportPartitionUtils throw Exception(ErrorCodes::CORRUPTED_DATA, "ExportPartition: Reached the commit phase, but exported paths size is less than the number of parts, will not commit export. This might be a bug"); } - IStorage::IcebergCommitExportPartitionArguments iceberg_args; - - if (!manifest.iceberg_metadata_json.empty()) - { - iceberg_args.metadata_json_string = manifest.iceberg_metadata_json; - const auto source_metadata = source_storage.getInMemoryMetadataPtr(context, false); - if (source_metadata->hasPartitionKey()) - iceberg_args.partition_source_block = - getPartitionSourceBlockForIcebergCommit(source_storage, manifest.partition_id, manifest.parts); - } - - const auto destination_commit_info = destination_storage->commitExportPartitionTransaction( - manifest.transaction_id, manifest.partition_id, exported_paths, iceberg_args, context); + const auto destination_commit_info = commitExportedPaths( + manifest.transaction_id, + manifest.partition_id, + manifest.iceberg_metadata_json, + manifest.write_full_path_in_iceberg_metadata, + manifest.iceberg_partition_timezone, + exported_paths, + manifest.parts, + destination_storage, + source_storage, + context_in); /// Failpoint to simulate a crash after the Iceberg commit succeeds but before /// ZooKeeper is updated to COMPLETED. Used by idempotency integration tests. diff --git a/src/Storages/MergeTree/ExportPartitionUtils.h b/src/Storages/MergeTree/ExportPartitionUtils.h index cf5493cae675..14e954859794 100644 --- a/src/Storages/MergeTree/ExportPartitionUtils.h +++ b/src/Storages/MergeTree/ExportPartitionUtils.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -28,9 +29,53 @@ namespace ExportPartitionUtils { bool isNonRetryableExportError(int code); + /// Capped exponential back-off, matching the standard ClickHouse convention + /// (see ZooKeeperRetriesControl): delay = min(initial << (retry_count - 1), max). + /// `retry_count` is the number of failures so far (>= 1 when a retry is pending). + size_t computeRetryBackoffSeconds(size_t retry_count, size_t initial_backoff_seconds, size_t max_backoff_seconds); + std::vector getExportedPaths(const LoggerPtr & log, const zkutil::ZooKeeperPtr & zk, const std::string & export_path); - ContextPtr getContextCopyWithTaskSettings(const ContextPtr & context, const ExportReplicatedMergeTreePartitionManifest & manifest); + /// Build a query context carrying the export task's persisted settings. Templated on the + /// descriptor type so it serves both the replicated manifest (backed by ZooKeeper) and the + /// plain `MergeTreePartitionExportTask` (backed by disk); both expose the same setting fields. + template + ContextPtr getContextCopyWithTaskSettings(const ContextPtr & context, const ManifestT & manifest); + + /// Validates that `dest_storage` is a legal export target for the specific partition being + /// exported and, for Iceberg destinations, returns the serialized destination `metadata.json` + /// to persist in the task descriptor; returns an empty string for non-data-lake destinations. + /// + /// For data lakes this resolves the destination `IcebergMetadata`, gates on + /// `allow_insert_into_iceberg` and runs `verifyIcebergPartitionCompatibility`. For every other + /// destination it runs `verifyPlainPartitionCompatibility`. Because both checks fold the + /// exported parts' min/max index, this must be called after the part list is collected; the + /// destination checks that do not depend on the partition (self-export, `supportsImport`, + /// `verifyExportSchemaCastable`) are done by the caller up-front. Throws on any incompatibility. + std::string extractDestinationIcebergMetadataJson( + const StorageMetadataPtr & source_metadata, + const StorageMetadataPtr & destination_metadata, + const StoragePtr & dest_storage, + const MergeTreeData::DataPartsVector & parts, + const String & partition_id, + const ContextPtr & context); + + /// ZooKeeper-free commit core shared by the replicated and plain partition-export paths: + /// assembles the Iceberg commit arguments (deriving the partition source block from the + /// exported parts when the destination is a data lake with a partition key) and invokes + /// `commitExportPartitionTransaction` on the destination storage, returning its commit info so + /// the caller can persist it. + IStorage::ExportPartitionCommitInfo commitExportedPaths( + const String & transaction_id, + const String & partition_id, + const String & iceberg_metadata_json, + bool write_full_path_in_iceberg_metadata, + const std::optional & iceberg_partition_timezone, + const std::vector & exported_paths, + const std::vector & exported_part_names, + const StoragePtr & destination_storage, + MergeTreeData & source_storage, + const ContextPtr & context); /// Get the min/max values from the partition expression columns Block getPartitionSourceBlockForIcebergCommit( diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index d9baad80143d..a9e1b556f591 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -1111,6 +1112,10 @@ class MergeTreeData : public WithMutableContext, public IStorage, public IBackgr throw Exception(ErrorCodes::NOT_IMPLEMENTED, "EXPORT PARTITION is not implemented for engine {}", getName()); } + /// Snapshot of this table's partition-export tasks for `system.partition_exports`, taken from + /// an in-memory mirror: no disk or ZooKeeper I/O, so it is safe to call from query threads. + virtual std::vector getPartitionExportsInfo() const { return {}; } + /// Checks that Partition could be dropped right now /// Otherwise - throws an exception with detailed information. /// We do not use mutex because it is not very important that the size could change during the operation. diff --git a/src/Storages/MergeTree/MergeTreePartitionExportScheduler.cpp b/src/Storages/MergeTree/MergeTreePartitionExportScheduler.cpp new file mode 100644 index 000000000000..eb919f4cce29 --- /dev/null +++ b/src/Storages/MergeTree/MergeTreePartitionExportScheduler.cpp @@ -0,0 +1,732 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace fs = std::filesystem; + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int EXPORT_PARTITION_ALREADY_EXPORTED; + extern const int CORRUPTED_DATA; + extern const int QUERY_WAS_CANCELLED; + extern const int UNKNOWN_TABLE; + extern const int UNKNOWN_EXCEPTION; +} + +MergeTreePartitionExportScheduler::MergeTreePartitionExportScheduler(StorageMergeTree & storage_) + : storage(storage_) +{ +} + +String MergeTreePartitionExportScheduler::describeKey(const MergeTreePartitionExportTask & descriptor) +{ + return fmt::format( + "{} -> {}.{}", + descriptor.partition_id, + backQuoteIfNeed(descriptor.destination_database), + backQuoteIfNeed(descriptor.destination_table)); +} + +String MergeTreePartitionExportScheduler::getExportsRelativePath() const +{ + return fs::path(storage.getRelativeDataPath()) / "partition_exports"; +} + +String MergeTreePartitionExportScheduler::descriptorRelativePath(const String & composite_key) const +{ + return fs::path(getExportsRelativePath()) / (sipHash128String(composite_key) + ".json"); +} + +std::map::iterator +MergeTreePartitionExportScheduler::findByTransactionId(const String & transaction_id) +{ + for (auto it = tasks.begin(); it != tasks.end(); ++it) + if (it->second.getDescriptor().transaction_id == transaction_id) + return it; + return tasks.end(); +} + +void MergeTreePartitionExportScheduler::addTask( + MergeTreePartitionExportTask descriptor, std::vector part_references, bool force) +{ + const auto composite_key = ExportPartitionUtils::compositeKey(descriptor.partition_id, descriptor.destination_database, descriptor.destination_table); + /// Rendered before `descriptor` is moved into the entry below. + const auto key_description = describeKey(descriptor); + const auto transaction_id = descriptor.transaction_id; + + String previous_transaction_id; + + { + std::lock_guard lock(mutex); + + if (const auto it = tasks.find(composite_key); it != tasks.end()) + { + if (!force) + throw Exception(ErrorCodes::EXPORT_PARTITION_ALREADY_EXPORTED, + "Export with key {} already exported or it is being exported. " + "Set `export_merge_tree_partition_force_export` to overwrite it.", + key_description); + + previous_transaction_id = it->second.getDescriptor().transaction_id; + } + else + { + /// Unreadable entries are not loaded into memory, therefore we need to check the disk as well + const auto path = descriptorRelativePath(composite_key); + if (storage.getDisks().front()->existsFile(path)) + { + if (!force) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "A partition export descriptor for key {} exists on disk at {} but could not be loaded. " + "Inspect the file or set `export_merge_tree_partition_force_export` to overwrite it.", + key_description, path); + + LOG_WARNING(storage.log, + "ExportPartition: overwriting unread descriptor at {} for key {}", path, key_description); + } + } + + TaskEntry entry; + entry.part_references = std::move(part_references); + entry.setDescriptor(std::move(descriptor)); + persist(composite_key, entry.getDescriptor().toJsonString()); + tasks.insert_or_assign(composite_key, std::move(entry)); + } + + if (!previous_transaction_id.empty()) + { + LOG_INFO(storage.log, "ExportPartition: overwriting export with key {}", key_description); + storage.killExportPart(previous_transaction_id); + } + + LOG_INFO(storage.log, "ExportPartition: scheduled export task {} (key {})", transaction_id, key_description); + storage.triggerPartitionExportTask(); +} + +CancellationCode MergeTreePartitionExportScheduler::kill(const String & transaction_id) +{ + /// set the status to killed + { + std::lock_guard lock(mutex); + + auto it = findByTransactionId(transaction_id); + if (it == tasks.end()) + return CancellationCode::NotFound; + + auto & entry = it->second; + + if (entry.getDescriptor().status != MergeTreePartitionExportTask::Status::PENDING) + return CancellationCode::CancelCannotBeSent; + + if (entry.committing) + { + LOG_INFO(storage.log, "ExportPartition: commit in progress for {}, cannot cancel export partition task", transaction_id); + return CancellationCode::CancelCannotBeSent; + } + + auto updated = entry.getDescriptor(); + + updated.status = MergeTreePartitionExportTask::Status::KILLED; + persist(it->first, updated.toJsonString()); + entry.setDescriptor(std::move(updated)); + } + + /// cancel in-flight operations + storage.killExportPart(transaction_id); + + return CancellationCode::CancelSent; +} + +std::vector MergeTreePartitionExportScheduler::getInfo() const +{ + std::vector result; + std::lock_guard lock(mutex); + result.reserve(tasks.size()); + for (const auto & [key, entry] : tasks) + { + const auto & descriptor = entry.getDescriptor(); + PartitionExportInfo info; + info.destination_database = descriptor.destination_database; + info.destination_table = descriptor.destination_table; + info.create_time = descriptor.create_time; + info.partition_id = descriptor.partition_id; + info.transaction_id = descriptor.transaction_id; + info.query_id = descriptor.query_id; + info.parts = descriptor.partNames(); + info.parts_count = descriptor.partsCount(); + info.parts_to_do = descriptor.partsToDo(); + info.status = String(magic_enum::enum_name(descriptor.status)); + + /// A single node exports on its own, so there is at most one "per-replica" entry and it + /// carries an empty replica name. + if (descriptor.last_exception.count > 0) + { + info.last_exception_per_replica.push_back( + {/*replica*/ "", descriptor.last_exception.message, descriptor.last_exception.part, + descriptor.last_exception.time, descriptor.last_exception.count}); + } + info.exception_count = descriptor.last_exception.count; + + for (const auto & part : descriptor.parts) + if (!part.paths_in_destination.empty()) + info.destination_file_paths_per_part.emplace(part.part_name, part.paths_in_destination); + + info.backoff_per_part.reserve(entry.part_backoff.size()); + for (const auto & [part_name, backoff] : entry.part_backoff) + info.backoff_per_part.push_back({part_name, backoff.attempts, backoff.next_retry_time}); + + result.push_back(std::move(info)); + } + return result; +} + +namespace +{ + bool isTimedOut(const MergeTreePartitionExportTask & descriptor, time_t now) + { + return descriptor.task_timeout_seconds > 0 + && descriptor.create_time + static_cast(descriptor.task_timeout_seconds) < now; + } + + /// `UNKNOWN_TABLE` is retryable on the replicated path (another replica, or a later recreate, + /// might restore the destination). For a local export there is no such helper: a missing + /// destination at dispatch or commit cannot succeed by retrying. + bool isNonRetryablePlainExportError(int code) + { + return ExportPartitionUtils::isNonRetryableExportError(code) + || code == ErrorCodes::UNKNOWN_TABLE; + } +} + +bool MergeTreePartitionExportScheduler::tryPersistTimeoutKill(const String & composite_key, TaskEntry & entry, time_t now) +{ + const auto transaction_id = entry.getDescriptor().transaction_id; + const auto timeout_seconds = entry.getDescriptor().task_timeout_seconds; + + auto updated = entry.getDescriptor(); + updated.status = MergeTreePartitionExportTask::Status::KILLED; + updated.last_exception.message = fmt::format( + "Export partition task timed out: exceeded export_merge_tree_partition_task_timeout_seconds={} (created at {}, now {})", + timeout_seconds, entry.getDescriptor().create_time, now); + updated.last_exception.part = ""; + updated.last_exception.time = now; + updated.last_exception.count += 1; + + try + { + persist(composite_key, updated.toJsonString()); + } + catch (...) + { + tryLogCurrentException(storage.log, "ExportPartition: failed to persist timeout kill, will retry"); + return false; + } + + entry.setDescriptor(std::move(updated)); + LOG_WARNING(storage.log, + "ExportPartition: task {} exceeded task_timeout_seconds={}s, transitioned PENDING -> KILLED", + transaction_id, timeout_seconds); + return true; +} + +bool MergeTreePartitionExportScheduler::enforceTimeouts() +{ + std::vector timed_out_transactions; + bool any_pending = false; + + { + std::lock_guard lock(mutex); + const auto now = time(nullptr); + for (auto & [key, entry] : tasks) + { + if (entry.getDescriptor().status != MergeTreePartitionExportTask::Status::PENDING) + continue; + + if (!isTimedOut(entry.getDescriptor(), now)) + { + any_pending = true; + continue; + } + + /// A commit already in flight may have landed on the destination; do not overwrite + /// it with KILLED (same as a user KILL that sees `committing`). + if (entry.committing || !tryPersistTimeoutKill(key, entry, now)) + { + any_pending = true; + continue; + } + + timed_out_transactions.push_back(entry.getDescriptor().transaction_id); + } + } + + for (const auto & transaction_id : timed_out_transactions) + storage.killExportPart(transaction_id); + + return any_pending; +} + +bool MergeTreePartitionExportScheduler::run() +{ + /// Timeouts first, before the "cannot make progress this tick" early returns, so a wedged + /// task (no move executors, memory pressure, ...) still expires. + if (!enforceTimeouts()) + return false; + + /// There is pending work. Even if we cannot make progress this tick (no move executors, moves + /// stopped, or memory pressure), keep the scheduler awake so it retries on the next tick. + const auto available_move_executors = storage.background_moves_assignee.getAvailableMoveExecutors(); + if (available_move_executors == 0) + return true; + + if (storage.parts_mover.moves_blocker.isCancelled()) + return true; + + /// Respect the background memory soft-limit like the per-part export path does. + if (!canEnqueueBackgroundTask()) + return true; + + std::vector> parts_to_schedule; /// (transaction_id, part_name) + std::vector tasks_to_commit; + + { + std::lock_guard lock(mutex); + const auto now = time(nullptr); + size_t scheduled = 0; + for (auto & [key, entry] : tasks) + { + const auto & descriptor = entry.getDescriptor(); + if (descriptor.status != MergeTreePartitionExportTask::Status::PENDING) + continue; + + if (descriptor.allPartsDone()) + { + /// All parts exported: commit (or retry a previously-failed commit). tryCommit + /// itself takes the committing lease; skip if a commit is already in flight. + if (!entry.committing) + tasks_to_commit.push_back(descriptor.transaction_id); + continue; + } + + for (const auto & part : descriptor.parts) + { + if (scheduled >= available_move_executors) + break; + if (part.done || entry.in_flight_parts.contains(part.part_name)) + continue; + if (const auto backoff_it = entry.part_backoff.find(part.part_name); + backoff_it != entry.part_backoff.end() && now < backoff_it->second.next_retry_time) + continue; + + entry.in_flight_parts.insert(part.part_name); + parts_to_schedule.emplace_back(descriptor.transaction_id, part.part_name); + ++scheduled; + } + + if (scheduled >= available_move_executors) + break; + } + } + + for (const auto & [transaction_id, part_name] : parts_to_schedule) + scheduleOnePart(transaction_id, part_name); + + for (const auto & transaction_id : tasks_to_commit) + tryCommit(transaction_id); + + /// A task was PENDING this tick (either scheduled/committed above, or waiting on in-flight parts + /// or a retry). Keep polling until every task reaches a terminal state. + return true; +} + +void MergeTreePartitionExportScheduler::scheduleOnePart(const String & transaction_id, const String & part_name) +{ + MergeTreePartitionExportTask descriptor_copy; + { + std::lock_guard lock(mutex); + auto it = findByTransactionId(transaction_id); + if (it == tasks.end()) + return; + descriptor_copy = it->second.getDescriptor(); + } + + const StorageID destination_storage_id{descriptor_copy.destination_database, descriptor_copy.destination_table}; + + try + { + auto context = ExportPartitionUtils::getContextCopyWithTaskSettings(storage.getContext(), descriptor_copy); + + LOG_INFO(storage.log, "ExportPartition: scheduling part export {} for task {}", part_name, transaction_id); + + storage.exportPartToTable( + part_name, + destination_storage_id, + transaction_id, + context, + descriptor_copy.iceberg_metadata_json, + /*allow_outdated_parts*/ true, + [this, transaction_id, part_name](MergeTreePartExportManifest::CompletionCallbackResult result) + { + handlePartCompletion(transaction_id, part_name, result); + }); + } + catch (const Exception & e) + { + tryLogCurrentException(storage.log, __PRETTY_FUNCTION__); + /// Dispatch failed before a move-executor task was queued (destination dropped, schema + /// mismatch, executor busy, ...). Route through the same completion-failure transition as + /// an async export error so last_exception is persisted and non-retryable faults become + /// FAILED. handlePartCompletion also releases the in-flight marker. + handlePartCompletion( + transaction_id, + part_name, + MergeTreePartExportManifest::CompletionCallbackResult::createFailure(e)); + } + catch (...) + { + tryLogCurrentException(storage.log, __PRETTY_FUNCTION__); + handlePartCompletion( + transaction_id, + part_name, + MergeTreePartExportManifest::CompletionCallbackResult::createFailure( + Exception::createRuntime( + ErrorCodes::UNKNOWN_EXCEPTION, + getCurrentExceptionMessage(/*with_stacktrace=*/ false)))); + } +} + +void MergeTreePartitionExportScheduler::handlePartCompletion( + const String & transaction_id, const String & part_name, const MergeTreePartExportManifest::CompletionCallbackResult & result) +{ + bool ready_to_commit = false; + + { + std::lock_guard lock(mutex); + auto it = findByTransactionId(transaction_id); + if (it == tasks.end()) + return; + + auto & entry = it->second; + + entry.in_flight_parts.erase(part_name); + + /// Task already terminal (KILLED / FAILED / COMPLETED): ignore late completions. + if (entry.getDescriptor().status != MergeTreePartitionExportTask::Status::PENDING) + return; + + /// A cancelled export (KILL, or SYSTEM STOP MOVES) is not a real failure: leave the part + /// pending. If it was a KILL the status is already handled above; otherwise the next tick + /// retries it. + if (!result.success && result.exception && result.exception->code() == ErrorCodes::QUERY_WAS_CANCELLED) + return; + + /// Persist-then-apply under the lock: mutate a copy, write it durably, then swap it in. If + /// the write throws we leave the in-memory descriptor unchanged (so the part is retried) and + /// swallow the error -- a completion callback must not surface a local disk error as a part + /// failure. + auto updated = entry.getDescriptor(); + + if (result.success) + { + if (auto * part = updated.findPart(part_name)) + { + part->done = true; + part->paths_in_destination = result.relative_paths_in_destination_storage; + } + } + else + { + updated.last_exception.message = result.exception ? result.exception->message() : "Unknown export failure"; + updated.last_exception.part = part_name; + updated.last_exception.time = time(nullptr); + updated.last_exception.count += 1; + + if (result.exception && isNonRetryablePlainExportError(result.exception->code())) + updated.status = MergeTreePartitionExportTask::Status::FAILED; + } + + try + { + persist(it->first, updated.toJsonString()); + } + catch (...) + { + tryLogCurrentException(storage.log, "ExportPartition: failed to persist part-completion state, will retry"); + return; + } + + entry.setDescriptor(std::move(updated)); + + if (result.success) + { + entry.part_backoff.erase(part_name); + if (entry.getDescriptor().allPartsDone() && !entry.committing) + ready_to_commit = true; + } + else if (entry.getDescriptor().status == MergeTreePartitionExportTask::Status::FAILED) + { + LOG_WARNING(storage.log, "ExportPartition: task {} failed on part {} with non-retryable error", + transaction_id, part_name); + } + else + { + auto & backoff = entry.part_backoff[part_name]; + ++backoff.attempts; + const auto backoff_seconds = ExportPartitionUtils::computeRetryBackoffSeconds( + backoff.attempts, + entry.getDescriptor().retry_initial_backoff_seconds, + entry.getDescriptor().retry_max_backoff_seconds); + const auto now = time(nullptr); + const size_t headroom = static_cast(std::numeric_limits::max() - now); + backoff.next_retry_time = now + static_cast(std::min(backoff_seconds, headroom)); + + LOG_INFO(storage.log, "ExportPartition: task {} part {} failed with retryable error, will retry at {}", + transaction_id, part_name, backoff.next_retry_time); + } + } + + if (ready_to_commit) + tryCommit(transaction_id); +} + +void MergeTreePartitionExportScheduler::tryCommit(const String & transaction_id) +{ + MergeTreePartitionExportTask descriptor_copy; + String composite_key; + /// This call is the sole writer of `committing`. Drop the lease on every exit after we claimed + /// it, including exceptions that are not `DB::Exception` (otherwise the task is wedged until + /// restart: timeout and KILL also refuse to act while the flag is set). + bool claimed = false; + SCOPE_EXIT( + { + if (!claimed) + return; + std::lock_guard lock(mutex); + auto it = tasks.find(composite_key); + if (it != tasks.end() && it->second.getDescriptor().transaction_id == transaction_id) + it->second.committing = false; + }); + + { + std::lock_guard lock(mutex); + auto it = findByTransactionId(transaction_id); + if (it == tasks.end()) + return; + + auto & entry = it->second; + if (entry.committing) + return; + if (entry.getDescriptor().status != MergeTreePartitionExportTask::Status::PENDING || !entry.getDescriptor().allPartsDone()) + return; + + composite_key = it->first; + entry.committing = true; + claimed = true; + descriptor_copy = entry.getDescriptor(); + } + + const StorageID destination_storage_id{descriptor_copy.destination_database, descriptor_copy.destination_table}; + + bool success = false; + std::optional failure; + try + { + auto destination_storage = DatabaseCatalog::instance().tryGetTable(destination_storage_id, storage.getContext()); + if (!destination_storage) + throw Exception(ErrorCodes::UNKNOWN_TABLE, "Destination table {} not found for export commit", + destination_storage_id.getNameForLogs()); + + const auto exported_paths = descriptor_copy.collectExportedPaths(); + if (exported_paths.empty()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "No exported paths found for export {}, will not commit. This might be a bug", transaction_id); + + auto context = ExportPartitionUtils::getContextCopyWithTaskSettings(storage.getContext(), descriptor_copy); + + LOG_INFO(storage.log, "ExportPartition: all parts exported for task {}, committing", transaction_id); + + ExportPartitionUtils::commitExportedPaths( + descriptor_copy.transaction_id, + descriptor_copy.partition_id, + descriptor_copy.iceberg_metadata_json, + descriptor_copy.write_full_path_in_iceberg_metadata, + descriptor_copy.iceberg_partition_timezone, + exported_paths, + descriptor_copy.partNames(), + destination_storage, + storage, + context); + + success = true; + } + catch (const Exception & e) + { + failure = e; + LOG_WARNING(storage.log, "ExportPartition: commit for task {} failed: {}", transaction_id, e.message()); + } + catch (...) + { + tryLogCurrentException(storage.log, "ExportPartition: commit for task " + transaction_id + " failed"); + failure = Exception::createRuntime(ErrorCodes::UNKNOWN_EXCEPTION, getCurrentExceptionMessage(/*with_stacktrace=*/ false)); + } + + { + std::lock_guard lock(mutex); + auto it = tasks.find(composite_key); + if (it == tasks.end() || it->second.getDescriptor().transaction_id != transaction_id) + return; + + auto & entry = it->second; + + /// A concurrent KILL may have won the race while we were committing. Its terminal state is + /// already durable, so there is nothing to persist here. `SCOPE_EXIT` still drops the lease. + if (entry.getDescriptor().status != MergeTreePartitionExportTask::Status::PENDING) + return; + + /// Persist-then-apply under the lock. Note the destination commit above already happened + /// (effect-first): if this local write throws, we leave the task PENDING with all parts done + /// so run() retries the commit -- idempotent thanks to the transaction id / commit file. + auto updated = entry.getDescriptor(); + if (success) + { + updated.status = MergeTreePartitionExportTask::Status::COMPLETED; + } + else + { + updated.last_exception.message = failure ? failure->message() : "Unknown commit failure"; + updated.last_exception.part = ""; + updated.last_exception.time = time(nullptr); + updated.last_exception.count += 1; + + if (failure && isNonRetryablePlainExportError(failure->code())) + updated.status = MergeTreePartitionExportTask::Status::FAILED; + /// Otherwise leave PENDING: run() will retry the commit on the next tick. + } + + try + { + persist(composite_key, updated.toJsonString()); + } + catch (...) + { + tryLogCurrentException(storage.log, "ExportPartition: failed to persist commit result, will retry"); + return; + } + + entry.setDescriptor(std::move(updated)); + } +} + +void MergeTreePartitionExportScheduler::persist(const String & composite_key, const String & descriptor_json) +{ + /// Always called while holding the registry `mutex`, so writes are already serialized. The file + /// is named by a 128-bit hash of the composite key (a fixed-length, filesystem-safe, opaque + /// handle), so a force-replace overwrites the previous record in place. The authoritative key is + /// recomputed from the JSON body in loadFromDisk, so the name only needs to be deterministic and + /// collision-free -- sipHash128 (matching FileCacheKey) provides both. + auto disk = storage.getDisks().front(); + disk->createDirectories(getExportsRelativePath()); + + const auto final_path = descriptorRelativePath(composite_key); + const auto tmp_path = final_path + ".tmp"; + + { + auto out = disk->writeFile(tmp_path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite, storage.getContext()->getWriteSettings()); + writeString(descriptor_json, *out); + out->finalize(); + } + + disk->replaceFile(tmp_path, final_path); +} + +void MergeTreePartitionExportScheduler::loadFromDisk() +{ + auto disk = storage.getDisks().front(); + const auto directory = getExportsRelativePath(); + + if (!disk->existsDirectory(directory)) + return; + + std::lock_guard lock(mutex); + + for (auto it = disk->iterateDirectory(directory); it->isValid(); it->next()) + { + const auto file_name = it->name(); + /// Skip stale temporary files left by an interrupted write. + if (!endsWith(file_name, ".json")) + continue; + + try + { + auto buf = disk->readFile(fs::path(directory) / file_name, getReadSettings()); + String content; + readStringUntilEOF(content, *buf); + + auto descriptor = MergeTreePartitionExportTask::fromJsonString(content); + const auto composite_key = ExportPartitionUtils::compositeKey(descriptor.partition_id, descriptor.destination_database, descriptor.destination_table); + /// Rendered before `descriptor` is moved into the entry below. + const auto key_description = describeKey(descriptor); + + TaskEntry entry; + + /// Re-pin every source part of a resumable task, including already-exported ones. + /// Unfinished parts still need to be read; Iceberg commit also derives partition + /// values from the original parts, so they must survive until COMPLETED/FAILED/KILLED. + if (descriptor.status == MergeTreePartitionExportTask::Status::PENDING) + { + for (const auto & part : descriptor.parts) + { + if (auto data_part = storage.getPartIfExists( + part.part_name, {MergeTreeDataPartState::Active, MergeTreeDataPartState::Outdated})) + entry.part_references.push_back(data_part); + } + } + + entry.setDescriptor(std::move(descriptor)); + + /// The file name is a pure function of the composite key, so this code can never write + /// two records for one key. A duplicate means the directory was tampered with or holds + /// records written by an incompatible version; keeping an arbitrary one of them would + /// silently resurrect stale state, so report it instead. + if (!tasks.emplace(composite_key, std::move(entry)).second) + { + LOG_ERROR(storage.log, "ExportPartition: ignoring {}, another record already describes key {}", + file_name, key_description); + continue; + } + + LOG_INFO(storage.log, "ExportPartition: loaded export task from disk (key {})", key_description); + } + catch (...) + { + tryLogCurrentException(storage.log, "Failed to load partition export descriptor " + file_name); + } + } +} + +} diff --git a/src/Storages/MergeTree/MergeTreePartitionExportScheduler.h b/src/Storages/MergeTree/MergeTreePartitionExportScheduler.h new file mode 100644 index 000000000000..b81546a69d65 --- /dev/null +++ b/src/Storages/MergeTree/MergeTreePartitionExportScheduler.h @@ -0,0 +1,116 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +class StorageMergeTree; + + +class MergeTreePartitionExportScheduler +{ +public: + explicit MergeTreePartitionExportScheduler(StorageMergeTree & storage_); + + using DataPartPtr = MergeTreePartExportManifest::DataPartPtr; + + /// Registers and persists a new task, then triggers the scheduler. + /// Throws if a task with the same (partition, destination) key already exists; + void addTask(MergeTreePartitionExportTask descriptor, std::vector part_references, bool force); + + CancellationCode kill(const String & transaction_id); + + std::vector getInfo() const; + + /// Scheduler tick: schedule pending parts of PENDING tasks and commit tasks whose parts are all + /// exported. Invoked from the storage's schedule-pool task. Returns true if at least one task is + /// still PENDING (so the caller should keep polling); false when there is no work left, letting + /// the task go idle until the next addTask/startup trigger. + bool run(); + + void loadFromDisk(); + +private: + StorageMergeTree & storage; + + struct TaskEntry + { + /// Pins the source parts so they are not physically removed before the export finishes, + /// including already-exported parts that Iceberg commit still reads for partition values. + /// Cleared by `setDescriptor` when the installed snapshot is terminal. + std::vector part_references; + /// Parts currently scheduled on the background move executor (avoids double scheduling). + std::unordered_set in_flight_parts; + /// True while `tryCommit` is in the destination-commit / persist window. Callers must not + /// write this flag; overlapping tryCommit calls no-op when it is already set. + bool committing = false; + /// In-memory per-part retry back-off. Keyed by part name so a force-replace of the same + /// composite key does not inherit a prior instance's delay. Not persisted: after restart + /// the first retry is immediate, then back-off resumes from subsequent failures. + struct PartBackoff + { + size_t attempts = 0; + time_t next_retry_time = 0; + }; + std::unordered_map part_backoff; + + const MergeTreePartitionExportTask & getDescriptor() const { return descriptor; } + + /// Install a (typically just-persisted) snapshot. A terminal status drops the source-part + /// pins so outdated parts can be physically removed without a restart. + void setDescriptor(MergeTreePartitionExportTask new_descriptor) + { + descriptor = std::move(new_descriptor); + if (descriptor.status != MergeTreePartitionExportTask::Status::PENDING) + part_references.clear(); + } + + private: + MergeTreePartitionExportTask descriptor; + }; + + mutable std::mutex mutex; + std::map tasks; + + /// Renders a task key for logs and error messages. Unlike `compositeKey` this is not injective, + /// so it must never be used as an identity. + static String describeKey(const MergeTreePartitionExportTask & descriptor); + + void scheduleOnePart(const String & transaction_id, const String & part_name); + void handlePartCompletion(const String & transaction_id, const String & part_name, const MergeTreePartExportManifest::CompletionCallbackResult & result); + void tryCommit(const String & transaction_id); + + /// Caller must hold `mutex`. `tasks.end()` if no task has this transaction id. + std::map::iterator findByTransactionId(const String & transaction_id); + + /// Wall-clock timeout pass. Transitions expired PENDING tasks to KILLED (unless a commit is + /// already in flight) and cancels their in-flight parts. Returns true if any PENDING work + /// remains, so the caller should keep polling. + bool enforceTimeouts(); + + /// Caller must hold `mutex`. Persist `KILLED` with a timeout reason. Returns false if the + /// local write fails (descriptor is left unchanged so the next tick retries). + bool tryPersistTimeoutKill(const String & composite_key, TaskEntry & entry, time_t now); + + /// Atomically write `descriptor_json` to the task's on-disk file (tmp + replace), named by the + /// composite key so a force-replace naturally overwrites the previous record. Always invoked + /// while holding the registry `mutex` (write-through), so writes are serialized by that lock. + void persist(const String & composite_key, const String & descriptor_json); + + /// Relative path of the descriptor file for `composite_key` (hash + `.json`). + String descriptorRelativePath(const String & composite_key) const; + + String getExportsRelativePath() const; +}; + +} diff --git a/src/Storages/MergeTree/MergeTreePartitionExportTask.h b/src/Storages/MergeTree/MergeTreePartitionExportTask.h new file mode 100644 index 000000000000..7e3e6273f819 --- /dev/null +++ b/src/Storages/MergeTree/MergeTreePartitionExportTask.h @@ -0,0 +1,307 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; +} + +/// On-disk descriptor for a plain (non-replicated) `MergeTree` partition export task. +/// +/// Unlike the replicated variant, there is a single node and no cross-replica coordination, +/// so this descriptor is the whole source of truth. It is persisted as one JSON file per task +/// under the table data directory and rewritten on every state transition (part done, status +/// change). The scheduler keeps an in-memory copy guarded by a mutex; the descriptor is small +/// and self-contained so it can be reloaded verbatim after a server restart. +struct MergeTreePartitionExportTask +{ + using FileAlreadyExistsPolicy = MergeTreePartExportManifest::FileAlreadyExistsPolicy; + + enum class Status + { + PENDING, + COMPLETED, + FAILED, + KILLED, + }; + + struct PartProgress + { + String part_name; + bool done = false; + std::vector paths_in_destination; + }; + + /// Best-effort record of the most recent failure observed for this task. `count` is a running + /// total of failures and matches the semantics documented for `system.partition_exports`. + struct LastException + { + String message; + String part; /// empty for task-level exceptions (commit failure) + time_t time = 0; + size_t count = 0; + }; + + /// Identity + String transaction_id; + String query_id; + String partition_id; + String source_database; + String source_table; + String destination_database; + String destination_table; + time_t create_time = 0; + + /// Work + progress + std::vector parts; + Status status = Status::PENDING; + LastException last_exception; + + size_t retry_initial_backoff_seconds = 5; + size_t retry_max_backoff_seconds = 300; + size_t task_timeout_seconds = 86400; + + /// Export settings. Field names are intentionally identical to + /// `ExportReplicatedMergeTreePartitionManifest` so that + /// `ExportPartitionUtils::getContextCopyWithTaskSettings` works for both descriptors. + size_t max_threads = 0; + bool parallel_formatting = false; + bool parquet_parallel_encoding = false; + size_t max_bytes_per_file = 0; + size_t max_rows_per_file = 0; + FileAlreadyExistsPolicy file_already_exists_policy = FileAlreadyExistsPolicy::skip; + String filename_pattern; + bool write_full_path_in_iceberg_metadata = false; + bool allow_lossy_cast = false; + String iceberg_metadata_json; + + /// Optional for backwards compatibility with descriptors written before these + /// settings were persisted (same shape as ExportReplicatedMergeTreePartitionManifest). + std::optional parquet_compression_method; + std::optional output_format_compression_level; + std::optional parquet_row_group_size; + std::optional parquet_row_group_size_bytes; + std::optional schema_match_mode; + std::optional ignore_extra_source_columns; + std::optional iceberg_partition_timezone; + + size_t partsCount() const { return parts.size(); } + + size_t partsToDo() const + { + size_t to_do = 0; + for (const auto & part : parts) + if (!part.done) + ++to_do; + return to_do; + } + + bool allPartsDone() const + { + for (const auto & part : parts) + if (!part.done) + return false; + return true; + } + + std::vector partNames() const + { + std::vector names; + names.reserve(parts.size()); + for (const auto & part : parts) + names.push_back(part.part_name); + return names; + } + + /// Flattened list of every destination path produced by the already-exported parts. + std::vector collectExportedPaths() const + { + std::vector paths; + for (const auto & part : parts) + for (const auto & path : part.paths_in_destination) + paths.push_back(path); + return paths; + } + + PartProgress * findPart(const String & part_name) + { + for (auto & part : parts) + if (part.part_name == part_name) + return ∂ + return nullptr; + } + + std::string toJsonString() const + { + Poco::JSON::Object json; + json.set("transaction_id", transaction_id); + json.set("query_id", query_id); + json.set("partition_id", partition_id); + json.set("source_database", source_database); + json.set("source_table", source_table); + json.set("destination_database", destination_database); + json.set("destination_table", destination_table); + json.set("create_time", create_time); + json.set("status", String(magic_enum::enum_name(status))); + + Poco::JSON::Array::Ptr parts_array = new Poco::JSON::Array(); + for (const auto & part : parts) + { + Poco::JSON::Object::Ptr part_object = new Poco::JSON::Object(); + part_object->set("part_name", part.part_name); + part_object->set("done", part.done); + Poco::JSON::Array::Ptr paths_array = new Poco::JSON::Array(); + for (const auto & path : part.paths_in_destination) + paths_array->add(path); + part_object->set("paths_in_destination", paths_array); + parts_array->add(part_object); + } + json.set("parts", parts_array); + + Poco::JSON::Object::Ptr exception_object = new Poco::JSON::Object(); + exception_object->set("message", last_exception.message); + exception_object->set("part", last_exception.part); + exception_object->set("time", last_exception.time); + exception_object->set("count", last_exception.count); + json.set("last_exception", exception_object); + + json.set("retry_initial_backoff_seconds", retry_initial_backoff_seconds); + json.set("retry_max_backoff_seconds", retry_max_backoff_seconds); + json.set("task_timeout_seconds", task_timeout_seconds); + json.set("max_threads", max_threads); + json.set("parallel_formatting", parallel_formatting); + json.set("parquet_parallel_encoding", parquet_parallel_encoding); + json.set("max_bytes_per_file", max_bytes_per_file); + json.set("max_rows_per_file", max_rows_per_file); + json.set("file_already_exists_policy", String(magic_enum::enum_name(file_already_exists_policy))); + json.set("filename_pattern", filename_pattern); + json.set("write_full_path_in_iceberg_metadata", write_full_path_in_iceberg_metadata); + json.set("allow_lossy_cast", allow_lossy_cast); + if (!iceberg_metadata_json.empty()) + json.set("iceberg_metadata_json", iceberg_metadata_json); + if (parquet_compression_method) + json.set("parquet_compression_method", *parquet_compression_method); + if (output_format_compression_level) + json.set("output_format_compression_level", *output_format_compression_level); + if (parquet_row_group_size) + json.set("parquet_row_group_size", *parquet_row_group_size); + if (parquet_row_group_size_bytes) + json.set("parquet_row_group_size_bytes", *parquet_row_group_size_bytes); + if (iceberg_partition_timezone) + json.set("iceberg_partition_timezone", *iceberg_partition_timezone); + if (schema_match_mode) + json.set("schema_match_mode", String(magic_enum::enum_name(*schema_match_mode))); + if (ignore_extra_source_columns) + json.set("ignore_extra_source_columns", *ignore_extra_source_columns); + + std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + oss.exceptions(std::ios::failbit); + Poco::JSON::Stringifier::stringify(json, oss); + return oss.str(); + } + + static MergeTreePartitionExportTask fromJsonString(const std::string & json_string) + { + Poco::JSON::Parser parser; + auto json = parser.parse(json_string).extract(); + chassert(json); + + MergeTreePartitionExportTask task; + task.transaction_id = json->getValue("transaction_id"); + task.query_id = json->getValue("query_id"); + task.partition_id = json->getValue("partition_id"); + task.source_database = json->getValue("source_database"); + task.source_table = json->getValue("source_table"); + task.destination_database = json->getValue("destination_database"); + task.destination_table = json->getValue("destination_table"); + task.create_time = json->getValue("create_time"); + + const auto status_str = json->getValue("status"); + if (const auto status = magic_enum::enum_cast(status_str)) + task.status = status.value(); + else + throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown status '{}' in partition export descriptor", status_str); + + const auto parts_array = json->getArray("parts"); + for (size_t i = 0; i < parts_array->size(); ++i) + { + const auto part_object = parts_array->getObject(static_cast(i)); + PartProgress part; + part.part_name = part_object->getValue("part_name"); + part.done = part_object->getValue("done"); + const auto paths_array = part_object->getArray("paths_in_destination"); + for (size_t j = 0; j < paths_array->size(); ++j) + part.paths_in_destination.push_back(paths_array->getElement(static_cast(j))); + task.parts.push_back(std::move(part)); + } + + if (json->has("last_exception")) + { + const auto exception_object = json->getObject("last_exception"); + task.last_exception.message = exception_object->getValue("message"); + task.last_exception.part = exception_object->getValue("part"); + task.last_exception.time = exception_object->getValue("time"); + task.last_exception.count = exception_object->getValue("count"); + } + + task.retry_initial_backoff_seconds = json->getValue("retry_initial_backoff_seconds"); + task.retry_max_backoff_seconds = json->getValue("retry_max_backoff_seconds"); + task.task_timeout_seconds = json->getValue("task_timeout_seconds"); + task.max_threads = json->getValue("max_threads"); + task.parallel_formatting = json->getValue("parallel_formatting"); + task.parquet_parallel_encoding = json->getValue("parquet_parallel_encoding"); + task.max_bytes_per_file = json->getValue("max_bytes_per_file"); + task.max_rows_per_file = json->getValue("max_rows_per_file"); + + const auto policy_str = json->getValue("file_already_exists_policy"); + if (const auto policy = magic_enum::enum_cast(policy_str)) + task.file_already_exists_policy = policy.value(); + else + throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown file_already_exists_policy '{}' in partition export descriptor", policy_str); + + task.filename_pattern = json->getValue("filename_pattern"); + task.write_full_path_in_iceberg_metadata = json->getValue("write_full_path_in_iceberg_metadata"); + task.allow_lossy_cast = json->getValue("allow_lossy_cast"); + if (json->has("iceberg_metadata_json")) + task.iceberg_metadata_json = json->getValue("iceberg_metadata_json"); + if (json->has("parquet_compression_method")) + task.parquet_compression_method = json->getValue("parquet_compression_method"); + if (json->has("output_format_compression_level")) + task.output_format_compression_level = json->getValue("output_format_compression_level"); + if (json->has("parquet_row_group_size")) + task.parquet_row_group_size = json->getValue("parquet_row_group_size"); + if (json->has("parquet_row_group_size_bytes")) + task.parquet_row_group_size_bytes = json->getValue("parquet_row_group_size_bytes"); + if (json->has("iceberg_partition_timezone")) + task.iceberg_partition_timezone = json->getValue("iceberg_partition_timezone"); + /// Left unset (nullopt) for tasks created before these fields existed - such tasks were + /// always scheduled under the old, strict column-matching check, so callers should treat + /// an absent value as `POSITION` with `ignore_extra_source_columns = false`. + if (json->has("schema_match_mode")) + { + const auto mode_str = json->getValue("schema_match_mode"); + if (const auto mode = magic_enum::enum_cast(mode_str)) + task.schema_match_mode = mode; + else + throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown schema_match_mode '{}' in partition export descriptor", mode_str); + } + if (json->has("ignore_extra_source_columns")) + task.ignore_extra_source_columns = json->getValue("ignore_extra_source_columns"); + + return task; + } +}; + +} diff --git a/src/Storages/MergeTree/PartitionExportInfo.h b/src/Storages/MergeTree/PartitionExportInfo.h new file mode 100644 index 000000000000..bbde641f6c2f --- /dev/null +++ b/src/Storages/MergeTree/PartitionExportInfo.h @@ -0,0 +1,77 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +/// One `EXPORT PARTITION` task as exposed by `system.partition_exports`, for both plain +/// `MergeTree` and `Replicated*MergeTree` sources. Producers fill it from their own in-memory +/// mirror, so building it never touches disk or ZooKeeper. +/// +/// Fields that only a replicated source can populate are documented below; a plain `MergeTree` +/// leaves them empty rather than inventing a value. +/// +/// `source_database` / `source_table` are absent on purpose: the system table emits the names it +/// iterated in `DatabaseCatalog`, which stay correct after a `RENAME`. +struct PartitionExportInfo +{ + /// Most recent exception recorded for this task by one replica. A plain `MergeTree` reports a + /// single entry with an empty `replica`. `count` is best-effort: concurrent failing writers on + /// the same replica may under-count by one. + struct LastException + { + String replica; /// empty for plain MergeTree + String message; + String part; /// empty for task-level exceptions (commit failure, timeout) + time_t time = 0; + size_t count = 0; + }; + + /// A part waiting before its next export attempt. Local to the node that reports it and never + /// shared across replicas. + struct PartBackoff + { + String part; + size_t attempts = 0; + time_t next_retry_time = 0; + }; + + String destination_database; + String destination_table; + String partition_id; + String transaction_id; + String query_id; + time_t create_time = 0; + /// Replica that received the export command. Empty for plain MergeTree. + String source_replica; + size_t parts_count = 0; + size_t parts_to_do = 0; + std::vector parts; + String status; + + /// One entry per replica that has recorded at least one exception for this task. + std::vector last_exception_per_replica; + /// Sum of every `count` in `last_exception_per_replica`. + size_t exception_count = 0; + + /// Destination file paths produced by each exported part, keyed by part name. Empty until parts + /// complete, partial while the task is PENDING. On the replicated path a value may be the + /// sentinel "" when a Keeper refresh was incomplete. + std::map> destination_file_paths_per_part; + + /// Commit-time paths reported by the destination storage. All empty before the commit lands. + /// Plain MergeTree does not persist them yet, so they stay empty there. + String committed_metadata_file; + String committed_manifest_list; + String committed_manifest_file; + String committed_marker_file; + + /// Parts of this task currently backing off on this node. Empty if none. + std::vector backoff_per_part; +}; + +} diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 7425488403b8..fe163365bed9 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -50,6 +52,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -108,6 +113,36 @@ namespace Setting extern const SettingsBool throw_on_unsupported_query_inside_transaction; extern const SettingsUInt64 max_parts_to_move; extern const SettingsUpdateParallelMode update_parallel_mode; + extern const SettingsBool export_merge_tree_partition_force_export; + extern const SettingsUInt64 export_merge_tree_partition_retry_initial_backoff_seconds; + extern const SettingsUInt64 export_merge_tree_partition_retry_max_backoff_seconds; + extern const SettingsUInt64 export_merge_tree_partition_task_timeout_seconds; + extern const SettingsBool output_format_parallel_formatting; + extern const SettingsBool output_format_parquet_parallel_encoding; + extern const SettingsParquetCompression output_format_parquet_compression_method; + extern const SettingsUInt64 output_format_compression_level; + extern const SettingsUInt64 output_format_parquet_row_group_size; + extern const SettingsUInt64 output_format_parquet_row_group_size_bytes; + extern const SettingsMaxThreads max_threads; + extern const SettingsMergeTreePartExportFileAlreadyExistsPolicy export_merge_tree_part_file_already_exists_policy; + extern const SettingsUInt64 export_merge_tree_part_max_bytes_per_file; + extern const SettingsUInt64 export_merge_tree_part_max_rows_per_file; + extern const SettingsBool export_merge_tree_part_throw_on_pending_mutations; + extern const SettingsBool export_merge_tree_part_throw_on_pending_patch_parts; + extern const SettingsBool export_merge_tree_part_allow_lossy_cast; + extern const SettingsExportPartitionAllOnError export_merge_tree_partition_all_on_error; + extern const SettingsString export_merge_tree_part_filename_pattern; + extern const SettingsBool write_full_path_in_iceberg_metadata; + extern const SettingsUInt64 iceberg_insert_max_bytes_in_data_file; + extern const SettingsUInt64 iceberg_insert_max_rows_in_data_file; + extern const SettingsTimezone iceberg_partition_timezone; + extern const SettingsMergeTreePartExportSchemaMatchMode export_merge_tree_part_schema_match_mode; + extern const SettingsBool export_merge_tree_part_ignore_extra_source_columns; +} + +namespace ServerSetting +{ + extern const ServerSettingsBool allow_experimental_export_merge_tree_partition; } namespace MergeTreeSetting @@ -150,6 +185,9 @@ namespace ErrorCodes extern const int PART_IS_TEMPORARILY_LOCKED; extern const int FAULT_INJECTED; extern const int INCOMPATIBLE_COLUMNS; + extern const int EXPORT_PARTITION_ALREADY_EXPORTED; + extern const int PARTITION_EXPORT_FAILED; + extern const int PENDING_MUTATIONS_NOT_ALLOWED; } namespace ActionLocks @@ -232,6 +270,19 @@ StorageMergeTree::StorageMergeTree( loadMutations(); loadDeduplicationLog(); prewarmCaches(getActivePartsLoadingThreadPool().get(), getCachesToPrewarm(0)); + + if (getContext()->getServerSettings()[ServerSetting::allow_experimental_export_merge_tree_partition]) + { + partition_export_scheduler = std::make_shared(*this); + + partition_export_task = getContext()->getSchedulePool().createTask( + getStorageID(), + getStorageID().getFullTableName() + " (StorageMergeTree::partition_export_task)", + [this] { partitionExportTask(); }); + + /// Activated in startup(); deactivated during shutdown. + partition_export_task->deactivate(); + } } @@ -251,6 +302,14 @@ void StorageMergeTree::startup() try { + /// Reload persisted partition-export tasks (and re-pin their parts) before background merges + /// can start removing parts, then activate the scheduler task so PENDING tasks resume. + if (partition_export_scheduler) + { + partition_export_scheduler->loadFromDisk(); + partition_export_task->activateAndSchedule(); + } + cleanup_thread.start(); background_operations_assignee.start(); background_streaming_assignee.start(); @@ -287,6 +346,9 @@ void StorageMergeTree::flushAndPrepareForShutdown() merger_mutator.merges_blocker.cancelForever(); parts_mover.moves_blocker.cancelForever(); + if (partition_export_task) + partition_export_task->deactivate(); + background_operations_assignee.finish(); background_moves_assignee.finish(); background_streaming_assignee.finish(); @@ -3654,4 +3716,248 @@ CommittingBlocksSet StorageMergeTree::getCommittingBlocks() const std::lock_guard lock(committing_blocks_mutex); return committing_blocks; } + +void StorageMergeTree::exportPartitionToTable(const PartitionCommand & command, ContextPtr query_context) +{ + if (!query_context->getServerSettings()[ServerSetting::allow_experimental_export_merge_tree_partition]) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "Exporting merge tree partition is experimental. Set the server setting `allow_experimental_export_merge_tree_partition` to enable it.\n" + "If you are exporting to an Apache Iceberg table, you also need to enable the setting `allow_insert_into_iceberg`."); + + /// The scheduler is created in the constructor whenever the server setting above is enabled, so + /// this should always hold here. + if (!partition_export_scheduler) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "Partition export is not initialized for table {}", getStorageID().getNameForLogs()); + + /// EXPORT PARTITION ALL: expand into one sub-call per active partition id. + /// Failure handling is controlled by `export_merge_tree_partition_all_on_error`. + if (const auto * partition_ast = command.partition->as(); partition_ast && partition_ast->all) + { + auto partition_id_set = getAllPartitionIds(); + if (partition_id_set.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Table {} has no active partitions to export", getStorageID().getNameForLogs()); + + std::vector partition_ids(partition_id_set.begin(), partition_id_set.end()); + std::sort(partition_ids.begin(), partition_ids.end()); + + const auto & on_error_setting = query_context->getSettingsRef()[Setting::export_merge_tree_partition_all_on_error]; + const ExportPartitionAllOnError on_error = on_error_setting.value; + + LOG_INFO(log, "EXPORT PARTITION ALL: scheduling export for {} partitions, on_error={}", + partition_ids.size(), on_error_setting.toString()); + + std::vector> failures; /// (partition_id, message) + size_t skipped_conflicts = 0; + + for (const auto & partition_id : partition_ids) + { + PartitionCommand sub = command; + auto synthetic = make_intrusive(); + synthetic->setPartitionID(make_intrusive(partition_id)); + sub.partition = synthetic; + + try + { + exportPartitionToTable(sub, query_context); + } + catch (const Exception & e) + { + switch (on_error) + { + case ExportPartitionAllOnError::throw_first: + throw; + case ExportPartitionAllOnError::skip_conflicts: + if (e.code() == ErrorCodes::EXPORT_PARTITION_ALREADY_EXPORTED) + { + ++skipped_conflicts; + LOG_INFO(log, "EXPORT PARTITION ALL: skipping partition {} (already exported): {}", + partition_id, e.message()); + break; + } + throw; + case ExportPartitionAllOnError::collect: + LOG_WARNING(log, "EXPORT PARTITION ALL: partition {} failed: {}", partition_id, e.message()); + failures.emplace_back(partition_id, e.message()); + break; + } + } + } + + if (!failures.empty()) + { + String aggregated = fmt::format( + "EXPORT PARTITION ALL: {}/{} partitions failed to schedule. Per-partition errors:", + failures.size(), partition_ids.size()); + for (const auto & [pid, msg] : failures) + aggregated += fmt::format("\n {}: {}", pid, msg); + throw Exception(ErrorCodes::PARTITION_EXPORT_FAILED, "{}", aggregated); + } + + if (skipped_conflicts > 0) + LOG_INFO(log, "EXPORT PARTITION ALL: skipped {} partitions due to existing exports", skipped_conflicts); + + return; + } + + const auto dest_database = query_context->resolveDatabase(command.to_database); + const auto dest_table = command.to_table; + const auto dest_storage_id = StorageID(dest_database, dest_table); + auto dest_storage = DatabaseCatalog::instance().getTable({dest_database, dest_table}, query_context); + + if (dest_storage->getStorageID() == this->getStorageID()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Exporting to the same table is not allowed"); + } + + if (!dest_storage->supportsImport(query_context)) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Destination storage {} does not support MergeTree parts or uses unsupported partitioning", dest_storage->getName()); + + auto src_snapshot = getInMemoryMetadataPtr(query_context, false); + auto destination_snapshot = dest_storage->getInMemoryMetadataPtr(query_context, false); + + /// Positional CAST matching, like `INSERT INTO dest SELECT * FROM src`. + ExportPartitionUtils::verifyExportSchemaCastable( + src_snapshot, destination_snapshot, dest_storage->getStorageID(), query_context); + + const String partition_id = getPartitionIDFromQuery(command.partition, query_context); + + const bool force = query_context->getSettingsRef()[Setting::export_merge_tree_partition_force_export]; + + DataPartsVector parts; + { + auto data_parts_lock = lockParts(); + parts = getDataPartsVectorInPartitionForInternalUsage(MergeTreeDataPartState::Active, partition_id, data_parts_lock); + } + + if (parts.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Partition {} doesn't exist", partition_id); + + const bool throw_on_pending_mutations = query_context->getSettingsRef()[Setting::export_merge_tree_part_throw_on_pending_mutations]; + const bool throw_on_pending_patch_parts = query_context->getSettingsRef()[Setting::export_merge_tree_part_throw_on_pending_patch_parts]; + + MergeTreeData::IMutationsSnapshot::Params mutations_snapshot_params + { + .metadata_version = src_snapshot->getMetadataVersion(), + .min_part_metadata_version = MergeTreeData::getPartsSnapshotInfo(parts).min_metadata_version, + .need_data_mutations = throw_on_pending_mutations, + .need_alter_mutations = throw_on_pending_mutations || throw_on_pending_patch_parts, + .need_patch_parts = throw_on_pending_patch_parts, + }; + + const auto mutations_snapshot = getMutationsSnapshot(mutations_snapshot_params); + + for (const auto & part : parts) + { + const auto alter_conversions = getAlterConversionsForPart(part, mutations_snapshot, query_context); + + /// re-check `throw_on_pending_mutations` because `pending_mutations` might have been filled due to `throw_on_pending_patch_parts` + if (alter_conversions->hasMutations() && throw_on_pending_mutations) + throw Exception(ErrorCodes::PENDING_MUTATIONS_NOT_ALLOWED, + "Partition {} can not be exported because the part {} has pending mutations. Either wait for the mutations to be applied or set `export_merge_tree_part_throw_on_pending_mutations` to false", + partition_id, part->name); + + if (alter_conversions->hasPatches()) + throw Exception(ErrorCodes::PENDING_MUTATIONS_NOT_ALLOWED, + "Partition {} can not be exported because the part {} has pending patch parts. Either wait for the patch parts to be applied or set `export_merge_tree_part_throw_on_pending_patch_parts` to false", + partition_id, part->name); + } + + MergeTreePartitionExportTask descriptor; + descriptor.transaction_id = toString(UUIDHelpers::generateV4()); + descriptor.query_id = query_context->getCurrentQueryId(); + descriptor.partition_id = partition_id; + descriptor.source_database = getStorageID().database_name; + descriptor.source_table = getStorageID().table_name; + descriptor.destination_database = dest_database; + descriptor.destination_table = dest_table; + descriptor.create_time = time(nullptr); + descriptor.status = MergeTreePartitionExportTask::Status::PENDING; + + for (const auto & part : parts) + descriptor.parts.push_back({part->name, /*done*/ false, /*paths*/ {}}); + + descriptor.retry_initial_backoff_seconds = query_context->getSettingsRef()[Setting::export_merge_tree_partition_retry_initial_backoff_seconds]; + descriptor.retry_max_backoff_seconds = query_context->getSettingsRef()[Setting::export_merge_tree_partition_retry_max_backoff_seconds]; + descriptor.task_timeout_seconds = query_context->getSettingsRef()[Setting::export_merge_tree_partition_task_timeout_seconds]; + descriptor.max_threads = query_context->getSettingsRef()[Setting::max_threads]; + descriptor.parallel_formatting = query_context->getSettingsRef()[Setting::output_format_parallel_formatting]; + descriptor.parquet_parallel_encoding = query_context->getSettingsRef()[Setting::output_format_parquet_parallel_encoding]; + descriptor.parquet_compression_method = query_context->getSettingsRef()[Setting::output_format_parquet_compression_method].toString(); + descriptor.output_format_compression_level = query_context->getSettingsRef()[Setting::output_format_compression_level]; + descriptor.parquet_row_group_size = query_context->getSettingsRef()[Setting::output_format_parquet_row_group_size]; + descriptor.parquet_row_group_size_bytes = query_context->getSettingsRef()[Setting::output_format_parquet_row_group_size_bytes]; + descriptor.max_bytes_per_file = query_context->getSettingsRef()[Setting::export_merge_tree_part_max_bytes_per_file]; + descriptor.max_rows_per_file = query_context->getSettingsRef()[Setting::export_merge_tree_part_max_rows_per_file]; + descriptor.file_already_exists_policy = query_context->getSettingsRef()[Setting::export_merge_tree_part_file_already_exists_policy].value; + descriptor.filename_pattern = query_context->getSettingsRef()[Setting::export_merge_tree_part_filename_pattern].value; + descriptor.write_full_path_in_iceberg_metadata = query_context->getSettingsRef()[Setting::write_full_path_in_iceberg_metadata]; + descriptor.allow_lossy_cast = query_context->getSettingsRef()[Setting::export_merge_tree_part_allow_lossy_cast]; + descriptor.iceberg_partition_timezone = query_context->getSettingsRef()[Setting::iceberg_partition_timezone].toString(); + descriptor.schema_match_mode = query_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; + descriptor.ignore_extra_source_columns = query_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns].value; + + /// Validate the destination against the partition being exported (Iceberg partition-spec + /// compatibility, or a matching partition key for plain destinations) and, for Iceberg + /// destinations, capture the destination metadata.json to persist in the descriptor. + /// Shared with the `Replicated*MergeTree` export path. + descriptor.iceberg_metadata_json = ExportPartitionUtils::extractDestinationIcebergMetadataJson( + src_snapshot, + destination_snapshot, + dest_storage, + parts, + partition_id, + query_context); + + if (dest_storage->isDataLake()) + { + descriptor.max_bytes_per_file = query_context->getSettingsRef()[Setting::iceberg_insert_max_bytes_in_data_file]; + descriptor.max_rows_per_file = query_context->getSettingsRef()[Setting::iceberg_insert_max_rows_in_data_file]; + } + + std::vector part_references(parts.begin(), parts.end()); + partition_export_scheduler->addTask(std::move(descriptor), std::move(part_references), force); +} + +CancellationCode StorageMergeTree::killExportPartition(const String & transaction_id) +{ + if (!partition_export_scheduler) + return CancellationCode::NotFound; + return partition_export_scheduler->kill(transaction_id); +} + +std::vector StorageMergeTree::getPartitionExportsInfo() const +{ + if (!partition_export_scheduler) + return {}; + return partition_export_scheduler->getInfo(); +} + +void StorageMergeTree::partitionExportTask() +{ + /// Reschedule only while there is pending export work. When the scheduler reports no pending + /// tasks the schedule-pool task goes idle (no periodic wakeups per table); it is re-armed by + /// triggerPartitionExportTask() on a new EXPORT PARTITION and by loadFromDisk() at startup. + bool has_pending_work = true; + try + { + has_pending_work = partition_export_scheduler->run(); + } + catch (...) + { + tryLogCurrentException(log, __PRETTY_FUNCTION__); + /// On an unexpected error keep polling so we do not get stuck idle with pending work. + has_pending_work = true; + } + + if (has_pending_work) + partition_export_task->scheduleAfter(5000); +} + +void StorageMergeTree::triggerPartitionExportTask() +{ + if (partition_export_task) + partition_export_task->schedule(); +} } diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 15db160702c3..d6c8c03c39db 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -1,10 +1,12 @@ #pragma once #include +#include #include #include #include #include +#include #include #include #include @@ -126,7 +128,28 @@ class StorageMergeTree final : public MergeTreeData MergeTreeDeduplicationLog * getDeduplicationLog() { return deduplication_log.get(); } + /// EXPORT PARTITION for a plain (non-replicated) MergeTree table. Coordinated locally by + /// `partition_export_scheduler`; the task descriptor is persisted on disk (no ZooKeeper). + void exportPartitionToTable(const PartitionCommand & command, ContextPtr query_context) override; + + CancellationCode killExportPartition(const String & transaction_id) override; + + /// Snapshot of local partition-export tasks for `system.partition_exports`. No disk I/O. + std::vector getPartitionExportsInfo() const override; + private: + friend class MergeTreePartitionExportScheduler; + + /// Local coordinator + on-disk state for EXPORT PARTITION. Only created when the server setting + /// `allow_experimental_export_merge_tree_partition` is enabled. + std::shared_ptr partition_export_scheduler; + BackgroundSchedulePoolTaskHolder partition_export_task; + + /// Schedule-pool task body: drives partition_export_scheduler->run() periodically. + void partitionExportTask(); + /// Wakes up the partition-export schedule-pool task (no-op when the feature is disabled). + void triggerPartitionExportTask(); + /// Mutex and condvar for synchronous mutations wait std::mutex mutation_wait_mutex; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 880dae5cd904..b53b79d35b1b 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -79,11 +79,11 @@ #include #include #include -#include #include #include #include #include +#include #include #include @@ -4825,7 +4825,7 @@ void StorageReplicatedMergeTree::exportMergeTreePartitionStatusHandlingTask() } } -std::vector StorageReplicatedMergeTree::getPartitionExportsInfo() const +std::vector StorageReplicatedMergeTree::getPartitionExportsInfo() const { return export_merge_tree_partition_manifest_updater->getPartitionExportsInfo(); } @@ -8652,7 +8652,8 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & const auto exports_path = fs::path(zookeeper_path) / "exports"; - const auto export_key = partition_id + "_" + dest_storage_id.getQualifiedName().getFullName(); + const auto export_key = ExportPartitionUtils::compositeKey( + partition_id, dest_storage_id.getDatabaseName(), dest_storage_id.getTableName()); const auto partition_exports_path = fs::path(exports_path) / export_key; @@ -8767,65 +8768,22 @@ void StorageReplicatedMergeTree::exportPartitionToTable(const PartitionCommand & manifest.schema_match_mode = query_context->getSettingsRef()[Setting::export_merge_tree_part_schema_match_mode].value; manifest.ignore_extra_source_columns = query_context->getSettingsRef()[Setting::export_merge_tree_part_ignore_extra_source_columns].value; + /// Validate the destination against the partition being exported (Iceberg partition-spec + /// compatibility, or a matching partition key for plain destinations) and, for Iceberg + /// destinations, capture the destination metadata.json to persist in the manifest. + /// Shared with the plain `MergeTree` export path. + manifest.iceberg_metadata_json = ExportPartitionUtils::extractDestinationIcebergMetadataJson( + src_snapshot, + destination_snapshot, + dest_storage, + parts, + partition_id, + query_context); + if (dest_storage->isDataLake()) { -#if USE_AVRO - auto * object_storage = dynamic_cast(dest_storage.get()); - auto * object_storage_cluster = dynamic_cast(dest_storage.get()); - - /// in theory this should never happen, but just in case - if (!object_storage && !object_storage_cluster) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Destination storage {} is not a StorageObjectStorage", dest_storage->getName()); - } - - IcebergMetadata * iceberg_metadata = nullptr; - if (object_storage) - iceberg_metadata = dynamic_cast(object_storage->getExternalMetadata(query_context)); - else if (object_storage_cluster) - iceberg_metadata = dynamic_cast(object_storage_cluster->getExternalMetadata(query_context)); - if (!iceberg_metadata) - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Destination storage {} is a data lake but not an iceberg table", dest_storage->getName()); - } - - if (!query_context->getSettingsRef()[Setting::allow_insert_into_iceberg]) - { - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, - "Iceberg writes are experimental. " - "To allow its usage, enable the setting `allow_insert_into_iceberg` on the initiator (query, session or profile) - replicas inherit it from the scheduled task."); - } - - const auto metadata_object = iceberg_metadata->getMetadataJSON(query_context); - - ExportPartitionUtils::verifyIcebergPartitionCompatibility( - metadata_object, - src_snapshot, - destination_snapshot, - parts, - partition_id, - query_context); - - std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - oss.exceptions(std::ios::failbit); - metadata_object->stringify(oss); - manifest.iceberg_metadata_json = oss.str(); - manifest.max_bytes_per_file = query_context->getSettingsRef()[Setting::iceberg_insert_max_bytes_in_data_file]; manifest.max_rows_per_file = query_context->getSettingsRef()[Setting::iceberg_insert_max_rows_in_data_file]; - -#else - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Data lake export requires Avro support"); -#endif - } - else - { - ExportPartitionUtils::verifyPlainPartitionCompatibility( - src_snapshot, - destination_snapshot, - parts, - partition_id, - query_context); } ops.emplace_back(zkutil::makeCreateRequest( diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 73cbb0e1f594..64e1b122e1c8 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -100,8 +100,6 @@ namespace DB class ZooKeeperWithFaultInjection; using ZooKeeperWithFaultInjectionPtr = std::shared_ptr; -struct ReplicatedPartitionExportInfo; - class StorageReplicatedMergeTree final : public MergeTreeData { public: @@ -378,7 +376,7 @@ class StorageReplicatedMergeTree final : public MergeTreeData using ShutdownDeadline = std::chrono::time_point; void waitForUniquePartsToBeFetchedByOtherReplicas(ShutdownDeadline shutdown_deadline); - std::vector getPartitionExportsInfo() const; + std::vector getPartitionExportsInfo() const override; private: std::atomic_bool are_restoring_replica {false}; diff --git a/src/Storages/System/StorageSystemReplicatedPartitionExports.cpp b/src/Storages/System/StorageSystemPartitionExports.cpp similarity index 79% rename from src/Storages/System/StorageSystemReplicatedPartitionExports.cpp rename to src/Storages/System/StorageSystemPartitionExports.cpp index 6370e0f99433..8a9767b92c6f 100644 --- a/src/Storages/System/StorageSystemReplicatedPartitionExports.cpp +++ b/src/Storages/System/StorageSystemPartitionExports.cpp @@ -1,23 +1,21 @@ -#include +#include #include -#include -#include -#include -#include +#include #include #include #include +#include #include +#include +#include #include -#include -#include "Columns/ColumnString.h" -#include "Storages/VirtualColumnUtils.h" - +#include +#include namespace DB { -ColumnsDescription StorageSystemReplicatedPartitionExports::getColumnsDescription() +ColumnsDescription StorageSystemPartitionExports::getColumnsDescription() { auto last_exception_tuple = std::make_shared( DataTypes{ @@ -47,37 +45,38 @@ ColumnsDescription StorageSystemReplicatedPartitionExports::getColumnsDescriptio {"partition_id", std::make_shared(), "ID of the partition"}, {"transaction_id", std::make_shared(), "ID of the transaction."}, {"query_id", std::make_shared(), "Query ID of the export operation."}, - {"source_replica", std::make_shared(), "Name of the source replica."}, + {"source_replica", std::make_shared(), + "Name of the replica that received the export command. Empty for plain (non-replicated) MergeTree, which exports on a single node."}, {"parts", std::make_shared(std::make_shared()), "List of part names to be exported."}, {"parts_count", std::make_shared(), "Number of parts in the export."}, {"parts_to_do", std::make_shared(), "Number of parts pending to be exported."}, {"status", std::make_shared(), "Status of the export."}, {"last_exception_per_replica", std::make_shared(last_exception_tuple), - "Per-replica last exception entries. Each tuple records the most recent exception observed by that replica plus a best-effort within-replica count. Empty array if no replica has reported an exception for this task."}, + "Per-replica last exception entries. Each tuple records the most recent exception observed by that replica plus a best-effort within-replica count. Empty array if no replica has reported an exception for this task. A plain (non-replicated) MergeTree reports at most one entry, with an empty replica name."}, {"exception_count", std::make_shared(), "Sum of per-replica exception counts. Each replica owns its own count, so the sum is exact w.r.t. the in-memory snapshot; within-replica updates remain best-effort and may under-count by one under concurrent failures."}, {"destination_file_paths", std::make_shared(std::make_shared(), std::make_shared(std::make_shared())), - "Per-part destination file paths written to the destination object storage. Keyed by part name; values are the file paths produced by exporting that part. Mirrored from ZooKeeper on every poll while PENDING; partial during in-flight tasks. When the in-memory mirror could not fully refresh from Keeper (or a processed leaf is unreadable), the map may contain the key and/or path value '' for the affected part (or for the whole field if listing processed leaves failed); replaced on the next successful poll."}, + "Per-part destination file paths written to the destination object storage. Keyed by part name; values are the file paths produced by exporting that part. Empty until parts complete; partial during in-flight tasks. On a Replicated*MergeTree this mirrors ZooKeeper on every poll while PENDING, and when the in-memory mirror could not fully refresh from Keeper (or a processed leaf is unreadable), the map may contain the key and/or path value '' for the affected part (or for the whole field if listing processed leaves failed); replaced on the next successful poll."}, {"committed_metadata_file", std::make_shared(), - "For Iceberg destinations: path of the new metadata JSON file written at commit time. Empty for non-Iceberg destinations and for tasks that have not committed yet. May also be empty if the committing replica crashed between writing the object-storage files and persisting commit_info. If the export was already committed by a previous run (detected via the transaction id stored in the snapshot summary), this column holds a human-readable note instead of a path since the original committer's paths are not trivially recoverable."}, + "For Iceberg destinations: path of the new metadata JSON file written at commit time. Empty for non-Iceberg destinations and for tasks that have not committed yet. May also be empty if the committing replica crashed between writing the object-storage files and persisting commit_info. If the export was already committed by a previous run (detected via the transaction id stored in the snapshot summary), this column holds a human-readable note instead of a path since the original committer's paths are not trivially recoverable. Always empty for plain (non-replicated) MergeTree, which does not persist commit paths."}, {"committed_manifest_list", std::make_shared(), "For Iceberg destinations: path of the manifest list file (snap-*.avro) referenced by the new snapshot. Empty under the same conditions as committed_metadata_file."}, {"committed_manifest_file", std::make_shared(), "For Iceberg destinations: path of the manifest file referenced by committed_manifest_list. Empty under the same conditions as committed_metadata_file."}, {"committed_marker_file", std::make_shared(), - "For plain object storage destinations: path of the per-transaction commit marker file written by the destination. Empty for Iceberg destinations and for tasks that have not committed yet."}, + "For plain object storage destinations: path of the per-transaction commit marker file written by the destination. Empty for Iceberg destinations and for tasks that have not committed yet. Always empty for plain (non-replicated) MergeTree, which does not persist commit paths."}, {"local_backoff_per_part", std::make_shared(backoff_tuple), - "Per-part retry back-off local to this replica: parts currently waiting before their next attempt, with attempt count and the next eligible time. Not shared across replicas; empty if no part is backing off."}, + "Per-part retry back-off local to this node: parts currently waiting before their next attempt, with attempt count and the next eligible time. Not shared across replicas; empty if no part is backing off."}, }; } -void StorageSystemReplicatedPartitionExports::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector) const +void StorageSystemPartitionExports::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector) const { const auto access = context->getAccess(); const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES); - std::map> replicated_merge_tree_tables; - for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false})) + std::map> merge_tree_tables; + for (const auto & db : DatabaseCatalog::instance().getDatabases(GetDatabasesOptions{.with_datalake_catalogs = false, .with_remote_databases = false})) { /// skip data lakes if (db.second->isExternal()) @@ -91,21 +90,20 @@ void StorageSystemReplicatedPartitionExports::fillData(MutableColumns & res_colu if (!table) continue; - StorageReplicatedMergeTree * table_replicated = dynamic_cast(table.get()); - if (!table_replicated) + if (!dynamic_cast(table.get())) continue; if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name())) continue; - replicated_merge_tree_tables[db.first][iterator->name()] = table; + merge_tree_tables[db.first][iterator->name()] = table; } } MutableColumnPtr col_database_mut = ColumnString::create(); MutableColumnPtr col_table_mut = ColumnString::create(); - for (auto & db : replicated_merge_tree_tables) + for (auto & db : merge_tree_tables) { for (auto & table : db.second) { @@ -139,14 +137,14 @@ void StorageSystemReplicatedPartitionExports::fillData(MutableColumns & res_colu const auto database = (*col_database)[i_storage].safeGet(); const auto table = (*col_table)[i_storage].safeGet(); - std::vector partition_exports_info; + std::vector partition_exports_info; { - const IStorage * storage = replicated_merge_tree_tables[database][table].get(); - if (const auto * replicated_merge_tree = dynamic_cast(storage)) - partition_exports_info = replicated_merge_tree->getPartitionExportsInfo(); + const IStorage * storage = merge_tree_tables[database][table].get(); + if (const auto * merge_tree = dynamic_cast(storage)) + partition_exports_info = merge_tree->getPartitionExportsInfo(); } - for (const ReplicatedPartitionExportInfo & info : partition_exports_info) + for (const PartitionExportInfo & info : partition_exports_info) { std::size_t i = 0; res_columns[i++]->insert(database); @@ -158,11 +156,13 @@ void StorageSystemReplicatedPartitionExports::fillData(MutableColumns & res_colu res_columns[i++]->insert(info.transaction_id); res_columns[i++]->insert(info.query_id); res_columns[i++]->insert(info.source_replica); + Array parts_array; parts_array.reserve(info.parts.size()); for (const auto & part : info.parts) parts_array.push_back(part); res_columns[i++]->insert(parts_array); + res_columns[i++]->insert(info.parts_count); res_columns[i++]->insert(info.parts_to_do); res_columns[i++]->insert(info.status); @@ -188,9 +188,7 @@ void StorageSystemReplicatedPartitionExports::fillData(MutableColumns & res_colu res_columns[i++]->insert(info.committed_metadata_file); res_columns[i++]->insert(info.committed_manifest_list); - res_columns[i++]->insert(info.committed_manifest_file); - res_columns[i++]->insert(info.committed_marker_file); Array backoff_array; diff --git a/src/Storages/System/StorageSystemPartitionExports.h b/src/Storages/System/StorageSystemPartitionExports.h new file mode 100644 index 000000000000..d7f8e5242618 --- /dev/null +++ b/src/Storages/System/StorageSystemPartitionExports.h @@ -0,0 +1,30 @@ +#pragma once + +#include + +namespace DB +{ + +class Context; + +/// system.partition_exports: progress of EXPORT PARTITION tasks of every MergeTree-family table, +/// both plain `MergeTree` (backed by on-disk task descriptors) and `Replicated*MergeTree` (backed +/// by the ZooKeeper manifest mirror). Both are read from memory, so querying it touches neither +/// disk nor ZooKeeper. Each export task is represented by a single row. +/// +/// Also attached as `system.replicated_partition_exports`, a backwards-compatible alias from when +/// the two engines had separate tables. +class StorageSystemPartitionExports final : public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemPartitionExports"; } + + static ColumnsDescription getColumnsDescription(); + +protected: + using IStorageSystemOneBlock::IStorageSystemOneBlock; + + void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node *, std::vector) const override; +}; + +} diff --git a/src/Storages/System/StorageSystemReplicatedPartitionExports.h b/src/Storages/System/StorageSystemReplicatedPartitionExports.h deleted file mode 100644 index 768516c0a2ea..000000000000 --- a/src/Storages/System/StorageSystemReplicatedPartitionExports.h +++ /dev/null @@ -1,76 +0,0 @@ -#pragma once - -#include -#include - -namespace DB -{ - -class Context; - -struct ReplicatedPartitionExportInfo -{ - String destination_database; - String destination_table; - String partition_id; - String transaction_id; - String query_id; - time_t create_time; - String source_replica; - size_t parts_count; - size_t parts_to_do; - std::vector parts; - String status; - /// One entry per replica that has recorded at least one exception for this task. - /// Sourced verbatim from the in-memory mirror; no ZooKeeper traffic. - std::vector last_exception_per_replica; - /// Sum of per-replica counts. Each replica owns its own count, so cross-replica - /// updates do not race; the sum is exact w.r.t. the in-memory snapshot. Within a - /// single replica the count is best-effort (concurrent failing writers may under- - /// count by one), matching the documented column semantics. - size_t exception_count = 0; - - /// Per-part destination file paths, keyed by part name. Mirrors the - /// /processed//paths_in_destination data from ZooKeeper. - /// Empty until parts complete; partial during PENDING. May contain - /// "" when a Keeper refresh was incomplete or a - /// processed leaf could not be parsed. - std::map> destination_file_paths_per_part; - - /// Iceberg commit-time paths surfaced from /commit_info. - /// All empty for non-Iceberg destinations or before commit lands. - String committed_metadata_file; - String committed_manifest_list; - String committed_manifest_file; - - /// Plain object storage commit marker file surfaced from - /// /commit_info. Empty for Iceberg destinations or before - /// commit lands. - String committed_marker_file; - - struct PartBackoffEntry - { - String part; - size_t attempts = 0; - time_t next_retry_time = 0; - }; - - /// Parts of this task currently backing off (local to this replica). Empty if none. - std::vector backoff_per_part; -}; - -class StorageSystemReplicatedPartitionExports final : public IStorageSystemOneBlock -{ -public: - - std::string getName() const override { return "SystemReplicatedPartitionExports"; } - - static ColumnsDescription getColumnsDescription(); - -protected: - using IStorageSystemOneBlock::IStorageSystemOneBlock; - - void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node *, std::vector) const override; -}; - -} diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 3ed91453bfe7..f75b7dcae4bb 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include "config.h" #include @@ -269,7 +269,9 @@ void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, b attach(context, system_database, "exports", "Contains a list of exports currently executing exports of MergeTree tables and their progress. Each export operation is represented by a single row."); if (context->getServerSettings()[ServerSetting::allow_experimental_export_merge_tree_partition]) { - attach(context, system_database, "replicated_partition_exports", "Contains a list of partition exports of ReplicatedMergeTree tables and their progress. Each export operation is represented by a single row."); + attach(context, system_database, "partition_exports", "Contains a list of partition exports of MergeTree tables, both plain and replicated, and their progress. Each export operation is represented by a single row."); + /// Backwards-compatible alias from when plain and replicated exports had separate tables. + attach(context, system_database, "replicated_partition_exports", "Alias of system.partition_exports, kept for backwards compatibility. Returns the same rows, including exports of plain (non-replicated) MergeTree tables."); } attach(context, system_database, "mutations", "Contains a list of mutations and their progress. Each mutation command is represented by a single row."); attachNoDescription(context, system_database, "replicas", "Contains information and status of all table replicas on current server. Each replica is represented by a single row."); diff --git a/tests/integration/helpers/export_partition_helpers.py b/tests/integration/helpers/export_partition_helpers.py index 04c9cb244757..3643f2e32455 100644 --- a/tests/integration/helpers/export_partition_helpers.py +++ b/tests/integration/helpers/export_partition_helpers.py @@ -7,11 +7,37 @@ import time import uuid +from typing import NamedTuple + +import pytest MINIO_USER = "minio" MINIO_PASS = "ClickHouse_Minio_P@ssw0rd" +# The `EXPORT PARTITION` implementation is shared between the two MergeTree flavours, so the +# scenarios that do not depend on cross-replica coordination run once per engine. Suites expose +# this through the `source_engine` fixture; a test that requests the fixture runs twice. +SOURCE_ENGINES = ["MergeTree", "ReplicatedMergeTree"] +SOURCE_ENGINE_IDS = ["mt", "rmt"] + + +def is_replicated_engine(engine): + return engine == "ReplicatedMergeTree" + + +def skip_if_remote_database_disk_enabled(cluster): + """Skip the test if any instance in the cluster has remote database disk enabled. + + Tests that block MinIO cannot run when remote database disk is enabled, as the database + metadata is stored on MinIO and blocking it would break the database. + """ + for instance in cluster.instances.values(): + if instance.with_remote_database_disk: + pytest.skip( + "Test cannot run with remote database disk enabled (db disk), as it blocks MinIO which stores database metadata" + ) + def wait_for_export_status( node, @@ -21,8 +47,9 @@ def wait_for_export_status( expected_status="COMPLETED", timeout=60, poll_interval=0.5, + system_table="partition_exports", ): - """Poll system.replicated_partition_exports until status matches. + """Poll a partition-exports system table until status matches. *dest_table* may be ``None`` to skip filtering by destination table (useful for catalog-based tests where the destination is a database-qualified path). @@ -34,7 +61,7 @@ def wait_for_export_status( f" AND destination_table = '{dest_table}'" if dest_table else "" ) status = node.query( - f"SELECT status FROM system.replicated_partition_exports" + f"SELECT status FROM system.{system_table}" f" WHERE source_table = '{source_table}'" f"{dest_filter}" f" AND partition_id = '{partition_id}'" @@ -59,12 +86,13 @@ def wait_for_export_to_start( partition_id, timeout=10, poll_interval=0.2, + system_table="partition_exports", ): - """Poll until at least one row exists in system.replicated_partition_exports.""" + """Poll until at least one row exists in the given partition-exports system table.""" start_time = time.time() while time.time() - start_time < timeout: count = node.query( - f"SELECT count() FROM system.replicated_partition_exports" + f"SELECT count() FROM system.{system_table}" f" WHERE source_table = '{source_table}'" f" AND destination_table = '{dest_table}'" f" AND partition_id = '{partition_id}'" @@ -88,23 +116,24 @@ def wait_for_exception_count( min_exception_count=1, timeout=60, poll_interval=0.5, + system_table="partition_exports", ): """Wait for exception_count to reach at least *min_exception_count*. The default timeout is intentionally larger than one manifest-updater poll cycle (~30s, see StorageReplicatedMergeTree::exportMergeTreePartitionUpdatingTask). - system.replicated_partition_exports is served from the in-memory mirror, which - is refreshed on (a) the periodic poll tick and (b) status changes. While the - task is still PENDING (e.g. transient part-export failures with a generous - max_retries), no status watch fires, so newly written per-replica exception - leaves only become visible on the next poll. Allow at least one full cycle - plus headroom so the test is not racing the cadence. + For a ReplicatedMergeTree source, system.partition_exports is served from the + in-memory mirror, which is refreshed on (a) the periodic poll tick and (b) + status changes. While the task is still PENDING (e.g. transient part-export + failures with a generous max_retries), no status watch fires, so newly written + per-replica exception leaves only become visible on the next poll. Allow at + least one full cycle plus headroom so the test is not racing the cadence. """ start_time = time.time() last_exception_count = None while time.time() - start_time < timeout: exception_count_str = node.query( - f"SELECT exception_count FROM system.replicated_partition_exports" + f"SELECT exception_count FROM system.{system_table}" f" WHERE source_table = '{source_table}'" f" AND destination_table = '{dest_table}'" f" AND partition_id = '{partition_id}'" @@ -158,19 +187,102 @@ def make_mt( columns, partition_by, order_by="tuple()", + extra_settings="", ): """Create a MergeTree table with block-number settings.""" + settings = f"{_BLOCK_SETTINGS}, {extra_settings}" if extra_settings else _BLOCK_SETTINGS node.query( f""" CREATE TABLE {name} ({columns}) ENGINE = MergeTree() PARTITION BY {partition_by} ORDER BY {order_by} - SETTINGS {_BLOCK_SETTINGS} + SETTINGS {settings} """ ) +def make_source( + node, + name, + columns, + partition_by, + engine="ReplicatedMergeTree", + order_by="tuple()", + replica_name="r1", + extra_settings="", +): + """Create an export source table of the given MergeTree flavour. + + *replica_name* is ignored for a plain MergeTree, which has no replicas. + """ + if is_replicated_engine(engine): + make_rmt( + node, + name, + columns, + partition_by, + replica_name=replica_name, + order_by=order_by, + extra_settings=extra_settings, + ) + else: + make_mt( + node, + name, + columns, + partition_by, + order_by=order_by, + extra_settings=extra_settings, + ) + + +def setup_source_tables( + nodes, + name, + columns, + partition_by, + engine, + insert_values=None, + order_by="tuple()", + extra_settings="", + replica_names=None, +): + """Create the export source table on the nodes that can host it and insert the initial data. + + A `ReplicatedMergeTree` source is created on every node, and the insert made on the first + node is synced to the others. A plain `MergeTree` only exists on the first node, which is + the only node that can drive its export, so there is nothing to create or sync elsewhere. + + This is the single place that branches on the engine, so the tests themselves stay free of + engine conditionals. Returns the nodes hosting the source table. + """ + nodes = list(nodes) + hosts = nodes if is_replicated_engine(engine) else nodes[:1] + + if replica_names is None: + replica_names = [node.name for node in hosts] + + for node, replica_name in zip(hosts, replica_names): + make_source( + node, + name, + columns, + partition_by, + engine=engine, + order_by=order_by, + replica_name=replica_name, + extra_settings=extra_settings, + ) + + if insert_values: + hosts[0].query(f"INSERT INTO {name} VALUES {insert_values}") + for node in hosts[1:]: + node.query(f"SYSTEM SYNC REPLICA {name}") + + return hosts + + def make_iceberg_s3( node, name, @@ -210,3 +322,60 @@ def first_partition_id(node, table): def unique_suffix(): """Return a UUID with hyphens replaced by underscores, suitable for table names.""" return str(uuid.uuid4()).replace("-", "_") + + +# -- how a destination whose column count exceeds the source's is matched +EXTRA_SOURCE_COLUMN_MODES = [ + pytest.param("POSITION", id="by-position"), + pytest.param("NAME", id="by-name"), +] + + +class RejectedPartitionExportCase(NamedTuple): + src_columns: str + src_partition_by: str + dst_columns: str + dst_partition_by: str + insert_values: str + error_substrings: tuple = () + + +# Partition keys that must be rejected regardless of the destination kind. A destination-specific +# suite may append its own cases (see the Iceberg transform case). +REJECTED_PARTITION_EXPORT_CASES = [ + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32", + src_partition_by="a", + dst_columns="b Int32, a Int32", + dst_partition_by="a", + insert_values="(1, 1), (1, 2)", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_single_column", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b, c)", + dst_columns="c Int32, b Int32, a Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 1, 1, 'x'), (1, 1, 1, 'y')", + error_substrings=("partition key column",), + ), + id="same_partition_key_different_column_order_multi_column", + ), + pytest.param( + RejectedPartitionExportCase( + src_columns="a Int32, b Int32, c Int32, val String", + src_partition_by="(a, b)", + dst_columns="a Int32, b Int32, c Int32, val String", + dst_partition_by="(a, b, c)", + insert_values="(1, 2, 3, 'x')", + error_substrings=( + "column 'c', which is not part of the source MergeTree partition key", + ), + ), + id="multi_column_partition_key_more_in_destination", + ), +] diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/__init__.py b/tests/integration/test_export_mt_partition_to_object_storage/__init__.py similarity index 100% rename from tests/integration/test_export_replicated_mt_partition_to_iceberg/__init__.py rename to tests/integration/test_export_mt_partition_to_object_storage/__init__.py diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/configs/allow_experimental_export_partition.xml b/tests/integration/test_export_mt_partition_to_object_storage/configs/allow_experimental_export_partition.xml similarity index 100% rename from tests/integration/test_export_replicated_mt_partition_to_iceberg/configs/allow_experimental_export_partition.xml rename to tests/integration/test_export_mt_partition_to_object_storage/configs/allow_experimental_export_partition.xml diff --git a/tests/integration/test_export_mt_partition_to_object_storage/configs/named_collections.xml b/tests/integration/test_export_mt_partition_to_object_storage/configs/named_collections.xml new file mode 100644 index 000000000000..573822539c50 --- /dev/null +++ b/tests/integration/test_export_mt_partition_to_object_storage/configs/named_collections.xml @@ -0,0 +1,9 @@ + + + + http://minio1:9001/root/data + minio + ClickHouse_Minio_P@ssw0rd + + + diff --git a/tests/integration/test_export_mt_partition_to_object_storage/configs/users.d/profile.xml b/tests/integration/test_export_mt_partition_to_object_storage/configs/users.d/profile.xml new file mode 100644 index 000000000000..8f9cbd70675d --- /dev/null +++ b/tests/integration/test_export_mt_partition_to_object_storage/configs/users.d/profile.xml @@ -0,0 +1,7 @@ + + + + 3 + + + diff --git a/tests/integration/test_export_mt_partition_to_object_storage/test.py b/tests/integration/test_export_mt_partition_to_object_storage/test.py new file mode 100644 index 000000000000..43ba733b4685 --- /dev/null +++ b/tests/integration/test_export_mt_partition_to_object_storage/test.py @@ -0,0 +1,157 @@ +import logging + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.export_partition_helpers import ( + make_mt, + unique_suffix, + wait_for_export_status, + wait_for_export_to_start, +) +from helpers.network import PartitionManager + +# What is left here is the plain-`MergeTree` behavior that the unified suites cannot express: +# +# * this cluster has no ZooKeeper at all, which is what proves a plain `MergeTree` export needs no +# Keeper ensemble - `test_export_partition_to_object_storage` runs inside a Keeper-backed cluster, +# * the task descriptor lives on the table's disk instead of in Keeper, so it must survive a hard +# restart and resume on its own. +# +# Everything else that used to live here now runs as the `mt` parameter of +# `test_export_partition_to_object_storage`. + + +def skip_if_remote_database_disk_enabled(cluster): + for instance in cluster.instances.values(): + if instance.with_remote_database_disk: + pytest.skip( + "Test cannot run with remote database disk enabled, as it blocks MinIO which stores database metadata" + ) + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance( + "node", + main_configs=[ + "configs/named_collections.xml", + "configs/allow_experimental_export_partition.xml", + ], + user_configs=["configs/users.d/profile.xml"], + with_minio=True, + stay_alive=True, + ) + logging.info("Starting cluster...") + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def drop_tables_after_test(cluster): + yield + for instance_name, instance in cluster.instances.items(): + try: + tables_str = instance.query( + "SELECT name FROM system.tables WHERE database = 'default' FORMAT TabSeparated" + ).strip() + if not tables_str: + continue + for table in tables_str.split("\n"): + table = table.strip() + if table: + instance.query(f"DROP TABLE IF EXISTS default.`{table}` SYNC") + except Exception as e: + logging.warning(f"drop_tables_after_test: cleanup failed on {instance_name}: {e}") + + +def create_tables_and_insert_data(node, mt_table, s3_table): + node.query(f"DROP TABLE IF EXISTS {mt_table} SYNC") + make_mt(node, mt_table, "id UInt64, year UInt16", "year") + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020), (3, 2020), (4, 2021)") + node.query( + f"CREATE TABLE {s3_table} (id UInt64, year UInt16) " + f"ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') " + f"PARTITION BY year" + ) + + +def test_export_partition_without_keeper(cluster): + """A plain MergeTree export is driven entirely by the local scheduler, so it must work on a node + that has no ZooKeeper configured at all.""" + node = cluster.instances["node"] + + # Asserted against the cluster definition rather than a system table: this is a property of + # how the instance was built, and it holds on every server version. + assert not node.with_zookeeper, ( + "This suite must run without ZooKeeper, otherwise it proves nothing about a Keeper-less " + "export" + ) + + postfix = unique_suffix() + mt_table = f"basic_mt_{postfix}" + s3_table = f"basic_s3_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table) + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}") + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") + + assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") == "3\n" + assert ( + node.query( + f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/commit_2020_*', format=LineAsString)" + ) + != "0\n" + ), "Commit file missing for partition 2020" + + +def test_export_partition_resumes_after_restart(cluster): + """The distinguishing feature of the plain MergeTree implementation: the on-disk task + descriptor must let an in-flight export resume after a hard restart.""" + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["node"] + + postfix = unique_suffix() + mt_table = f"restart_mt_{postfix}" + s3_table = f"restart_s3_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table) + + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + with PartitionManager() as pm: + pm.add_rule({ + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + pm.add_rule({ + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}") + wait_for_export_to_start(node, mt_table, s3_table, "2020") + + # Kill the server while the export is still in flight (S3 blocked, nothing committed yet). + node.stop_clickhouse(kill=True) + + # We cannot observe the "nothing committed before the crash" invariant on a single node: while + # the only node is down there is nothing to query S3 with, and once it restarts (S3 now + # reachable) the persisted PENDING task resumes immediately. So we only assert the actual + # restart-resume behavior: the task must resume from its on-disk descriptor and complete. + node.start_clickhouse() + + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED", timeout=90) + assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") == "3\n" diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/__init__.py b/tests/integration/test_export_partition_to_iceberg/__init__.py similarity index 100% rename from tests/integration/test_export_replicated_mt_partition_to_object_storage/__init__.py rename to tests/integration/test_export_partition_to_iceberg/__init__.py diff --git a/tests/integration/test_export_partition_to_iceberg/common.py b/tests/integration/test_export_partition_to_iceberg/common.py new file mode 100644 index 000000000000..313695118511 --- /dev/null +++ b/tests/integration/test_export_partition_to_iceberg/common.py @@ -0,0 +1,112 @@ +import time + +from helpers.export_partition_helpers import ( + make_iceberg_s3, + setup_source_tables, +) + +# Table factories and metadata assertions shared by the `EXPORT PARTITION` Iceberg test modules. + + +def create_iceberg_s3_table(node, iceberg_table: str, if_not_exists: bool = False, + s3_retry_attempts: int = 3): + """Create (or attach to an existing) IcebergS3 table at a per-test MinIO prefix.""" + make_iceberg_s3( + node, iceberg_table, "id Int64, year Int32", + partition_by="year", if_not_exists=if_not_exists, + s3_retry_attempts=s3_retry_attempts, + ) + + +def setup_tables(cluster, mt_table: str, iceberg_table: str, nodes: list | None = None, + s3_retry_attempts: int = 3, engine: str = "ReplicatedMergeTree"): + """ + Create the source table on the given nodes, insert data on the first node, wait for + replication, then create the Iceberg destination table on each node. + + A plain `MergeTree` source only exists on the first node (see `setup_source_tables`), but the + destination is still created everywhere the caller asked for, so a test can read it back from + any node. The Iceberg table is created on the first node (which initialises the S3 metadata); + subsequent nodes attach to the same path with IF NOT EXISTS. + + `nodes` defaults to every instance the test module asked for in `CLUSTER_INSTANCES`, so it + can never name an instance this cluster did not start. + """ + if nodes is None: + nodes = list(cluster.instances) + + instances = [cluster.instances[n] for n in nodes] + primary = instances[0] + + setup_source_tables( + instances, + mt_table, + "id Int64, year Int32", + "year", + engine, + insert_values="(1, 2020), (2, 2020), (3, 2020), (4, 2021)", + replica_names=nodes, + ) + + create_iceberg_s3_table(primary, iceberg_table, s3_retry_attempts=s3_retry_attempts) + for instance in instances[1:]: + create_iceberg_s3_table(instance, iceberg_table, if_not_exists=True, + s3_retry_attempts=s3_retry_attempts) + + +def _destination_paths_has_sync_failed_marker(node, source_table, dest_table, partition_id): + """True when destination_file_paths contains the Keeper sync-failed marker value.""" + result = node.query( + f"SELECT has(arrayFlatten(mapValues(destination_file_paths)), '')" + f" FROM system.partition_exports" + f" WHERE source_table = '{source_table}'" + f" AND destination_table = '{dest_table}'" + f" AND partition_id = '{partition_id}'" + ).strip() + return result == "1" + + +def wait_for_destination_paths_sync_failed_marker( + node, source_table, dest_table, partition_id, expect_marker, timeout=90, poll_interval=0.5 +): + """Wait until destination_file_paths does/does not contain the sync-failed marker. + + The in-memory mirror refreshes on the manifest-updater poll (~30s), so the + default timeout allows at least one full cycle plus headroom. + """ + start_time = time.time() + last = None + while time.time() - start_time < timeout: + last = _destination_paths_has_sync_failed_marker( + node, source_table, dest_table, partition_id + ) + if last == expect_marker: + return + time.sleep(poll_interval) + + raise TimeoutError( + f"destination_file_paths sync-failed marker did not become {expect_marker}" + f" within {timeout}s (last={last})" + ) + + +def data_file_partition_records(entries): + """Partition dicts of the non-delete data files described by manifest entries.""" + records = [] + for entry in entries: + data_file = entry.get("data_file") or {} + if data_file.get("content", 0) not in (0, None): + continue + partition = data_file.get("partition") + if partition is not None: + records.append(partition) + return records + + +def partition_scalar(partition, field): + """Read a partition field value, tolerating an Avro-union ``{type: value}`` wrapper.""" + value = partition.get(field) + if isinstance(value, dict): + assert len(value) == 1, f"Unexpected partition union shape for {field!r}: {value!r}" + value = next(iter(value.values())) + return value diff --git a/tests/integration/test_export_partition_to_iceberg/configs/allow_experimental_export_partition.xml b/tests/integration/test_export_partition_to_iceberg/configs/allow_experimental_export_partition.xml new file mode 100644 index 000000000000..514cd710836a --- /dev/null +++ b/tests/integration/test_export_partition_to_iceberg/configs/allow_experimental_export_partition.xml @@ -0,0 +1,3 @@ + + 1 + diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/configs/config.d/metadata_log.xml b/tests/integration/test_export_partition_to_iceberg/configs/config.d/metadata_log.xml similarity index 100% rename from tests/integration/test_export_replicated_mt_partition_to_iceberg/configs/config.d/metadata_log.xml rename to tests/integration/test_export_partition_to_iceberg/configs/config.d/metadata_log.xml diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/configs/users.d/profile.xml b/tests/integration/test_export_partition_to_iceberg/configs/users.d/profile.xml similarity index 100% rename from tests/integration/test_export_replicated_mt_partition_to_iceberg/configs/users.d/profile.xml rename to tests/integration/test_export_partition_to_iceberg/configs/users.d/profile.xml diff --git a/tests/integration/test_export_partition_to_iceberg/conftest.py b/tests/integration/test_export_partition_to_iceberg/conftest.py new file mode 100644 index 000000000000..dcab2dd0953d --- /dev/null +++ b/tests/integration/test_export_partition_to_iceberg/conftest.py @@ -0,0 +1,77 @@ +import logging + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.export_partition_helpers import SOURCE_ENGINE_IDS, SOURCE_ENGINES + +# `EXPORT PARTITION` to an Iceberg destination, split across several modules so the harness can +# spread them over xdist workers (`--dist=loadfile` assigns a whole module to one worker). +# +# Each module declares the instances it needs in `CLUSTER_INSTANCES` and gets a cluster with only +# those, so modules that never touch a second replica do not start one. + +REPLICA = dict( + main_configs=[ + "configs/allow_experimental_export_partition.xml", + "configs/config.d/metadata_log.xml", + ], + user_configs=["configs/users.d/profile.xml"], + with_minio=True, + stay_alive=True, + with_zookeeper=True, + keeper_required_feature_flags=["multi_read"], +) + +INSTANCES = {"replica1": REPLICA, "replica2": REPLICA} + + +@pytest.fixture(scope="module") +def cluster(request): + instance_names = getattr(request.module, "CLUSTER_INSTANCES", list(INSTANCES)) + try: + cluster = ClickHouseCluster(__file__) + for name in instance_names: + cluster.add_instance(name, **INSTANCES[name]) + logging.info("Starting cluster with instances %s...", instance_names) + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def drop_tables_after_test(cluster): + """Drop all tables in the default database after every test. + + Without this, ReplicatedMergeTree tables from completed tests remain alive and keep + running ZooKeeper background threads. With many tables alive simultaneously the + ZooKeeper session becomes overwhelmed and subsequent tests start seeing + operation-timeout / session-expired errors. + """ + yield + for instance_name, instance in cluster.instances.items(): + try: + tables_str = instance.query( + "SELECT name FROM system.tables WHERE database = 'default' FORMAT TabSeparated" + ).strip() + if not tables_str: + continue + for table in tables_str.split("\n"): + table = table.strip() + if table: + instance.query(f"DROP TABLE IF EXISTS default.`{table}` SYNC") + except Exception as e: + logging.warning( + f"drop_tables_after_test: cleanup failed on {instance_name}: {e}" + ) + + +@pytest.fixture(params=SOURCE_ENGINES, ids=SOURCE_ENGINE_IDS) +def source_engine(request): + """The MergeTree flavour of the export source table. + + A test that requests this fixture runs once per engine; the scenarios that only make sense + with cross-replica coordination do not request it and stay on `ReplicatedMergeTree`. + """ + return request.param diff --git a/tests/integration/test_export_partition_to_iceberg/test_failures.py b/tests/integration/test_export_partition_to_iceberg/test_failures.py new file mode 100644 index 000000000000..75c52c1fe60f --- /dev/null +++ b/tests/integration/test_export_partition_to_iceberg/test_failures.py @@ -0,0 +1,743 @@ +import time + +from helpers.export_partition_helpers import ( + make_iceberg_s3, + make_source, + unique_suffix, + wait_for_exception_count, + wait_for_export_status, + wait_for_export_to_start, +) +from helpers.iceberg_export_stats import fetch_manifest_entries +from helpers.network import PartitionManager + +from .common import ( + data_file_partition_records, + partition_scalar, + setup_tables, +) + +CLUSTER_INSTANCES = ["replica1", "replica2"] + +# `EXPORT PARTITION` into Iceberg under injected failure: commit-path failpoints, retryable errors, +# stopped moves, killed and timed-out tasks, and the guarantee that a failed commit leaves the +# destination snapshot intact. Slow and timing sensitive - kept out of the parallel batch. + + +def test_failure_is_logged_in_system_table(cluster, source_engine): + """ + When a part export fails with a non-retryable error the export must be marked + FAILED in system.partition_exports with a non-zero exception_count. + + Uses the export_part_non_retryable_throw failpoint (throws BAD_ARGUMENTS, a + denylisted code) so the task fails fast without consuming any timeout budget. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"], engine=source_engine) + + node.query("SYSTEM ENABLE FAILPOINT export_part_non_retryable_throw") + try: + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + + # short timeout to exercise the fast fail path for non retryable errors + wait_for_export_status(node, mt_table, iceberg_table, "2020", "FAILED", timeout=20) + finally: + node.query("SYSTEM DISABLE FAILPOINT export_part_non_retryable_throw") + + status = node.query( + f""" + SELECT status FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{iceberg_table}' + AND partition_id = '2020' + """ + ).strip() + assert status == "FAILED", f"Expected FAILED status, got: {status!r}" + + exception_count = int(node.query( + f""" + SELECT any(exception_count) FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{iceberg_table}' + AND partition_id = '2020' + """ + ).strip()) + assert exception_count > 0, "Expected non-zero exception_count in system.partition_exports" + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after a failed export, got {count}" + + +def test_inject_short_living_failures(cluster): + """ + Transient S3 failures must not prevent the export from completing: after the + network is restored the export should retry and eventually land COMPLETED. + """ + node = cluster.instances["replica1"] + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"], + s3_retry_attempts=1) + + node.query(f"SYSTEM STOP MOVES {mt_table}") + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table} SETTINGS allow_insert_into_iceberg = 1") + + with PartitionManager() as pm: + pm.add_rule({ + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + pm.add_rule({ + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + + node.query(f"SYSTEM START MOVES {mt_table}") + + # Let at least one retry happen before restoring the network. + time.sleep(15) + + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) + assert count == 3, f"Expected 3 rows after retry, got {count}" + + status = node.query( + f""" + SELECT status FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{iceberg_table}' + AND partition_id = '2020' + """ + ).strip() + assert status == "COMPLETED", f"Expected COMPLETED in system table, got: {status!r}" + + exception_count = int(node.query( + f""" + SELECT exception_count FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{iceberg_table}' + AND partition_id = '2020' + """ + ).strip()) + assert exception_count >= 1, "Expected at least one transient exception to be recorded" + + +def test_export_partition_retryable_error_killed_on_timeout(cluster, source_engine): + """ + A retryable part-export error (here FAULT_INJECTED via export_part_retryable_throw) + must NOT fail the task on a retry budget: there is no retry budget anymore, so the + part keeps retrying until the absolute task timeout fires and the task is KILLED. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"], engine=source_engine) + + node.query("SYSTEM ENABLE FAILPOINT export_part_retryable_throw") + try: + # Under the old budget model a small retry budget would fail the task after the + # first retry. With the new model there is no budget and only the 5s timeout fails it. + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}" + f" SETTINGS export_merge_tree_partition_task_timeout_seconds = 5," + f" allow_insert_into_iceberg = 1" + ) + + # Give the scheduler time to attempt and fail the part several times. The old + # budget would already have transitioned the task to FAILED by now. + time.sleep(15) + status = node.query( + f"SELECT status FROM system.partition_exports" + f" WHERE source_table = '{mt_table}'" + f" AND destination_table = '{iceberg_table}'" + f" AND partition_id = '2020'" + ).strip() + assert status != "FAILED", ( + f"Retryable failures must not fail the task on a budget, got status {status!r}" + ) + + # The timeout (5s) is past; KILLED fires on the next manifest-updater poll cycle. + wait_for_export_status( + node, mt_table, iceberg_table, "2020", "KILLED", timeout=90 + ) + finally: + node.query("SYSTEM DISABLE FAILPOINT export_part_retryable_throw") + + exception_count = int(node.query( + f"SELECT any(exception_count) FROM system.partition_exports" + f" WHERE source_table = '{mt_table}'" + f" AND destination_table = '{iceberg_table}'" + f" AND partition_id = '2020'" + ).strip()) + assert exception_count > 0, "Expected at least one retryable exception to be recorded" + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table after a killed export, got {count}" + + +def test_export_partition_retryable_error_recovers_after_failpoint_cleared(cluster, source_engine): + """ + A retryable part-export error must keep the task PENDING (not FAILED) while the + failure persists, applying a per-replica back-off between attempts. Once the + failure clears the export completes successfully — proving the back-off only + spaces retries out and never permanently blocks progress. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"], engine=source_engine) + + node.query("SYSTEM ENABLE FAILPOINT export_part_retryable_throw") + try: + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}" + f" SETTINGS export_merge_tree_partition_retry_initial_backoff_seconds = 1," + f" export_merge_tree_partition_retry_max_backoff_seconds = 2," + f" allow_insert_into_iceberg = 1" + ) + + # Wait until at least one retryable failure has been recorded; the task must + # still be PENDING (retrying), never FAILED. + wait_for_exception_count(node, mt_table, iceberg_table, "2020", + min_exception_count=1, timeout=60) + status = node.query( + f"SELECT status FROM system.partition_exports" + f" WHERE source_table = '{mt_table}'" + f" AND destination_table = '{iceberg_table}'" + f" AND partition_id = '2020'" + ).strip() + assert status == "PENDING", ( + f"Retryable failures must keep the task PENDING, got status {status!r}" + ) + finally: + node.query("SYSTEM DISABLE FAILPOINT export_part_retryable_throw") + + # With the failpoint cleared the next retry succeeds and the export completes. + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED", timeout=90) + + count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) + assert count == 3, f"Expected 3 rows after recovery, got {count}" + + +def test_export_partition_local_backoff_does_not_block_other_replica(cluster): + """ + Back-off is per-replica and in-memory: a part that one replica keeps failing on + (and therefore puts into its local back-off) must NOT be prevented from being + exported by another replica. This is the whole reason the back-off is local + rather than distributed in ZooKeeper. + + replica1 is given a persistent *retryable* failure (export_part_retryable_throw) + and is the only replica scheduling at first (moves are stopped on replica2). Once + replica1 has recorded a failure and a local back-off entry, replica2's scheduler + is enabled. Because the failpoint stays active on replica1 the whole time, the + only way the export can reach COMPLETED is replica2 picking up the very part that + replica1 keeps failing — proving the back-off does not leak across replicas. + """ + replica1 = cluster.instances["replica1"] + replica2 = cluster.instances["replica2"] + + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1", "replica2"]) + + # Phase 1: only replica1 schedules. Stop the export scheduler on replica2 so the + # part is guaranteed to be attempted (and fail) on replica1 first. + replica2.query(f"SYSTEM STOP MOVES {mt_table}") + + replica1.query("SYSTEM ENABLE FAILPOINT export_part_retryable_throw") + try: + replica1.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}" + f" SETTINGS export_merge_tree_partition_retry_initial_backoff_seconds = 1," + f" export_merge_tree_partition_retry_max_backoff_seconds = 2," + f" allow_insert_into_iceberg = 1" + ) + + # replica1 attempts the part, fails (retryable), and enters local back-off. + # The task must stay PENDING — there is no retry budget to fail it. + wait_for_exception_count(replica1, mt_table, iceberg_table, "2020", + min_exception_count=1, timeout=60) + + wait_for_export_status(replica1, mt_table, iceberg_table, "2020", "PENDING", timeout=60) + + # The back-off entry must be observable on replica1 (the failing replica). + deadline = time.time() + 90 + backoff_replica1 = "0" + while time.time() < deadline: + backoff_replica1 = replica1.query( + f"SELECT length(local_backoff_per_part) FROM system.partition_exports" + f" WHERE source_table = '{mt_table}'" + f" AND destination_table = '{iceberg_table}'" + f" AND partition_id = '2020'" + ).strip() + if backoff_replica1 not in ("", "0"): + break + time.sleep(0.5) + assert backoff_replica1 not in ("", "0"), ( + "Expected replica1 to carry a local back-off entry for the failing part, " + f"got {backoff_replica1!r}" + ) + + # ... and it must NOT have leaked to replica2, which never attempted the part. + # This is the core assertion: local back-off state is not shared across replicas. + backoff_replica2 = replica2.query( + f"SELECT length(local_backoff_per_part) FROM system.partition_exports" + f" WHERE source_table = '{mt_table}'" + f" AND destination_table = '{iceberg_table}'" + f" AND partition_id = '2020'" + ).strip() + + assert backoff_replica2 in ("", "0"), ( + f"replica2 must not carry replica1's local back-off, got {backoff_replica2!r}" + ) + + # Phase 2: enable replica2's scheduler. replica1 keeps failing (the failpoint + # is still active), so completion can only come from replica2 exporting the + # part that replica1 is backing off on. + replica2.query(f"SYSTEM START MOVES {mt_table}") + + wait_for_export_status(replica2, mt_table, iceberg_table, "2020", "COMPLETED", timeout=60) + finally: + replica1.query("SYSTEM DISABLE FAILPOINT export_part_retryable_throw") + + count = int(replica2.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) + assert count == 3, f"Expected 3 rows after replica2 completed the export, got {count}" + + +def test_export_partition_scheduler_skipped_when_moves_stopped(cluster, source_engine): + """ + Verify that selectPartsToExport() skips the scheduler entirely when moves + are stopped (moves_blocker guard at the top of the function). + + No ZK locks are acquired and no background tasks are submitted, so the + Iceberg table must remain empty across multiple scheduler cycles. Once moves + are re-enabled the export completes and rows appear in the Iceberg table. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"], engine=source_engine) + + node.query(f"SYSTEM STOP MOVES {mt_table}") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + + wait_for_export_to_start(node, mt_table, iceberg_table, "2020") + + # Wait for several scheduler cycles (each fires every 5 s). + # If the guard is absent the scheduler would run and rows would appear in the Iceberg table. + time.sleep(12) + + status = node.query( + f"SELECT status FROM system.partition_exports" + f" WHERE source_table = '{mt_table}' AND destination_table = '{iceberg_table}'" + f" AND partition_id = '2020'" + ).strip() + + assert status == "PENDING", f"Expected PENDING while moves are stopped, got '{status}'" + + count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table while scheduler is skipped, got {count}" + + node.query(f"SYSTEM START MOVES {mt_table}") + + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) + assert count == 3, f"Expected 3 rows in Iceberg table after export completed, got {count}" + + +def test_export_partition_resumes_after_stop_moves(cluster, source_engine): + """ + Verify that SYSTEM STOP MOVES before EXPORT PARTITION does not permanently + orphan the ZooKeeper part lock for Iceberg destinations. + + When moves are stopped the scheduler still picks parts up and submits them to + the background executor, but ExportPartTask::isCancelled() returns true (via + moves_blocker), causing QUERY_WAS_CANCELLED before any data is written. The + fix in handlePartExportFailure must release the ZK lock so the part is retried + once moves are restarted. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"], engine=source_engine) + + node.query(f"SYSTEM STOP MOVES {mt_table}") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}" + f" SETTINGS allow_insert_into_iceberg = 1" + ) + + wait_for_export_to_start(node, mt_table, iceberg_table, "2020") + + # Give the scheduler enough time to attempt (and cancel) the part task at least once. + time.sleep(5) + + status = node.query( + f"SELECT status FROM system.partition_exports" + f" WHERE source_table = '{mt_table}' AND destination_table = '{iceberg_table}'" + f" AND partition_id = '2020'" + ).strip() + assert status == "PENDING", f"Expected PENDING while moves are stopped, got '{status}'" + + count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) + assert count == 0, f"Expected 0 rows in Iceberg table while moves are stopped, got {count}" + + node.query(f"SYSTEM START MOVES {mt_table}") + + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) + assert count == 3, f"Expected 3 rows in Iceberg table after export completed, got {count}" + + +def test_export_partition_resumes_after_stop_moves_during_export(cluster, source_engine): + """ + Verify that SYSTEM STOP MOVES issued while an Iceberg export is actively + retrying (S3 blocked) does not permanently orphan the ZooKeeper part lock. + """ + node = cluster.instances["replica1"] + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"], engine=source_engine) + + node.query(f"SYSTEM STOP MOVES {mt_table}") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}" + f" SETTINGS allow_insert_into_iceberg = 1") + + wait_for_export_to_start(node, mt_table, iceberg_table, "2020") + + with PartitionManager() as pm: + pm.add_rule({ + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + pm.add_rule({ + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + + node.query(f"SYSTEM STOP MOVES {mt_table}") + + time.sleep(3) + + status = node.query( + f"SELECT status FROM system.partition_exports" + f" WHERE source_table = '{mt_table}' AND destination_table = '{iceberg_table}'" + f" AND partition_id = '2020'" + ).strip() + assert status == "PENDING", ( + f"Expected PENDING while moves are stopped and S3 is blocked, got '{status}'" + ) + + node.query(f"SYSTEM START MOVES {mt_table}") + + # MinIO is now unblocked; the next scheduler cycle should succeed. + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) + assert count == 3, f"Expected 3 rows in Iceberg table after export completed, got {count}" + + +def test_export_data_files_are_not_cleaned_up_on_commit_failure(cluster): + """ + Verify that a commit failure does not delete the already-written data files. + `cleanup` only removes the manifest entry / manifest list, never the data files + (a peer replica might still commit the same transaction). This guards against + data loss / dangling references. + + The iceberg_writes_non_retry_cleanup failpoint throws BAD_ARGUMENTS while writing + the manifest entry, after the data files have been written. BAD_ARGUMENTS is a + non-retryable error code, so the task transitions to FAILED; we then confirm the + exported data files are still physically present in object storage by reading + them directly (the Iceberg manifests were removed by cleanup, so we glob the raw + parquet data files instead). + """ + node = cluster.instances["replica1"] + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) + + node.query("SYSTEM ENABLE FAILPOINT iceberg_writes_non_retry_cleanup") + try: + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + # BAD_ARGUMENTS from the commit phase is non-retryable -> the task fails fast. + wait_for_export_status(node, mt_table, iceberg_table, "2020", "FAILED", timeout=60) + finally: + node.query("SYSTEM DISABLE FAILPOINT iceberg_writes_non_retry_cleanup") + + # The data files were written before the commit failure; cleanup must have left + # them intact. Read them straight from object storage (bypassing the Iceberg + # metadata, which cleanup removed) and confirm all 3 exported rows survive. + rows = int(node.query( + f"SELECT count() FROM s3(" + f"'http://minio1:9001/root/data/{iceberg_table}/**.parquet', " + f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet')" + ).strip()) + assert rows == 3, ( + f"Expected the 3 exported rows to still exist as data files after a failed " + f"commit (data files must not be cleaned up), got {rows}" + ) + + +def test_post_publish_exception_preserves_snapshot(cluster): + """ + Regression test for the post-publish exception-safety bug in + commitImportPartitionTransactionImpl. + + Before the fix, any exception thrown after the Iceberg snapshot was published + (e.g. from metadata-cache invalidation) would fall through to the outer + `catch (...)` and invoke `cleanup(false)`, which unconditionally removed the + manifest entry and manifest list referenced by the just-published snapshot. + A subsequent read would then fail because the live snapshot points to deleted + files. + + The failpoint `iceberg_writes_post_publish_throw` is placed inside the + post-publish region (after both the metadata file is written and + `published = true` is set). With the fix in place: + - the commit stays durable (snapshot is readable, manifests are intact); + - the export is marked COMPLETED because the outer `catch (...)` sees + `published == true` and returns the populated commit info with the real + paths produced by this attempt (no retry needed); + - all exported rows are visible through the Iceberg table. + """ + node = cluster.instances["replica1"] + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) + + node.query("SYSTEM ENABLE FAILPOINT iceberg_writes_post_publish_throw") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) + assert count == 3, ( + f"Snapshot must remain readable after a post-publish exception, " + f"expected 3 rows but got {count} (manifest files likely deleted by " + f"over-broad cleanup)" + ) + + result = node.query( + f"SELECT id, year FROM {iceberg_table} WHERE year = 2020 ORDER BY id" + ).strip() + assert result == "1\t2020\n2\t2020\n3\t2020", ( + f"Unexpected data after post-publish exception recovery:\n{result}" + ) + + # After a post-publish exception the catch handler with published==true returns + # the populated commit info (real metadata / manifest list / manifest file paths). + # ExportPartitionUtils::commit persists it to the commit_info znode, so the system + # table should show a real metadata path here, not the already-committed sentinel. + committed_metadata_file = node.query( + f""" + SELECT committed_metadata_file FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{iceberg_table}' + AND partition_id = '2020' + """ + ).strip() + assert committed_metadata_file, ( + "committed_metadata_file should be populated after a successful post-publish-catch return" + ) + assert not committed_metadata_file.startswith("<"), ( + f"committed_metadata_file should be a real metadata path, got the already-committed sentinel: {committed_metadata_file!r}" + ) + assert committed_metadata_file.endswith(".metadata.json"), ( + f"Expected a *.metadata.json path in committed_metadata_file, got: {committed_metadata_file!r}" + ) + + +def test_export_task_timeout_kills_stuck_pending_task(cluster): + """ + Verify that export_merge_tree_partition_task_timeout_seconds auto-kills a task + that remains PENDING past the deadline, transitioning it to KILLED with a + descriptive last_exception. + + The export_partition_commit_always_throw failpoint wedges the task in the + commit retry loop (REGULAR failpoint, fires on every commit attempt) with a + retryable error, so the task never fails on its own and the timeout branch in + tryCleanup is the actual mechanism under test. + + Replicated-only: the failpoint lives in `ExportPartitionUtils::commit`, the + ZooKeeper-coordinated commit routine. A plain MergeTree commits through + `MergeTreePartitionExportScheduler::tryCommit`, which the failpoint does not reach, so the + export simply completes and there is nothing for the timeout to kill. + """ + node = cluster.instances["replica1"] + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) + + node.query("SYSTEM ENABLE FAILPOINT export_partition_commit_always_throw") + + try: + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}" + f" SETTINGS export_merge_tree_partition_task_timeout_seconds = 5," + f" allow_insert_into_iceberg = 1" + ) + + # Timeout budget must cover: the 5s task timeout + one manifest-updating + # poll cycle (~30s) + watch propagation. 90s is safe. + wait_for_export_status( + node, mt_table, iceberg_table, "2020", + expected_status="KILLED", + timeout=90, + ) + + # The KILL transition writes a per-replica last_exception leaf in the same + # ZK multi as the status flip; handleStatusChanges then mirrors it into + # memory together with the status. Poll briefly to allow that watch -> + # mirror hop. We use arrayJoin to flatten the per-replica array column; + # any replica reporting the timeout reason is sufficient. + deadline = time.time() + 30 + last_exception = "" + while time.time() < deadline: + last_exception = node.query( + f""" + SELECT arrayStringConcat( + arrayMap(x -> x.message, last_exception_per_replica), + '\\n' + ) + FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{iceberg_table}' + AND partition_id = '2020' + """ + ).strip() + if "timed out" in last_exception: + break + time.sleep(0.5) + assert "timed out" in last_exception, ( + f"Expected last_exception_per_replica column to mention the timeout reason, got: {last_exception!r}" + ) + finally: + node.query("SYSTEM DISABLE FAILPOINT export_partition_commit_always_throw") + + +def test_export_partition_commit_uses_exported_parts_not_new_inserts(cluster): + """The deferred commit derives the Iceberg partition value only from the exact exported parts + recorded in the manifest, never from parts inserted/merged into the source partition after + scheduling. A month-partitioned source exports one day into a day-partitioned destination (a + data-dependent acceptance); while the commit is wedged, an earlier day is inserted and merged in, + so the only active part now spans both days with its min at the new day. The commit must still + stamp the exported day (the exported part is found among Outdated parts by name), not the merged-in + earlier day, so the metadata matches the exported data files. + + Replicated-only for the same reason as test_export_task_timeout_kills_stuck_pending_task: the + commit failpoint this test wedges the commit with only exists on the ZooKeeper-coordinated + commit path.""" + node = cluster.instances["replica1"] + uid = unique_suffix() + mt_table = f"mt_commit_parts_{uid}" + iceberg_table = f"iceberg_commit_parts_{uid}" + + make_source(node, mt_table, "id Int64, event_date Date", "toYYYYMM(event_date)", replica_name="replica1") + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-20'), (2, '2024-03-20')") + make_iceberg_s3(node, iceberg_table, "id Int64, event_date Date", + partition_by="toRelativeDayNum(event_date)") + + exported_day = int(node.query("SELECT toRelativeDayNum(toDate('2024-03-20'))").strip()) + injected_day = int(node.query("SELECT toRelativeDayNum(toDate('2024-03-05'))").strip()) + + node.query("SYSTEM ENABLE FAILPOINT export_partition_commit_always_throw") + try: + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '202403' TO TABLE {iceberg_table}" + f" SETTINGS allow_insert_into_iceberg = 1" + ) + # The commit is attempted only after every part is exported, so a non-zero exception count + # means the data files are written and the commit is now wedged by the failpoint. + wait_for_exception_count(node, mt_table, iceberg_table, "202403", min_exception_count=1, timeout=90) + + # Insert an earlier day into the same month partition and merge: the merged active part spans + # both days with min = the injected (earlier) day, while the exported part becomes Outdated. + node.query(f"INSERT INTO {mt_table} VALUES (3, '2024-03-05')") + node.query(f"OPTIMIZE TABLE {mt_table} PARTITION ID '202403' FINAL") + finally: + node.query("SYSTEM DISABLE FAILPOINT export_partition_commit_always_throw") + + wait_for_export_status(node, mt_table, iceberg_table, "202403", "COMPLETED", timeout=90) + + # The exported data files hold only 2024-03-20; the metadata day must match them. + query_id = f"commit_parts_{uid}" + node.query( + f"SELECT * FROM {iceberg_table}", + query_id=query_id, + settings={"iceberg_metadata_log_level": "manifest_file_entry"}, + ) + entries = fetch_manifest_entries(node, query_id) + partitions = data_file_partition_records(entries) + assert partitions, "No data-file partition records found in manifest entries" + meta_days = {int(partition_scalar(p, "event_date")) for p in partitions} + assert meta_days == {exported_day}, ( + f"Metadata day {meta_days} must equal the exported day {exported_day} (2024-03-20), " + f"not the injected day {injected_day} (2024-03-05)." + ) + + assert int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) == 2, ( + "Only the two exported rows must be present in the destination." + ) diff --git a/tests/integration/test_export_partition_to_iceberg/test_lifecycle.py b/tests/integration/test_export_partition_to_iceberg/test_lifecycle.py new file mode 100644 index 000000000000..b5667d6c016a --- /dev/null +++ b/tests/integration/test_export_partition_to_iceberg/test_lifecycle.py @@ -0,0 +1,165 @@ +from helpers.export_partition_helpers import ( + make_iceberg_s3, + make_source, + unique_suffix, + wait_for_export_status, +) +from helpers.iceberg_export_stats import ( + assert_exported_stats, + fetch_manifest_entries, +) + +from .common import setup_tables + +CLUSTER_INSTANCES = ["replica1"] + +# The happy paths of `EXPORT PARTITION` into an Iceberg destination: one partition, several +# partitions, all of them, and the column statistics carried by the resulting manifest entry. + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +def test_export_partition_to_iceberg(cluster, source_engine): + """ + Basic happy path: export a single partition and verify row count and content. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"], engine=source_engine) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 3, f"Expected 3 rows in Iceberg table after export, got {count}" + + result = node.query(f"SELECT id, year FROM {iceberg_table} ORDER BY id").strip() + assert result == "1\t2020\n2\t2020\n3\t2020", ( + f"Unexpected data in Iceberg table:\n{result}" + ) + + +def test_export_two_partitions_to_iceberg(cluster, source_engine): + """ + Export two partitions in a single ALTER TABLE statement and verify that both + land in the Iceberg table with correct row counts. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"], engine=source_engine) + + node.query( + f""" + ALTER TABLE {mt_table} + EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}, + EXPORT PARTITION ID '2021' TO TABLE {iceberg_table} + """, + settings={"allow_insert_into_iceberg": 1}, + ) + + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + wait_for_export_status(node, mt_table, iceberg_table, "2021", "COMPLETED") + + count_2020 = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) + count_2021 = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2021").strip()) + + assert count_2020 == 3, f"Expected 3 rows for year=2020, got {count_2020}" + assert count_2021 == 1, f"Expected 1 row for year=2021, got {count_2021}" + + +def test_export_partition_all_to_iceberg(cluster, source_engine): + """ + `ALTER TABLE ... EXPORT PARTITION ALL TO TABLE ...` schedules every active partition + in one statement and exercises the Iceberg-specific destination compatibility checks + (which are repeated per sub-call inside the loop). + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_{uid}" + iceberg_table = f"iceberg_{uid}" + + setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"], engine=source_engine) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + wait_for_export_status(node, mt_table, iceberg_table, "2021", "COMPLETED") + + count_2020 = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) + count_2021 = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2021").strip()) + + assert count_2020 == 3, f"Expected 3 rows for year=2020, got {count_2020}" + assert count_2021 == 1, f"Expected 1 row for year=2021, got {count_2021}" + + +def setup_stats_tables(node, mt_table: str, iceberg_table: str, engine: str = "ReplicatedMergeTree"): + """Local variant of setup_tables using the wider schema with a Nullable column.""" + columns = "id Int32, name String, tag Nullable(String), year Int32" + + make_source( + node, mt_table, columns, "year", + order_by="id", replica_name="replica1", +engine=engine) + node.query( + f""" + INSERT INTO {mt_table} (id, name, tag, year) VALUES + (1, 'aaa', 'x', 2020), + (2, 'mmm', NULL, 2020), + (3, 'zzz', 'y', 2020), + (4, 'kkk', 'z', 2021) + """ + ) + + make_iceberg_s3(node, iceberg_table, columns, partition_by="year") + + +def test_export_partition_writes_column_statistics(cluster, source_engine): + """ + Export a whole partition (EXPORT PARTITION ID '2020') that contains one NULL + and verify that the resulting Iceberg manifest entry carries accurate per-file + column statistics: record_count, file_size_in_bytes, column_sizes, + null_value_counts, and lower/upper bounds. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_stats_{uid}" + iceberg_table = f"iceberg_stats_{uid}" + + setup_stats_tables(node, mt_table, iceberg_table, engine=source_engine) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 3, f"Expected 3 rows in Iceberg table after export, got {count}" + + query_id = f"stats_partition_{uid}" + node.query( + f"SELECT * FROM {iceberg_table} ORDER BY id", + query_id=query_id, + settings={"iceberg_metadata_log_level": "manifest_file_entry"}, + ) + + entries = fetch_manifest_entries(node, query_id) + assert_exported_stats(entries) diff --git a/tests/integration/test_export_partition_to_iceberg/test_partition_key_compat.py b/tests/integration/test_export_partition_to_iceberg/test_partition_key_compat.py new file mode 100644 index 000000000000..cdb0abe940de --- /dev/null +++ b/tests/integration/test_export_partition_to_iceberg/test_partition_key_compat.py @@ -0,0 +1,1077 @@ +import pytest + +from helpers.export_partition_helpers import ( + REJECTED_PARTITION_EXPORT_CASES as SHARED_REJECTED_PARTITION_EXPORT_CASES, +) +from helpers.export_partition_helpers import ( + RejectedPartitionExportCase, + first_partition_id, + make_iceberg_s3, + make_source, + unique_suffix, + wait_for_export_status, +) +from helpers.iceberg_export_stats import fetch_manifest_entries + +from .common import ( + data_file_partition_records, + partition_scalar, +) + +CLUSTER_INSTANCES = ["replica1"] + +# The Iceberg partition-compatibility gate: which source partition keys may be exported into which +# destination transform, and whether the partition metadata the commit writes matches the data. +# Rejections are synchronous. + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def create_source_mt(node, mt_table: str, replica_name: str, engine: str = "ReplicatedMergeTree"): + make_source(node, mt_table, "id Int64, year Int32", "year", + engine=engine, replica_name=replica_name) + + +def test_partition_transform_compatibility_accepted(cluster, source_engine): + """ + Verify that EXPORT PARTITION is accepted (no BAD_ARGUMENTS) for every + supported transform when the MergeTree and Iceberg partition specs match. + + Cases covered: + 1. Compound identity (year, region), exported to a spec that lists the fields in reverse order + 2. Year transform – toYearNumSinceEpoch(event_date) + 3. Month transform – toMonthNumSinceEpoch(event_date) + 4. truncate[4] – icebergTruncate(4, category) + 5. bucket[8] – icebergBucket(8, user_id) + 6. Compound mixed – (toYearNumSinceEpoch(event_date), icebergBucket(16, user_id)) + """ + node = cluster.instances["replica1"] + uid = unique_suffix() + + def check_accepted(mt, iceberg, description): + pid = first_partition_id(node, mt) + node.query( + f"ALTER TABLE {mt} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg}", + settings={"allow_insert_into_iceberg": 1}, + ) + return pid + + # 1. Compound identity, with the destination listing the fields in the opposite order: the + # source key pins both columns, so the partition is single-valued for either field order. + cols = "id Int64, year Int32, region String" + t = f"mt_acc_1_{uid}"; i = f"iceberg_acc_1_{uid}" + make_source(node, t, cols, "(year, region)", engine=source_engine) + node.query(f"INSERT INTO {t} VALUES (1, 2023, 'EU')") + make_iceberg_s3(node, i, cols, "(region, year)") + pid = check_accepted(t, i, "compound identity (year, region)") + wait_for_export_status(node, t, i, pid, "COMPLETED") + count = int(node.query(f"SELECT count() FROM {i}").strip()) + assert count == 1, f"[compound identity (year, region)] Expected 1 row in Iceberg table, got {count}" + result = node.query(f"SELECT id, year, region FROM {i}").strip() + assert result == "1\t2023\tEU", f"[compound identity (year, region)] Unexpected exported data:\n{result}" + + # 2. Year transform + cols = "id Int64, event_date Date" + t = f"mt_acc_2_{uid}"; i = f"iceberg_acc_2_{uid}" + make_source(node, t, cols, "toYearNumSinceEpoch(event_date)", engine=source_engine) + node.query(f"INSERT INTO {t} VALUES (1, '2020-06-15')") + make_iceberg_s3(node, i, cols, "toYearNumSinceEpoch(event_date)") + check_accepted(t, i, "year transform") + + # 3. Month transform + cols = "id Int64, event_date Date" + t = f"mt_acc_3_{uid}"; i = f"iceberg_acc_3_{uid}" + make_source(node, t, cols, "toMonthNumSinceEpoch(event_date)", engine=source_engine) + node.query(f"INSERT INTO {t} VALUES (1, '2020-06-15')") + make_iceberg_s3(node, i, cols, "toMonthNumSinceEpoch(event_date)") + check_accepted(t, i, "month transform") + + # 4. truncate[4] + cols = "id Int64, category String" + t = f"mt_acc_4_{uid}"; i = f"iceberg_acc_4_{uid}" + make_source(node, t, cols, "icebergTruncate(4, category)", engine=source_engine) + node.query(f"INSERT INTO {t} VALUES (1, 'clickhouse')") + make_iceberg_s3(node, i, cols, "icebergTruncate(4, category)") + check_accepted(t, i, "truncate[4]") + + # 5. bucket[8] + cols = "id Int64, user_id Int64" + t = f"mt_acc_5_{uid}"; i = f"iceberg_acc_5_{uid}" + make_source(node, t, cols, "icebergBucket(8, user_id)", engine=source_engine) + node.query(f"INSERT INTO {t} VALUES (1, 42)") + make_iceberg_s3(node, i, cols, "icebergBucket(8, user_id)") + check_accepted(t, i, "bucket[8]") + + # 6. Compound mixed: year(event_date) + bucket[16](user_id) + cols = "id Int64, event_date Date, user_id Int64" + t = f"mt_acc_6_{uid}"; i = f"iceberg_acc_6_{uid}" + make_source(node, t, cols, "(toYearNumSinceEpoch(event_date), icebergBucket(16, user_id))", engine=source_engine) + node.query(f"INSERT INTO {t} VALUES (1, '2021-03-01', 99)") + make_iceberg_s3(node, i, cols, "(toYearNumSinceEpoch(event_date), icebergBucket(16, user_id))") + check_accepted(t, i, "compound year+bucket[16]") + + +def test_partition_transform_compatibility_rejected(cluster, source_engine): + """ + Verify that partition specs that cannot be exported are rejected with BAD_ARGUMENTS. + + Acceptance is data-dependent: a source partition must map to a single Iceberg partition. The + mismatch cases below therefore use data that makes the source partition span several + destination partitions (a single-row partition would be trivially single-valued and accepted). + + Cases covered: + 1. Transform mismatch on the same column: year-transform source vs identity destination, where + the year partition contains several distinct dates. + 2. Bucket count mismatch: bucket[8] vs bucket[16] (bucket is non-monotonic, always structural). + 3. Truncate width mismatch: truncate[4] source vs truncate[8] destination, with values sharing + the 4-char prefix but differing within the first 8 chars. + 4. Unsupported MergeTree expression (intDiv) vs identity, with one bucket spanning several years. + 5. Destination partitions by a column that is not in the source partition key. + """ + node = cluster.instances["replica1"] + uid = unique_suffix() + + def assert_rejected(mt, iceberg, description): + pid = first_partition_id(node, mt) + error = node.query_and_get_error( + f"ALTER TABLE {mt} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, ( + f"[{description}] Expected BAD_ARGUMENTS, got: {error!r}" + ) + + # 1. Transform mismatch: MergeTree year-transform, Iceberg identity on same Date col + cols = "id Int64, event_date Date" + t = f"mt_rej_1_{uid}"; i = f"iceberg_rej_1_{uid}" + make_source(node, t, cols, "toYearNumSinceEpoch(event_date)", engine=source_engine) + node.query(f"INSERT INTO {t} VALUES (1, '2020-01-01'), (2, '2020-12-31')") + make_iceberg_s3(node, i, cols, "event_date") # identity, not year-transform + assert_rejected(t, i, "year-transform source vs identity destination") + + # 2. Bucket count mismatch: bucket[8] vs bucket[16] + cols = "id Int64, user_id Int64" + t = f"mt_rej_2_{uid}"; i = f"iceberg_rej_2_{uid}" + make_source(node, t, cols, "icebergBucket(8, user_id)", engine=source_engine) + node.query(f"INSERT INTO {t} VALUES (1, 42)") + make_iceberg_s3(node, i, cols, "icebergBucket(16, user_id)") + assert_rejected(t, i, "bucket[8] vs bucket[16]") + + # 3. Truncate width mismatch: values share the 4-char prefix but differ within 8 chars. + cols = "id Int64, category String" + t = f"mt_rej_3_{uid}"; i = f"iceberg_rej_3_{uid}" + make_source(node, t, cols, "icebergTruncate(4, category)", engine=source_engine) + node.query(f"INSERT INTO {t} VALUES (1, 'clickhouse'), (2, 'clickfmt')") + make_iceberg_s3(node, i, cols, "icebergTruncate(8, category)") + assert_rejected(t, i, "truncate[4] source vs truncate[8] destination") + + # 4. Unsupported MergeTree expression vs identity: one intDiv bucket spans several years. + cols = "id Int64, year Int32" + t = f"mt_rej_4_{uid}"; i = f"iceberg_rej_4_{uid}" + make_source(node, t, cols, "intDiv(year, 100)", engine=source_engine) + node.query(f"INSERT INTO {t} VALUES (1, 2000), (2, 2099)") + make_iceberg_s3(node, i, cols, "year") + assert_rejected(t, i, "intDiv source vs identity destination") + + # 5. Destination partitions by a column absent from the source partition key. + cols = "id Int64, year Int32" + t = f"mt_rej_5_{uid}"; i = f"iceberg_rej_5_{uid}" + make_source(node, t, cols, "year", engine=source_engine) + node.query(f"INSERT INTO {t} VALUES (1, 2020)") + make_iceberg_s3(node, i, cols, "id") # identity on id, which the source does not partition by + assert_rejected(t, i, "destination partitions by a non-source-key column") + + +def test_partition_key_compatibility_check(cluster, source_engine): + """ + Verify that EXPORT PARTITION throws BAD_ARGUMENTS synchronously when the + MergeTree partition key does not match the Iceberg table's partition spec, + and is accepted without error when the destination is satisfiable. + + Three cases: + 1. Column mismatch – MergeTree PARTITION BY year, Iceberg PARTITION BY id (must be rejected) + 2. Unpartitioned dst – MergeTree PARTITION BY year, Iceberg unpartitioned (accepted: the source is + flattened into the single empty Iceberg partition) + 3. Matching keys – both PARTITION BY year (must be accepted) + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_{uid}" + + create_source_mt(node, mt_table, "replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020), (3, 2021)") + + # --- Case 1: Iceberg partitioned by 'id' but MergeTree by 'year' --- + iceberg_col_mismatch = f"iceberg_col_mismatch_{uid}" + node.query( + f""" + CREATE TABLE {iceberg_col_mismatch} + (id Int64, year Int32) + ENGINE = IcebergS3( + 'http://minio1:9001/root/data/{iceberg_col_mismatch}/', + 'minio', + 'ClickHouse_Minio_P@ssw0rd' + ) + PARTITION BY id SETTINGS s3_retry_attempts = 3 + """ + ) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_col_mismatch}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, ( + f"Expected BAD_ARGUMENTS for partition column mismatch, got: {error!r}" + ) + + # --- Case 2: Iceberg unpartitioned, MergeTree PARTITION BY year --- + # An unpartitioned Iceberg table has a single (empty) partition, so a partitioned source is + # flattened into it and the export is accepted; the partition-column values survive as data. + iceberg_unpartitioned = f"iceberg_unpartitioned_{uid}" + node.query( + f""" + CREATE TABLE {iceberg_unpartitioned} + (id Int64, year Int32) + ENGINE = IcebergS3( + 'http://minio1:9001/root/data/{iceberg_unpartitioned}/', + 'minio', + 'ClickHouse_Minio_P@ssw0rd' + ) + SETTINGS s3_retry_attempts = 3 + """ + ) + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_unpartitioned}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_unpartitioned, "2020", "COMPLETED") + count = int(node.query(f"SELECT count() FROM {iceberg_unpartitioned}").strip()) + assert count == 2, f"Expected 2 rows in unpartitioned Iceberg table after export, got {count}" + result = node.query(f"SELECT id, year FROM {iceberg_unpartitioned} ORDER BY id").strip() + assert result == "1\t2020\n2\t2020", f"Unexpected data in unpartitioned Iceberg table:\n{result}" + + # --- Case 3: Matching partition keys (both PARTITION BY year) --- + iceberg_match = f"iceberg_match_{uid}" + node.query( + f""" + CREATE TABLE {iceberg_match} + (id Int64, year Int32) + ENGINE = IcebergS3( + 'http://minio1:9001/root/data/{iceberg_match}/', + 'minio', + 'ClickHouse_Minio_P@ssw0rd' + ) + PARTITION BY year SETTINGS s3_retry_attempts = 3 + """ + ) + # Should not raise — the check passes so the export is accepted synchronously + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_match}", + settings={"allow_insert_into_iceberg": 1}, + ) + + +def test_partition_transform_equivalence_gate(cluster, source_engine): + """ + The Iceberg partition-compatibility gate accepts a source partition key whose transform is + equivalent to (or finer than) the destination Iceberg transform when the exported partition is + provably single-valued for every destination field, and rejects it otherwise. Accept cases are + verified end-to-end (data + metadata); reject cases must throw BAD_ARGUMENTS synchronously. + """ + node = cluster.instances["replica1"] + dt = "id Int64, event_time DateTime" + yr = "id Int64, year Int32, region String" + + cases = [ + # toDate -> day: rows within one day map to a single Iceberg day partition. + {"name": "todate_day", "columns": dt, "source_key": "toDate(event_time)", + "dest_key": "toRelativeDayNum(event_time)", + "rows": "(1, '2024-03-05 01:00:00'), (2, '2024-03-05 20:00:00')", "expect_ok": True}, + # toYYYYMM -> month: different days of the same month map to a single month partition. + {"name": "toyyyymm_month", "columns": dt, "source_key": "toYYYYMM(event_time)", + "dest_key": "toMonthNumSinceEpoch(event_time)", + "rows": "(1, '2024-03-01 00:00:00'), (2, '2024-03-20 00:00:00')", "expect_ok": True}, + # toStartOfHour -> hour. + {"name": "startofhour_hour", "columns": dt, "source_key": "toStartOfHour(event_time)", + "dest_key": "toRelativeHourNum(event_time)", + "rows": "(1, '2024-03-05 12:00:00'), (2, '2024-03-05 12:59:00')", "expect_ok": True}, + # Finer source (day + country) into a day-partitioned destination: extra column allowed. + {"name": "finer_day", "columns": "id Int64, event_time DateTime, country String", + "source_key": "(toDate(event_time), country)", "dest_key": "toRelativeDayNum(event_time)", + "rows": "(1, '2024-03-05 01:00:00', 'US'), (2, '2024-03-05 20:00:00', 'US')", + "expect_ok": True}, + # Compound field order reversed: matching is by column; the destination defines tuple order. + {"name": "reversed_order", "columns": yr, "source_key": "(year, region)", + "dest_key": "(region, year)", "rows": "(1, 2020, 'EU')", "expect_ok": True, + "verify": [("region", "region"), ("year", "year")]}, + # Superset source: (year, region) into a year-only destination is finer, so accepted. + {"name": "superset", "columns": yr, "source_key": "(year, region)", "dest_key": "year", + "rows": "(1, 2020, 'EU')", "expect_ok": True, "verify": [("year", "year")]}, + # Coarser source: a month partition spans several days, so it cannot map to one day. + {"name": "coarser_day", "columns": dt, "source_key": "toYYYYMM(event_time)", + "dest_key": "toRelativeDayNum(event_time)", + "rows": "(1, '2024-03-01 00:00:00'), (2, '2024-03-20 00:00:00')", "expect_ok": False}, + # A hash is never monotonic, so min/max cannot prove anything about it, but an identity source key + # pins k within the partition and a bucket of a single value is a single bucket. + {"name": "bucket_from_identity_source", "columns": "id Int64, k Int64", "source_key": "k", + "dest_key": "icebergBucket(8, k)", "rows": "(1, 10), (2, 10)", "expect_ok": True, + "verify": [("k", "icebergBucket(8, k)")]}, + # The same bucket destination over a source key that does not pin k: nothing proves the rows of one + # source partition hash into the same bucket. + {"name": "bucket_needs_structural", "columns": "id Int64, k Int64", + "source_key": "intDiv(k, 100)", "dest_key": "icebergBucket(8, k)", + "rows": "(1, 10), (2, 20)", "expect_ok": False}, + # Identical expressions on a Nullable column: accepted structurally. The min/max proof refuses + # Nullable (a NULL forms its own destination partition and the endpoints cannot rule it out), + # so this only passes because the source already groups by exactly this transform. DateTime64(6) + # round-trips through the Iceberg schema unchanged, which the structural type check requires. + {"name": "nullable_exact_day", "columns": "id Int64, event_time Nullable(DateTime64(6))", + "source_key": "toRelativeDayNum(event_time)", "dest_key": "toRelativeDayNum(event_time)", + "rows": "(1, '2024-03-05 01:00:00'), (2, '2024-03-05 20:00:00')", + "source_settings": "allow_nullable_key = 1", "expect_ok": True}, + # Same, for identity, which is exempt from the structural type check. + {"name": "nullable_exact_identity", "columns": "id Int64, k Nullable(Int64)", + "source_key": "k", "dest_key": "k", "rows": "(1, 10), (2, 10)", + "source_settings": "allow_nullable_key = 1", "expect_ok": True, + "verify": [("k", "k")]}, + # A Nullable column without identical expressions falls to the min/max proof, which cannot see + # NULLs, so it is rejected. + {"name": "nullable_no_match", "columns": "id Int64, event_time Nullable(DateTime64(6))", + "source_key": "toYYYYMM(event_time)", "dest_key": "toRelativeDayNum(event_time)", + "rows": "(1, '2024-03-05 01:00:00'), (2, '2024-03-05 20:00:00')", + "source_settings": "allow_nullable_key = 1", "expect_ok": False}, + ] + run_partition_compat_cases(node, cases, engine=source_engine) + + +def test_partition_transform_granularity_matrix(cluster, source_engine): + """ + Exercise the common ClickHouse temporal partition keys and the granularity relationships between + the source key and the destination Iceberg transform. Acceptance is data-dependent (a source + partition must be single-valued for every destination field), so a coarser source can still be + accepted when a particular partition does not actually repartition. Accept cases are verified + end-to-end (data + metadata); reject cases must throw BAD_ARGUMENTS. + """ + node = cluster.instances["replica1"] + dt = "id Int64, event_time DateTime" + same_day = "(1, '2024-03-05 01:00:00'), (2, '2024-03-05 20:00:00')" + same_month = "(1, '2024-03-01 00:00:00'), (2, '2024-03-20 00:00:00')" + same_year = "(1, '2024-03-05 00:00:00'), (2, '2024-09-10 00:00:00')" + + def case(name, source_key, dest_key, rows, expect_ok): + return {"name": name, "columns": dt, "source_key": source_key, "dest_key": dest_key, + "rows": rows, "expect_ok": expect_ok} + + cases = [ + # Common temporal keys at the same granularity as the destination transform. + case("startofmonth_month", "toStartOfMonth(event_time)", "toMonthNumSinceEpoch(event_time)", same_month, True), + case("yyyymmdd_day", "toYYYYMMDD(event_time)", "toRelativeDayNum(event_time)", same_day, True), + case("startofday_day", "toStartOfDay(event_time)", "toRelativeDayNum(event_time)", same_day, True), + case("toyear_year", "toYear(event_time)", "toYearNumSinceEpoch(event_time)", same_year, True), + case("startofyear_year", "toStartOfYear(event_time)", "toYearNumSinceEpoch(event_time)", same_year, True), + # Finer source into a coarser destination: a finer partition sits inside one coarser bucket. + case("day_into_month", "toDate(event_time)", "toMonthNumSinceEpoch(event_time)", same_day, True), + case("day_into_year", "toDate(event_time)", "toYearNumSinceEpoch(event_time)", same_day, True), + case("hour_into_day", "toStartOfHour(event_time)", "toRelativeDayNum(event_time)", + "(1, '2024-03-05 12:00:00'), (2, '2024-03-05 12:30:00')", True), + case("month_into_year", "toYYYYMM(event_time)", "toYearNumSinceEpoch(event_time)", same_month, True), + # Coarser source into a finer destination: the partition spans several destination buckets. + case("year_into_month", "toYear(event_time)", "toMonthNumSinceEpoch(event_time)", + "(1, '2020-01-15 00:00:00'), (2, '2020-06-15 00:00:00')", False), + case("year_into_day", "toYear(event_time)", "toRelativeDayNum(event_time)", + "(1, '2020-01-01 00:00:00'), (2, '2020-12-31 00:00:00')", False), + # Same coarse/fine pair, but this year partition holds a single day, so it does not + # repartition and is accepted - acceptance depends on the data, not the structure. + case("year_into_day_single_day", "toYear(event_time)", "toRelativeDayNum(event_time)", same_day, True), + # Weekly has no Iceberg equivalent: a week partition holding two days cannot map to one day. + case("week_into_day", "toMonday(event_time)", "toRelativeDayNum(event_time)", + "(1, '2024-03-05 00:00:00'), (2, '2024-03-07 00:00:00')", False), + ] + run_partition_compat_cases(node, cases, engine=source_engine) + + +def test_partition_multicolumn_subset(cluster, source_engine): + """ + Destination partition columns must be a subset of the source partition-key columns. A wide + source whose partition key is a superset of the destination's is accepted (and its multi-column + data plus per-field metadata verified); a destination partitioning by a column absent from the + source partition key is rejected. + """ + node = cluster.instances["replica1"] + wide = "id Int64, event_time DateTime, region String, tenant Int32, v1 Float64, v2 String" + + cases = [ + # Destination partition columns {event_time, region} are a strict subset of the source's + # {event_time, region, tenant}: accepted, with multi-column data and per-field metadata. + {"name": "subset_ok", "columns": wide, + "source_key": "(toDate(event_time), region, tenant)", + "dest_key": "(toRelativeDayNum(event_time), region)", + "rows": "(1, '2024-03-05 01:00:00', 'US', 7, 1.5, 'a'), " + "(2, '2024-03-05 20:00:00', 'US', 7, 2.5, 'b')", + "expect_ok": True, + "verify": [("event_time", "toRelativeDayNum(event_time)"), ("region", "region")]}, + # Destination partitions by 'region', which is not in the source partition key: rejected. + {"name": "not_subset", "columns": "id Int64, event_time DateTime, region String", + "source_key": "toDate(event_time)", + "dest_key": "(toRelativeDayNum(event_time), region)", + "rows": "(1, '2024-03-05 01:00:00', 'US'), (2, '2024-03-05 20:00:00', 'EU')", + "expect_ok": False}, + ] + run_partition_compat_cases(node, cases, engine=source_engine) + + +def test_export_partition_todate_source_matches_day_metadata(cluster, source_engine): + """ + End-to-end: a source partitioned by toDate(event_time) exports into a day-partitioned Iceberg + table through the min/max refinement, and the day value written to the Iceberg metadata matches + the exported data. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_todate_{uid}" + iceberg_table = f"iceberg_todate_{uid}" + + make_source(node, mt_table, "id Int64, event_time DateTime", "toDate(event_time)", + replica_name="replica1", engine=source_engine) + node.query( + f"INSERT INTO {mt_table} VALUES " + f"(1, '2024-03-05 01:00:00'), (2, '2024-03-05 12:00:00'), (3, '2024-03-05 23:00:00')" + ) + make_iceberg_s3(node, iceberg_table, "id Int64, event_time DateTime", + partition_by="toRelativeDayNum(event_time)") + + pid = first_partition_id(node, mt_table) + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 3, f"Expected 3 rows after export, got {count}" + + expected_day = int(node.query( + f"SELECT DISTINCT toRelativeDayNum(event_time) FROM {iceberg_table}" + ).strip()) + + query_id = f"todate_{uid}" + node.query( + f"SELECT * FROM {iceberg_table}", + query_id=query_id, + settings={"iceberg_metadata_log_level": "manifest_file_entry"}, + ) + entries = fetch_manifest_entries(node, query_id) + partitions = data_file_partition_records(entries) + assert partitions, "No data-file partition records found in manifest entries" + meta_days = {int(partition_scalar(p, "event_time")) for p in partitions} + assert meta_days == {expected_day}, ( + f"Metadata day {meta_days} must equal toRelativeDayNum {expected_day}." + ) + + +def test_export_partition_day_source_into_year_metadata(cluster, source_engine): + """ + End-to-end: a source partitioned by toDate(event_time) (finer) exports into a year-partitioned + Iceberg destination (coarser). The value written to the Iceberg metadata is the year computed by + the destination transform over the data, not the source day. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_day_year_{uid}" + iceberg_table = f"iceberg_day_year_{uid}" + + make_source(node, mt_table, "id Int64, event_time DateTime", "toDate(event_time)", + replica_name="replica1", engine=source_engine) + node.query( + f"INSERT INTO {mt_table} VALUES " + f"(1, '2024-03-05 01:00:00'), (2, '2024-03-05 12:00:00'), (3, '2024-03-05 23:00:00')" + ) + make_iceberg_s3(node, iceberg_table, "id Int64, event_time DateTime", + partition_by="toYearNumSinceEpoch(event_time)") + + pid = first_partition_id(node, mt_table) + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 3, f"Expected 3 rows after export, got {count}" + + expected_year = int(node.query( + f"SELECT DISTINCT toYearNumSinceEpoch(event_time) FROM {iceberg_table}" + ).strip()) + + query_id = f"day_year_{uid}" + node.query( + f"SELECT * FROM {iceberg_table}", + query_id=query_id, + settings={"iceberg_metadata_log_level": "manifest_file_entry"}, + ) + entries = fetch_manifest_entries(node, query_id) + partitions = data_file_partition_records(entries) + assert partitions, "No data-file partition records found in manifest entries" + meta_years = {int(partition_scalar(p, "event_time")) for p in partitions} + assert meta_years == {expected_year}, ( + f"Metadata year {meta_years} must equal toYearNumSinceEpoch {expected_year}." + ) + + +def test_export_partition_lossy_cast_dynamic_accept(cluster, source_engine): + """ + A lossy Int64 -> Int32 partition-column cast is accepted by the dynamic proof when the + partition's values fit the destination type and map to a single Iceberg bucket. Source and + destination use different truncate widths, so the field is proven via min/max rather than a + structural match. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_lossy_{uid}" + iceberg_table = f"iceberg_lossy_{uid}" + + make_source(node, mt_table, "id Int64, val Int64", "icebergTruncate(10, val)", + replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 100), (2, 109)") + make_iceberg_s3(node, iceberg_table, "id Int64, val Int32", + partition_by="icebergTruncate(1000000, val)") + + pid = first_partition_id(node, mt_table) + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={ + "allow_insert_into_iceberg": 1, + "export_merge_tree_part_allow_lossy_cast": 1, + }, + ) + wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") + assert int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) == 2 + + +# The shared cases plus one that only an Iceberg destination can express: a transform whose +# partition-key columns are listed in a different order on either side. +REJECTED_PARTITION_EXPORT_CASES = SHARED_REJECTED_PARTITION_EXPORT_CASES + [ + pytest.param( + RejectedPartitionExportCase( + src_columns="other_id Int64, user_id Int64", + src_partition_by="icebergBucket(8, user_id)", + dst_columns="user_id Int64, other_id Int64", + dst_partition_by="icebergBucket(8, user_id)", + insert_values="(1, 42)", + error_substrings=("partition key column",), + ), + id="transform_partition_key_different_column_order", + ), +] + + +@pytest.mark.parametrize("case", REJECTED_PARTITION_EXPORT_CASES) +def test_export_partition_partition_key_mismatch_variants_are_rejected(cluster, case, source_engine): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_rejected_{uid}" + iceberg_table = f"iceberg_rejected_{uid}" + + make_source(node, mt_table, case.src_columns, case.src_partition_by, replica_name="replica1", engine=source_engine) + make_iceberg_s3(node, iceberg_table, case.dst_columns, partition_by=case.dst_partition_by) + + node.query(f"INSERT INTO {mt_table} VALUES {case.insert_values}") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + for substring in case.error_substrings: + assert substring in error, f"Expected {substring!r} in error, got: {error}" + + error_all = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +def test_export_partition_multi_column_partition_key_success_all(cluster, source_engine): + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_multi_pkey_ok_all_{uid}" + iceberg_table = f"iceberg_multi_pkey_ok_all_{uid}" + + cols = "a Int32, b Int32, c Int32, val String" + make_source(node, mt_table, cols, "(a, b, c)", replica_name="replica1", engine=source_engine) + make_iceberg_s3(node, iceberg_table, cols, partition_by="(a, b, c)") + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (4, 5, 6, 'y')") + + partition_ids = node.query( + f"SELECT DISTINCT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY partition_id" + ).strip().split("\n") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + + for pid in partition_ids: + wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 2, f"Expected 2 rows in destination after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {iceberg_table} ORDER BY val").strip() + assert result == "1\t2\t3\tx\n4\t5\t6\ty", f"Unexpected exported data:\n{result}" + + +def assert_iceberg_partition_metadata(node, iceberg_table, uid, fields): + """Assert every data-file partition record's field equals the single DISTINCT value of the + corresponding expression over the exported destination data. `fields` is a list of + (metadata_field_name, value_expr). String-normalized so integer transforms and identity + string/int fields compare uniformly.""" + query_id = f"verify_{uid}" + node.query( + f"SELECT * FROM {iceberg_table}", + query_id=query_id, + settings={"iceberg_metadata_log_level": "manifest_file_entry"}, + ) + entries = fetch_manifest_entries(node, query_id) + partitions = data_file_partition_records(entries) + assert partitions, "No data-file partition records found in manifest entries" + for field_name, value_expr in fields: + expected = node.query( + f"SELECT DISTINCT toString({value_expr}) FROM {iceberg_table}" + ).strip() + got = {str(partition_scalar(p, field_name)) for p in partitions} + assert got == {expected}, ( + f"metadata field {field_name!r} = {got}, expected {{{expected!r}}}" + ) + + +def run_partition_compat_cases(node, cases, engine: str = "ReplicatedMergeTree"): + """Run partition-compatibility cases against the Iceberg export gate. + + Reject cases (``expect_ok=False``) are checked synchronously - the gate fires while scheduling, + so the ALTER throws immediately. Accept cases are dispatched together, then awaited, then their + data (full ordered row comparison against the exported source partition) and Iceberg partition + metadata are verified. Each case is a dict: name, columns, source_key, dest_key, rows, expect_ok, + and optional verify (list of (metadata_field_name, value_expr); defaults to + [("event_time", dest_key)]) and source_settings (extra MergeTree settings).""" + settings = {"allow_insert_into_iceberg": 1} + + def setup(case): + uid = unique_suffix() + mt_table = f"mt_{case['name']}_{uid}" + iceberg_table = f"iceberg_{case['name']}_{uid}" + make_source(node, mt_table, case["columns"], case["source_key"], replica_name="replica1", + extra_settings=case.get("source_settings", ""), engine=engine) + node.query(f"INSERT INTO {mt_table} VALUES {case['rows']}") + make_iceberg_s3(node, iceberg_table, case["columns"], partition_by=case["dest_key"]) + pid = first_partition_id(node, mt_table) + return uid, mt_table, iceberg_table, pid + + for case in cases: + if case["expect_ok"]: + continue + _uid, mt_table, iceberg_table, pid = setup(case) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings=settings, + ) + assert "BAD_ARGUMENTS" in error, f"{case['name']}: expected BAD_ARGUMENTS, got: {error!r}" + + dispatched = [] + for case in cases: + if not case["expect_ok"]: + continue + uid, mt_table, iceberg_table, pid = setup(case) + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings=settings, + ) + dispatched.append((case, uid, mt_table, iceberg_table, pid)) + + for case, uid, mt_table, iceberg_table, pid in dispatched: + wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") + + for case, uid, mt_table, iceberg_table, pid in dispatched: + # Export is a positional cast into the destination schema, so verify the destination equals + # the source cast into the destination column types. Normalizing to the destination types + # tolerates legitimate Iceberg type promotion (e.g. DateTime is stored as a microsecond + # timestamp and returns as DateTime64(6)) while preserving destination precision, so a + # spurious sub-second value would still surface as a mismatch. + col_defs = node.query( + f"SELECT name, type FROM system.columns " + f"WHERE database = currentDatabase() AND table = '{iceberg_table}' ORDER BY position" + ).strip().split("\n") + projection = ", ".join( + f"CAST({name} AS {ctype})" for name, ctype in (c.split("\t") for c in col_defs) + ) + src = node.query(f"SELECT {projection} FROM {mt_table} ORDER BY id") + dst = node.query(f"SELECT {projection} FROM {iceberg_table} ORDER BY id") + assert src == dst, f"{case['name']}: destination rows differ from source" + fields = case.get("verify") or [("event_time", case["dest_key"])] + assert_iceberg_partition_metadata(node, iceberg_table, f"{case['name']}_{uid}", fields) + + +def test_export_partition_bucket_type_change_rejected(cluster, source_engine): + """A bucket[N] partition column whose type changes (Int64 -> String) is rejected. The source + hashLong grouping differs from the destination murmur(String) grouping, so a single source bucket + can fan out across several destination buckets; bucket is not order-preserving, so this cannot be + proven dynamically and must be rejected. This previously slipped through the structural fast path, + which matched on transform name and width while ignoring the pre-transform cast.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_bucket_xform_{uid}" + iceberg_table = f"iceberg_bucket_xform_{uid}" + + make_source(node, mt_table, "id Int64, key Int64", "icebergBucket(16, key)", + replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 42), (2, 42)") + + make_iceberg_s3(node, iceberg_table, "id Int64, key String", + partition_by="icebergBucket(16, key)") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, ( + f"Expected BAD_ARGUMENTS for a type-changing bucket transform, got: {error!r}" + ) + + +def test_export_partition_truncate_type_change_rejected(cluster, source_engine): + """icebergTruncate with the same width but a changed column type (Int64 -> String) is rejected. + Truncate is numeric on integers (120..129 -> 120) but byte-wise on strings ('120'..'129' stay + distinct), so one source truncate bucket can map to several destination buckets. The structural + fast path must not accept it on matching transform name and width; the dynamic proof rejects it + because the endpoints do not collapse to a single destination value.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_trunc_xform_{uid}" + iceberg_table = f"iceberg_trunc_xform_{uid}" + + # 120 and 129 are one Int64 truncate[10] bucket (120) but two distinct string truncations. + make_source(node, mt_table, "id Int64, key Int64", "icebergTruncate(10, key)", + replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 120), (2, 129)") + + make_iceberg_s3(node, iceberg_table, "id Int64, key String", + partition_by="icebergTruncate(10, key)") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, ( + f"Expected BAD_ARGUMENTS for a type-changing truncate transform, got: {error!r}" + ) + + +def test_export_partition_value_preserving_cast_not_order_preserving_rejected(cluster, source_engine): + """Int64 -> String keeps every value, but not their order: 2 and 29 are the endpoints of the + source partition, yet the interior value 10 casts to a string that sorts outside them. The + endpoints truncate to '2' while 10 truncates to '1', so the partition spans two destination + buckets and must be rejected instead of being waved through as a lossless cast.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_cast_order_{uid}" + iceberg_table = f"iceberg_cast_order_{uid}" + + make_source(node, mt_table, "id Int64, k Int64", "intDiv(k, 100)", + replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2), (2, 10), (3, 29)") + + make_iceberg_s3(node, iceberg_table, "id Int64, k String", + partition_by="icebergTruncate(1, k)") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, ( + f"Expected BAD_ARGUMENTS for a non-order-preserving cast, got: {error!r}" + ) + + +def test_export_partition_order_preserving_cast_accepted(cluster, source_engine): + """The same shape as the rejected case, but with all values sharing a digit count: Int64 -> + String is order-preserving over [20, 29], so the endpoints do bound the interior and the whole + source partition truncates to the single destination bucket '2'.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_cast_order_ok_{uid}" + iceberg_table = f"iceberg_cast_order_ok_{uid}" + + make_source(node, mt_table, "id Int64, k Int64", "intDiv(k, 100)", + replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 20), (2, 25), (3, 29)") + + make_iceberg_s3(node, iceberg_table, "id Int64, k String", + partition_by="icebergTruncate(1, k)") + + pid = first_partition_id(node, mt_table) + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") + + src = node.query(f"SELECT id, toString(k) FROM {mt_table} ORDER BY id").strip() + dst = node.query(f"SELECT id, k FROM {iceberg_table} ORDER BY id").strip() + assert src == dst, f"destination rows differ from source:\n{src}\n---\n{dst}" + + assert_iceberg_partition_metadata(node, iceberg_table, uid, [("k", "icebergTruncate(1, k)")]) + + +def test_export_partition_timezone_mismatch_rejected(cluster, source_engine): + """A source partitioned by day in one timezone must not be treated as structurally identical to a + destination day computed in another timezone. The source uses Asia/Tokyo (UTC+9) and the + destination UTC; the exported part spans a UTC-day boundary while staying within one Tokyo day, so + it maps to two destination partitions and must be rejected.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_tzmismatch_{uid}" + iceberg_table = f"iceberg_tzmismatch_{uid}" + + make_source(node, mt_table, "id Int64, event_time DateTime('UTC')", + "toRelativeDayNum(event_time, 'Asia/Tokyo')", replica_name="replica1", engine=source_engine) + # Both instants are 2024-03-05 in Tokyo (UTC+9) but 2024-03-04 and 2024-03-05 in UTC. + node.query( + f"INSERT INTO {mt_table} VALUES (1, '2024-03-04 16:00:00'), (2, '2024-03-05 10:00:00')" + ) + + make_iceberg_s3(node, iceberg_table, "id Int64, event_time DateTime('UTC')", + partition_by="toRelativeDayNum(event_time)") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1, "iceberg_partition_timezone": "UTC"}, + ) + assert "BAD_ARGUMENTS" in error, ( + f"Expected BAD_ARGUMENTS for a source/destination timezone mismatch, got: {error!r}" + ) + + +def test_export_partition_column_timezone_mismatch_rejected(cluster, source_engine): + """The same mismatch as above, but with the timezone carried by the column type instead of the + partition expression. Both sides read `toRelativeDayNum(event_time)`, so the terms are identical and + only the types differ - and DateTime types with different timezones compare equal, so the structural + match must not be decided by type equality alone. The part stays within one Tokyo day while spanning + two UTC days, so it maps to two destination partitions and must be rejected. + + `iceberg_partition_timezone` is deliberately left unset: setting it stamps a timezone onto the + destination term, which alone makes the terms differ and hides what this test covers.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_coltz_{uid}" + iceberg_table = f"iceberg_coltz_{uid}" + + make_source(node, mt_table, "id Int64, event_time DateTime('Asia/Tokyo')", + "toRelativeDayNum(event_time)", replica_name="replica1", engine=source_engine) + # Both literals are 2024-03-05 in Tokyo (the column's timezone) but 2024-03-04 and 2024-03-05 in UTC. + node.query( + f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 01:00:00'), (2, '2024-03-05 18:00:00')" + ) + + make_iceberg_s3(node, iceberg_table, "id Int64, event_time DateTime('UTC')", + partition_by="toRelativeDayNum(event_time)") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "BAD_ARGUMENTS" in error, ( + f"Expected BAD_ARGUMENTS for a partition-column timezone mismatch, got: {error!r}" + ) + + +def test_export_partition_month_transform_metadata_matches_data(cluster, source_engine): + """A month-transform partition records a months-since-epoch value in metadata that + matches the value derived from the exported data, and a transform-filtered read + returns the rows.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_month_xform_{uid}" + iceberg_table = f"iceberg_month_xform_{uid}" + + make_source(node, mt_table, "id Int64, event_date Date", + "toMonthNumSinceEpoch(event_date)", replica_name="replica1", engine=source_engine) + node.query( + f"INSERT INTO {mt_table} VALUES " + f"(1, '2024-03-05'), (2, '2024-03-20'), (3, '2024-03-31')" + ) + + make_iceberg_s3(node, iceberg_table, "id Int64, event_date Date", + partition_by="toMonthNumSinceEpoch(event_date)") + + pid = first_partition_id(node, mt_table) + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 3, f"Expected 3 rows after export, got {count}" + + month_num = int(node.query( + f"SELECT DISTINCT toMonthNumSinceEpoch(event_date) FROM {iceberg_table}" + ).strip()) + + query_id = f"month_xform_{uid}" + node.query( + f"SELECT * FROM {iceberg_table}", + query_id=query_id, + settings={"iceberg_metadata_log_level": "manifest_file_entry"}, + ) + entries = fetch_manifest_entries(node, query_id) + partitions = data_file_partition_records(entries) + assert partitions, "No data-file partition records found in manifest entries" + meta_values = {int(partition_scalar(p, "event_date")) for p in partitions} + assert meta_values == {month_num}, ( + f"Metadata month {meta_values} must equal toMonthNumSinceEpoch over the data " + f"({month_num})." + ) + + filtered = int(node.query( + f"SELECT count() FROM {iceberg_table} " + f"WHERE toMonthNumSinceEpoch(event_date) = {month_num}" + ).strip()) + assert filtered == 3, f"Transform-filtered read expected 3 rows, got {filtered}" + + +def test_export_partition_identity_type_change_metadata_matches_data(cluster, source_engine): + """An identity partition column whose type changes UInt16 -> String records the + destination String value in the Iceberg metadata, matching the exported data.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_identity_xform_{uid}" + iceberg_table = f"iceberg_identity_xform_{uid}" + + make_source(node, mt_table, "id Int32, year UInt16", "year", replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2024), (2, 2024)") + + make_iceberg_s3(node, iceberg_table, "id Int32, year String", partition_by="year") + + pid = first_partition_id(node, mt_table) + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 2, f"Expected 2 rows after export, got {count}" + + data_year = node.query(f"SELECT DISTINCT year FROM {iceberg_table}").strip() + assert data_year == "2024", f"Expected exported year '2024' (String), got {data_year!r}" + + query_id = f"identity_xform_{uid}" + node.query( + f"SELECT * FROM {iceberg_table}", + query_id=query_id, + settings={"iceberg_metadata_log_level": "manifest_file_entry"}, + ) + entries = fetch_manifest_entries(node, query_id) + partitions = data_file_partition_records(entries) + assert partitions, "No data-file partition records found in manifest entries" + meta_values = {str(partition_scalar(p, "year")) for p in partitions} + assert meta_values == {"2024"}, ( + f"Metadata partition {meta_values} must equal the destination String value " + f"'2024' (not the source integer representation)." + ) + + +def test_export_partition_multicolumn_identity_metadata_matches_data(cluster, source_engine): + """A multi-column identity partition (event_date Date, retention UInt64 -> Int64) + records per-column values in the Iceberg metadata that match the exported data.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_multicol_{uid}" + iceberg_table = f"iceberg_multicol_{uid}" + + # Iceberg has no unsigned types, so retention widens UInt64 -> Int64; the cast is + # not value-preserving per canBeSafelyCast, hence the lossy opt-in below. + make_source(node, mt_table, "id Int64, event_date Date, retention UInt64", + "(event_date, retention)", replica_name="replica1", engine=source_engine) + node.query( + f"INSERT INTO {mt_table} VALUES " + f"(1, '2024-03-05', 30), (2, '2024-03-05', 30), (3, '2024-03-05', 30)" + ) + + make_iceberg_s3(node, iceberg_table, "id Int64, event_date Date, retention Int64", + partition_by="(event_date, retention)") + + pid = first_partition_id(node, mt_table) + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", + settings={ + "allow_insert_into_iceberg": 1, + "export_merge_tree_part_allow_lossy_cast": 1, + }, + ) + wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 3, f"Expected 3 rows after export, got {count}" + + data_retention = int(node.query( + f"SELECT DISTINCT retention FROM {iceberg_table}" + ).strip()) + assert data_retention == 30, f"Expected exported retention 30, got {data_retention}" + + days = int(node.query( + f"SELECT DISTINCT toInt64(event_date) FROM {iceberg_table}" + ).strip()) + + query_id = f"multicol_{uid}" + node.query( + f"SELECT * FROM {iceberg_table}", + query_id=query_id, + settings={"iceberg_metadata_log_level": "manifest_file_entry"}, + ) + entries = fetch_manifest_entries(node, query_id) + partitions = data_file_partition_records(entries) + assert partitions, "No data-file partition records found in manifest entries" + + meta_dates = {int(partition_scalar(p, "event_date")) for p in partitions} + assert meta_dates == {days}, ( + f"Metadata event_date {meta_dates} must equal days-since-epoch {days}." + ) + meta_retentions = {int(partition_scalar(p, "retention")) for p in partitions} + assert meta_retentions == {30}, ( + f"Metadata retention {meta_retentions} must equal the exported value 30." + ) + + filtered = int(node.query( + f"SELECT count() FROM {iceberg_table} " + f"WHERE event_date = '2024-03-05' AND retention = 30" + ).strip()) + assert filtered == 3, f"Partition-filtered read expected 3 rows, got {filtered}" diff --git a/tests/integration/test_export_partition_to_iceberg/test_schema_match.py b/tests/integration/test_export_partition_to_iceberg/test_schema_match.py new file mode 100644 index 000000000000..755b1095816c --- /dev/null +++ b/tests/integration/test_export_partition_to_iceberg/test_schema_match.py @@ -0,0 +1,893 @@ +import pytest + +from helpers.export_partition_helpers import ( + EXTRA_SOURCE_COLUMN_MODES, + make_iceberg_s3, + make_source, + unique_suffix, + wait_for_export_status, +) + +CLUSTER_INSTANCES = ["replica1"] + +# Destination schema matching for `EXPORT PARTITION` into Iceberg: column counts, by-name versus +# by-position matching, the extra-source-column opt-in, and value-preserving versus lossy casts. + + +def test_export_partition_column_count_mismatch_source_more_is_rejected(cluster, source_engine): + """ + Source has 3 columns (id, year, extra), destination has 2 (id, year). + The ALTER must be rejected synchronously with NUMBER_OF_COLUMNS_DOESNT_MATCH, + nothing must be scheduled in system.partition_exports, and the + Iceberg table must remain empty. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_count_more_{uid}" + iceberg_table = f"iceberg_count_more_{uid}" + + make_source(node, mt_table, "id Int64, year Int32, extra String", "year", + replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020, 'foo'), (2, 2020, 'bar')") + + make_iceberg_s3(node, iceberg_table, "id Int64, year Int32", partition_by="year") + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in error, ( + f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH for source>dest column count, " + f"got: {error!r}" + ) + + rows_in_system_view = node.query( + f"SELECT count() FROM system.partition_exports " + f"WHERE source_table = '{mt_table}' " + f" AND destination_table = '{iceberg_table}' " + f" AND partition_id = '2020'" + ).strip() + assert rows_in_system_view == "0", ( + f"Expected no row in system.partition_exports after a " + f"synchronously-rejected export, got {rows_in_system_view}." + ) + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, ( + f"Expected 0 rows in Iceberg table after rejected export, got {count}" + ) + + +def test_export_partition_column_count_mismatch_source_fewer_is_rejected(cluster, source_engine): + """ + Source has 2 columns (id, year), destination has 3 (id, year, extra). + Same expected synchronous rejection as the source>dest case. + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_count_fewer_{uid}" + iceberg_table = f"iceberg_count_fewer_{uid}" + + make_source(node, mt_table, "id Int64, year Int32", "year", replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020)") + + make_iceberg_s3(node, iceberg_table, "id Int64, year Int32, extra String", + partition_by="year") + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in error, ( + f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH for source Int64) and the + partition column (year Int32 -> Int64) round-trips.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_widen_{uid}" + iceberg_table = f"iceberg_widen_{uid}" + + make_source(node, mt_table, "id Int32, year Int32", "year", replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020)") + + make_iceberg_s3(node, iceberg_table, "id Int64, year Int64", partition_by="year") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 2, f"Expected 2 rows in Iceberg table after export, got {count}" + + result = node.query( + f"SELECT id, toTypeName(id), year, toTypeName(year) FROM {iceberg_table} ORDER BY id" + ).strip() + assert result == "1\tInt64\t2020\tInt64\n2\tInt64\t2020\tInt64", ( + f"Unexpected widened data:\n{result}" + ) + + +def test_export_partition_with_castable_narrowing_values_fit(cluster, source_engine): + """A lossy narrowing (id Int64 -> Int32) succeeds once the user opts in via + export_merge_tree_part_allow_lossy_cast.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_narrow_fit_{uid}" + iceberg_table = f"iceberg_narrow_fit_{uid}" + + make_source(node, mt_table, "id Int64, year Int32", "year", replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020)") + + make_iceberg_s3(node, iceberg_table, "id Int32, year Int32", partition_by="year") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", + settings={ + "allow_insert_into_iceberg": 1, + "export_merge_tree_part_allow_lossy_cast": 1, + }, + ) + wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 2, f"Expected 2 rows in Iceberg table after export, got {count}" + + result = node.query( + f"SELECT id, toTypeName(id), year FROM {iceberg_table} ORDER BY id" + ).strip() + assert result == "1\tInt32\t2020\n2\tInt32\t2020", ( + f"Unexpected narrowed data:\n{result}" + ) + + +def test_export_partition_lossy_cast_rejected_without_optin(cluster, source_engine): + """A lossy narrowing (id Int64 -> Int32) is rejected synchronously with + INCOMPATIBLE_COLUMNS unless export_merge_tree_part_allow_lossy_cast is set.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_lossy_reject_{uid}" + iceberg_table = f"iceberg_lossy_reject_{uid}" + + make_source(node, mt_table, "id Int64, year Int32", "year", replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020)") + + make_iceberg_s3(node, iceberg_table, "id Int32, year Int32", partition_by="year") + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table} " + f"SETTINGS allow_insert_into_iceberg = 1" + ) + assert "INCOMPATIBLE_COLUMNS" in error, f"Expected INCOMPATIBLE_COLUMNS, got: {error!r}" + assert "lossy cast" in error, f"Expected 'lossy cast' in error, got: {error!r}" + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, f"Expected no rows after a rejected export, got {count}" + + +def test_export_partition_runtime_cast_failure_propagates_async(cluster, source_engine): + """A String value that cannot be parsed as the destination Int32 passes the + synchronous lossy-cast gate (with export_merge_tree_part_allow_lossy_cast = 1) but + fails at runtime in the async worker with CANNOT_PARSE_TEXT. That is a deterministic + value-conversion error on the part's immutable data — retrying the same part can + never succeed — so it is classified as non-retryable and fails the whole task fast, + without waiting for the absolute task timeout, leaving Iceberg empty. + + The task timeout is left at its large default, so reaching FAILED quickly proves the + transition is driven by error classification rather than by a timeout. + + (Integer overflow is not used because the internal cast uses CastType::nonAccurate, + which wraps rather than throwing.) + """ + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_runtime_cast_fail_{uid}" + iceberg_table = f"iceberg_runtime_cast_fail_{uid}" + + make_source(node, mt_table, "id String, year Int32", "year", replica_name="replica1", engine=source_engine) + node.query(f"INSERT INTO {mt_table} VALUES ('not a number', 2020)") + + make_iceberg_s3(node, iceberg_table, "id Int32, year Int32", partition_by="year") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table} " + f"SETTINGS allow_insert_into_iceberg = 1, export_merge_tree_part_allow_lossy_cast = 1" + ) + + # The runtime parse error (CANNOT_PARSE_TEXT) is non-retryable, so the task fails fast. + # No short timeout is set; FAILED within this window can only come from the + # non-retryable classification, not from the (default, ~1 day) task timeout. + wait_for_export_status(node, mt_table, iceberg_table, "2020", "FAILED", timeout=60) + + exception_count = int(node.query( + f"SELECT any(exception_count) FROM system.partition_exports " + f"WHERE source_table = '{mt_table}' " + f" AND destination_table = '{iceberg_table}' " + f" AND partition_id = '2020'" + ).strip()) + assert exception_count > 0, ( + "Expected non-zero exception_count after a failed runtime cast" + ) + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 0, ( + f"Expected 0 rows in Iceberg table after failed export, got {count}" + ) + + +def test_export_partition_all_iceberg_types(cluster, source_engine): + """Every getIcebergType-supported type round-trips through an EXPORT PARTITION: + scalars use narrower source types (explicit lossless widening CASTs), plus + Array/Map/Tuple nested columns.""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_all_types_{uid}" + iceberg_table = f"iceberg_all_types_{uid}" + + # Scalar source types are strictly narrower than the destination; the export inserts + # a positional widening CAST per column (Int8->Int16, UInt32->UInt64, ...). Nested + # columns keep the same type on both sides. + source_columns = ( + "i16 Int8, u16 UInt8, u32 UInt16, u64 UInt32, " + "id Int16, big Int32, f32 Float32, f64 Float64, " + "d Date, d32 Date32, dt DateTime, dt64 DateTime64(6), " + "s String, uid UUID, " + "arr Array(Int32), m Map(String, Int64), tup Tuple(a Int32, b String), " + "year Int32" + ) + dest_columns = ( + "i16 Int16, u16 UInt16, u32 UInt32, u64 UInt64, " + "id Int32, big Int64, f32 Float32, f64 Float64, " + "d Date, d32 Date32, dt DateTime, dt64 DateTime64(6), " + "s String, uid UUID, " + "arr Array(Int32), m Map(String, Int64), tup Tuple(a Int32, b String), " + "year Int32" + ) + + make_source(node, mt_table, source_columns, "year", replica_name="replica1", engine=source_engine) + make_iceberg_s3(node, iceberg_table, dest_columns, partition_by="year") + + node.query( + f""" + INSERT INTO {mt_table} + (i16, u16, u32, u64, id, big, f32, f64, d, d32, dt, dt64, s, uid, arr, m, tup, year) + VALUES ( + -100, 200, 50000, 4000000000, + 12345, 1000000000, 3.14, 2.718281828459045, + '2024-01-15', '2024-01-15', '2024-01-15 12:30:45', '2024-01-15 12:30:45.123456', + 'hello iceberg', '550e8400-e29b-41d4-a716-446655440000', + [1, 2, 3], {{'a': 10, 'b': 20}}, (7, 'seven'), 2024 + ) + """ + ) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2024' TO TABLE {iceberg_table}", + settings={"allow_insert_into_iceberg": 1}, + ) + wait_for_export_status(node, mt_table, iceberg_table, "2024", "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 1, f"Expected 1 row in Iceberg table, got {count}" + + result = node.query( + f""" + SELECT + i16, u16, u32, u64, id, big, + toString(d), toString(d32), toString(dt), + s, toString(uid), + arr, m['a'], m['b'], tup.a, tup.b, year + FROM {iceberg_table} + """ + ).strip() + expected = "\t".join([ + "-100", "200", "50000", "4000000000", + "12345", "1000000000", + "2024-01-15", "2024-01-15", "2024-01-15 12:30:45.000000", + "hello iceberg", "550e8400-e29b-41d4-a716-446655440000", + "[1,2,3]", "10", "20", "7", "seven", "2024", + ]) + assert result == expected, f"Unexpected round-trip data:\n{result!r}\nexpected:\n{expected!r}" + + # Floats compared with a tolerance to avoid formatting flakiness. + floats_ok = node.query( + f"SELECT abs(f32 - 3.14) < 1e-4 AND abs(f64 - 2.718281828459045) < 1e-12 FROM {iceberg_table}" + ).strip() + assert floats_ok == "1", f"Float round-trip outside tolerance: {floats_ok!r}" + + # DateTime64 sub-second component: assert the date part is preserved (exact format varies). + ts_result = node.query(f"SELECT dt64 FROM {iceberg_table}").strip() + assert "2024-01-15" in ts_result, f"DateTime64 date component missing: {ts_result!r}" + + +def test_export_partition_all_iceberg_types_lossy(cluster, source_engine): + """Lossy narrowing casts across types succeed with the opt-in flag: values that + fit round-trip, Float64 -> Float32 loses precision, and Nullable columns carry + both NULL and non-NULL (the latter via a lossy Nullable(Int64) -> Nullable(Int32)).""" + node = cluster.instances["replica1"] + + uid = unique_suffix() + mt_table = f"mt_lossy_types_{uid}" + iceberg_table = f"iceberg_lossy_types_{uid}" + + # Each source column is wider than the destination, so the export inserts a lossy + # narrowing CAST (allowed only because export_merge_tree_part_allow_lossy_cast=1). + # Int8/UInt8 are not Iceberg-representable, so the narrowest integer dest is Int16. + source_columns = ( + "big Int64, ubig UInt64, mid Int32, " + "f Float64, dt DateTime64(6), d Date32, " + "opt_s Nullable(String), opt_i Nullable(Int64), year Int32" + ) + dest_columns = ( + "big Int32, ubig UInt32, mid Int16, " + "f Float32, dt DateTime, d Date, " + "opt_s Nullable(String), opt_i Nullable(Int32), year Int32" + ) + + make_source(node, mt_table, source_columns, "year", replica_name="replica1", engine=source_engine) + make_iceberg_s3(node, iceberg_table, dest_columns, partition_by="year") + + # Values chosen to fit the destination types (the async cast wraps on overflow + # rather than throwing, so out-of-range values would silently corrupt instead). + # opt_s is NULL and opt_i is set, covering both nullable paths in one row. + node.query( + f""" + INSERT INTO {mt_table} (big, ubig, mid, f, dt, d, opt_s, opt_i, year) + VALUES ( + 1000000, 2000000000, 30000, + 2.718281828459045, '2024-01-15 12:30:45.123456', '2024-01-15', + NULL, 100, 2024 + ) + """ + ) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2024' TO TABLE {iceberg_table}", + settings={ + "allow_insert_into_iceberg": 1, + "export_merge_tree_part_allow_lossy_cast": 1, + }, + ) + wait_for_export_status(node, mt_table, iceberg_table, "2024", "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) + assert count == 1, f"Expected 1 row in Iceberg table, got {count}" + + result = node.query( + f"SELECT big, ubig, mid, toString(d), toString(dt), opt_s, opt_i, year FROM {iceberg_table}" + ).strip() + expected = "\t".join([ + "1000000", "2000000000", "30000", + "2024-01-15", "2024-01-15 12:30:45.000000", "\\N", "100", "2024", + ]) + assert result == expected, f"Unexpected lossy round-trip data:\n{result!r}\nexpected:\n{expected!r}" + + # Float64 -> Float32 stays within Float32 precision but is no longer exact. + f_checks = node.query( + f"SELECT abs(f - 2.718281828459045) < 1e-6, abs(f - 2.718281828459045) > 1e-9 FROM {iceberg_table}" + ).strip() + assert f_checks == "1\t1", f"Expected Float32 precision loss within tolerance, got: {f_checks!r}" diff --git a/tests/integration/test_export_partition_to_object_storage/__init__.py b/tests/integration/test_export_partition_to_object_storage/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_export_partition_to_object_storage/common.py b/tests/integration/test_export_partition_to_object_storage/common.py new file mode 100644 index 000000000000..cad61fbeedd2 --- /dev/null +++ b/tests/integration/test_export_partition_to_object_storage/common.py @@ -0,0 +1,26 @@ +from helpers.export_partition_helpers import ( + is_replicated_engine, + make_source, +) + +# Table factories shared by the `EXPORT PARTITION` object-storage test modules. + + +def create_s3_table(node, s3_table): + node.query(f"CREATE TABLE {s3_table} (id UInt64, year UInt16) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') PARTITION BY year") + + +def source_engine_clause(engine, table, replica_name="replica1"): + """The `ENGINE` clause of a source table, for tests that spell out their own `CREATE`.""" + if is_replicated_engine(engine): + return f"ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica_name}')" + return "MergeTree()" + + +def create_tables_and_insert_data(node, mt_table, s3_table, replica_name, engine="ReplicatedMergeTree"): + node.query(f"DROP TABLE IF EXISTS {mt_table} SYNC") + # enable_block_number_column and enable_block_offset_column are needed for patch parts support + make_source(node, mt_table, "id UInt64, year UInt16", "year", engine=engine, replica_name=replica_name) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020), (3, 2020), (4, 2021)") + + create_s3_table(node, s3_table) diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/configs/allow_experimental_export_partition.xml b/tests/integration/test_export_partition_to_object_storage/configs/allow_experimental_export_partition.xml similarity index 100% rename from tests/integration/test_export_replicated_mt_partition_to_object_storage/configs/allow_experimental_export_partition.xml rename to tests/integration/test_export_partition_to_object_storage/configs/allow_experimental_export_partition.xml diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/configs/disable_experimental_export_partition.xml b/tests/integration/test_export_partition_to_object_storage/configs/disable_experimental_export_partition.xml similarity index 100% rename from tests/integration/test_export_replicated_mt_partition_to_object_storage/configs/disable_experimental_export_partition.xml rename to tests/integration/test_export_partition_to_object_storage/configs/disable_experimental_export_partition.xml diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/configs/macros_shard1_replica1.xml b/tests/integration/test_export_partition_to_object_storage/configs/macros_shard1_replica1.xml similarity index 100% rename from tests/integration/test_export_replicated_mt_partition_to_object_storage/configs/macros_shard1_replica1.xml rename to tests/integration/test_export_partition_to_object_storage/configs/macros_shard1_replica1.xml diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/configs/macros_shard2_replica1.xml b/tests/integration/test_export_partition_to_object_storage/configs/macros_shard2_replica1.xml similarity index 100% rename from tests/integration/test_export_replicated_mt_partition_to_object_storage/configs/macros_shard2_replica1.xml rename to tests/integration/test_export_partition_to_object_storage/configs/macros_shard2_replica1.xml diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/configs/named_collections.xml b/tests/integration/test_export_partition_to_object_storage/configs/named_collections.xml similarity index 100% rename from tests/integration/test_export_replicated_mt_partition_to_object_storage/configs/named_collections.xml rename to tests/integration/test_export_partition_to_object_storage/configs/named_collections.xml diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/configs/users.d/profile.xml b/tests/integration/test_export_partition_to_object_storage/configs/users.d/profile.xml similarity index 100% rename from tests/integration/test_export_replicated_mt_partition_to_object_storage/configs/users.d/profile.xml rename to tests/integration/test_export_partition_to_object_storage/configs/users.d/profile.xml diff --git a/tests/integration/test_export_partition_to_object_storage/conftest.py b/tests/integration/test_export_partition_to_object_storage/conftest.py new file mode 100644 index 000000000000..4c8686ce08f9 --- /dev/null +++ b/tests/integration/test_export_partition_to_object_storage/conftest.py @@ -0,0 +1,101 @@ +import logging + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.export_partition_helpers import SOURCE_ENGINE_IDS, SOURCE_ENGINES + +# `EXPORT PARTITION` to a plain object-storage destination, split across several modules so the +# harness can spread them over xdist workers (`--dist=loadfile` assigns a whole module to one +# worker). +# +# Each module declares the instances it needs in `CLUSTER_INSTANCES` and gets a cluster with only +# those. That matters because several of these modules can be in flight on different workers at +# once, and every instance is a full server - starting the whole set six times over saturates the +# host under a sanitizer build. + +NAMED_COLLECTIONS = "configs/named_collections.xml" +EXPORT_ENABLED = "configs/allow_experimental_export_partition.xml" +PROFILE = "configs/users.d/profile.xml" + +_REPLICA = dict( + main_configs=[NAMED_COLLECTIONS, EXPORT_ENABLED], + user_configs=[PROFILE], + with_minio=True, + stay_alive=True, + with_zookeeper=True, + keeper_required_feature_flags=["multi_read"], +) + +INSTANCES = { + "replica1": _REPLICA, + "replica2": _REPLICA, + # Takes no part in the export, but has visibility over the destination table. + "watcher_node": dict( + main_configs=[NAMED_COLLECTIONS], + user_configs=[], + with_minio=True, + ), + "replica_with_export_disabled": dict( + _REPLICA, + main_configs=[NAMED_COLLECTIONS, "configs/disable_experimental_export_partition.xml"], + ), + # Sharded instances for the filename pattern tests. + "shard1_replica1": dict( + _REPLICA, + main_configs=[NAMED_COLLECTIONS, EXPORT_ENABLED, "configs/macros_shard1_replica1.xml"], + ), + "shard2_replica1": dict( + _REPLICA, + main_configs=[NAMED_COLLECTIONS, EXPORT_ENABLED, "configs/macros_shard2_replica1.xml"], + ), +} + + +@pytest.fixture(scope="module") +def cluster(request): + instance_names = getattr(request.module, "CLUSTER_INSTANCES", list(INSTANCES)) + try: + cluster = ClickHouseCluster(__file__) + for name in instance_names: + cluster.add_instance(name, **INSTANCES[name]) + logging.info("Starting cluster with instances %s...", instance_names) + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def drop_tables_after_test(cluster): + """Drop all tables in the default database after every test. + + Without this, ReplicatedMergeTree tables from completed tests remain alive and keep + running ZooKeeper background threads (merge selector, queue log, cleanup, export manifest + updater). With many tables alive simultaneously the ZooKeeper session becomes overwhelmed + and subsequent tests start seeing operation-timeout / session-expired errors. + """ + yield + for instance_name, instance in cluster.instances.items(): + try: + tables_str = instance.query( + "SELECT name FROM system.tables WHERE database = 'default' FORMAT TabSeparated" + ).strip() + if not tables_str: + continue + for table in tables_str.split('\n'): + table = table.strip() + if table: + instance.query(f"DROP TABLE IF EXISTS default.`{table}` SYNC") + except Exception as e: + logging.warning(f"drop_tables_after_test: cleanup failed on {instance_name}: {e}") + + +@pytest.fixture(params=SOURCE_ENGINES, ids=SOURCE_ENGINE_IDS) +def source_engine(request): + """The MergeTree flavour of the export source table. + + A test that requests this fixture runs once per engine; the scenarios that only make sense + with cross-replica coordination do not request it and stay on `ReplicatedMergeTree`. + """ + return request.param diff --git a/tests/integration/test_export_partition_to_object_storage/test_failures.py b/tests/integration/test_export_partition_to_object_storage/test_failures.py new file mode 100644 index 000000000000..47d20591a64c --- /dev/null +++ b/tests/integration/test_export_partition_to_object_storage/test_failures.py @@ -0,0 +1,893 @@ +import time +import uuid + +from helpers.export_partition_helpers import ( + setup_source_tables, + skip_if_remote_database_disk_enabled, + wait_for_exception_count, + wait_for_export_status, + wait_for_export_to_start, +) +from helpers.network import PartitionManager + +from .common import ( + create_s3_table, + create_tables_and_insert_data, +) + +CLUSTER_INSTANCES = ["replica1", "replica2", "watcher_node"] + +# `EXPORT PARTITION` under injected failure: object storage cut off with `PartitionManager`, +# failpoints, stopped moves, killed and timed-out tasks. These pace themselves against retry +# back-off and scheduler ticks, so they are slow and timing sensitive - kept out of the parallel +# batch. + + +def test_kill_export(cluster, source_engine): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + node2 = cluster.instances["replica2"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"kill_export_mt_table_{postfix}" + s3_table = f"kill_export_s3_table_{postfix}" + + hosts = setup_source_tables( + [node, node2], + mt_table, + "id UInt64, year UInt16", + "year", + source_engine, + insert_values="(1, 2020), (2, 2020), (3, 2020), (4, 2021)", + ) + for host in hosts: + create_s3_table(host, s3_table) + + # Block S3/MinIO requests to keep exports alive via retry mechanism + # This allows ZooKeeper operations (KILL) to proceed quickly + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + with PartitionManager() as pm: + # Every host that can export a part has to be cut off from MinIO, otherwise it would + # finish the export before the KILL is processed. + for host in hosts: + # Block responses from MinIO (source_port matches MinIO service) + pm.add_rule({ + "instance": host, + "destination": host.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + + # Block requests to MinIO (destination: MinIO, destination_port: minio_port) + pm.add_rule({ + "instance": host, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + + export_queries = f""" + ALTER TABLE {mt_table} + EXPORT PARTITION ID '2020' TO TABLE {s3_table}; + ALTER TABLE {mt_table} + EXPORT PARTITION ID '2021' TO TABLE {s3_table}; + """ + + node.query(export_queries) + + # Kill only 2020 while S3 is blocked - retry mechanism keeps exports alive + # ZooKeeper operations (KILL) proceed quickly since only S3 is blocked + node.query(f"KILL EXPORT PARTITION WHERE partition_id = '2020' and source_table = '{mt_table}' and destination_table = '{s3_table}'") + + # sleep for a while to let the kill to be processed + time.sleep(2) + + # wait for 2021 to finish + wait_for_export_status(node, mt_table, s3_table, "2021", "COMPLETED") + + # checking for the commit file because maybe the data file was too fast? + assert node.query(f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/commit_2020_*', format=LineAsString)") == '0\n', "Partition 2020 was written to S3, it was not killed as expected" + assert node.query(f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/commit_2021_*', format=LineAsString)") != f'0\n', "Partition 2021 was not written to S3, but it should have been" + + # check system.partition_exports for the export, status should be KILLED + assert node.query(f"SELECT status FROM system.partition_exports WHERE partition_id = '2020' and source_table = '{mt_table}' and destination_table = '{s3_table}'") == 'KILLED\n', "Partition 2020 was not killed as expected" + assert node.query(f"SELECT status FROM system.partition_exports WHERE partition_id = '2021' and source_table = '{mt_table}' and destination_table = '{s3_table}'") == 'COMPLETED\n', "Partition 2021 was not completed, this is unexpected" + + # check the data did not land on s3 + assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") == '0\n', "Partition 2020 was written to S3, it was not killed as expected" + + +def test_kill_export_resilient_to_status_handling_failure(cluster): + """KILL EXPORT PARTITION must eventually take effect even when the first + attempt to handle the ZK status-change event throws (simulated via a ONCE + failpoint). The re-queue + reschedule mechanism retries after ~5 s and + the second attempt succeeds because the ONCE failpoint has already fired.""" + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"kill_resilient_mt_{postfix}" + s3_table = f"kill_resilient_s3_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1") + + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + with PartitionManager() as pm: + pm.add_rule({ + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + + pm.add_rule({ + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + ) + + node.query("SYSTEM ENABLE FAILPOINT export_partition_status_change_throw") + + node.query( + f"KILL EXPORT PARTITION WHERE partition_id = '2020'" + f" AND source_table = '{mt_table}' AND destination_table = '{s3_table}'") + + # sleep for a while to let the kill to be processed + time.sleep(5) + + # The ONCE failpoint makes the first handleStatusChanges() throw. + # The catch re-queues the key and scheduleAfter(5000) arms a retry. + # Wait up to 15 s (5 s retry delay + margin) for the kill to propagate. + wait_for_export_status(node, mt_table, s3_table, "2020", "KILLED", timeout=15) + + assert ( + node.query( + f"SELECT status FROM system.partition_exports" + f" WHERE partition_id = '2020'" + f" AND source_table = '{mt_table}'" + f" AND destination_table = '{s3_table}'" + ).strip() == "KILLED" + ), "Export was not killed — status change was lost after the injected failure" + + +def test_drop_source_table_during_export(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + # node2 = cluster.instances["replica2"] + watcher_node = cluster.instances["watcher_node"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"drop_source_table_during_export_mt_table_{postfix}" + s3_table = f"drop_source_table_during_export_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1") + # create_tables_and_insert_data(node2, mt_table, s3_table, "replica2") + create_s3_table(watcher_node, s3_table) + + # Block S3/MinIO requests to keep exports alive via retry mechanism + # This allows ZooKeeper operations (KILL) to proceed quickly + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + with PartitionManager() as pm: + # Block responses from MinIO (source_port matches MinIO service) + pm_rule_reject_responses = { + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_responses) + + # Block requests to MinIO (destination: MinIO, destination_port: minio_port) + pm_rule_reject_requests = { + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_requests) + + export_queries = f""" + ALTER TABLE {mt_table} + EXPORT PARTITION ID '2020' TO TABLE {s3_table} SETTINGS s3_retry_attempts = 500; + ALTER TABLE {mt_table} + EXPORT PARTITION ID '2021' TO TABLE {s3_table} SETTINGS s3_retry_attempts = 500; + """ + + node.query(export_queries) + + wait_for_export_status(node, mt_table, s3_table, "2020", "PENDING") + wait_for_export_status(node, mt_table, s3_table, "2021", "PENDING") + + # This should kill the background operations and drop the table + node.query(f"DROP TABLE {mt_table}") + + # Sleep some time to let the export finish (assuming it was not properly cancelled) + time.sleep(10) + + assert node.query(f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/commit_*', format=LineAsString)") == '0\n', "Background operations completed even with the table dropped" + + +def test_concurrent_exports_to_different_targets(cluster): + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"concurrent_diff_targets_mt_table_{postfix}" + s3_table_a = f"concurrent_diff_targets_s3_a_{postfix}" + s3_table_b = f"concurrent_diff_targets_s3_b_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table_a, "replica1") + create_s3_table(node, s3_table_b) + + # Launch two exports of the same partition to two different S3 tables concurrently + with PartitionManager() as pm: + pm.add_network_delay(node, delay_ms=1000) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table_a}" + ) + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table_b}" + ) + + wait_for_export_status(node, mt_table, s3_table_a, "2020", "COMPLETED") + wait_for_export_status(node, mt_table, s3_table_b, "2020", "COMPLETED") + + # Both targets should receive the same data independently + assert node.query(f"SELECT count() FROM {s3_table_a} WHERE year = 2020") == '3\n', "First target did not receive expected rows" + assert node.query(f"SELECT count() FROM {s3_table_b} WHERE year = 2020") == '3\n', "Second target did not receive expected rows" + + # And both should have a commit marker + assert node.query( + f"SELECT count() FROM s3(s3_conn, filename='{s3_table_a}/commit_2020_*', format=LineAsString)" + ) != '0\n', "Commit file missing for first target" + assert node.query( + f"SELECT count() FROM s3(s3_conn, filename='{s3_table_b}/commit_2020_*', format=LineAsString)" + ) != '0\n', "Commit file missing for second target" + + +def test_failure_is_logged_in_system_table(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"failure_is_logged_in_system_table_mt_table_{postfix}" + s3_table = f"failure_is_logged_in_system_table_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1") + + # Block traffic to/from MinIO to force upload errors and retries, following existing S3 tests style + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + with PartitionManager() as pm: + # Block responses from MinIO (source_port matches MinIO service) + pm_rule_reject_responses = { + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_responses) + + # Also block requests to MinIO (destination: MinIO, destination_port: 9001) with REJECT to fail fast + pm_rule_reject_requests = { + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_requests) + + # Blocked MinIO produces transient (retryable) S3 errors. There is no retry + # budget anymore, so the task keeps retrying and is only torn down once the + # absolute task timeout fires (transitioning to KILLED). Use a small timeout + # so the test does not wait for the default (a day). + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + f" SETTINGS export_merge_tree_partition_task_timeout_seconds = 5;" + ) + + # Wait for the timeout to kill the stuck task. The KILL is a Keeper operation + # (MinIO being blocked does not affect it); the status mirror needs roughly one + # manifest-updater poll cycle (~30s) plus watch propagation on top of the 5s + # timeout, so allow a generous budget. + wait_for_export_status(node, mt_table, s3_table, "2020", "KILLED", timeout=90) + + # Network restored; verify the export is marked as KILLED in the system table + # Also verify we captured at least one exception and no commit file exists + status = node.query( + f""" + SELECT status FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '2020' + """ + ) + + assert status.strip() == "KILLED", f"Expected KILLED status, got: {status!r}" + + exception_count = node.query( + f""" + SELECT any(exception_count) FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '2020' + """ + ) + assert int(exception_count.strip()) > 0, "Expected non-zero exception_count in system.partition_exports" + + # No commit should have been produced for this partition + assert node.query( + f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/commit_2020_*', format=LineAsString)" + ) == '0\n', "Commit file exists despite forced S3 failures" + + +def test_inject_short_living_failures(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"inject_short_living_failures_mt_table_{postfix}" + s3_table = f"inject_short_living_failures_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1") + + # Block traffic to/from MinIO to force upload errors and retries, following existing S3 tests style + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + with PartitionManager() as pm: + # Block responses from MinIO (source_port matches MinIO service) + pm_rule_reject_responses = { + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_responses) + + # Also block requests to MinIO (destination: MinIO, destination_port: 9001) with REJECT to fail fast + pm_rule_reject_requests = { + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_requests) + + # Transient (retryable) failures never fail the task on a budget; it keeps + # retrying until the network is restored and the export completes. + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table};" + ) + + # wait for at least one exception to occur, but not enough to finish the export. + # Use the helper default (>= one manifest-updater poll cycle): system.partition_exports + # is served from the in-memory mirror, and while the task stays PENDING the mirror only + # picks up new exception leaves on the next poll tick (~30s) — see helper docstring. + wait_for_exception_count(node, mt_table, s3_table, "2020", min_exception_count=1) + + # wait for the export to finish + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") + + # Assert the export succeeded + assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") == '3\n', "Export did not succeed" + assert node.query(f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/commit_2020_*', format=LineAsString)") == '1\n', "Export did not succeed" + + # check system.partition_exports for the export + assert node.query( + f""" + SELECT status FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '2020' + """ + ) == "COMPLETED\n", "Export should be marked as COMPLETED" + + exception_count = node.query( + f""" + SELECT exception_count FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '2020' + """ + ) + assert int(exception_count.strip()) >= 1, "Expected at least one exception" + + +def test_export_partition_retry_backoff(cluster, source_engine): + """Verify the per-replica in-memory exponential back-off between failed part exports. + + The back-off is local in-memory state (no ZooKeeper retry_count / next_retry_time + anymore), so it is not directly observable; instead we observe its effect. With a + large back-off, a part that keeps failing (object storage blocked) is parked for the + back-off window after its first failure and must NOT be retried on every ~5s + scheduler tick. We assert that exception_count stays low across a window that spans + several ticks. Once the network is restored and the back-off elapses, the export + completes (there is no retry budget to exhaust).""" + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"retry_backoff_mt_table_{postfix}" + s3_table = f"retry_backoff_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + # Large back-off so a single failed attempt parks the part well beyond the + # ~5s scheduler tick. Kept moderate so the export can still complete promptly + # once the network is restored. + initial_backoff_seconds = 30 + max_backoff_seconds = 30 + + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + with PartitionManager() as pm: + # Block responses from MinIO (source_port matches MinIO service) + pm.add_rule({ + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + # Also block requests to MinIO to fail fast + pm.add_rule({ + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " + f"SETTINGS export_merge_tree_partition_retry_initial_backoff_seconds = {initial_backoff_seconds}, " + f"export_merge_tree_partition_retry_max_backoff_seconds = {max_backoff_seconds}" + ) + + # Wait until the first failure is recorded. + count_after_first = wait_for_exception_count( + node, mt_table, s3_table, "2020", min_exception_count=1, timeout=60 + ) + + # While the part is backing off (~30s) it must not be retried again. Observe + # across a window that spans several scheduler ticks: without back-off the + # ~5s tick would add roughly five more failures, so a small increase proves + # the back-off is pacing retries. + time.sleep(25) + count_during_backoff = int(node.query( + f"SELECT exception_count FROM system.partition_exports" + f" WHERE source_table = '{mt_table}'" + f" AND destination_table = '{s3_table}'" + f" AND partition_id = '2020'" + ).strip()) + assert count_during_backoff - count_after_first <= 2, ( + f"exception_count jumped during the back-off window: " + f"{count_after_first} -> {count_during_backoff}; back-off was not applied" + ) + + # Network restored; once the back-off elapses the export should complete because + # there is no retry budget to exhaust. + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED", timeout=120) + assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") == "3\n", "Export did not succeed" + + +def test_mutations_after_export_partition_started(cluster, source_engine): + """Test that mutations applied after export partition starts don't affect the exported data.""" + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"mutations_after_export_partition_mt_table_{postfix}" + s3_table = f"mutations_after_export_partition_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + # Block traffic to MinIO to delay export + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + with PartitionManager() as pm: + pm_rule_reject_responses = { + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_responses) + + pm_rule_reject_requests = { + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_requests) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " + f"SETTINGS export_merge_tree_part_throw_on_pending_mutations=true" + ) + + # Wait for export to start + wait_for_export_to_start(node, mt_table, s3_table, "2020") + + node.query(f"ALTER TABLE {mt_table} UPDATE id = id + 100 WHERE year = 2020") + + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") + + result = node.query(f"SELECT id FROM {s3_table} WHERE year = 2020 ORDER BY id") + assert "1\n2\n3" in result, "Export should contain original data before mutation" + assert "101" not in result, "Export should not contain mutated data" + + +def test_patch_parts_after_export_partition_started(cluster, source_engine): + """Test that patch parts created after export partition starts don't affect the exported data.""" + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"patches_after_export_partition_mt_table_{postfix}" + s3_table = f"patches_after_export_partition_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + # Block traffic to MinIO to delay export + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + with PartitionManager() as pm: + pm_rule_reject_responses = { + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_responses) + + pm_rule_reject_requests = { + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_requests) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + ) + + # Wait for export to start + wait_for_export_to_start(node, mt_table, s3_table, "2020") + + node.query(f"UPDATE {mt_table} SET id = id + 100 WHERE year = 2020") + + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") + + result = node.query(f"SELECT id FROM {s3_table} WHERE year = 2020 ORDER BY id") + assert "1\n2\n3" in result, "Export should contain original data before patch" + assert "101" not in result, "Export should not contain patched data" + + node.query(f"DROP TABLE {mt_table}") + + +def test_export_partition_scheduler_skipped_when_moves_stopped(cluster, source_engine): + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"sched_skip_mt_{uid}" + s3_table = f"sched_skip_s3_{uid}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + node.query(f"SYSTEM STOP MOVES {mt_table}") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + ) + + wait_for_export_to_start(node, mt_table, s3_table, "2020") + + # Wait for several scheduler cycles (each fires every 5 s). + # If the guard is missing the scheduler would run and data would land in S3. + time.sleep(10) + + status = node.query( + f"SELECT status FROM system.partition_exports" + f" WHERE source_table = '{mt_table}' AND destination_table = '{s3_table}'" + f" AND partition_id = '2020'" + ).strip() + + assert status == "PENDING", ( + f"Expected PENDING while moves are stopped, got '{status}'" + ) + + row_count = int(node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip()) + assert row_count == 0, ( + f"Expected 0 rows in S3 while scheduler is skipped, got {row_count}" + ) + + node.query(f"SYSTEM START MOVES {mt_table}") + + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED", timeout=60) + + row_count = int(node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip()) + assert row_count == 3, f"Expected 3 rows in S3 after export completed, got {row_count}" + + +def test_export_partition_resumes_after_stop_moves(cluster, source_engine): + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"stop_moves_before_mt_{uid}" + s3_table = f"stop_moves_before_s3_{uid}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + node.query(f"SYSTEM STOP MOVES {mt_table}") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + ) + + wait_for_export_to_start(node, mt_table, s3_table, "2020") + + # Give the scheduler enough time to attempt (and cancel) the part task at + # least once, exercising the lock-release code path. + time.sleep(5) + + status = node.query( + f"SELECT status FROM system.partition_exports" + f" WHERE source_table = '{mt_table}' AND destination_table = '{s3_table}'" + f" AND partition_id = '2020'" + ).strip() + assert status == "PENDING", f"Expected PENDING while moves are stopped, got '{status}'" + + row_count = int(node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip()) + assert row_count == 0, f"Expected 0 rows in S3 while moves are stopped, got {row_count}" + + node.query(f"SYSTEM START MOVES {mt_table}") + + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED", timeout=60) + + row_count = int(node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip()) + assert row_count == 3, f"Expected 3 rows in S3 after export completed, got {row_count}" + + +def test_export_partition_resumes_after_stop_moves_during_export(cluster, source_engine): + skip_if_remote_database_disk_enabled(cluster) + + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"stop_moves_during_mt_{uid}" + s3_table = f"stop_moves_during_s3_{uid}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + with PartitionManager() as pm: + pm.add_rule({ + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + pm.add_rule({ + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + ) + + wait_for_export_to_start(node, mt_table, s3_table, "2020") + + # Let the tasks start executing and failing against the blocked S3. + time.sleep(2) + + node.query(f"SYSTEM STOP MOVES {mt_table}") + + # Give the cancel callback time to fire and the lock-release path to run. + time.sleep(3) + + status = node.query( + f"SELECT status FROM system.partition_exports" + f" WHERE source_table = '{mt_table}' AND destination_table = '{s3_table}'" + f" AND partition_id = '2020'" + ).strip() + + assert status == "PENDING", ( + f"Expected PENDING while moves are stopped and S3 is blocked, got '{status}'" + ) + + node.query(f"SYSTEM START MOVES {mt_table}") + + # MinIO is now unblocked; the next scheduler cycle should succeed. + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED", timeout=60) + + row_count = int(node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip()) + assert row_count == 3, f"Expected 3 rows in S3 after export completed, got {row_count}" + + +# ---- Dispatch-time destination validation ------------------------------------------------------ +# +# The destination is resolved when a part task is dispatched, not when the export is scheduled, so a +# destination that disappears or changes shape in between must fail the task instead of leaving it +# PENDING until the task timeout. Moves are stopped so the export stays scheduled but undispatched +# while the destination is being tampered with. +# +# Plain-MergeTree-only. Both scenarios leave a `Replicated*MergeTree` task PENDING instead of +# failing it, which is the behavior these tests were originally written against and which the plain +# scheduler fixed. For the dropped destination the mechanism is visible in +# `ExportPartitionUtils::isNonRetryableExportError`: `UNKNOWN_TABLE` is not in the non-retryable +# set, so the task retries until `export_merge_tree_partition_task_timeout_seconds` (a day by +# default). The schema-mismatch case is less clear - `INCOMPATIBLE_COLUMNS` *is* non-retryable, so +# the replicated dispatch appears not to reach the classification at all. Tracked as a gap in +# antalya/docs/design/alter-table-export-part-partition.md; parametrize these over `source_engine` +# once the replicated path fails the task too. + + +def test_dispatch_fails_when_destination_dropped(cluster): + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"dispatch_drop_mt_{postfix}" + s3_table = f"dispatch_drop_s3_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine="MergeTree") + + node.query(f"SYSTEM STOP MOVES {mt_table}") + try: + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}") + wait_for_export_to_start(node, mt_table, s3_table, "2020") + + status = node.query( + f"SELECT status FROM system.partition_exports" + f" WHERE source_table = '{mt_table}' AND destination_table = '{s3_table}'" + f" AND partition_id = '2020'" + ).strip() + assert status == "PENDING", f"Expected PENDING while moves are stopped, got {status!r}" + + node.query(f"DROP TABLE {s3_table} SYNC") + node.query(f"SYSTEM START MOVES {mt_table}") + + wait_for_export_status(node, mt_table, s3_table, "2020", "FAILED", timeout=60) + + last_exceptions = node.query( + f"SELECT last_exception_per_replica FROM system.partition_exports" + f" WHERE source_table = '{mt_table}'" + f" AND destination_table = '{s3_table}'" + f" AND partition_id = '2020'" + ).strip() + assert last_exceptions not in ("", "[]"), ( + "Expected an exception to be recorded for the dropped destination" + ) + finally: + node.query(f"SYSTEM START MOVES {mt_table}") + + +def test_dispatch_fails_when_destination_schema_incompatible(cluster): + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"dispatch_schema_mt_{postfix}" + s3_table = f"dispatch_schema_s3_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine="MergeTree") + + node.query(f"SYSTEM STOP MOVES {mt_table}") + try: + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}") + wait_for_export_to_start(node, mt_table, s3_table, "2020") + + node.query(f"DROP TABLE {s3_table} SYNC") + # An extra destination column is always rejected (the source has only id and year). + node.query( + f"CREATE TABLE {s3_table} (id UInt64, year UInt16, extra String) " + f"ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') " + f"PARTITION BY year" + ) + node.query(f"SYSTEM START MOVES {mt_table}") + + wait_for_export_status(node, mt_table, s3_table, "2020", "FAILED", timeout=60) + + last_exceptions = node.query( + f"SELECT last_exception_per_replica FROM system.partition_exports" + f" WHERE source_table = '{mt_table}'" + f" AND destination_table = '{s3_table}'" + f" AND partition_id = '2020'" + ).strip() + assert last_exceptions not in ("", "[]"), ( + "Expected an exception to be recorded for the schema mismatch" + ) + finally: + node.query(f"SYSTEM START MOVES {mt_table}") + + +def test_export_task_timeout_kills_stuck_pending_task(cluster, source_engine): + """A task that cannot make progress is torn down by the absolute task timeout, and the reason + is reported through the system table rather than being silently dropped.""" + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"timeout_mt_{postfix}" + s3_table = f"timeout_s3_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + with PartitionManager() as pm: + pm.add_rule({ + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + pm.add_rule({ + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + }) + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + f" SETTINGS export_merge_tree_partition_task_timeout_seconds = 5" + ) + + wait_for_export_status(node, mt_table, s3_table, "2020", "KILLED", timeout=90) + + last_exceptions = node.query( + f"SELECT last_exception_per_replica FROM system.partition_exports" + f" WHERE source_table = '{mt_table}'" + f" AND destination_table = '{s3_table}'" + f" AND partition_id = '2020'" + ).strip() + assert "timed out" in last_exceptions, ( + f"Expected the recorded exception to mention the timeout reason, got: {last_exceptions!r}" + ) + + assert node.query( + f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/commit_2020_*', format=LineAsString)" + ) == "0\n", "Commit file exists despite the task timeout" + assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") == "0\n" diff --git a/tests/integration/test_export_partition_to_object_storage/test_lifecycle.py b/tests/integration/test_export_partition_to_object_storage/test_lifecycle.py new file mode 100644 index 000000000000..1cc6c0722bfa --- /dev/null +++ b/tests/integration/test_export_partition_to_object_storage/test_lifecycle.py @@ -0,0 +1,577 @@ +import uuid + +from helpers.export_partition_helpers import ( + is_replicated_engine, + wait_for_export_status, +) + +from .common import ( + create_s3_table, + create_tables_and_insert_data, + source_engine_clause, +) + +CLUSTER_INSTANCES = ["replica1", "replica_with_export_disabled"] + +# The happy paths and the user-facing guards of `EXPORT PARTITION` into a plain object-storage +# destination: exporting one partition or all of them, the already-exists policies, permissions, +# and the pending mutation / patch part gates. + + +def test_export_partition_file_already_exists_policy(cluster, source_engine): + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"export_partition_file_already_exists_policy_mt_table_{postfix}" + s3_table = f"export_partition_file_already_exists_policy_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + # stop merges so part names remain stable. it is important for the test. + node.query(f"SYSTEM STOP MERGES {mt_table}") + + # Export all parts + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}", + ) + + # check system.partition_exports for the export + assert node.query( + f""" + SELECT status FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '2020' + """ + ) == "COMPLETED\n", "Export should be marked as COMPLETED" + + # wait for the exports to finish + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") + + # plain object storage destinations surface the commit marker file path via + # system.partition_exports.committed_marker_file + committed_marker_file = node.query( + f""" + SELECT committed_marker_file FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '2020' + """ + ).strip() + if is_replicated_engine(source_engine): + # `committed_marker_file` is the absolute key in the bucket (same convention as + # `destination_file_paths`); it may carry the s3_conn URL's in-bucket prefix on + # top of the table's `filename` argument, so use a "contains" check that does + # not depend on knowing that prefix. + assert f"{s3_table}/commit_2020_" in committed_marker_file, \ + f"Expected committed_marker_file under {s3_table}/, got: {committed_marker_file!r}" + # Path relative to the `s3_conn` URL, derived from the absolute key without + # assuming a particular URL prefix. + marker_relative_path = committed_marker_file[committed_marker_file.index(f"{s3_table}/"):] + assert node.query( + f"SELECT count() FROM s3(s3_conn, filename='{marker_relative_path}', format=LineAsString)" + ) == '1\n', f"Commit marker file does not exist at {committed_marker_file!r}" + else: + # A plain MergeTree does not persist the commit paths, so the column stays empty even + # after a successful commit (documented in docs/en/antalya/partition_export.md). The + # marker itself is still written - the commit-file assertions elsewhere cover that. + assert committed_marker_file == "", ( + f"Expected an empty committed_marker_file for a plain MergeTree source, " + f"got: {committed_marker_file!r}" + ) + + # try to export the partition + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} SETTINGS export_merge_tree_partition_force_export=1" + ) + + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") + + assert node.query( + f""" + SELECT count() FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '2020' + AND status = 'COMPLETED' + """ + ) == '1\n', "Expected the export to be marked as COMPLETED" + + # overwrite policy + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} SETTINGS export_merge_tree_partition_force_export=1, export_merge_tree_part_file_already_exists_policy='overwrite'" + ) + + # wait for the export to finish + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") + + # check system.partition_exports for the export + # ideally we would make sure the transaction id is different, but I do not have the time to do that now + assert node.query( + f""" + SELECT count() FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '2020' + AND status = 'COMPLETED' + """ + ) == '1\n', "Expected the export to be marked as COMPLETED" + + # last but not least, let's try with the error policy. FILE_ALREADY_EXISTS is a + # non-retryable error (retrying always hits the same existing file), so the task + # fails fast without needing a retry budget. + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} SETTINGS export_merge_tree_partition_force_export=1, export_merge_tree_part_file_already_exists_policy='error'", + ) + + # wait for the export to finish + wait_for_export_status(node, mt_table, s3_table, "2020", "FAILED") + + # check system.partition_exports for the export + assert node.query( + f""" + SELECT count() FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '2020' + AND status = 'FAILED' + """ + ) == '1\n', "Expected the export to be marked as FAILED" + + +def test_export_partition_feature_is_disabled(cluster, source_engine): + replica_with_export_disabled = cluster.instances["replica_with_export_disabled"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"export_partition_feature_is_disabled_mt_table_{postfix}" + s3_table = f"export_partition_feature_is_disabled_s3_table_{postfix}" + + create_tables_and_insert_data(replica_with_export_disabled, mt_table, s3_table, "replica1", engine=source_engine) + + error = replica_with_export_disabled.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table};") + assert "experimental" in error, "Expected error about disabled feature" + + # make sure kill operation also throws + error = replica_with_export_disabled.query_and_get_error(f"KILL EXPORT PARTITION WHERE partition_id = '2020' and source_table = '{mt_table}' and destination_table = '{s3_table}'") + assert "experimental" in error, "Expected error about disabled feature" + + +def test_export_partition_permissions(cluster, source_engine): + """Test that export partition validates permissions correctly: + - User needs ALTER permission on source table + - User needs INSERT permission on destination table + """ + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"permissions_mt_table_{postfix}" + s3_table = f"permissions_s3_table_{postfix}" + + # Create tables as default user + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + # Create test users with specific permissions + node.query("CREATE USER IF NOT EXISTS user_no_alter IDENTIFIED WITH no_password") + node.query("CREATE USER IF NOT EXISTS user_no_insert IDENTIFIED WITH no_password") + node.query("CREATE USER IF NOT EXISTS user_with_permissions IDENTIFIED WITH no_password") + + # Grant basic access to all users + node.query(f"GRANT SELECT ON {mt_table} TO user_no_alter") + node.query(f"GRANT SELECT ON {s3_table} TO user_no_alter") + + # user_no_insert has ALTER on source but no INSERT on destination + node.query(f"GRANT ALTER ON {mt_table} TO user_no_insert") + node.query(f"GRANT SELECT ON {s3_table} TO user_no_insert") + + # user_with_permissions has both ALTER and INSERT + node.query(f"GRANT ALTER ON {mt_table} TO user_with_permissions") + node.query(f"GRANT INSERT ON {s3_table} TO user_with_permissions") + + # Test 1: User without ALTER permission should fail + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}", + user="user_no_alter" + ) + + assert "ACCESS_DENIED" in error or "Not enough privileges" in error, \ + f"Expected ACCESS_DENIED error for user without ALTER, got: {error}" + + # Test 2: User with ALTER but without INSERT permission should fail + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}", + user="user_no_insert" + ) + + assert "ACCESS_DENIED" in error or "Not enough privileges" in error, \ + f"Expected ACCESS_DENIED error for user without INSERT, got: {error}" + + # Test 3: User with both ALTER and INSERT should succeed + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}", + user="user_with_permissions" + ) + + # Wait for export to complete + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") + + # Verify the export succeeded + result = node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") + assert result.strip() == "3", f"Expected 3 rows exported, got: {result}" + + # Verify system table shows COMPLETED status + status = node.query( + f""" + SELECT status FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '2020' + """ + ) + assert status.strip() == "COMPLETED", f"Expected COMPLETED status, got: {status}" + + +# assert multiple exports within a single query are executed. They all share the same query id +# and previously the transaction id was the query id, which would cause problems +def test_multiple_exports_within_a_single_query(cluster, source_engine): + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multiple_exports_within_a_single_query_mt_table_{postfix}" + s3_table = f"multiple_exports_within_a_single_query_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}, EXPORT PARTITION ID '2021' TO TABLE {s3_table};") + + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") + wait_for_export_status(node, mt_table, s3_table, "2021", "COMPLETED") + + # assert the exports have been executed + assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") == '3\n', "Export did not succeed" + assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2021") == '1\n', "Export did not succeed" + + # check system.partition_exports for the exports + assert node.query( + f""" + SELECT status FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '2020' + """ + ) == "COMPLETED\n", "Export should be marked as COMPLETED" + + assert node.query( + f""" + SELECT status FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '2021' + """ + ) == "COMPLETED\n", "Export should be marked as COMPLETED" + + +def test_pending_mutations_throw_before_export_partition(cluster, source_engine): + """Test that pending mutations before export partition throw an error.""" + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"pending_mutations_throw_partition_mt_table_{postfix}" + s3_table = f"pending_mutations_throw_partition_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + node.query(f"SYSTEM STOP MERGES {mt_table}") + + node.query(f"ALTER TABLE {mt_table} UPDATE id = id + 100 WHERE year = 2020") + + mutations = node.query(f"SELECT count() FROM system.mutations WHERE table = '{mt_table}' AND is_done = 0") + assert mutations.strip() != '0', "Mutation should be pending" + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " + f"SETTINGS export_merge_tree_part_throw_on_pending_mutations=true" + ) + + assert "PENDING_MUTATIONS_NOT_ALLOWED" in error, f"Expected error about pending mutations, got: {error}" + + +def test_pending_mutations_skip_before_export_partition(cluster, source_engine): + """Test that pending mutations before export partition are skipped with throw_on_pending_mutations=false.""" + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"pending_mutations_skip_partition_mt_table_{postfix}" + s3_table = f"pending_mutations_skip_partition_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + node.query(f"SYSTEM STOP MERGES {mt_table}") + + node.query(f"ALTER TABLE {mt_table} UPDATE id = id + 100 WHERE year = 2020") + + mutations = node.query(f"SELECT count() FROM system.mutations WHERE table = '{mt_table}' AND is_done = 0") + assert mutations.strip() != '0', "Mutation should be pending" + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " + f"SETTINGS export_merge_tree_part_throw_on_pending_mutations=false" + ) + + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") + + result = node.query(f"SELECT id FROM {s3_table} WHERE year = 2020 ORDER BY id") + assert "101" not in result and "102" not in result and "103" not in result, \ + "Export should contain original data before mutation" + assert "1\n2\n3" in result, "Export should contain original data" + + +def test_pending_patch_parts_throw_before_export_partition(cluster, source_engine): + """Test that pending patch parts before export partition throw an error with default settings.""" + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"pending_patches_throw_partition_mt_table_{postfix}" + s3_table = f"pending_patches_throw_partition_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + node.query(f"SYSTEM STOP MERGES {mt_table}") + + node.query(f"UPDATE {mt_table} SET id = id + 100 WHERE year = 2020") + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + ) + + node.query(f"DROP TABLE {mt_table}") + + assert "PENDING_MUTATIONS_NOT_ALLOWED" in error or "pending patch parts" in error.lower(), \ + f"Expected error about pending patch parts, got: {error}" + + +def test_pending_patch_parts_skip_before_export_partition(cluster, source_engine): + """Test that pending patch parts before export partition are skipped with throw_on_pending_patch_parts=false.""" + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"pending_patches_skip_partition_mt_table_{postfix}" + s3_table = f"pending_patches_skip_partition_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1", engine=source_engine) + + node.query(f"SYSTEM STOP MERGES {mt_table}") + + node.query(f"UPDATE {mt_table} SET id = id + 100 WHERE year = 2020") + + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " + f"SETTINGS export_merge_tree_part_throw_on_pending_patch_parts=false" + ) + + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") + + result = node.query(f"SELECT id FROM {s3_table} WHERE year = 2020 ORDER BY id") + assert "1\n2\n3" in result, "Export should contain original data before patch" + + node.query(f"DROP TABLE {mt_table}") + + +def test_mutation_in_partition_clause(cluster): + """Test that mutations limited to specific partitions using IN PARTITION clause + allow exports of unaffected partitions to succeed. + + Replicated-only: a plain MergeTree's mutations snapshot is not partition-scoped, so a + mutation confined to one partition still marks parts of every other partition as having + pending mutations and the export of an unaffected partition is refused. See "Pending + mutations" under Plain (non-replicated) MergeTree in docs/en/antalya/partition_export.md. + """ + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"mutation_in_partition_clause_mt_table_{postfix}" + s3_table = f"mutation_in_partition_clause_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1") + + node.query(f"SYSTEM STOP MERGES {mt_table}") + + # Issue a mutation that uses IN PARTITION to limit it to partition 2020 + node.query(f"ALTER TABLE {mt_table} UPDATE id = id + 100 IN PARTITION '2020' WHERE year = 2020") + + # Verify mutation is pending for 2020 + mutations = node.query( + f"SELECT count() FROM system.mutations WHERE table = '{mt_table}' AND is_done = 0" + ) + assert mutations.strip() != '0', "Mutation should be pending" + + # Export of 2020 should fail (it has pending mutations) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " + f"SETTINGS export_merge_tree_part_throw_on_pending_mutations=true" + ) + assert "PENDING_MUTATIONS_NOT_ALLOWED" in error, f"Expected error about pending mutations for partition 2020, got: {error}" + + # Export of 2021 should succeed (no mutations affecting it) + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2021' TO TABLE {s3_table} " + f"SETTINGS export_merge_tree_part_throw_on_pending_mutations=true" + ) + + wait_for_export_status(node, mt_table, s3_table, "2021", "COMPLETED") + + result = node.query(f"SELECT id FROM {s3_table} WHERE year = 2021 ORDER BY id") + assert "4" in result, "Export of partition 2021 should contain original data" + + +def test_export_partition_with_mixed_computed_columns(cluster, source_engine): + """Test export partition with ALIAS, MATERIALIZED, and EPHEMERAL columns.""" + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"mixed_computed_mt_table_{postfix}" + s3_table = f"mixed_computed_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} ( + id UInt32, + value UInt32, + tag_input String EPHEMERAL, + doubled UInt64 ALIAS value * 2, + tripled UInt64 MATERIALIZED value * 3, + tag String DEFAULT upper(tag_input) + ) ENGINE = {source_engine_clause(source_engine, mt_table)} + PARTITION BY id + ORDER BY id + SETTINGS index_granularity = 1 + """) + + # Create S3 destination table with regular columns (no EPHEMERAL) + node.query(f""" + CREATE TABLE {s3_table} ( + id UInt32, + value UInt32, + doubled UInt64, + tripled UInt64, + tag String + ) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY id + """) + + node.query(f"INSERT INTO {mt_table} (id, value, tag_input) VALUES (1, 5, 'test'), (1, 10, 'prod')") + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {s3_table}") + + wait_for_export_status(node, mt_table, s3_table, "1", "COMPLETED") + + # Verify source data (ALIAS computed, EPHEMERAL not stored) + source_result = node.query(f"SELECT id, value, doubled, tripled, tag FROM {mt_table} ORDER BY value") + expected = "1\t5\t10\t15\tTEST\n1\t10\t20\t30\tPROD\n" + assert source_result == expected, f"Source table data mismatch. Expected:\n{expected}\nGot:\n{source_result}" + + dest_result = node.query(f"SELECT id, value, doubled, tripled, tag FROM {s3_table} ORDER BY value") + assert dest_result == expected, f"Exported data mismatch. Expected:\n{expected}\nGot:\n{dest_result}" + + status = node.query(f""" + SELECT status FROM system.partition_exports + WHERE source_table = '{mt_table}' + AND destination_table = '{s3_table}' + AND partition_id = '1' + """) + assert status.strip() == "COMPLETED", f"Expected COMPLETED status, got: {status}" + + +def test_export_partition_all(cluster, source_engine): + """Happy path for `ALTER TABLE ... EXPORT PARTITION ALL TO TABLE ...`. + + Schedules one export task per active partition in a single ALTER, then + verifies every partition lands in the destination S3 table. + """ + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"export_all_mt_{uid}" + s3_table = f"export_all_s3_{uid}" + + node.query( + f"CREATE TABLE {mt_table} (id UInt64, year UInt16)" + f" ENGINE = {source_engine_clause(source_engine, mt_table)}" + f" PARTITION BY year ORDER BY tuple()" + ) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2021), (3, 2022)") + create_s3_table(node, s3_table) + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + + for partition_id in ("2020", "2021", "2022"): + wait_for_export_status(node, mt_table, s3_table, partition_id, "COMPLETED", timeout=60) + + row_count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert row_count == 3, f"Expected 3 rows in S3 after EXPORT PARTITION ALL, got {row_count}" + + +def test_export_partition_all_failure_modes(cluster, source_engine): + """Cover the three values of `export_merge_tree_partition_all_on_error`. + + Set up an already-fully-exported source table, then re-run EXPORT PARTITION ALL + with each failure mode and assert the documented behavior. + """ + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"export_all_modes_mt_{uid}" + s3_table = f"export_all_modes_s3_{uid}" + empty_mt = f"export_all_empty_mt_{uid}" + + node.query( + f"CREATE TABLE {mt_table} (id UInt64, year UInt16)" + f" ENGINE = {source_engine_clause(source_engine, mt_table)}" + f" PARTITION BY year ORDER BY tuple()" + ) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2021), (3, 2022)") + create_s3_table(node, s3_table) + + # First run: schedule + wait for all partitions to complete. + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + for partition_id in ("2020", "2021", "2022"): + wait_for_export_status(node, mt_table, s3_table, partition_id, "COMPLETED", timeout=60) + + # Empty table: throws BAD_ARGUMENTS (no active partitions). + node.query( + f"CREATE TABLE {empty_mt} (id UInt64, year UInt16)" + f" ENGINE = {source_engine_clause(source_engine, empty_mt)}" + f" PARTITION BY year ORDER BY tuple()" + ) + error = node.query_and_get_error( + f"ALTER TABLE {empty_mt} EXPORT PARTITION ALL TO TABLE {s3_table}" + ) + assert "no active partitions to export" in error, ( + f"Expected 'no active partitions' error, got: {error}" + ) + + # throw_first (default): re-run aborts on the first conflicting partition. + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}" + f" SETTINGS export_merge_tree_partition_all_on_error = 'throw_first'" + ) + assert "EXPORT_PARTITION_ALREADY_EXPORTED" in error, ( + f"Expected EXPORT_PARTITION_ALREADY_EXPORTED in error, got: {error}" + ) + + # collect: aggregated PARTITION_EXPORT_FAILED message lists every conflicting partition. + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}" + f" SETTINGS export_merge_tree_partition_all_on_error = 'collect'" + ) + assert "PARTITION_EXPORT_FAILED" in error, ( + f"Expected PARTITION_EXPORT_FAILED in error, got: {error}" + ) + for partition_id in ("2020", "2021", "2022"): + assert partition_id in error, ( + f"Expected aggregated error to mention partition {partition_id}, got: {error}" + ) + + # skip_conflicts: succeeds silently because every partition conflicts and is skipped. + node.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}" + f" SETTINGS export_merge_tree_partition_all_on_error = 'skip_conflicts'" + ) diff --git a/tests/integration/test_export_partition_to_object_storage/test_replication.py b/tests/integration/test_export_partition_to_object_storage/test_replication.py new file mode 100644 index 000000000000..998c71a4f071 --- /dev/null +++ b/tests/integration/test_export_partition_to_object_storage/test_replication.py @@ -0,0 +1,427 @@ +import uuid + +import pytest + +from helpers.export_partition_helpers import ( + EXTRA_SOURCE_COLUMN_MODES, + make_rmt, + skip_if_remote_database_disk_enabled, + wait_for_export_status, + wait_for_export_to_start, +) +from helpers.network import PartitionManager + +from .common import ( + create_s3_table, + create_tables_and_insert_data, +) + +CLUSTER_INSTANCES = ["replica1", "replica2", "watcher_node", "shard1_replica1", "shard2_replica1"] + +# `EXPORT PARTITION` behavior that only exists with cross-replica coordination: assisting and +# non-initiating replicas, restarts mid-export, sharded destinations and the macros resolved from +# a `Replicated` database. These always use a `ReplicatedMergeTree` source. + + +def create_sharded_tables_and_insert_data(node, mt_table, s3_table, replica_name): + """Create sharded ReplicatedMergeTree table with {shard} macro in ZooKeeper path.""" + node.query(f"CREATE TABLE {mt_table} (id UInt64, year UInt16) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{mt_table}', '{replica_name}') PARTITION BY year ORDER BY tuple()") + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020), (3, 2020), (4, 2021)") + + create_s3_table(node, s3_table) + + +def test_restart_nodes_during_export(cluster): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + node2 = cluster.instances["replica2"] + watcher_node = cluster.instances["watcher_node"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"disaster_mt_table_{postfix}" + s3_table = f"disaster_s3_table_{postfix}" + + create_tables_and_insert_data(node, mt_table, s3_table, "replica1") + create_tables_and_insert_data(node2, mt_table, s3_table, "replica2") + create_s3_table(watcher_node, s3_table) + + # Block S3/MinIO requests to keep exports alive via retry mechanism + # This allows ZooKeeper operations to proceed quickly + minio_ip = cluster.minio_ip + minio_port = cluster.minio_port + + with PartitionManager() as pm: + # Block responses from MinIO (source_port matches MinIO service) + pm_rule_reject_responses_node1 = { + "instance": node, + "destination": node.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_responses_node1) + + pm_rule_reject_responses_node2 = { + "instance": node2, + "destination": node2.ip_address, + "protocol": "tcp", + "source_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_responses_node2) + + # Block requests to MinIO (destination: MinIO, destination_port: minio_port) + pm_rule_reject_requests_node1 = { + "instance": node, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_requests_node1) + + pm_rule_reject_requests_node2 = { + "instance": node2, + "destination": minio_ip, + "protocol": "tcp", + "destination_port": minio_port, + "action": "REJECT --reject-with tcp-reset", + } + pm.add_rule(pm_rule_reject_requests_node2) + + export_queries = f""" + ALTER TABLE {mt_table} + EXPORT PARTITION ID '2020' TO TABLE {s3_table}; + ALTER TABLE {mt_table} + EXPORT PARTITION ID '2021' TO TABLE {s3_table}; + """ + + node.query(export_queries) + + # wait for the exports to start + wait_for_export_to_start(node, mt_table, s3_table, "2020") + wait_for_export_to_start(node, mt_table, s3_table, "2021") + + node.stop_clickhouse(kill=True) + node2.stop_clickhouse(kill=True) + + assert watcher_node.query(f"SELECT count() FROM {s3_table} where year = 2020") == '0\n', "Partition 2020 was written to S3 during network delay crash" + + assert watcher_node.query(f"SELECT count() FROM {s3_table} where year = 2021") == '0\n', "Partition 2021 was written to S3 during network delay crash" + + # start the nodes, they should finish the export + node.start_clickhouse() + node2.start_clickhouse() + + wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") + wait_for_export_status(node, mt_table, s3_table, "2021", "COMPLETED") + + assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") != f'0\n', "Export of partition 2020 did not resume after crash" + + assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2021") != f'0\n', "Export of partition 2021 did not resume after crash" + + +def test_sharded_export_partition_with_filename_pattern(cluster): + """Test that export partition with filename pattern prevents collisions in sharded setup.""" + shard1_r1 = cluster.instances["shard1_replica1"] + shard2_r1 = cluster.instances["shard2_replica1"] + watcher_node = cluster.instances["watcher_node"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"sharded_mt_table_{postfix}" + s3_table = f"sharded_s3_table_{postfix}" + + # Create sharded tables on all shards with same partition data (same part names) + # Each shard uses different ZooKeeper path via {shard} macro + create_sharded_tables_and_insert_data(shard1_r1, mt_table, s3_table, "replica1") + create_sharded_tables_and_insert_data(shard2_r1, mt_table, s3_table, "replica1") + create_s3_table(watcher_node, s3_table) + + # Export partition from both shards with filename pattern including shard + # This should prevent filename collisions + shard1_r1.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " + f"SETTINGS export_merge_tree_part_filename_pattern = '{{part_name}}_{{shard}}_{{replica}}_{{checksum}}'" + ) + shard2_r1.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " + f"SETTINGS export_merge_tree_part_filename_pattern = '{{part_name}}_{{shard}}_{{replica}}_{{checksum}}'" + ) + + # Wait for exports to complete + wait_for_export_status(shard1_r1, mt_table, s3_table, "2020", "COMPLETED") + wait_for_export_status(shard2_r1, mt_table, s3_table, "2020", "COMPLETED") + + total_count = watcher_node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip() + assert total_count == "6", f"Expected 6 total rows (3 from each shard), got {total_count}" + + # Verify filenames contain shard information (check via S3 directly) + # Get all files from S3 - query from watcher_node since S3 is shared + files_shard1 = watcher_node.query( + f"SELECT _file FROM s3(s3_conn, filename='{s3_table}/**', format='One') WHERE _file LIKE '%shard1%' LIMIT 1" + ).strip() + files_shard2 = watcher_node.query( + f"SELECT _file FROM s3(s3_conn, filename='{s3_table}/**', format='One') WHERE _file LIKE '%shard2%' LIMIT 1" + ).strip() + + # Both shards should have files with their shard names + assert "shard1" in files_shard1 or files_shard1 == "", f"Expected shard1 in filenames, got: {files_shard1}" + assert "shard2" in files_shard2 or files_shard2 == "", f"Expected shard2 in filenames, got: {files_shard2}" + + +def test_export_partition_from_replicated_database_uses_db_shard_replica_macros(cluster): + """Test that {shard} and {replica} in the filename pattern are expanded from the + DatabaseReplicated identity, NOT from server config macros. + + replica1 has no / entries in its server config section. + Without the fix buildDestinationFilename() leaves macro_info.shard/replica unset, so + Macros::expand() falls through to the config-macros lookup and throws NO_ELEMENTS_IN_CONFIG. + With the fix the DatabaseReplicated shard_name / replica_name are injected into macro_info + before the expand call, and the pattern resolves correctly. + """ + + # The remote disk test suite sets the shard and replica macros in https://github.com/Altinity/ClickHouse/blob/bbabcaa96e8b7fe8f70ecd0bd4f76fb0f76f2166/tests/integration/helpers/cluster.py#L4356 + # When expanding the macros, the configured ones are preferred over the ones from the DatabaseReplicated definition. + # Therefore, this test fails. It is easier to skip it than to fix it. + skip_if_remote_database_disk_enabled(cluster) + + node = cluster.instances["replica1"] + watcher_node = cluster.instances["watcher_node"] + + postfix = str(uuid.uuid4()).replace("-", "_") + db_name = f"repdb_{postfix}" + table_name = "mt_table" + s3_table = f"s3_dbreplicated_{postfix}" + + # These values exist only in the DatabaseReplicated definition – they are NOT + # present anywhere in replica1's server config . + db_shard = "db_shard_x" + db_replica = "db_replica_y" + + node.query( + f"CREATE DATABASE {db_name} " + f"ENGINE = Replicated('/clickhouse/databases/{db_name}', '{db_shard}', '{db_replica}')") + + node.query(f""" + CREATE TABLE {db_name}.{table_name} + (id UInt64, year UInt16) + ENGINE = ReplicatedMergeTree() + PARTITION BY year ORDER BY tuple()""") + + node.query(f"INSERT INTO {db_name}.{table_name} VALUES (1, 2020), (2, 2020), (3, 2020)") + # Stop merges so part names stay stable during the test. + node.query(f"SYSTEM STOP MERGES {db_name}.{table_name}") + + node.query( + f"CREATE TABLE {s3_table} (id UInt64, year UInt16) " + f"ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') " + f"PARTITION BY year") + + watcher_node.query( + f"CREATE TABLE {s3_table} (id UInt64, year UInt16) " + f"ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') " + f"PARTITION BY year") + + # Export with {shard} and {replica} in the pattern. + # Before the fix: Macros::expand throws NO_ELEMENTS_IN_CONFIG because replica1 has + # no / server config macros. + # After the fix: DatabaseReplicated's shard_name/replica_name are wired into + # macro_info before the expand call, so this succeeds and produces the right names. + node.query( + f"ALTER TABLE {db_name}.{table_name} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " + f"SETTINGS export_merge_tree_part_filename_pattern = " + f"'{{part_name}}_{{shard}}_{{replica}}_{{checksum}}'") + + # A FAILED status here almost certainly means the macro expansion threw + # NO_ELEMENTS_IN_CONFIG (i.e. the fix is missing or broken). + wait_for_export_status(node, table_name, s3_table, "2020", "COMPLETED") + + # Data should have landed in S3. + count = watcher_node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip() + assert count == "3", f"Expected 3 exported rows, got {count}" + + # The exported filename must contain the exact shard and replica names from the + # DatabaseReplicated definition, proving the fix injected them (not server config macros). + filename = watcher_node.query( + f"SELECT _file FROM s3(s3_conn, filename='{s3_table}/**/*.parquet', format='One') LIMIT 1" + ).strip() + + assert db_shard in filename, ( + f"Expected filename to contain DatabaseReplicated shard '{db_shard}', got: {filename!r}. " + "Suggests {shard} was not expanded from the DatabaseReplicated identity.") + + assert db_replica in filename, ( + f"Expected filename to contain DatabaseReplicated replica '{db_replica}', got: {filename!r}. " + "Suggests {replica} was not expanded from the DatabaseReplicated identity.") + + +def test_sharded_export_partition_default_pattern(cluster): + shard1_r1 = cluster.instances["shard1_replica1"] + shard2_r1 = cluster.instances["shard2_replica1"] + watcher_node = cluster.instances["watcher_node"] + + mt_table = "sharded_mt_table_default" + s3_table = "sharded_s3_table_default" + + # Create sharded tables with different ZooKeeper paths per shard + create_sharded_tables_and_insert_data(shard1_r1, mt_table, s3_table, "replica1") + create_sharded_tables_and_insert_data(shard2_r1, mt_table, s3_table, "replica1") + create_s3_table(watcher_node, s3_table) + + # Export with default pattern ({part_name}_{checksum}) - may cause collisions if parts have same name and the same checksum + shard1_r1.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + ) + shard2_r1.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + ) + + wait_for_export_status(shard1_r1, mt_table, s3_table, "2020", "COMPLETED") + wait_for_export_status(shard2_r1, mt_table, s3_table, "2020", "COMPLETED") + + # Both exports should complete (even if there are collisions, the overwrite policy handles it) + # S3 tables are shared, so query from watcher_node + total_count = watcher_node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip() + + # only one file with 3 rows should be present + assert int(total_count) == 3, f"Expected 3 rows, got {total_count}" + + +@pytest.mark.parametrize("schema_match_mode", EXTRA_SOURCE_COLUMN_MODES) +def test_export_partition_schema_match_mode_honored_by_non_initiating_replica(cluster, schema_match_mode): + replica1 = cluster.instances["replica1"] + replica2 = cluster.instances["replica2"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"schema_mode_cross_replica_mt_{postfix}" + s3_table = f"schema_mode_cross_replica_s3_{postfix}" + + make_rmt(node=replica1, name=mt_table, columns="id UInt64, year UInt16, extra String", + partition_by="year", replica_name="replica1") + make_rmt(node=replica2, name=mt_table, columns="id UInt64, year UInt16, extra String", + partition_by="year", replica_name="replica2") + replica1.query(f"INSERT INTO {mt_table} VALUES (1, 2020, 'foo'), (2, 2020, 'bar'), (3, 2020, 'baz')") + replica2.query(f"SYSTEM SYNC REPLICA {mt_table}") + + create_s3_table(node=replica1, s3_table=s3_table) + create_s3_table(node=replica2, s3_table=s3_table) + + replica1.query(f"SYSTEM STOP MOVES {mt_table}") + + replica1.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + f" SETTINGS export_merge_tree_part_schema_match_mode = '{schema_match_mode}'," + f" export_merge_tree_part_ignore_extra_source_columns = 1" + ) + + wait_for_export_status(node=replica1, source_table=mt_table, dest_table=s3_table, + partition_id="2020", expected_status="COMPLETED", timeout=60) + + count = int(replica1.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 3, f"Expected 3 rows in destination table after export, got {count}" + + result = replica1.query(f"SELECT id, year FROM {s3_table} ORDER BY id").strip() + assert result == "1\t2020\n2\t2020\n3\t2020", f"Unexpected data:\n{result}" + + replica1.query(f"SYSTEM START MOVES {mt_table}") + + +def test_export_partition_match_by_name_honored_by_non_initiating_replica(cluster): + replica1 = cluster.instances["replica1"] + replica2 = cluster.instances["replica2"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"match_by_name_cross_replica_mt_{postfix}" + s3_table = f"match_by_name_cross_replica_s3_{postfix}" + + source_columns = "id UInt64, year UInt16, omitted String, payload String" + make_rmt( + node=replica1, + name=mt_table, + columns=source_columns, + partition_by="year", + replica_name="replica1", + ) + make_rmt( + node=replica2, + name=mt_table, + columns=source_columns, + partition_by="year", + replica_name="replica2", + ) + replica1.query( + f"INSERT INTO {mt_table} VALUES " + f"(1, 2020, 'left', 'first'), (2, 2020, 'right', 'second')" + ) + replica2.query(f"SYSTEM SYNC REPLICA {mt_table}") + + for replica in (replica1, replica2): + replica.query( + f"CREATE TABLE {s3_table} (payload String, year UInt16, id UInt64) " + f"ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, " + f"partition_strategy='hive') PARTITION BY year" + ) + + replica1.query(f"SYSTEM STOP MOVES {mt_table}") + + replica1.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + f" SETTINGS export_merge_tree_part_schema_match_mode = 'NAME'," + f" export_merge_tree_part_ignore_extra_source_columns = 1" + ) + + wait_for_export_status( + node=replica1, + source_table=mt_table, + dest_table=s3_table, + partition_id="2020", + expected_status="COMPLETED", + timeout=60, + ) + + result = replica1.query( + f"SELECT payload, year, id FROM {s3_table} ORDER BY id" + ).strip() + assert result == "first\t2020\t1\nsecond\t2020\t2", f"Unexpected data:\n{result}" + + replica1.query(f"SYSTEM START MOVES {mt_table}") + + +def test_export_partition_match_by_name_with_equal_column_count_reordered(cluster): + """Test that match_by_name matches columns by name even with an equal source/destination column count.""" + replica1 = cluster.instances["replica1"] + replica2 = cluster.instances["replica2"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"match_by_name_equal_count_mt_{postfix}" + s3_table = f"match_by_name_equal_count_s3_{postfix}" + + source_columns = "id UInt64, year UInt16, payload String" + make_rmt(node=replica1, name=mt_table, columns=source_columns, + partition_by="year", replica_name="replica1") + make_rmt(node=replica2, name=mt_table, columns=source_columns, + partition_by="year", replica_name="replica2") + replica1.query(f"INSERT INTO {mt_table} VALUES (1, 2020, 'foo'), (2, 2020, 'bar')") + replica2.query(f"SYSTEM SYNC REPLICA {mt_table}") + + for replica in (replica1, replica2): + replica.query( + f"CREATE TABLE {s3_table} (payload String, id UInt64, year UInt16) " + f"ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, " + f"partition_strategy='hive') PARTITION BY year" + ) + + replica1.query(f"SYSTEM STOP MOVES {mt_table}") + + replica1.query( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" + f" SETTINGS export_merge_tree_part_schema_match_mode = 'NAME'" + ) + + wait_for_export_status(node=replica1, source_table=mt_table, dest_table=s3_table, + partition_id="2020", expected_status="COMPLETED", timeout=60) + + result = replica1.query(f"SELECT id, year, payload FROM {s3_table} ORDER BY id").strip() + assert result == "1\t2020\tfoo\n2\t2020\tbar", f"Unexpected data:\n{result}" + + replica1.query(f"SYSTEM START MOVES {mt_table}") diff --git a/tests/integration/test_export_partition_to_object_storage/test_validation.py b/tests/integration/test_export_partition_to_object_storage/test_validation.py new file mode 100644 index 000000000000..1e40a46a6de3 --- /dev/null +++ b/tests/integration/test_export_partition_to_object_storage/test_validation.py @@ -0,0 +1,566 @@ +import uuid + +import pytest + +from helpers.export_partition_helpers import ( + REJECTED_PARTITION_EXPORT_CASES, + first_partition_id, + skip_if_remote_database_disk_enabled, + wait_for_export_status, +) + +from .common import source_engine_clause + +CLUSTER_INSTANCES = ["replica1"] + +# Schedule-time validation of `EXPORT PARTITION` into a plain object-storage destination: the +# partition-key compatibility gate and the hive partition-value rendering. Rejections are +# synchronous, so these tests are cheap and touch Keeper only to create the source table. + + +def test_export_partition_partition_column_castable_type_mismatch(cluster, source_engine): + """A lossy partition-column cast (year String -> UInt16) is rejected synchronously + when export_merge_tree_part_allow_lossy_cast is off, scheduling nothing.""" + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"pkey_cast_mismatch_partition_mt_{postfix}" + s3_table = f"pkey_cast_mismatch_partition_s3_{postfix}" + + # Source: year String; destination: year UInt16. PARTITION BY year on + # both sides — same AST text — to defeat the AST equivalence check. + node.query( + f"CREATE TABLE {mt_table} (id UInt64, year String) " + f"ENGINE = {source_engine_clause(source_engine, mt_table)} " + f"PARTITION BY year " + f"ORDER BY tuple()" + ) + node.query( + f"CREATE TABLE {s3_table} (id UInt64, year UInt16) " + f"ENGINE = S3(s3_conn, filename='{s3_table}', " + f"format=Parquet, partition_strategy='hive') " + f"PARTITION BY year" + ) + + node.query( + f"INSERT INTO {mt_table} VALUES (1, '2020'), (2, '2020'), (3, '2020')" + ) + + # With a String partition column the partition_id is the SipHash of the + # value rather than the textual representation — look it up so we can + # reference the partition explicitly in EXPORT PARTITION ID and in + # subsequent system.partition_exports queries. + partition_id = node.query( + f"SELECT partition_id FROM system.parts " + f"WHERE database = currentDatabase() AND table = '{mt_table}' " + f" AND active " + f"ORDER BY name LIMIT 1" + ).strip() + assert partition_id, ( + "Expected one active part on the source table after INSERT; " + "system.parts returned nothing." + ) + + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' " + f"TO TABLE {s3_table}" + ) + assert "INCOMPATIBLE_COLUMNS" in error, ( + f"Expected INCOMPATIBLE_COLUMNS for a lossy partition-column cast, " + f"got: {error!r}" + ) + assert "requires a lossy cast" in error and "'year'" in error, ( + f"Expected the error message to report the lossy cast on column " + f"'year', got: {error!r}" + ) + + # Nothing scheduled: no row in system.partition_exports. + rows_in_system_view = node.query( + f"SELECT count() FROM system.partition_exports " + f"WHERE source_table = '{mt_table}' " + f" AND destination_table = '{s3_table}' " + f" AND partition_id = '{partition_id}'" + ).strip() + assert rows_in_system_view == "0", ( + f"Expected no row in system.partition_exports after a " + f"synchronously-rejected export, got {rows_in_system_view}." + ) + + # Nothing written: no parquet file under any year=*/ partition prefix. + files_in_s3 = node.query( + f"SELECT count() FROM s3(s3_conn, " + f"filename='{s3_table}/year=*/*.parquet', format='One')" + ).strip() + assert files_in_s3 == "0", ( + f"Expected no Parquet files in S3 after a synchronously-rejected " + f"export, found {files_in_s3}." + ) + + +# ---- Partition-key compatibility gate (unified with the Iceberg gate) -------------------------- +# +# Plain (hive) object storage writes every row of a part to the single directory computed from the +# destination PARTITION BY, so each source partition must map to exactly one destination partition. +# The gate accepts equivalent or finer source keys (e.g. a source that adds partition columns on top +# of the destination's) and rejects source partitions that would span several destination partitions +# or that do not cover the destination partition column. Hive destinations partition by bare columns +# only, so these cases exercise the column-subset and single-value paths. + + +def _run_subset_accept(node, source_key, engine): + """Export a source partitioned by *source_key* (a superset of the destination key ``year``) into a + hive destination partitioned by ``year``, then verify the full dataset, the hive directory layout, + and a round-trip back into MergeTree.""" + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"subset_mt_{uid}" + s3_table = f"subset_s3_{uid}" + roundtrip = f"subset_roundtrip_{uid}" + + node.query( + f"CREATE TABLE {mt_table} (id UInt64, year UInt16, country String)" + f" ENGINE = {source_engine_clause(engine, mt_table)}" + f" PARTITION BY {source_key} ORDER BY tuple()" + ) + node.query( + f"INSERT INTO {mt_table} VALUES (1, 2020, 'US'), (2, 2020, 'FR'), (3, 2021, 'US')" + ) + node.query( + f"CREATE TABLE {s3_table} (id UInt64, year UInt16, country String)" + f" ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive')" + f" PARTITION BY year" + ) + + partition_ids = node.query( + f"SELECT DISTINCT partition_id FROM system.parts" + f" WHERE database = currentDatabase() AND table = '{mt_table}' AND active" + ).strip().split("\n") + assert len(partition_ids) == 3, f"expected 3 source partitions, got {partition_ids}" + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + for pid in partition_ids: + wait_for_export_status(node, mt_table, s3_table, pid, "COMPLETED", timeout=90) + + src = node.query(f"SELECT id, year, country FROM {mt_table} ORDER BY id") + dst = node.query(f"SELECT id, year, country FROM {s3_table} ORDER BY id") + assert dst == src, f"destination rows differ from source:\nsrc={src!r}\ndst={dst!r}" + + # The destination partitions by year only: rows land in the year= hive directory. + rows_2020 = node.query( + f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/year=2020/*.parquet', format='Parquet')" + ).strip() + rows_2021 = node.query( + f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/year=2021/*.parquet', format='Parquet')" + ).strip() + assert rows_2020 == "2", f"expected 2 rows under year=2020, got {rows_2020}" + assert rows_2021 == "1", f"expected 1 row under year=2021, got {rows_2021}" + + node.query( + f"CREATE TABLE {roundtrip} (id UInt64, year UInt16, country String)" + f" ENGINE = {source_engine_clause(engine, roundtrip)}" + f" PARTITION BY {source_key} ORDER BY tuple()" + ) + node.query(f"INSERT INTO {roundtrip} SELECT * FROM {s3_table}") + rt = node.query(f"SELECT id, year, country FROM {roundtrip} ORDER BY id") + assert rt == src, f"round-trip rows differ from source:\nsrc={src!r}\nrt={rt!r}" + + +def test_export_partition_multicolumn_subset_accepted(cluster, source_engine): + """Source partitions by (year, country); destination by year only - a coarser key that is covered + by the source key, so every source partition has a single year and maps to exactly one destination + partition. Accepted (this was rejected as a partition-key mismatch before the plain gate was + unified with the Iceberg one).""" + node = cluster.instances["replica1"] + _run_subset_accept(node, "(year, country)", source_engine) + + +def test_export_partition_subset_reversed_order_accepted(cluster, source_engine): + """The subset match is order-independent: a source keyed by (country, year) still covers a + destination keyed by year.""" + node = cluster.instances["replica1"] + _run_subset_accept(node, "(country, year)", source_engine) + + +def test_export_partition_coarser_source_rejected(cluster, source_engine): + """Source partitions monthly (toYYYYMM(dt)); destination by the raw date. A single source part + holding two different days would map to two destination partitions, so the gate rejects the + export synchronously with BAD_ARGUMENTS and schedules nothing.""" + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"coarser_mt_{uid}" + s3_table = f"coarser_s3_{uid}" + + node.query( + f"CREATE TABLE {mt_table} (id UInt64, dt Date)" + f" ENGINE = {source_engine_clause(source_engine, mt_table)}" + f" PARTITION BY toYYYYMM(dt) ORDER BY tuple()" + ) + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05'), (2, '2024-03-20')") + node.query( + f"CREATE TABLE {s3_table} (id UInt64, dt Date)" + f" ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive')" + f" PARTITION BY dt" + ) + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error, f"expected BAD_ARGUMENTS, got: {error!r}" + + scheduled = node.query( + f"SELECT count() FROM system.partition_exports" + f" WHERE source_table = '{mt_table}' AND destination_table = '{s3_table}'" + ).strip() + assert scheduled == "0", f"expected nothing scheduled after a synchronous reject, got {scheduled}" + + +def test_export_partition_dest_column_not_in_source_key_rejected(cluster, source_engine): + """Destination partitions by a column that is not part of the source partition key; the gate + rejects the export synchronously with BAD_ARGUMENTS naming the uncovered column.""" + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"nocover_mt_{uid}" + s3_table = f"nocover_s3_{uid}" + + node.query( + f"CREATE TABLE {mt_table} (id UInt64, year UInt16, country String)" + f" ENGINE = {source_engine_clause(source_engine, mt_table)}" + f" PARTITION BY year ORDER BY tuple()" + ) + node.query(f"INSERT INTO {mt_table} VALUES (1, 2020, 'US'), (2, 2020, 'FR')") + node.query( + f"CREATE TABLE {s3_table} (id UInt64, year UInt16, country String)" + f" ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive')" + f" PARTITION BY country" + ) + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error, f"expected BAD_ARGUMENTS, got: {error!r}" + assert "country" in error, f"expected the error to name column 'country', got: {error!r}" + + +def test_export_partition_column_timezone_rendered_in_destination_zone(cluster, source_engine): + """A hive partition value lives as text in the object path and is read back in the destination + column's time zone, so the export has to spell it the way the destination would. Spelling it in the + source's zone names a different instant and the row reads back shifted by the offset between the + two zones. INSERT SELECT into an identical table is the reference behavior.""" + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"tz_mt_{uid}" + s3_export = f"tz_export_s3_{uid}" + s3_insert = f"tz_insert_s3_{uid}" + + node.query( + f"CREATE TABLE {mt_table} (id UInt64, ts DateTime('UTC'))" + f" ENGINE = {source_engine_clause(source_engine, mt_table)}" + f" PARTITION BY toDate(ts) ORDER BY tuple()" + ) + node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") + for table in (s3_export, s3_insert): + node.query( + f"CREATE TABLE {table} (id UInt64, ts DateTime('Asia/Tokyo'))" + f" ENGINE = S3(s3_conn, filename='{table}', format=Parquet, partition_strategy='hive')" + f" PARTITION BY ts" + ) + + pid = first_partition_id(node, mt_table) + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_export}") + wait_for_export_status(node, mt_table, s3_export, pid, "COMPLETED", timeout=90) + + node.query(f"INSERT INTO {s3_insert} SELECT * FROM {mt_table}") + + source_instant = node.query(f"SELECT toUnixTimestamp(ts) FROM {mt_table}").strip() + exported_instant = node.query(f"SELECT toUnixTimestamp(ts) FROM {s3_export}").strip() + inserted_instant = node.query(f"SELECT toUnixTimestamp(ts) FROM {s3_insert}").strip() + assert exported_instant == source_instant, ( + f"the exported row moved in time: source {source_instant}, destination {exported_instant}" + ) + assert inserted_instant == source_instant, ( + f"INSERT SELECT must not move it either: source {source_instant}," + f" destination {inserted_instant}" + ) + + # 2024-03-05 15:00:00 UTC is 2024-03-06 00:00:00 in Tokyo. + exported_directory = node.query( + f"SELECT DISTINCT extract(_path, 'ts=[^/]*') FROM {s3_export}" + ).strip() + inserted_directory = node.query( + f"SELECT DISTINCT extract(_path, 'ts=[^/]*') FROM {s3_insert}" + ).strip() + assert exported_directory == "ts=2024-03-06 00:00:00", ( + f"unexpected hive directory: {exported_directory!r}" + ) + assert inserted_directory == exported_directory, ( + f"export and INSERT SELECT disagree on the partition directory:" + f" {exported_directory!r} vs {inserted_directory!r}" + ) + + +def create_wildcard_destination(node, table, columns, partition_key): + """A wildcard destination, the only partition strategy that accepts an expression as its + partition key: the hive strategy allows storage columns only.""" + node.query( + f"CREATE TABLE {table} ({columns})" + f" ENGINE = S3(s3_conn, filename='{table}/{{_partition_id}}/{{_file}}.parquet'," + f" format=Parquet, partition_strategy='wildcard')" + f" PARTITION BY {partition_key}" + ) + + +def test_export_partition_dest_argument_order_rejected(cluster, source_engine): + """The destination key intDiv(x, 100) has to be validated as written. This source part holds + x in [201, 350], which covers the destination partitions 2 and 3, so the export must be rejected. + Reading the arguments in the reverse order would validate intDiv(100, x) instead, which is 0 at + both endpoints and would silently write both destination partitions into one directory.""" + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"argorder_mt_{uid}" + s3_table = f"argorder_s3_{uid}" + + node.query( + f"CREATE TABLE {mt_table} (id UInt64, x UInt64)" + f" ENGINE = {source_engine_clause(source_engine, mt_table)}" + f" PARTITION BY intDiv(x, 1000) ORDER BY tuple()" + ) + node.query(f"INSERT INTO {mt_table} VALUES (1, 201), (2, 350)") + create_wildcard_destination(node, s3_table, "id UInt64, x UInt64", "intDiv(x, 100)") + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error, f"expected BAD_ARGUMENTS, got: {error!r}" + + +def test_export_partition_dest_finer_expression_single_partition_accepted(cluster, source_engine): + """The same shape as the rejected case, with x in [100, 150]: the whole source partition maps to + the single destination partition 1, so it is accepted and every row lands in one directory. The + swapped-argument reading would refuse this one, since intDiv(100, 100) != intDiv(100, 150).""" + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"argorder_ok_mt_{uid}" + s3_table = f"argorder_ok_s3_{uid}" + + node.query( + f"CREATE TABLE {mt_table} (id UInt64, x UInt64)" + f" ENGINE = {source_engine_clause(source_engine, mt_table)}" + f" PARTITION BY intDiv(x, 1000) ORDER BY tuple()" + ) + node.query(f"INSERT INTO {mt_table} VALUES (1, 100), (2, 150)") + create_wildcard_destination(node, s3_table, "id UInt64, x UInt64", "intDiv(x, 100)") + + pid = first_partition_id(node, mt_table) + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_table}") + wait_for_export_status(node, mt_table, s3_table, pid, "COMPLETED", timeout=90) + + # A wildcard destination cannot be read as a table, so read the objects it wrote. + exported = f"s3(s3_conn, filename='{s3_table}/**/*.parquet', format='Parquet', structure='id UInt64, x UInt64')" + src = node.query(f"SELECT id, x FROM {mt_table} ORDER BY id") + dst = node.query(f"SELECT id, x FROM {exported} ORDER BY id") + assert dst == src, f"destination rows differ from source:\nsrc={src!r}\ndst={dst!r}" + + directories = node.query( + f"SELECT DISTINCT extract(_path, '{s3_table}/[^/]*') FROM {exported}" + ).strip() + assert directories == f"{s3_table}/1", f"unexpected destination directories: {directories!r}" + + +def test_export_partition_dest_nested_expression_accepted(cluster, source_engine): + """A destination key that wraps the source key in a coarser transform - toYYYYMM(toDate(ts)) over + a source keyed by toDate(ts) - is a function of the source key, so every source partition sits + inside one destination partition whatever the data is.""" + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"nested_mt_{uid}" + s3_table = f"nested_s3_{uid}" + + node.query( + f"CREATE TABLE {mt_table} (id UInt64, ts DateTime)" + f" ENGINE = {source_engine_clause(source_engine, mt_table)}" + f" PARTITION BY toDate(ts) ORDER BY tuple()" + ) + node.query( + f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 01:00:00'), (2, '2024-03-05 20:00:00')" + ) + create_wildcard_destination(node, s3_table, "id UInt64, ts DateTime", "toYYYYMM(toDate(ts))") + + pid = first_partition_id(node, mt_table) + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_table}") + wait_for_export_status(node, mt_table, s3_table, pid, "COMPLETED", timeout=90) + + exported = f"s3(s3_conn, filename='{s3_table}/**/*.parquet', format='Parquet', structure='id UInt64, ts DateTime')" + src = node.query(f"SELECT id, ts FROM {mt_table} ORDER BY id") + dst = node.query(f"SELECT id, ts FROM {exported} ORDER BY id") + assert dst == src, f"destination rows differ from source:\nsrc={src!r}\ndst={dst!r}" + + directories = node.query( + f"SELECT DISTINCT extract(_path, '{s3_table}/[^/]*') FROM {exported}" + ).strip() + assert directories == f"{s3_table}/202403", ( + f"unexpected destination directories: {directories!r}" + ) + + +def test_export_partition_dest_term_over_two_columns_rejected(cluster, source_engine): + """A destination expression over two columns is only single-valued when the source key pins both. + This source pins b but only intDiv(a, 100), so a spans [10, 90] within one source partition and + intDiv(a + b, 100) takes both 0 and 1 there. Per-column min/max cannot bound such an expression, + so it is rejected; a source keyed by (a, b) would be accepted, since it pins both columns.""" + node = cluster.instances["replica1"] + + uid = str(uuid.uuid4()).replace("-", "_") + mt_table = f"twocol_mt_{uid}" + s3_table = f"twocol_s3_{uid}" + + node.query( + f"CREATE TABLE {mt_table} (id UInt64, a UInt64, b UInt64)" + f" ENGINE = {source_engine_clause(source_engine, mt_table)}" + f" PARTITION BY (intDiv(a, 100), b) ORDER BY tuple()" + ) + node.query(f"INSERT INTO {mt_table} VALUES (1, 10, 20), (2, 90, 20)") + create_wildcard_destination( + node, s3_table, "id UInt64, a UInt64, b UInt64", "intDiv(a + b, 100)" + ) + + pid = first_partition_id(node, mt_table) + error = node.query_and_get_error( + f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_table}" + ) + assert "BAD_ARGUMENTS" in error, f"expected BAD_ARGUMENTS, got: {error!r}" + + +@pytest.mark.parametrize("case", REJECTED_PARTITION_EXPORT_CASES) +def test_export_partition_partition_key_mismatch_variants_are_rejected(cluster, case, source_engine): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"rejected_mt_table_{postfix}" + s3_table = f"rejected_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} ({case.src_columns}) + ENGINE = {source_engine_clause(source_engine, mt_table)} + PARTITION BY {case.src_partition_by} + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} ({case.dst_columns}) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY {case.dst_partition_by} + """) + + node.query(f"INSERT INTO {mt_table} VALUES {case.insert_values}") + + partition_id = node.query( + f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" + for substring in case.error_substrings: + assert substring in error, f"Expected {substring!r} in error, got: {error}" + + error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" + + +@pytest.mark.parametrize( + "dst_partition_by", + ["(a, b, c)", "(c, b, a)", "(a, b)"], + ids=["same", "reordered", "coarser"], +) +def test_export_partition_multi_column_partition_key_success(cluster, dst_partition_by, source_engine): + """The source key pins every column the destination partitions by, so the destination may + also name them in another order or leave some out: each destination expression is still + single-valued over a source partition.""" + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_pkey_ok_mt_table_{postfix}" + s3_table = f"multi_pkey_ok_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = {source_engine_clause(source_engine, mt_table)} + PARTITION BY (a, b, c) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY {dst_partition_by} + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (1, 2, 3, 'y')") + + partition_id = node.query( + f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" + ).strip() + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") + wait_for_export_status(node, mt_table, s3_table, partition_id, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 2, f"Expected 2 rows in destination after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() + assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" + + +def test_export_partition_multi_column_partition_key_success_all(cluster, source_engine): + skip_if_remote_database_disk_enabled(cluster) + node = cluster.instances["replica1"] + + postfix = str(uuid.uuid4()).replace("-", "_") + mt_table = f"multi_pkey_ok_all_mt_table_{postfix}" + s3_table = f"multi_pkey_ok_all_s3_table_{postfix}" + + node.query(f""" + CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) + ENGINE = {source_engine_clause(source_engine, mt_table)} + PARTITION BY (a, b, c) + ORDER BY tuple() + """) + + node.query(f""" + CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) + ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') + PARTITION BY (a, b, c) + """) + + node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (4, 5, 6, 'y')") + + partition_ids = node.query( + f"SELECT DISTINCT partition_id FROM system.parts WHERE database = currentDatabase() " + f"AND table = '{mt_table}' AND active ORDER BY partition_id" + ).strip().split("\n") + + node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") + + for pid in partition_ids: + wait_for_export_status(node, mt_table, s3_table, pid, "COMPLETED") + + count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) + assert count == 2, f"Expected 2 rows in destination after export, got {count}" + + result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() + assert result == "1\t2\t3\tx\n4\t5\t6\ty", f"Unexpected exported data:\n{result}" diff --git a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py b/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py deleted file mode 100644 index 4c746bc26607..000000000000 --- a/tests/integration/test_export_replicated_mt_partition_to_iceberg/test.py +++ /dev/null @@ -1,3022 +0,0 @@ -import io -import json -import logging -import re -import time -from typing import NamedTuple - -import pytest -from avro.datafile import DataFileReader -from avro.io import DatumReader - -from helpers.cluster import ClickHouseCluster -from helpers.export_partition_helpers import ( - first_partition_id, - make_iceberg_s3, - make_rmt, - unique_suffix, - wait_for_exception_count, - wait_for_export_status, - wait_for_export_to_start, -) -from helpers.iceberg_export_stats import ( - assert_exported_stats, - fetch_manifest_entries, -) -from helpers.network import PartitionManager - - -EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("POSITION", id="by-position"), - pytest.param("NAME", id="by-name"), -] - - -@pytest.fixture(scope="module") -def cluster(): - try: - cluster = ClickHouseCluster(__file__) - cluster.add_instance( - "replica1", - main_configs=[ - "configs/allow_experimental_export_partition.xml", - "configs/config.d/metadata_log.xml", - ], - user_configs=["configs/users.d/profile.xml"], - with_minio=True, - stay_alive=True, - with_zookeeper=True, - keeper_required_feature_flags=["multi_read"], - ) - cluster.add_instance( - "replica2", - main_configs=[ - "configs/allow_experimental_export_partition.xml", - "configs/config.d/metadata_log.xml", - ], - user_configs=["configs/users.d/profile.xml"], - with_minio=True, - stay_alive=True, - with_zookeeper=True, - keeper_required_feature_flags=["multi_read"], - ) - logging.info("Starting cluster...") - cluster.start() - yield cluster - finally: - cluster.shutdown() - - -@pytest.fixture(autouse=True) -def drop_tables_after_test(cluster): - """Drop all tables in the default database after every test. - - Without this, ReplicatedMergeTree tables from completed tests remain alive and keep - running ZooKeeper background threads. With many tables alive simultaneously the - ZooKeeper session becomes overwhelmed and subsequent tests start seeing - operation-timeout / session-expired errors. - """ - yield - for instance_name, instance in cluster.instances.items(): - try: - tables_str = instance.query( - "SELECT name FROM system.tables WHERE database = 'default' FORMAT TabSeparated" - ).strip() - if not tables_str: - continue - for table in tables_str.split("\n"): - table = table.strip() - if table: - instance.query(f"DROP TABLE IF EXISTS default.`{table}` SYNC") - except Exception as e: - logging.warning( - f"drop_tables_after_test: cleanup failed on {instance_name}: {e}" - ) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def create_replicated_mt(node, mt_table: str, replica_name: str): - make_rmt(node, mt_table, "id Int64, year Int32", "year", - replica_name=replica_name) - - -def create_iceberg_s3_table(node, iceberg_table: str, if_not_exists: bool = False, - s3_retry_attempts: int = 3): - """Create (or attach to an existing) IcebergS3 table at a per-test MinIO prefix.""" - make_iceberg_s3( - node, iceberg_table, "id Int64, year Int32", - partition_by="year", if_not_exists=if_not_exists, - s3_retry_attempts=s3_retry_attempts, - ) - - -def setup_tables(cluster, mt_table: str, iceberg_table: str, nodes: list | None = None, - s3_retry_attempts: int = 3): - """ - Create the ReplicatedMergeTree table on the given nodes, insert data on the first - node, wait for replication, then create the Iceberg destination table on each node. - - The Iceberg table is created on the first node (which initialises the S3 metadata). - Subsequent nodes attach to the same path with IF NOT EXISTS. - - `nodes` defaults to ["replica1", "replica2"]. - """ - if nodes is None: - nodes = ["replica1", "replica2"] - - instances = [cluster.instances[n] for n in nodes] - primary = instances[0] - - for i, instance in enumerate(instances): - create_replicated_mt(instance, mt_table, nodes[i]) - - primary.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020), (3, 2020), (4, 2021)") - for instance in instances[1:]: - instance.query(f"SYSTEM SYNC REPLICA {mt_table}") - - create_iceberg_s3_table(primary, iceberg_table, s3_retry_attempts=s3_retry_attempts) - for instance in instances[1:]: - create_iceberg_s3_table(instance, iceberg_table, if_not_exists=True, - s3_retry_attempts=s3_retry_attempts) - - -# --------------------------------------------------------------------------- -# Tests -# --------------------------------------------------------------------------- - -def test_export_partition_to_iceberg(cluster): - """ - Basic happy path: export a single partition and verify row count and content. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 3, f"Expected 3 rows in Iceberg table after export, got {count}" - - result = node.query(f"SELECT id, year FROM {iceberg_table} ORDER BY id").strip() - assert result == "1\t2020\n2\t2020\n3\t2020", ( - f"Unexpected data in Iceberg table:\n{result}" - ) - - -def _destination_paths_has_sync_failed_marker(node, source_table, dest_table, partition_id): - """True when destination_file_paths contains the Keeper sync-failed marker value.""" - result = node.query( - f"SELECT has(arrayFlatten(mapValues(destination_file_paths)), '')" - f" FROM system.replicated_partition_exports" - f" WHERE source_table = '{source_table}'" - f" AND destination_table = '{dest_table}'" - f" AND partition_id = '{partition_id}'" - ).strip() - return result == "1" - - -def wait_for_destination_paths_sync_failed_marker( - node, source_table, dest_table, partition_id, expect_marker, timeout=90, poll_interval=0.5 -): - """Wait until destination_file_paths does/does not contain the sync-failed marker. - - The in-memory mirror refreshes on the manifest-updater poll (~30s), so the - default timeout allows at least one full cycle plus headroom. - """ - start_time = time.time() - last = None - while time.time() - start_time < timeout: - last = _destination_paths_has_sync_failed_marker( - node, source_table, dest_table, partition_id - ) - if last == expect_marker: - return - time.sleep(poll_interval) - - raise TimeoutError( - f"destination_file_paths sync-failed marker did not become {expect_marker}" - f" within {timeout}s (last={last})" - ) - - -def test_export_two_partitions_to_iceberg(cluster): - """ - Export two partitions in a single ALTER TABLE statement and verify that both - land in the Iceberg table with correct row counts. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) - - node.query( - f""" - ALTER TABLE {mt_table} - EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}, - EXPORT PARTITION ID '2021' TO TABLE {iceberg_table} - """, - settings={"allow_insert_into_iceberg": 1}, - ) - - wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") - wait_for_export_status(node, mt_table, iceberg_table, "2021", "COMPLETED") - - count_2020 = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) - count_2021 = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2021").strip()) - - assert count_2020 == 3, f"Expected 3 rows for year=2020, got {count_2020}" - assert count_2021 == 1, f"Expected 1 row for year=2021, got {count_2021}" - - -def test_export_partition_all_to_iceberg(cluster): - """ - `ALTER TABLE ... EXPORT PARTITION ALL TO TABLE ...` schedules every active partition - in one statement and exercises the Iceberg-specific destination compatibility checks - (which are repeated per sub-call inside the loop). - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - - wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") - wait_for_export_status(node, mt_table, iceberg_table, "2021", "COMPLETED") - - count_2020 = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) - count_2021 = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2021").strip()) - - assert count_2020 == 3, f"Expected 3 rows for year=2020, got {count_2020}" - assert count_2021 == 1, f"Expected 1 row for year=2021, got {count_2021}" - - -def test_failure_is_logged_in_system_table(cluster): - """ - When a part export fails with a non-retryable error the export must be marked - FAILED in system.replicated_partition_exports with a non-zero exception_count. - - Uses the export_part_non_retryable_throw failpoint (throws BAD_ARGUMENTS, a - denylisted code) so the task fails fast without consuming any timeout budget. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) - - node.query("SYSTEM ENABLE FAILPOINT export_part_non_retryable_throw") - try: - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - - # short timeout to exercise the fast fail path for non retryable errors - wait_for_export_status(node, mt_table, iceberg_table, "2020", "FAILED", timeout=20) - finally: - node.query("SYSTEM DISABLE FAILPOINT export_part_non_retryable_throw") - - status = node.query( - f""" - SELECT status FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{iceberg_table}' - AND partition_id = '2020' - """ - ).strip() - assert status == "FAILED", f"Expected FAILED status, got: {status!r}" - - exception_count = int(node.query( - f""" - SELECT any(exception_count) FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{iceberg_table}' - AND partition_id = '2020' - """ - ).strip()) - assert exception_count > 0, "Expected non-zero exception_count in system.replicated_partition_exports" - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table after a failed export, got {count}" - - -def test_inject_short_living_failures(cluster): - """ - Transient S3 failures must not prevent the export from completing: after the - network is restored the export should retry and eventually land COMPLETED. - """ - node = cluster.instances["replica1"] - minio_ip = cluster.minio_ip - minio_port = cluster.minio_port - - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"], - s3_retry_attempts=1) - - node.query(f"SYSTEM STOP MOVES {mt_table}") - - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table} SETTINGS allow_insert_into_iceberg = 1") - - with PartitionManager() as pm: - pm.add_rule({ - "instance": node, - "destination": node.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - }) - pm.add_rule({ - "instance": node, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - }) - - node.query(f"SYSTEM START MOVES {mt_table}") - - # Let at least one retry happen before restoring the network. - time.sleep(15) - - wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) - assert count == 3, f"Expected 3 rows after retry, got {count}" - - status = node.query( - f""" - SELECT status FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{iceberg_table}' - AND partition_id = '2020' - """ - ).strip() - assert status == "COMPLETED", f"Expected COMPLETED in system table, got: {status!r}" - - exception_count = int(node.query( - f""" - SELECT exception_count FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{iceberg_table}' - AND partition_id = '2020' - """ - ).strip()) - assert exception_count >= 1, "Expected at least one transient exception to be recorded" - - -def test_export_partition_retryable_error_killed_on_timeout(cluster): - """ - A retryable part-export error (here FAULT_INJECTED via export_part_retryable_throw) - must NOT fail the task on a retry budget: there is no retry budget anymore, so the - part keeps retrying until the absolute task timeout fires and the task is KILLED. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) - - node.query("SYSTEM ENABLE FAILPOINT export_part_retryable_throw") - try: - # Under the old budget model a small retry budget would fail the task after the - # first retry. With the new model there is no budget and only the 5s timeout fails it. - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}" - f" SETTINGS export_merge_tree_partition_task_timeout_seconds = 5," - f" allow_insert_into_iceberg = 1" - ) - - # Give the scheduler time to attempt and fail the part several times. The old - # budget would already have transitioned the task to FAILED by now. - time.sleep(15) - status = node.query( - f"SELECT status FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}'" - f" AND destination_table = '{iceberg_table}'" - f" AND partition_id = '2020'" - ).strip() - assert status != "FAILED", ( - f"Retryable failures must not fail the task on a budget, got status {status!r}" - ) - - # The timeout (5s) is past; KILLED fires on the next manifest-updater poll cycle. - wait_for_export_status( - node, mt_table, iceberg_table, "2020", "KILLED", timeout=90 - ) - finally: - node.query("SYSTEM DISABLE FAILPOINT export_part_retryable_throw") - - exception_count = int(node.query( - f"SELECT any(exception_count) FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}'" - f" AND destination_table = '{iceberg_table}'" - f" AND partition_id = '2020'" - ).strip()) - assert exception_count > 0, "Expected at least one retryable exception to be recorded" - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table after a killed export, got {count}" - - -def test_export_partition_retryable_error_recovers_after_failpoint_cleared(cluster): - """ - A retryable part-export error must keep the task PENDING (not FAILED) while the - failure persists, applying a per-replica back-off between attempts. Once the - failure clears the export completes successfully — proving the back-off only - spaces retries out and never permanently blocks progress. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) - - node.query("SYSTEM ENABLE FAILPOINT export_part_retryable_throw") - try: - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}" - f" SETTINGS export_merge_tree_partition_retry_initial_backoff_seconds = 1," - f" export_merge_tree_partition_retry_max_backoff_seconds = 2," - f" allow_insert_into_iceberg = 1" - ) - - # Wait until at least one retryable failure has been recorded; the task must - # still be PENDING (retrying), never FAILED. - wait_for_exception_count(node, mt_table, iceberg_table, "2020", - min_exception_count=1, timeout=60) - status = node.query( - f"SELECT status FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}'" - f" AND destination_table = '{iceberg_table}'" - f" AND partition_id = '2020'" - ).strip() - assert status == "PENDING", ( - f"Retryable failures must keep the task PENDING, got status {status!r}" - ) - finally: - node.query("SYSTEM DISABLE FAILPOINT export_part_retryable_throw") - - # With the failpoint cleared the next retry succeeds and the export completes. - wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED", timeout=90) - - count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) - assert count == 3, f"Expected 3 rows after recovery, got {count}" - - -def test_export_partition_local_backoff_does_not_block_other_replica(cluster): - """ - Back-off is per-replica and in-memory: a part that one replica keeps failing on - (and therefore puts into its local back-off) must NOT be prevented from being - exported by another replica. This is the whole reason the back-off is local - rather than distributed in ZooKeeper. - - replica1 is given a persistent *retryable* failure (export_part_retryable_throw) - and is the only replica scheduling at first (moves are stopped on replica2). Once - replica1 has recorded a failure and a local back-off entry, replica2's scheduler - is enabled. Because the failpoint stays active on replica1 the whole time, the - only way the export can reach COMPLETED is replica2 picking up the very part that - replica1 keeps failing — proving the back-off does not leak across replicas. - """ - replica1 = cluster.instances["replica1"] - replica2 = cluster.instances["replica2"] - - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1", "replica2"]) - - # Phase 1: only replica1 schedules. Stop the export scheduler on replica2 so the - # part is guaranteed to be attempted (and fail) on replica1 first. - replica2.query(f"SYSTEM STOP MOVES {mt_table}") - - replica1.query("SYSTEM ENABLE FAILPOINT export_part_retryable_throw") - try: - replica1.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}" - f" SETTINGS export_merge_tree_partition_retry_initial_backoff_seconds = 1," - f" export_merge_tree_partition_retry_max_backoff_seconds = 2," - f" allow_insert_into_iceberg = 1" - ) - - # replica1 attempts the part, fails (retryable), and enters local back-off. - # The task must stay PENDING — there is no retry budget to fail it. - wait_for_exception_count(replica1, mt_table, iceberg_table, "2020", - min_exception_count=1, timeout=60) - - wait_for_export_status(replica1, mt_table, iceberg_table, "2020", "PENDING", timeout=60) - - # The back-off entry must be observable on replica1 (the failing replica). - deadline = time.time() + 90 - backoff_replica1 = "0" - while time.time() < deadline: - backoff_replica1 = replica1.query( - f"SELECT length(local_backoff_per_part) FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}'" - f" AND destination_table = '{iceberg_table}'" - f" AND partition_id = '2020'" - ).strip() - if backoff_replica1 not in ("", "0"): - break - time.sleep(0.5) - assert backoff_replica1 not in ("", "0"), ( - "Expected replica1 to carry a local back-off entry for the failing part, " - f"got {backoff_replica1!r}" - ) - - # ... and it must NOT have leaked to replica2, which never attempted the part. - # This is the core assertion: local back-off state is not shared across replicas. - backoff_replica2 = replica2.query( - f"SELECT length(local_backoff_per_part) FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}'" - f" AND destination_table = '{iceberg_table}'" - f" AND partition_id = '2020'" - ).strip() - - assert backoff_replica2 in ("", "0"), ( - f"replica2 must not carry replica1's local back-off, got {backoff_replica2!r}" - ) - - # Phase 2: enable replica2's scheduler. replica1 keeps failing (the failpoint - # is still active), so completion can only come from replica2 exporting the - # part that replica1 is backing off on. - replica2.query(f"SYSTEM START MOVES {mt_table}") - - wait_for_export_status(replica2, mt_table, iceberg_table, "2020", "COMPLETED", timeout=60) - finally: - replica1.query("SYSTEM DISABLE FAILPOINT export_part_retryable_throw") - - count = int(replica2.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) - assert count == 3, f"Expected 3 rows after replica2 completed the export, got {count}" - - -def test_export_partition_scheduler_skipped_when_moves_stopped(cluster): - """ - Verify that selectPartsToExport() skips the scheduler entirely when moves - are stopped (moves_blocker guard at the top of the function). - - No ZK locks are acquired and no background tasks are submitted, so the - Iceberg table must remain empty across multiple scheduler cycles. Once moves - are re-enabled the export completes and rows appear in the Iceberg table. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) - - node.query(f"SYSTEM STOP MOVES {mt_table}") - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - - wait_for_export_to_start(node, mt_table, iceberg_table, "2020") - - # Wait for several scheduler cycles (each fires every 5 s). - # If the guard is absent the scheduler would run and rows would appear in the Iceberg table. - time.sleep(12) - - status = node.query( - f"SELECT status FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}' AND destination_table = '{iceberg_table}'" - f" AND partition_id = '2020'" - ).strip() - - assert status == "PENDING", f"Expected PENDING while moves are stopped, got '{status}'" - - count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table while scheduler is skipped, got {count}" - - node.query(f"SYSTEM START MOVES {mt_table}") - - wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) - assert count == 3, f"Expected 3 rows in Iceberg table after export completed, got {count}" - - -def test_export_partition_resumes_after_stop_moves(cluster): - """ - Verify that SYSTEM STOP MOVES before EXPORT PARTITION does not permanently - orphan the ZooKeeper part lock for Iceberg destinations. - - When moves are stopped the scheduler still picks parts up and submits them to - the background executor, but ExportPartTask::isCancelled() returns true (via - moves_blocker), causing QUERY_WAS_CANCELLED before any data is written. The - fix in handlePartExportFailure must release the ZK lock so the part is retried - once moves are restarted. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) - - node.query(f"SYSTEM STOP MOVES {mt_table}") - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}" - f" SETTINGS allow_insert_into_iceberg = 1" - ) - - wait_for_export_to_start(node, mt_table, iceberg_table, "2020") - - # Give the scheduler enough time to attempt (and cancel) the part task at least once. - time.sleep(5) - - status = node.query( - f"SELECT status FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}' AND destination_table = '{iceberg_table}'" - f" AND partition_id = '2020'" - ).strip() - assert status == "PENDING", f"Expected PENDING while moves are stopped, got '{status}'" - - count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) - assert count == 0, f"Expected 0 rows in Iceberg table while moves are stopped, got {count}" - - node.query(f"SYSTEM START MOVES {mt_table}") - - wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) - assert count == 3, f"Expected 3 rows in Iceberg table after export completed, got {count}" - - -def test_export_partition_resumes_after_stop_moves_during_export(cluster): - """ - Verify that SYSTEM STOP MOVES issued while an Iceberg export is actively - retrying (S3 blocked) does not permanently orphan the ZooKeeper part lock. - """ - node = cluster.instances["replica1"] - minio_ip = cluster.minio_ip - minio_port = cluster.minio_port - - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) - - node.query(f"SYSTEM STOP MOVES {mt_table}") - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}" - f" SETTINGS allow_insert_into_iceberg = 1") - - wait_for_export_to_start(node, mt_table, iceberg_table, "2020") - - with PartitionManager() as pm: - pm.add_rule({ - "instance": node, - "destination": node.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - }) - pm.add_rule({ - "instance": node, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - }) - - node.query(f"SYSTEM STOP MOVES {mt_table}") - - time.sleep(3) - - status = node.query( - f"SELECT status FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}' AND destination_table = '{iceberg_table}'" - f" AND partition_id = '2020'" - ).strip() - assert status == "PENDING", ( - f"Expected PENDING while moves are stopped and S3 is blocked, got '{status}'" - ) - - node.query(f"SYSTEM START MOVES {mt_table}") - - # MinIO is now unblocked; the next scheduler cycle should succeed. - wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) - assert count == 3, f"Expected 3 rows in Iceberg table after export completed, got {count}" - - -def test_partition_transform_compatibility_accepted(cluster): - """ - Verify that EXPORT PARTITION is accepted (no BAD_ARGUMENTS) for every - supported transform when the MergeTree and Iceberg partition specs match. - - Cases covered: - 1. Compound identity (year, region), exported to a spec that lists the fields in reverse order - 2. Year transform – toYearNumSinceEpoch(event_date) - 3. Month transform – toMonthNumSinceEpoch(event_date) - 4. truncate[4] – icebergTruncate(4, category) - 5. bucket[8] – icebergBucket(8, user_id) - 6. Compound mixed – (toYearNumSinceEpoch(event_date), icebergBucket(16, user_id)) - """ - node = cluster.instances["replica1"] - uid = unique_suffix() - - def check_accepted(mt, iceberg, description): - pid = first_partition_id(node, mt) - node.query( - f"ALTER TABLE {mt} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg}", - settings={"allow_insert_into_iceberg": 1}, - ) - return pid - - # 1. Compound identity, with the destination listing the fields in the opposite order: the - # source key pins both columns, so the partition is single-valued for either field order. - cols = "id Int64, year Int32, region String" - t = f"mt_acc_1_{uid}"; i = f"iceberg_acc_1_{uid}" - make_rmt(node, t, cols, "(year, region)") - node.query(f"INSERT INTO {t} VALUES (1, 2023, 'EU')") - make_iceberg_s3(node, i, cols, "(region, year)") - pid = check_accepted(t, i, "compound identity (year, region)") - wait_for_export_status(node, t, i, pid, "COMPLETED") - count = int(node.query(f"SELECT count() FROM {i}").strip()) - assert count == 1, f"[compound identity (year, region)] Expected 1 row in Iceberg table, got {count}" - result = node.query(f"SELECT id, year, region FROM {i}").strip() - assert result == "1\t2023\tEU", f"[compound identity (year, region)] Unexpected exported data:\n{result}" - - # 2. Year transform - cols = "id Int64, event_date Date" - t = f"mt_acc_2_{uid}"; i = f"iceberg_acc_2_{uid}" - make_rmt(node, t, cols, "toYearNumSinceEpoch(event_date)") - node.query(f"INSERT INTO {t} VALUES (1, '2020-06-15')") - make_iceberg_s3(node, i, cols, "toYearNumSinceEpoch(event_date)") - check_accepted(t, i, "year transform") - - # 3. Month transform - cols = "id Int64, event_date Date" - t = f"mt_acc_3_{uid}"; i = f"iceberg_acc_3_{uid}" - make_rmt(node, t, cols, "toMonthNumSinceEpoch(event_date)") - node.query(f"INSERT INTO {t} VALUES (1, '2020-06-15')") - make_iceberg_s3(node, i, cols, "toMonthNumSinceEpoch(event_date)") - check_accepted(t, i, "month transform") - - # 4. truncate[4] - cols = "id Int64, category String" - t = f"mt_acc_4_{uid}"; i = f"iceberg_acc_4_{uid}" - make_rmt(node, t, cols, "icebergTruncate(4, category)") - node.query(f"INSERT INTO {t} VALUES (1, 'clickhouse')") - make_iceberg_s3(node, i, cols, "icebergTruncate(4, category)") - check_accepted(t, i, "truncate[4]") - - # 5. bucket[8] - cols = "id Int64, user_id Int64" - t = f"mt_acc_5_{uid}"; i = f"iceberg_acc_5_{uid}" - make_rmt(node, t, cols, "icebergBucket(8, user_id)") - node.query(f"INSERT INTO {t} VALUES (1, 42)") - make_iceberg_s3(node, i, cols, "icebergBucket(8, user_id)") - check_accepted(t, i, "bucket[8]") - - # 6. Compound mixed: year(event_date) + bucket[16](user_id) - cols = "id Int64, event_date Date, user_id Int64" - t = f"mt_acc_6_{uid}"; i = f"iceberg_acc_6_{uid}" - make_rmt(node, t, cols, "(toYearNumSinceEpoch(event_date), icebergBucket(16, user_id))") - node.query(f"INSERT INTO {t} VALUES (1, '2021-03-01', 99)") - make_iceberg_s3(node, i, cols, "(toYearNumSinceEpoch(event_date), icebergBucket(16, user_id))") - check_accepted(t, i, "compound year+bucket[16]") - - -def test_partition_transform_compatibility_rejected(cluster): - """ - Verify that partition specs that cannot be exported are rejected with BAD_ARGUMENTS. - - Acceptance is data-dependent: a source partition must map to a single Iceberg partition. The - mismatch cases below therefore use data that makes the source partition span several - destination partitions (a single-row partition would be trivially single-valued and accepted). - - Cases covered: - 1. Transform mismatch on the same column: year-transform source vs identity destination, where - the year partition contains several distinct dates. - 2. Bucket count mismatch: bucket[8] vs bucket[16] (bucket is non-monotonic, always structural). - 3. Truncate width mismatch: truncate[4] source vs truncate[8] destination, with values sharing - the 4-char prefix but differing within the first 8 chars. - 4. Unsupported MergeTree expression (intDiv) vs identity, with one bucket spanning several years. - 5. Destination partitions by a column that is not in the source partition key. - """ - node = cluster.instances["replica1"] - uid = unique_suffix() - - def assert_rejected(mt, iceberg, description): - pid = first_partition_id(node, mt) - error = node.query_and_get_error( - f"ALTER TABLE {mt} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error, ( - f"[{description}] Expected BAD_ARGUMENTS, got: {error!r}" - ) - - # 1. Transform mismatch: MergeTree year-transform, Iceberg identity on same Date col - cols = "id Int64, event_date Date" - t = f"mt_rej_1_{uid}"; i = f"iceberg_rej_1_{uid}" - make_rmt(node, t, cols, "toYearNumSinceEpoch(event_date)") - node.query(f"INSERT INTO {t} VALUES (1, '2020-01-01'), (2, '2020-12-31')") - make_iceberg_s3(node, i, cols, "event_date") # identity, not year-transform - assert_rejected(t, i, "year-transform source vs identity destination") - - # 2. Bucket count mismatch: bucket[8] vs bucket[16] - cols = "id Int64, user_id Int64" - t = f"mt_rej_2_{uid}"; i = f"iceberg_rej_2_{uid}" - make_rmt(node, t, cols, "icebergBucket(8, user_id)") - node.query(f"INSERT INTO {t} VALUES (1, 42)") - make_iceberg_s3(node, i, cols, "icebergBucket(16, user_id)") - assert_rejected(t, i, "bucket[8] vs bucket[16]") - - # 3. Truncate width mismatch: values share the 4-char prefix but differ within 8 chars. - cols = "id Int64, category String" - t = f"mt_rej_3_{uid}"; i = f"iceberg_rej_3_{uid}" - make_rmt(node, t, cols, "icebergTruncate(4, category)") - node.query(f"INSERT INTO {t} VALUES (1, 'clickhouse'), (2, 'clickfmt')") - make_iceberg_s3(node, i, cols, "icebergTruncate(8, category)") - assert_rejected(t, i, "truncate[4] source vs truncate[8] destination") - - # 4. Unsupported MergeTree expression vs identity: one intDiv bucket spans several years. - cols = "id Int64, year Int32" - t = f"mt_rej_4_{uid}"; i = f"iceberg_rej_4_{uid}" - make_rmt(node, t, cols, "intDiv(year, 100)") - node.query(f"INSERT INTO {t} VALUES (1, 2000), (2, 2099)") - make_iceberg_s3(node, i, cols, "year") - assert_rejected(t, i, "intDiv source vs identity destination") - - # 5. Destination partitions by a column absent from the source partition key. - cols = "id Int64, year Int32" - t = f"mt_rej_5_{uid}"; i = f"iceberg_rej_5_{uid}" - make_rmt(node, t, cols, "year") - node.query(f"INSERT INTO {t} VALUES (1, 2020)") - make_iceberg_s3(node, i, cols, "id") # identity on id, which the source does not partition by - assert_rejected(t, i, "destination partitions by a non-source-key column") - - -def test_partition_key_compatibility_check(cluster): - """ - Verify that EXPORT PARTITION throws BAD_ARGUMENTS synchronously when the - MergeTree partition key does not match the Iceberg table's partition spec, - and is accepted without error when the destination is satisfiable. - - Three cases: - 1. Column mismatch – MergeTree PARTITION BY year, Iceberg PARTITION BY id (must be rejected) - 2. Unpartitioned dst – MergeTree PARTITION BY year, Iceberg unpartitioned (accepted: the source is - flattened into the single empty Iceberg partition) - 3. Matching keys – both PARTITION BY year (must be accepted) - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_{uid}" - - create_replicated_mt(node, mt_table, "replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020), (3, 2021)") - node.query(f"SYSTEM SYNC REPLICA {mt_table}") - - # --- Case 1: Iceberg partitioned by 'id' but MergeTree by 'year' --- - iceberg_col_mismatch = f"iceberg_col_mismatch_{uid}" - node.query( - f""" - CREATE TABLE {iceberg_col_mismatch} - (id Int64, year Int32) - ENGINE = IcebergS3( - 'http://minio1:9001/root/data/{iceberg_col_mismatch}/', - 'minio', - 'ClickHouse_Minio_P@ssw0rd' - ) - PARTITION BY id SETTINGS s3_retry_attempts = 3 - """ - ) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_col_mismatch}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error, ( - f"Expected BAD_ARGUMENTS for partition column mismatch, got: {error!r}" - ) - - # --- Case 2: Iceberg unpartitioned, MergeTree PARTITION BY year --- - # An unpartitioned Iceberg table has a single (empty) partition, so a partitioned source is - # flattened into it and the export is accepted; the partition-column values survive as data. - iceberg_unpartitioned = f"iceberg_unpartitioned_{uid}" - node.query( - f""" - CREATE TABLE {iceberg_unpartitioned} - (id Int64, year Int32) - ENGINE = IcebergS3( - 'http://minio1:9001/root/data/{iceberg_unpartitioned}/', - 'minio', - 'ClickHouse_Minio_P@ssw0rd' - ) - SETTINGS s3_retry_attempts = 3 - """ - ) - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_unpartitioned}", - settings={"allow_insert_into_iceberg": 1}, - ) - wait_for_export_status(node, mt_table, iceberg_unpartitioned, "2020", "COMPLETED") - count = int(node.query(f"SELECT count() FROM {iceberg_unpartitioned}").strip()) - assert count == 2, f"Expected 2 rows in unpartitioned Iceberg table after export, got {count}" - result = node.query(f"SELECT id, year FROM {iceberg_unpartitioned} ORDER BY id").strip() - assert result == "1\t2020\n2\t2020", f"Unexpected data in unpartitioned Iceberg table:\n{result}" - - # --- Case 3: Matching partition keys (both PARTITION BY year) --- - iceberg_match = f"iceberg_match_{uid}" - node.query( - f""" - CREATE TABLE {iceberg_match} - (id Int64, year Int32) - ENGINE = IcebergS3( - 'http://minio1:9001/root/data/{iceberg_match}/', - 'minio', - 'ClickHouse_Minio_P@ssw0rd' - ) - PARTITION BY year SETTINGS s3_retry_attempts = 3 - """ - ) - # Should not raise — the check passes so the export is accepted synchronously - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_match}", - settings={"allow_insert_into_iceberg": 1}, - ) - - -def test_partition_transform_equivalence_gate(cluster): - """ - The Iceberg partition-compatibility gate accepts a source partition key whose transform is - equivalent to (or finer than) the destination Iceberg transform when the exported partition is - provably single-valued for every destination field, and rejects it otherwise. Accept cases are - verified end-to-end (data + metadata); reject cases must throw BAD_ARGUMENTS synchronously. - """ - node = cluster.instances["replica1"] - dt = "id Int64, event_time DateTime" - yr = "id Int64, year Int32, region String" - - cases = [ - # toDate -> day: rows within one day map to a single Iceberg day partition. - {"name": "todate_day", "columns": dt, "source_key": "toDate(event_time)", - "dest_key": "toRelativeDayNum(event_time)", - "rows": "(1, '2024-03-05 01:00:00'), (2, '2024-03-05 20:00:00')", "expect_ok": True}, - # toYYYYMM -> month: different days of the same month map to a single month partition. - {"name": "toyyyymm_month", "columns": dt, "source_key": "toYYYYMM(event_time)", - "dest_key": "toMonthNumSinceEpoch(event_time)", - "rows": "(1, '2024-03-01 00:00:00'), (2, '2024-03-20 00:00:00')", "expect_ok": True}, - # toStartOfHour -> hour. - {"name": "startofhour_hour", "columns": dt, "source_key": "toStartOfHour(event_time)", - "dest_key": "toRelativeHourNum(event_time)", - "rows": "(1, '2024-03-05 12:00:00'), (2, '2024-03-05 12:59:00')", "expect_ok": True}, - # Finer source (day + country) into a day-partitioned destination: extra column allowed. - {"name": "finer_day", "columns": "id Int64, event_time DateTime, country String", - "source_key": "(toDate(event_time), country)", "dest_key": "toRelativeDayNum(event_time)", - "rows": "(1, '2024-03-05 01:00:00', 'US'), (2, '2024-03-05 20:00:00', 'US')", - "expect_ok": True}, - # Compound field order reversed: matching is by column; the destination defines tuple order. - {"name": "reversed_order", "columns": yr, "source_key": "(year, region)", - "dest_key": "(region, year)", "rows": "(1, 2020, 'EU')", "expect_ok": True, - "verify": [("region", "region"), ("year", "year")]}, - # Superset source: (year, region) into a year-only destination is finer, so accepted. - {"name": "superset", "columns": yr, "source_key": "(year, region)", "dest_key": "year", - "rows": "(1, 2020, 'EU')", "expect_ok": True, "verify": [("year", "year")]}, - # Coarser source: a month partition spans several days, so it cannot map to one day. - {"name": "coarser_day", "columns": dt, "source_key": "toYYYYMM(event_time)", - "dest_key": "toRelativeDayNum(event_time)", - "rows": "(1, '2024-03-01 00:00:00'), (2, '2024-03-20 00:00:00')", "expect_ok": False}, - # A hash is never monotonic, so min/max cannot prove anything about it, but an identity source key - # pins k within the partition and a bucket of a single value is a single bucket. - {"name": "bucket_from_identity_source", "columns": "id Int64, k Int64", "source_key": "k", - "dest_key": "icebergBucket(8, k)", "rows": "(1, 10), (2, 10)", "expect_ok": True, - "verify": [("k", "icebergBucket(8, k)")]}, - # The same bucket destination over a source key that does not pin k: nothing proves the rows of one - # source partition hash into the same bucket. - {"name": "bucket_needs_structural", "columns": "id Int64, k Int64", - "source_key": "intDiv(k, 100)", "dest_key": "icebergBucket(8, k)", - "rows": "(1, 10), (2, 20)", "expect_ok": False}, - # Identical expressions on a Nullable column: accepted structurally. The min/max proof refuses - # Nullable (a NULL forms its own destination partition and the endpoints cannot rule it out), - # so this only passes because the source already groups by exactly this transform. DateTime64(6) - # round-trips through the Iceberg schema unchanged, which the structural type check requires. - {"name": "nullable_exact_day", "columns": "id Int64, event_time Nullable(DateTime64(6))", - "source_key": "toRelativeDayNum(event_time)", "dest_key": "toRelativeDayNum(event_time)", - "rows": "(1, '2024-03-05 01:00:00'), (2, '2024-03-05 20:00:00')", - "source_settings": "allow_nullable_key = 1", "expect_ok": True}, - # Same, for identity, which is exempt from the structural type check. - {"name": "nullable_exact_identity", "columns": "id Int64, k Nullable(Int64)", - "source_key": "k", "dest_key": "k", "rows": "(1, 10), (2, 10)", - "source_settings": "allow_nullable_key = 1", "expect_ok": True, - "verify": [("k", "k")]}, - # A Nullable column without identical expressions falls to the min/max proof, which cannot see - # NULLs, so it is rejected. - {"name": "nullable_no_match", "columns": "id Int64, event_time Nullable(DateTime64(6))", - "source_key": "toYYYYMM(event_time)", "dest_key": "toRelativeDayNum(event_time)", - "rows": "(1, '2024-03-05 01:00:00'), (2, '2024-03-05 20:00:00')", - "source_settings": "allow_nullable_key = 1", "expect_ok": False}, - ] - run_partition_compat_cases(node, cases) - - -def test_partition_transform_granularity_matrix(cluster): - """ - Exercise the common ClickHouse temporal partition keys and the granularity relationships between - the source key and the destination Iceberg transform. Acceptance is data-dependent (a source - partition must be single-valued for every destination field), so a coarser source can still be - accepted when a particular partition does not actually repartition. Accept cases are verified - end-to-end (data + metadata); reject cases must throw BAD_ARGUMENTS. - """ - node = cluster.instances["replica1"] - dt = "id Int64, event_time DateTime" - same_day = "(1, '2024-03-05 01:00:00'), (2, '2024-03-05 20:00:00')" - same_month = "(1, '2024-03-01 00:00:00'), (2, '2024-03-20 00:00:00')" - same_year = "(1, '2024-03-05 00:00:00'), (2, '2024-09-10 00:00:00')" - - def case(name, source_key, dest_key, rows, expect_ok): - return {"name": name, "columns": dt, "source_key": source_key, "dest_key": dest_key, - "rows": rows, "expect_ok": expect_ok} - - cases = [ - # Common temporal keys at the same granularity as the destination transform. - case("startofmonth_month", "toStartOfMonth(event_time)", "toMonthNumSinceEpoch(event_time)", same_month, True), - case("yyyymmdd_day", "toYYYYMMDD(event_time)", "toRelativeDayNum(event_time)", same_day, True), - case("startofday_day", "toStartOfDay(event_time)", "toRelativeDayNum(event_time)", same_day, True), - case("toyear_year", "toYear(event_time)", "toYearNumSinceEpoch(event_time)", same_year, True), - case("startofyear_year", "toStartOfYear(event_time)", "toYearNumSinceEpoch(event_time)", same_year, True), - # Finer source into a coarser destination: a finer partition sits inside one coarser bucket. - case("day_into_month", "toDate(event_time)", "toMonthNumSinceEpoch(event_time)", same_day, True), - case("day_into_year", "toDate(event_time)", "toYearNumSinceEpoch(event_time)", same_day, True), - case("hour_into_day", "toStartOfHour(event_time)", "toRelativeDayNum(event_time)", - "(1, '2024-03-05 12:00:00'), (2, '2024-03-05 12:30:00')", True), - case("month_into_year", "toYYYYMM(event_time)", "toYearNumSinceEpoch(event_time)", same_month, True), - # Coarser source into a finer destination: the partition spans several destination buckets. - case("year_into_month", "toYear(event_time)", "toMonthNumSinceEpoch(event_time)", - "(1, '2020-01-15 00:00:00'), (2, '2020-06-15 00:00:00')", False), - case("year_into_day", "toYear(event_time)", "toRelativeDayNum(event_time)", - "(1, '2020-01-01 00:00:00'), (2, '2020-12-31 00:00:00')", False), - # Same coarse/fine pair, but this year partition holds a single day, so it does not - # repartition and is accepted - acceptance depends on the data, not the structure. - case("year_into_day_single_day", "toYear(event_time)", "toRelativeDayNum(event_time)", same_day, True), - # Weekly has no Iceberg equivalent: a week partition holding two days cannot map to one day. - case("week_into_day", "toMonday(event_time)", "toRelativeDayNum(event_time)", - "(1, '2024-03-05 00:00:00'), (2, '2024-03-07 00:00:00')", False), - ] - run_partition_compat_cases(node, cases) - - -def test_partition_multicolumn_subset(cluster): - """ - Destination partition columns must be a subset of the source partition-key columns. A wide - source whose partition key is a superset of the destination's is accepted (and its multi-column - data plus per-field metadata verified); a destination partitioning by a column absent from the - source partition key is rejected. - """ - node = cluster.instances["replica1"] - wide = "id Int64, event_time DateTime, region String, tenant Int32, v1 Float64, v2 String" - - cases = [ - # Destination partition columns {event_time, region} are a strict subset of the source's - # {event_time, region, tenant}: accepted, with multi-column data and per-field metadata. - {"name": "subset_ok", "columns": wide, - "source_key": "(toDate(event_time), region, tenant)", - "dest_key": "(toRelativeDayNum(event_time), region)", - "rows": "(1, '2024-03-05 01:00:00', 'US', 7, 1.5, 'a'), " - "(2, '2024-03-05 20:00:00', 'US', 7, 2.5, 'b')", - "expect_ok": True, - "verify": [("event_time", "toRelativeDayNum(event_time)"), ("region", "region")]}, - # Destination partitions by 'region', which is not in the source partition key: rejected. - {"name": "not_subset", "columns": "id Int64, event_time DateTime, region String", - "source_key": "toDate(event_time)", - "dest_key": "(toRelativeDayNum(event_time), region)", - "rows": "(1, '2024-03-05 01:00:00', 'US'), (2, '2024-03-05 20:00:00', 'EU')", - "expect_ok": False}, - ] - run_partition_compat_cases(node, cases) - - -def test_export_partition_todate_source_matches_day_metadata(cluster): - """ - End-to-end: a source partitioned by toDate(event_time) exports into a day-partitioned Iceberg - table through the min/max refinement, and the day value written to the Iceberg metadata matches - the exported data. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_todate_{uid}" - iceberg_table = f"iceberg_todate_{uid}" - - make_rmt(node, mt_table, "id Int64, event_time DateTime", "toDate(event_time)", - replica_name="replica1") - node.query( - f"INSERT INTO {mt_table} VALUES " - f"(1, '2024-03-05 01:00:00'), (2, '2024-03-05 12:00:00'), (3, '2024-03-05 23:00:00')" - ) - make_iceberg_s3(node, iceberg_table, "id Int64, event_time DateTime", - partition_by="toRelativeDayNum(event_time)") - - pid = first_partition_id(node, mt_table) - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 3, f"Expected 3 rows after export, got {count}" - - expected_day = int(node.query( - f"SELECT DISTINCT toRelativeDayNum(event_time) FROM {iceberg_table}" - ).strip()) - - query_id = f"todate_{uid}" - node.query( - f"SELECT * FROM {iceberg_table}", - query_id=query_id, - settings={"iceberg_metadata_log_level": "manifest_file_entry"}, - ) - entries = fetch_manifest_entries(node, query_id) - partitions = _data_file_partition_records(entries) - assert partitions, "No data-file partition records found in manifest entries" - meta_days = {int(_partition_scalar(p, "event_time")) for p in partitions} - assert meta_days == {expected_day}, ( - f"Metadata day {meta_days} must equal toRelativeDayNum {expected_day}." - ) - - -def test_export_partition_day_source_into_year_metadata(cluster): - """ - End-to-end: a source partitioned by toDate(event_time) (finer) exports into a year-partitioned - Iceberg destination (coarser). The value written to the Iceberg metadata is the year computed by - the destination transform over the data, not the source day. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_day_year_{uid}" - iceberg_table = f"iceberg_day_year_{uid}" - - make_rmt(node, mt_table, "id Int64, event_time DateTime", "toDate(event_time)", - replica_name="replica1") - node.query( - f"INSERT INTO {mt_table} VALUES " - f"(1, '2024-03-05 01:00:00'), (2, '2024-03-05 12:00:00'), (3, '2024-03-05 23:00:00')" - ) - make_iceberg_s3(node, iceberg_table, "id Int64, event_time DateTime", - partition_by="toYearNumSinceEpoch(event_time)") - - pid = first_partition_id(node, mt_table) - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 3, f"Expected 3 rows after export, got {count}" - - expected_year = int(node.query( - f"SELECT DISTINCT toYearNumSinceEpoch(event_time) FROM {iceberg_table}" - ).strip()) - - query_id = f"day_year_{uid}" - node.query( - f"SELECT * FROM {iceberg_table}", - query_id=query_id, - settings={"iceberg_metadata_log_level": "manifest_file_entry"}, - ) - entries = fetch_manifest_entries(node, query_id) - partitions = _data_file_partition_records(entries) - assert partitions, "No data-file partition records found in manifest entries" - meta_years = {int(_partition_scalar(p, "event_time")) for p in partitions} - assert meta_years == {expected_year}, ( - f"Metadata year {meta_years} must equal toYearNumSinceEpoch {expected_year}." - ) - - -def test_export_partition_lossy_cast_dynamic_accept(cluster): - """ - A lossy Int64 -> Int32 partition-column cast is accepted by the dynamic proof when the - partition's values fit the destination type and map to a single Iceberg bucket. Source and - destination use different truncate widths, so the field is proven via min/max rather than a - structural match. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_lossy_{uid}" - iceberg_table = f"iceberg_lossy_{uid}" - - make_rmt(node, mt_table, "id Int64, val Int64", "icebergTruncate(10, val)", - replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 100), (2, 109)") - make_iceberg_s3(node, iceberg_table, "id Int64, val Int32", - partition_by="icebergTruncate(1000000, val)") - - pid = first_partition_id(node, mt_table) - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={ - "allow_insert_into_iceberg": 1, - "export_merge_tree_part_allow_lossy_cast": 1, - }, - ) - wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") - assert int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) == 2 - - -def test_export_data_files_are_not_cleaned_up_on_commit_failure(cluster): - """ - Verify that a commit failure does not delete the already-written data files. - `cleanup` only removes the manifest entry / manifest list, never the data files - (a peer replica might still commit the same transaction). This guards against - data loss / dangling references. - - The iceberg_writes_non_retry_cleanup failpoint throws BAD_ARGUMENTS while writing - the manifest entry, after the data files have been written. BAD_ARGUMENTS is a - non-retryable error code, so the task transitions to FAILED; we then confirm the - exported data files are still physically present in object storage by reading - them directly (the Iceberg manifests were removed by cleanup, so we glob the raw - parquet data files instead). - """ - node = cluster.instances["replica1"] - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) - - node.query("SYSTEM ENABLE FAILPOINT iceberg_writes_non_retry_cleanup") - try: - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - # BAD_ARGUMENTS from the commit phase is non-retryable -> the task fails fast. - wait_for_export_status(node, mt_table, iceberg_table, "2020", "FAILED", timeout=60) - finally: - node.query("SYSTEM DISABLE FAILPOINT iceberg_writes_non_retry_cleanup") - - # The data files were written before the commit failure; cleanup must have left - # them intact. Read them straight from object storage (bypassing the Iceberg - # metadata, which cleanup removed) and confirm all 3 exported rows survive. - rows = int(node.query( - f"SELECT count() FROM s3(" - f"'http://minio1:9001/root/data/{iceberg_table}/**.parquet', " - f"'minio', 'ClickHouse_Minio_P@ssw0rd', 'Parquet')" - ).strip()) - assert rows == 3, ( - f"Expected the 3 exported rows to still exist as data files after a failed " - f"commit (data files must not be cleaned up), got {rows}" - ) - - -def test_post_publish_exception_preserves_snapshot(cluster): - """ - Regression test for the post-publish exception-safety bug in - commitImportPartitionTransactionImpl. - - Before the fix, any exception thrown after the Iceberg snapshot was published - (e.g. from metadata-cache invalidation) would fall through to the outer - `catch (...)` and invoke `cleanup(false)`, which unconditionally removed the - manifest entry and manifest list referenced by the just-published snapshot. - A subsequent read would then fail because the live snapshot points to deleted - files. - - The failpoint `iceberg_writes_post_publish_throw` is placed inside the - post-publish region (after both the metadata file is written and - `published = true` is set). With the fix in place: - - the commit stays durable (snapshot is readable, manifests are intact); - - the export is marked COMPLETED because the outer `catch (...)` sees - `published == true` and returns the populated commit info with the real - paths produced by this attempt (no retry needed); - - all exported rows are visible through the Iceberg table. - """ - node = cluster.instances["replica1"] - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) - - node.query("SYSTEM ENABLE FAILPOINT iceberg_writes_post_publish_throw") - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table} WHERE year = 2020").strip()) - assert count == 3, ( - f"Snapshot must remain readable after a post-publish exception, " - f"expected 3 rows but got {count} (manifest files likely deleted by " - f"over-broad cleanup)" - ) - - result = node.query( - f"SELECT id, year FROM {iceberg_table} WHERE year = 2020 ORDER BY id" - ).strip() - assert result == "1\t2020\n2\t2020\n3\t2020", ( - f"Unexpected data after post-publish exception recovery:\n{result}" - ) - - # After a post-publish exception the catch handler with published==true returns - # the populated commit info (real metadata / manifest list / manifest file paths). - # ExportPartitionUtils::commit persists it to the commit_info znode, so the system - # table should show a real metadata path here, not the already-committed sentinel. - committed_metadata_file = node.query( - f""" - SELECT committed_metadata_file FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{iceberg_table}' - AND partition_id = '2020' - """ - ).strip() - assert committed_metadata_file, ( - "committed_metadata_file should be populated after a successful post-publish-catch return" - ) - assert not committed_metadata_file.startswith("<"), ( - f"committed_metadata_file should be a real metadata path, got the already-committed sentinel: {committed_metadata_file!r}" - ) - assert committed_metadata_file.endswith(".metadata.json"), ( - f"Expected a *.metadata.json path in committed_metadata_file, got: {committed_metadata_file!r}" - ) - - -def test_export_task_timeout_kills_stuck_pending_task(cluster): - """ - Verify that export_merge_tree_partition_task_timeout_seconds auto-kills a task - that remains PENDING past the deadline, transitioning it to KILLED with a - descriptive last_exception. - - The export_partition_commit_always_throw failpoint wedges the task in the - commit retry loop (REGULAR failpoint, fires on every commit attempt) with a - retryable error, so the task never fails on its own and the timeout branch in - tryCleanup is the actual mechanism under test. - """ - node = cluster.instances["replica1"] - uid = unique_suffix() - mt_table = f"mt_{uid}" - iceberg_table = f"iceberg_{uid}" - setup_tables(cluster, mt_table, iceberg_table, nodes=["replica1"]) - - node.query("SYSTEM ENABLE FAILPOINT export_partition_commit_always_throw") - - try: - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}" - f" SETTINGS export_merge_tree_partition_task_timeout_seconds = 5," - f" allow_insert_into_iceberg = 1" - ) - - # Timeout budget must cover: the 5s task timeout + one manifest-updating - # poll cycle (~30s) + watch propagation. 90s is safe. - wait_for_export_status( - node, mt_table, iceberg_table, "2020", - expected_status="KILLED", - timeout=90, - ) - - # The KILL transition writes a per-replica last_exception leaf in the same - # ZK multi as the status flip; handleStatusChanges then mirrors it into - # memory together with the status. Poll briefly to allow that watch -> - # mirror hop. We use arrayJoin to flatten the per-replica array column; - # any replica reporting the timeout reason is sufficient. - deadline = time.time() + 30 - last_exception = "" - while time.time() < deadline: - last_exception = node.query( - f""" - SELECT arrayStringConcat( - arrayMap(x -> x.message, last_exception_per_replica), - '\\n' - ) - FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{iceberg_table}' - AND partition_id = '2020' - """ - ).strip() - if "timed out" in last_exception: - break - time.sleep(0.5) - assert "timed out" in last_exception, ( - f"Expected last_exception_per_replica column to mention the timeout reason, got: {last_exception!r}" - ) - finally: - node.query("SYSTEM DISABLE FAILPOINT export_partition_commit_always_throw") - - -def setup_stats_tables(node, mt_table: str, iceberg_table: str): - """Local variant of setup_tables using the wider schema with a Nullable column.""" - columns = "id Int32, name String, tag Nullable(String), year Int32" - - make_rmt( - node, mt_table, columns, "year", - order_by="id", replica_name="replica1", - ) - node.query( - f""" - INSERT INTO {mt_table} (id, name, tag, year) VALUES - (1, 'aaa', 'x', 2020), - (2, 'mmm', NULL, 2020), - (3, 'zzz', 'y', 2020), - (4, 'kkk', 'z', 2021) - """ - ) - - make_iceberg_s3(node, iceberg_table, columns, partition_by="year") - - -def test_export_partition_writes_column_statistics(cluster): - """ - Export a whole partition (EXPORT PARTITION ID '2020') that contains one NULL - and verify that the resulting Iceberg manifest entry carries accurate per-file - column statistics: record_count, file_size_in_bytes, column_sizes, - null_value_counts, and lower/upper bounds. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_stats_{uid}" - iceberg_table = f"iceberg_stats_{uid}" - - setup_stats_tables(node, mt_table, iceberg_table) - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 3, f"Expected 3 rows in Iceberg table after export, got {count}" - - query_id = f"stats_partition_{uid}" - node.query( - f"SELECT * FROM {iceberg_table} ORDER BY id", - query_id=query_id, - settings={"iceberg_metadata_log_level": "manifest_file_entry"}, - ) - - entries = fetch_manifest_entries(node, query_id) - assert_exported_stats(entries) - - -def test_export_partition_column_count_mismatch_source_more_is_rejected(cluster): - """ - Source has 3 columns (id, year, extra), destination has 2 (id, year). - The ALTER must be rejected synchronously with NUMBER_OF_COLUMNS_DOESNT_MATCH, - nothing must be scheduled in system.replicated_partition_exports, and the - Iceberg table must remain empty. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_count_more_{uid}" - iceberg_table = f"iceberg_count_more_{uid}" - - make_rmt(node, mt_table, "id Int64, year Int32, extra String", "year", - replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 2020, 'foo'), (2, 2020, 'bar')") - - make_iceberg_s3(node, iceberg_table, "id Int64, year Int32", partition_by="year") - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in error, ( - f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH for source>dest column count, " - f"got: {error!r}" - ) - - rows_in_system_view = node.query( - f"SELECT count() FROM system.replicated_partition_exports " - f"WHERE source_table = '{mt_table}' " - f" AND destination_table = '{iceberg_table}' " - f" AND partition_id = '2020'" - ).strip() - assert rows_in_system_view == "0", ( - f"Expected no row in system.replicated_partition_exports after a " - f"synchronously-rejected export, got {rows_in_system_view}." - ) - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 0, ( - f"Expected 0 rows in Iceberg table after rejected export, got {count}" - ) - - -def test_export_partition_column_count_mismatch_source_fewer_is_rejected(cluster): - """ - Source has 2 columns (id, year), destination has 3 (id, year, extra). - Same expected synchronous rejection as the source>dest case. - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_count_fewer_{uid}" - iceberg_table = f"iceberg_count_fewer_{uid}" - - make_rmt(node, mt_table, "id Int64, year Int32", "year", replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020)") - - make_iceberg_s3(node, iceberg_table, "id Int64, year Int32, extra String", - partition_by="year") - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "NUMBER_OF_COLUMNS_DOESNT_MATCH" in error, ( - f"Expected NUMBER_OF_COLUMNS_DOESNT_MATCH for source Int64) and the - partition column (year Int32 -> Int64) round-trips.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_widen_{uid}" - iceberg_table = f"iceberg_widen_{uid}" - - make_rmt(node, mt_table, "id Int32, year Int32", "year", replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020)") - - make_iceberg_s3(node, iceberg_table, "id Int64, year Int64", partition_by="year") - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 2, f"Expected 2 rows in Iceberg table after export, got {count}" - - result = node.query( - f"SELECT id, toTypeName(id), year, toTypeName(year) FROM {iceberg_table} ORDER BY id" - ).strip() - assert result == "1\tInt64\t2020\tInt64\n2\tInt64\t2020\tInt64", ( - f"Unexpected widened data:\n{result}" - ) - - -def test_export_partition_with_castable_narrowing_values_fit(cluster): - """A lossy narrowing (id Int64 -> Int32) succeeds once the user opts in via - export_merge_tree_part_allow_lossy_cast.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_narrow_fit_{uid}" - iceberg_table = f"iceberg_narrow_fit_{uid}" - - make_rmt(node, mt_table, "id Int64, year Int32", "year", replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020)") - - make_iceberg_s3(node, iceberg_table, "id Int32, year Int32", partition_by="year") - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table}", - settings={ - "allow_insert_into_iceberg": 1, - "export_merge_tree_part_allow_lossy_cast": 1, - }, - ) - wait_for_export_status(node, mt_table, iceberg_table, "2020", "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 2, f"Expected 2 rows in Iceberg table after export, got {count}" - - result = node.query( - f"SELECT id, toTypeName(id), year FROM {iceberg_table} ORDER BY id" - ).strip() - assert result == "1\tInt32\t2020\n2\tInt32\t2020", ( - f"Unexpected narrowed data:\n{result}" - ) - - -def test_export_partition_lossy_cast_rejected_without_optin(cluster): - """A lossy narrowing (id Int64 -> Int32) is rejected synchronously with - INCOMPATIBLE_COLUMNS unless export_merge_tree_part_allow_lossy_cast is set.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_lossy_reject_{uid}" - iceberg_table = f"iceberg_lossy_reject_{uid}" - - make_rmt(node, mt_table, "id Int64, year Int32", "year", replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 2020)") - - make_iceberg_s3(node, iceberg_table, "id Int32, year Int32", partition_by="year") - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table} " - f"SETTINGS allow_insert_into_iceberg = 1" - ) - assert "INCOMPATIBLE_COLUMNS" in error, f"Expected INCOMPATIBLE_COLUMNS, got: {error!r}" - assert "lossy cast" in error, f"Expected 'lossy cast' in error, got: {error!r}" - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 0, f"Expected no rows after a rejected export, got {count}" - - -def test_export_partition_runtime_cast_failure_propagates_async(cluster): - """A String value that cannot be parsed as the destination Int32 passes the - synchronous lossy-cast gate (with export_merge_tree_part_allow_lossy_cast = 1) but - fails at runtime in the async worker with CANNOT_PARSE_TEXT. That is a deterministic - value-conversion error on the part's immutable data — retrying the same part can - never succeed — so it is classified as non-retryable and fails the whole task fast, - without waiting for the absolute task timeout, leaving Iceberg empty. - - The task timeout is left at its large default, so reaching FAILED quickly proves the - transition is driven by error classification rather than by a timeout. - - (Integer overflow is not used because the internal cast uses CastType::nonAccurate, - which wraps rather than throwing.) - """ - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_runtime_cast_fail_{uid}" - iceberg_table = f"iceberg_runtime_cast_fail_{uid}" - - make_rmt(node, mt_table, "id String, year Int32", "year", replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES ('not a number', 2020)") - - make_iceberg_s3(node, iceberg_table, "id Int32, year Int32", partition_by="year") - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {iceberg_table} " - f"SETTINGS allow_insert_into_iceberg = 1, export_merge_tree_part_allow_lossy_cast = 1" - ) - - # The runtime parse error (CANNOT_PARSE_TEXT) is non-retryable, so the task fails fast. - # No short timeout is set; FAILED within this window can only come from the - # non-retryable classification, not from the (default, ~1 day) task timeout. - wait_for_export_status(node, mt_table, iceberg_table, "2020", "FAILED", timeout=60) - - exception_count = int(node.query( - f"SELECT any(exception_count) FROM system.replicated_partition_exports " - f"WHERE source_table = '{mt_table}' " - f" AND destination_table = '{iceberg_table}' " - f" AND partition_id = '2020'" - ).strip()) - assert exception_count > 0, ( - "Expected non-zero exception_count after a failed runtime cast" - ) - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 0, ( - f"Expected 0 rows in Iceberg table after failed export, got {count}" - ) - - -def test_export_partition_all_iceberg_types(cluster): - """Every getIcebergType-supported type round-trips through an EXPORT PARTITION: - scalars use narrower source types (explicit lossless widening CASTs), plus - Array/Map/Tuple nested columns.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_all_types_{uid}" - iceberg_table = f"iceberg_all_types_{uid}" - - # Scalar source types are strictly narrower than the destination; the export inserts - # a positional widening CAST per column (Int8->Int16, UInt32->UInt64, ...). Nested - # columns keep the same type on both sides. - source_columns = ( - "i16 Int8, u16 UInt8, u32 UInt16, u64 UInt32, " - "id Int16, big Int32, f32 Float32, f64 Float64, " - "d Date, d32 Date32, dt DateTime, dt64 DateTime64(6), " - "s String, uid UUID, " - "arr Array(Int32), m Map(String, Int64), tup Tuple(a Int32, b String), " - "year Int32" - ) - dest_columns = ( - "i16 Int16, u16 UInt16, u32 UInt32, u64 UInt64, " - "id Int32, big Int64, f32 Float32, f64 Float64, " - "d Date, d32 Date32, dt DateTime, dt64 DateTime64(6), " - "s String, uid UUID, " - "arr Array(Int32), m Map(String, Int64), tup Tuple(a Int32, b String), " - "year Int32" - ) - - make_rmt(node, mt_table, source_columns, "year", replica_name="replica1") - make_iceberg_s3(node, iceberg_table, dest_columns, partition_by="year") - - node.query( - f""" - INSERT INTO {mt_table} - (i16, u16, u32, u64, id, big, f32, f64, d, d32, dt, dt64, s, uid, arr, m, tup, year) - VALUES ( - -100, 200, 50000, 4000000000, - 12345, 1000000000, 3.14, 2.718281828459045, - '2024-01-15', '2024-01-15', '2024-01-15 12:30:45', '2024-01-15 12:30:45.123456', - 'hello iceberg', '550e8400-e29b-41d4-a716-446655440000', - [1, 2, 3], {{'a': 10, 'b': 20}}, (7, 'seven'), 2024 - ) - """ - ) - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2024' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - wait_for_export_status(node, mt_table, iceberg_table, "2024", "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 1, f"Expected 1 row in Iceberg table, got {count}" - - result = node.query( - f""" - SELECT - i16, u16, u32, u64, id, big, - toString(d), toString(d32), toString(dt), - s, toString(uid), - arr, m['a'], m['b'], tup.a, tup.b, year - FROM {iceberg_table} - """ - ).strip() - expected = "\t".join([ - "-100", "200", "50000", "4000000000", - "12345", "1000000000", - "2024-01-15", "2024-01-15", "2024-01-15 12:30:45.000000", - "hello iceberg", "550e8400-e29b-41d4-a716-446655440000", - "[1,2,3]", "10", "20", "7", "seven", "2024", - ]) - assert result == expected, f"Unexpected round-trip data:\n{result!r}\nexpected:\n{expected!r}" - - # Floats compared with a tolerance to avoid formatting flakiness. - floats_ok = node.query( - f"SELECT abs(f32 - 3.14) < 1e-4 AND abs(f64 - 2.718281828459045) < 1e-12 FROM {iceberg_table}" - ).strip() - assert floats_ok == "1", f"Float round-trip outside tolerance: {floats_ok!r}" - - # DateTime64 sub-second component: assert the date part is preserved (exact format varies). - ts_result = node.query(f"SELECT dt64 FROM {iceberg_table}").strip() - assert "2024-01-15" in ts_result, f"DateTime64 date component missing: {ts_result!r}" - - -def test_export_partition_all_iceberg_types_lossy(cluster): - """Lossy narrowing casts across types succeed with the opt-in flag: values that - fit round-trip, Float64 -> Float32 loses precision, and Nullable columns carry - both NULL and non-NULL (the latter via a lossy Nullable(Int64) -> Nullable(Int32)).""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_lossy_types_{uid}" - iceberg_table = f"iceberg_lossy_types_{uid}" - - # Each source column is wider than the destination, so the export inserts a lossy - # narrowing CAST (allowed only because export_merge_tree_part_allow_lossy_cast=1). - # Int8/UInt8 are not Iceberg-representable, so the narrowest integer dest is Int16. - source_columns = ( - "big Int64, ubig UInt64, mid Int32, " - "f Float64, dt DateTime64(6), d Date32, " - "opt_s Nullable(String), opt_i Nullable(Int64), year Int32" - ) - dest_columns = ( - "big Int32, ubig UInt32, mid Int16, " - "f Float32, dt DateTime, d Date, " - "opt_s Nullable(String), opt_i Nullable(Int32), year Int32" - ) - - make_rmt(node, mt_table, source_columns, "year", replica_name="replica1") - make_iceberg_s3(node, iceberg_table, dest_columns, partition_by="year") - - # Values chosen to fit the destination types (the async cast wraps on overflow - # rather than throwing, so out-of-range values would silently corrupt instead). - # opt_s is NULL and opt_i is set, covering both nullable paths in one row. - node.query( - f""" - INSERT INTO {mt_table} (big, ubig, mid, f, dt, d, opt_s, opt_i, year) - VALUES ( - 1000000, 2000000000, 30000, - 2.718281828459045, '2024-01-15 12:30:45.123456', '2024-01-15', - NULL, 100, 2024 - ) - """ - ) - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2024' TO TABLE {iceberg_table}", - settings={ - "allow_insert_into_iceberg": 1, - "export_merge_tree_part_allow_lossy_cast": 1, - }, - ) - wait_for_export_status(node, mt_table, iceberg_table, "2024", "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 1, f"Expected 1 row in Iceberg table, got {count}" - - result = node.query( - f"SELECT big, ubig, mid, toString(d), toString(dt), opt_s, opt_i, year FROM {iceberg_table}" - ).strip() - expected = "\t".join([ - "1000000", "2000000000", "30000", - "2024-01-15", "2024-01-15 12:30:45.000000", "\\N", "100", "2024", - ]) - assert result == expected, f"Unexpected lossy round-trip data:\n{result!r}\nexpected:\n{expected!r}" - - # Float64 -> Float32 stays within Float32 precision but is no longer exact. - f_checks = node.query( - f"SELECT abs(f - 2.718281828459045) < 1e-6, abs(f - 2.718281828459045) > 1e-9 FROM {iceberg_table}" - ).strip() - assert f_checks == "1\t1", f"Expected Float32 precision loss within tolerance, got: {f_checks!r}" - - -def _data_file_partition_records(entries): - """Partition dicts of the non-delete data files described by manifest entries.""" - records = [] - for entry in entries: - data_file = entry.get("data_file") or {} - if data_file.get("content", 0) not in (0, None): - continue - partition = data_file.get("partition") - if partition is not None: - records.append(partition) - return records - - -def _partition_scalar(partition, field): - """Read a partition field value, tolerating an Avro-union ``{type: value}`` wrapper.""" - value = partition.get(field) - if isinstance(value, dict): - assert len(value) == 1, f"Unexpected partition union shape for {field!r}: {value!r}" - value = next(iter(value.values())) - return value - - -def assert_iceberg_partition_metadata(node, iceberg_table, uid, fields): - """Assert every data-file partition record's field equals the single DISTINCT value of the - corresponding expression over the exported destination data. `fields` is a list of - (metadata_field_name, value_expr). String-normalized so integer transforms and identity - string/int fields compare uniformly.""" - query_id = f"verify_{uid}" - node.query( - f"SELECT * FROM {iceberg_table}", - query_id=query_id, - settings={"iceberg_metadata_log_level": "manifest_file_entry"}, - ) - entries = fetch_manifest_entries(node, query_id) - partitions = _data_file_partition_records(entries) - assert partitions, "No data-file partition records found in manifest entries" - for field_name, value_expr in fields: - expected = node.query( - f"SELECT DISTINCT toString({value_expr}) FROM {iceberg_table}" - ).strip() - got = {str(_partition_scalar(p, field_name)) for p in partitions} - assert got == {expected}, ( - f"metadata field {field_name!r} = {got}, expected {{{expected!r}}}" - ) - - -def run_partition_compat_cases(node, cases): - """Run partition-compatibility cases against the Iceberg export gate. - - Reject cases (``expect_ok=False``) are checked synchronously - the gate fires while scheduling, - so the ALTER throws immediately. Accept cases are dispatched together, then awaited, then their - data (full ordered row comparison against the exported source partition) and Iceberg partition - metadata are verified. Each case is a dict: name, columns, source_key, dest_key, rows, expect_ok, - and optional verify (list of (metadata_field_name, value_expr); defaults to - [("event_time", dest_key)]) and source_settings (extra MergeTree settings).""" - settings = {"allow_insert_into_iceberg": 1} - - def setup(case): - uid = unique_suffix() - mt_table = f"mt_{case['name']}_{uid}" - iceberg_table = f"iceberg_{case['name']}_{uid}" - make_rmt(node, mt_table, case["columns"], case["source_key"], replica_name="replica1", - extra_settings=case.get("source_settings", "")) - node.query(f"INSERT INTO {mt_table} VALUES {case['rows']}") - make_iceberg_s3(node, iceberg_table, case["columns"], partition_by=case["dest_key"]) - pid = first_partition_id(node, mt_table) - return uid, mt_table, iceberg_table, pid - - for case in cases: - if case["expect_ok"]: - continue - _uid, mt_table, iceberg_table, pid = setup(case) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings=settings, - ) - assert "BAD_ARGUMENTS" in error, f"{case['name']}: expected BAD_ARGUMENTS, got: {error!r}" - - dispatched = [] - for case in cases: - if not case["expect_ok"]: - continue - uid, mt_table, iceberg_table, pid = setup(case) - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings=settings, - ) - dispatched.append((case, uid, mt_table, iceberg_table, pid)) - - for case, uid, mt_table, iceberg_table, pid in dispatched: - wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") - - for case, uid, mt_table, iceberg_table, pid in dispatched: - # Export is a positional cast into the destination schema, so verify the destination equals - # the source cast into the destination column types. Normalizing to the destination types - # tolerates legitimate Iceberg type promotion (e.g. DateTime is stored as a microsecond - # timestamp and returns as DateTime64(6)) while preserving destination precision, so a - # spurious sub-second value would still surface as a mismatch. - col_defs = node.query( - f"SELECT name, type FROM system.columns " - f"WHERE database = currentDatabase() AND table = '{iceberg_table}' ORDER BY position" - ).strip().split("\n") - projection = ", ".join( - f"CAST({name} AS {ctype})" for name, ctype in (c.split("\t") for c in col_defs) - ) - src = node.query(f"SELECT {projection} FROM {mt_table} ORDER BY id") - dst = node.query(f"SELECT {projection} FROM {iceberg_table} ORDER BY id") - assert src == dst, f"{case['name']}: destination rows differ from source" - fields = case.get("verify") or [("event_time", case["dest_key"])] - assert_iceberg_partition_metadata(node, iceberg_table, f"{case['name']}_{uid}", fields) - - -def test_export_partition_bucket_type_change_rejected(cluster): - """A bucket[N] partition column whose type changes (Int64 -> String) is rejected. The source - hashLong grouping differs from the destination murmur(String) grouping, so a single source bucket - can fan out across several destination buckets; bucket is not order-preserving, so this cannot be - proven dynamically and must be rejected. This previously slipped through the structural fast path, - which matched on transform name and width while ignoring the pre-transform cast.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_bucket_xform_{uid}" - iceberg_table = f"iceberg_bucket_xform_{uid}" - - make_rmt(node, mt_table, "id Int64, key Int64", "icebergBucket(16, key)", - replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 42), (2, 42)") - - make_iceberg_s3(node, iceberg_table, "id Int64, key String", - partition_by="icebergBucket(16, key)") - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error, ( - f"Expected BAD_ARGUMENTS for a type-changing bucket transform, got: {error!r}" - ) - - -def test_export_partition_truncate_type_change_rejected(cluster): - """icebergTruncate with the same width but a changed column type (Int64 -> String) is rejected. - Truncate is numeric on integers (120..129 -> 120) but byte-wise on strings ('120'..'129' stay - distinct), so one source truncate bucket can map to several destination buckets. The structural - fast path must not accept it on matching transform name and width; the dynamic proof rejects it - because the endpoints do not collapse to a single destination value.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_trunc_xform_{uid}" - iceberg_table = f"iceberg_trunc_xform_{uid}" - - # 120 and 129 are one Int64 truncate[10] bucket (120) but two distinct string truncations. - make_rmt(node, mt_table, "id Int64, key Int64", "icebergTruncate(10, key)", - replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 120), (2, 129)") - - make_iceberg_s3(node, iceberg_table, "id Int64, key String", - partition_by="icebergTruncate(10, key)") - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error, ( - f"Expected BAD_ARGUMENTS for a type-changing truncate transform, got: {error!r}" - ) - - -def test_export_partition_value_preserving_cast_not_order_preserving_rejected(cluster): - """Int64 -> String keeps every value, but not their order: 2 and 29 are the endpoints of the - source partition, yet the interior value 10 casts to a string that sorts outside them. The - endpoints truncate to '2' while 10 truncates to '1', so the partition spans two destination - buckets and must be rejected instead of being waved through as a lossless cast.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_cast_order_{uid}" - iceberg_table = f"iceberg_cast_order_{uid}" - - make_rmt(node, mt_table, "id Int64, k Int64", "intDiv(k, 100)", - replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 2), (2, 10), (3, 29)") - - make_iceberg_s3(node, iceberg_table, "id Int64, k String", - partition_by="icebergTruncate(1, k)") - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error, ( - f"Expected BAD_ARGUMENTS for a non-order-preserving cast, got: {error!r}" - ) - - -def test_export_partition_order_preserving_cast_accepted(cluster): - """The same shape as the rejected case, but with all values sharing a digit count: Int64 -> - String is order-preserving over [20, 29], so the endpoints do bound the interior and the whole - source partition truncates to the single destination bucket '2'.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_cast_order_ok_{uid}" - iceberg_table = f"iceberg_cast_order_ok_{uid}" - - make_rmt(node, mt_table, "id Int64, k Int64", "intDiv(k, 100)", - replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 20), (2, 25), (3, 29)") - - make_iceberg_s3(node, iceberg_table, "id Int64, k String", - partition_by="icebergTruncate(1, k)") - - pid = first_partition_id(node, mt_table) - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") - - src = node.query(f"SELECT id, toString(k) FROM {mt_table} ORDER BY id").strip() - dst = node.query(f"SELECT id, k FROM {iceberg_table} ORDER BY id").strip() - assert src == dst, f"destination rows differ from source:\n{src}\n---\n{dst}" - - assert_iceberg_partition_metadata(node, iceberg_table, uid, [("k", "icebergTruncate(1, k)")]) - - -def test_export_partition_timezone_mismatch_rejected(cluster): - """A source partitioned by day in one timezone must not be treated as structurally identical to a - destination day computed in another timezone. The source uses Asia/Tokyo (UTC+9) and the - destination UTC; the exported part spans a UTC-day boundary while staying within one Tokyo day, so - it maps to two destination partitions and must be rejected.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_tzmismatch_{uid}" - iceberg_table = f"iceberg_tzmismatch_{uid}" - - make_rmt(node, mt_table, "id Int64, event_time DateTime('UTC')", - "toRelativeDayNum(event_time, 'Asia/Tokyo')", replica_name="replica1") - # Both instants are 2024-03-05 in Tokyo (UTC+9) but 2024-03-04 and 2024-03-05 in UTC. - node.query( - f"INSERT INTO {mt_table} VALUES (1, '2024-03-04 16:00:00'), (2, '2024-03-05 10:00:00')" - ) - - make_iceberg_s3(node, iceberg_table, "id Int64, event_time DateTime('UTC')", - partition_by="toRelativeDayNum(event_time)") - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1, "iceberg_partition_timezone": "UTC"}, - ) - assert "BAD_ARGUMENTS" in error, ( - f"Expected BAD_ARGUMENTS for a source/destination timezone mismatch, got: {error!r}" - ) - - -def test_export_partition_column_timezone_mismatch_rejected(cluster): - """The same mismatch as above, but with the timezone carried by the column type instead of the - partition expression. Both sides read `toRelativeDayNum(event_time)`, so the terms are identical and - only the types differ - and DateTime types with different timezones compare equal, so the structural - match must not be decided by type equality alone. The part stays within one Tokyo day while spanning - two UTC days, so it maps to two destination partitions and must be rejected. - - `iceberg_partition_timezone` is deliberately left unset: setting it stamps a timezone onto the - destination term, which alone makes the terms differ and hides what this test covers.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_coltz_{uid}" - iceberg_table = f"iceberg_coltz_{uid}" - - make_rmt(node, mt_table, "id Int64, event_time DateTime('Asia/Tokyo')", - "toRelativeDayNum(event_time)", replica_name="replica1") - # Both literals are 2024-03-05 in Tokyo (the column's timezone) but 2024-03-04 and 2024-03-05 in UTC. - node.query( - f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 01:00:00'), (2, '2024-03-05 18:00:00')" - ) - - make_iceberg_s3(node, iceberg_table, "id Int64, event_time DateTime('UTC')", - partition_by="toRelativeDayNum(event_time)") - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - assert "BAD_ARGUMENTS" in error, ( - f"Expected BAD_ARGUMENTS for a partition-column timezone mismatch, got: {error!r}" - ) - - -def test_export_partition_commit_uses_exported_parts_not_new_inserts(cluster): - """The deferred commit derives the Iceberg partition value only from the exact exported parts - recorded in the manifest, never from parts inserted/merged into the source partition after - scheduling. A month-partitioned source exports one day into a day-partitioned destination (a - data-dependent acceptance); while the commit is wedged, an earlier day is inserted and merged in, - so the only active part now spans both days with its min at the new day. The commit must still - stamp the exported day (the exported part is found among Outdated parts by name), not the merged-in - earlier day, so the metadata matches the exported data files.""" - node = cluster.instances["replica1"] - uid = unique_suffix() - mt_table = f"mt_commit_parts_{uid}" - iceberg_table = f"iceberg_commit_parts_{uid}" - - make_rmt(node, mt_table, "id Int64, event_date Date", "toYYYYMM(event_date)", replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-20'), (2, '2024-03-20')") - make_iceberg_s3(node, iceberg_table, "id Int64, event_date Date", - partition_by="toRelativeDayNum(event_date)") - - exported_day = int(node.query("SELECT toRelativeDayNum(toDate('2024-03-20'))").strip()) - injected_day = int(node.query("SELECT toRelativeDayNum(toDate('2024-03-05'))").strip()) - - node.query("SYSTEM ENABLE FAILPOINT export_partition_commit_always_throw") - try: - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '202403' TO TABLE {iceberg_table}" - f" SETTINGS allow_insert_into_iceberg = 1" - ) - # The commit is attempted only after every part is exported, so a non-zero exception count - # means the data files are written and the commit is now wedged by the failpoint. - wait_for_exception_count(node, mt_table, iceberg_table, "202403", min_exception_count=1, timeout=90) - - # Insert an earlier day into the same month partition and merge: the merged active part spans - # both days with min = the injected (earlier) day, while the exported part becomes Outdated. - node.query(f"INSERT INTO {mt_table} VALUES (3, '2024-03-05')") - node.query(f"OPTIMIZE TABLE {mt_table} PARTITION ID '202403' FINAL") - finally: - node.query("SYSTEM DISABLE FAILPOINT export_partition_commit_always_throw") - - wait_for_export_status(node, mt_table, iceberg_table, "202403", "COMPLETED", timeout=90) - - # The exported data files hold only 2024-03-20; the metadata day must match them. - query_id = f"commit_parts_{uid}" - node.query( - f"SELECT * FROM {iceberg_table}", - query_id=query_id, - settings={"iceberg_metadata_log_level": "manifest_file_entry"}, - ) - entries = fetch_manifest_entries(node, query_id) - partitions = _data_file_partition_records(entries) - assert partitions, "No data-file partition records found in manifest entries" - meta_days = {int(_partition_scalar(p, "event_date")) for p in partitions} - assert meta_days == {exported_day}, ( - f"Metadata day {meta_days} must equal the exported day {exported_day} (2024-03-20), " - f"not the injected day {injected_day} (2024-03-05)." - ) - - assert int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) == 2, ( - "Only the two exported rows must be present in the destination." - ) - - -def test_export_partition_month_transform_metadata_matches_data(cluster): - """A month-transform partition records a months-since-epoch value in metadata that - matches the value derived from the exported data, and a transform-filtered read - returns the rows.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_month_xform_{uid}" - iceberg_table = f"iceberg_month_xform_{uid}" - - make_rmt(node, mt_table, "id Int64, event_date Date", - "toMonthNumSinceEpoch(event_date)", replica_name="replica1") - node.query( - f"INSERT INTO {mt_table} VALUES " - f"(1, '2024-03-05'), (2, '2024-03-20'), (3, '2024-03-31')" - ) - - make_iceberg_s3(node, iceberg_table, "id Int64, event_date Date", - partition_by="toMonthNumSinceEpoch(event_date)") - - pid = first_partition_id(node, mt_table) - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 3, f"Expected 3 rows after export, got {count}" - - month_num = int(node.query( - f"SELECT DISTINCT toMonthNumSinceEpoch(event_date) FROM {iceberg_table}" - ).strip()) - - query_id = f"month_xform_{uid}" - node.query( - f"SELECT * FROM {iceberg_table}", - query_id=query_id, - settings={"iceberg_metadata_log_level": "manifest_file_entry"}, - ) - entries = fetch_manifest_entries(node, query_id) - partitions = _data_file_partition_records(entries) - assert partitions, "No data-file partition records found in manifest entries" - meta_values = {int(_partition_scalar(p, "event_date")) for p in partitions} - assert meta_values == {month_num}, ( - f"Metadata month {meta_values} must equal toMonthNumSinceEpoch over the data " - f"({month_num})." - ) - - filtered = int(node.query( - f"SELECT count() FROM {iceberg_table} " - f"WHERE toMonthNumSinceEpoch(event_date) = {month_num}" - ).strip()) - assert filtered == 3, f"Transform-filtered read expected 3 rows, got {filtered}" - - -def test_export_partition_identity_type_change_metadata_matches_data(cluster): - """An identity partition column whose type changes UInt16 -> String records the - destination String value in the Iceberg metadata, matching the exported data.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_identity_xform_{uid}" - iceberg_table = f"iceberg_identity_xform_{uid}" - - make_rmt(node, mt_table, "id Int32, year UInt16", "year", replica_name="replica1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 2024), (2, 2024)") - - make_iceberg_s3(node, iceberg_table, "id Int32, year String", partition_by="year") - - pid = first_partition_id(node, mt_table) - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={"allow_insert_into_iceberg": 1}, - ) - wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 2, f"Expected 2 rows after export, got {count}" - - data_year = node.query(f"SELECT DISTINCT year FROM {iceberg_table}").strip() - assert data_year == "2024", f"Expected exported year '2024' (String), got {data_year!r}" - - query_id = f"identity_xform_{uid}" - node.query( - f"SELECT * FROM {iceberg_table}", - query_id=query_id, - settings={"iceberg_metadata_log_level": "manifest_file_entry"}, - ) - entries = fetch_manifest_entries(node, query_id) - partitions = _data_file_partition_records(entries) - assert partitions, "No data-file partition records found in manifest entries" - meta_values = {str(_partition_scalar(p, "year")) for p in partitions} - assert meta_values == {"2024"}, ( - f"Metadata partition {meta_values} must equal the destination String value " - f"'2024' (not the source integer representation)." - ) - - -def test_export_partition_multicolumn_identity_metadata_matches_data(cluster): - """A multi-column identity partition (event_date Date, retention UInt64 -> Int64) - records per-column values in the Iceberg metadata that match the exported data.""" - node = cluster.instances["replica1"] - - uid = unique_suffix() - mt_table = f"mt_multicol_{uid}" - iceberg_table = f"iceberg_multicol_{uid}" - - # Iceberg has no unsigned types, so retention widens UInt64 -> Int64; the cast is - # not value-preserving per canBeSafelyCast, hence the lossy opt-in below. - make_rmt(node, mt_table, "id Int64, event_date Date, retention UInt64", - "(event_date, retention)", replica_name="replica1") - node.query( - f"INSERT INTO {mt_table} VALUES " - f"(1, '2024-03-05', 30), (2, '2024-03-05', 30), (3, '2024-03-05', 30)" - ) - - make_iceberg_s3(node, iceberg_table, "id Int64, event_date Date, retention Int64", - partition_by="(event_date, retention)") - - pid = first_partition_id(node, mt_table) - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {iceberg_table}", - settings={ - "allow_insert_into_iceberg": 1, - "export_merge_tree_part_allow_lossy_cast": 1, - }, - ) - wait_for_export_status(node, mt_table, iceberg_table, pid, "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {iceberg_table}").strip()) - assert count == 3, f"Expected 3 rows after export, got {count}" - - data_retention = int(node.query( - f"SELECT DISTINCT retention FROM {iceberg_table}" - ).strip()) - assert data_retention == 30, f"Expected exported retention 30, got {data_retention}" - - days = int(node.query( - f"SELECT DISTINCT toInt64(event_date) FROM {iceberg_table}" - ).strip()) - - query_id = f"multicol_{uid}" - node.query( - f"SELECT * FROM {iceberg_table}", - query_id=query_id, - settings={"iceberg_metadata_log_level": "manifest_file_entry"}, - ) - entries = fetch_manifest_entries(node, query_id) - partitions = _data_file_partition_records(entries) - assert partitions, "No data-file partition records found in manifest entries" - - meta_dates = {int(_partition_scalar(p, "event_date")) for p in partitions} - assert meta_dates == {days}, ( - f"Metadata event_date {meta_dates} must equal days-since-epoch {days}." - ) - meta_retentions = {int(_partition_scalar(p, "retention")) for p in partitions} - assert meta_retentions == {30}, ( - f"Metadata retention {meta_retentions} must equal the exported value 30." - ) - - filtered = int(node.query( - f"SELECT count() FROM {iceberg_table} " - f"WHERE event_date = '2024-03-05' AND retention = 30" - ).strip()) - assert filtered == 3, f"Partition-filtered read expected 3 rows, got {filtered}" diff --git a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py b/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py deleted file mode 100644 index b6c721c87dbf..000000000000 --- a/tests/integration/test_export_replicated_mt_partition_to_object_storage/test.py +++ /dev/null @@ -1,2412 +0,0 @@ -import logging -import time -import uuid -from typing import NamedTuple - -import pytest - -from helpers.cluster import ClickHouseCluster -from helpers.export_partition_helpers import ( - first_partition_id, - make_rmt, - wait_for_exception_count, - wait_for_export_status, - wait_for_export_to_start, -) -from helpers.network import PartitionManager - - -EXTRA_SOURCE_COLUMN_MODES = [ - pytest.param("POSITION", id="by-position"), - pytest.param("NAME", id="by-name"), -] - - - -def skip_if_remote_database_disk_enabled(cluster): - """Skip test if any instance in the cluster has remote database disk enabled. - - Tests that block MinIO cannot run when remote database disk is enabled, - as the database metadata is stored on MinIO and blocking it would break the database. - """ - for instance in cluster.instances.values(): - if instance.with_remote_database_disk: - pytest.skip("Test cannot run with remote database disk enabled (db disk), as it blocks MinIO which stores database metadata") - - -@pytest.fixture(scope="module") -def cluster(): - try: - cluster = ClickHouseCluster(__file__) - cluster.add_instance( - "replica1", - main_configs=["configs/named_collections.xml", "configs/allow_experimental_export_partition.xml"], - user_configs=["configs/users.d/profile.xml"], - with_minio=True, - stay_alive=True, - with_zookeeper=True, - keeper_required_feature_flags=["multi_read"], - ) - cluster.add_instance( - "replica2", - main_configs=["configs/named_collections.xml", "configs/allow_experimental_export_partition.xml"], - user_configs=["configs/users.d/profile.xml"], - with_minio=True, - stay_alive=True, - with_zookeeper=True, - keeper_required_feature_flags=["multi_read"], - ) - # node that does not participate in the export, but will have visibility over the s3 table - cluster.add_instance( - "watcher_node", - main_configs=["configs/named_collections.xml"], - user_configs=[], - with_minio=True, - ) - cluster.add_instance( - "replica_with_export_disabled", - main_configs=["configs/named_collections.xml", "configs/disable_experimental_export_partition.xml"], - user_configs=["configs/users.d/profile.xml"], - with_minio=True, - stay_alive=True, - with_zookeeper=True, - keeper_required_feature_flags=["multi_read"], - ) - # Sharded instances for filename pattern tests - cluster.add_instance( - "shard1_replica1", - main_configs=["configs/named_collections.xml", "configs/allow_experimental_export_partition.xml", "configs/macros_shard1_replica1.xml"], - user_configs=["configs/users.d/profile.xml"], - with_minio=True, - stay_alive=True, - with_zookeeper=True, - keeper_required_feature_flags=["multi_read"], - ) - - cluster.add_instance( - "shard2_replica1", - main_configs=["configs/named_collections.xml", "configs/allow_experimental_export_partition.xml", "configs/macros_shard2_replica1.xml"], - user_configs=["configs/users.d/profile.xml"], - with_minio=True, - stay_alive=True, - with_zookeeper=True, - keeper_required_feature_flags=["multi_read"], - ) - logging.info("Starting cluster...") - cluster.start() - yield cluster - finally: - cluster.shutdown() - - -@pytest.fixture(autouse=True) -def drop_tables_after_test(cluster): - """Drop all tables in the default database after every test. - - Without this, ReplicatedMergeTree tables from completed tests remain alive and keep - running ZooKeeper background threads (merge selector, queue log, cleanup, export manifest - updater). With many tables alive simultaneously the ZooKeeper session becomes overwhelmed - and subsequent tests start seeing operation-timeout / session-expired errors. - """ - yield - for instance_name, instance in cluster.instances.items(): - try: - tables_str = instance.query( - "SELECT name FROM system.tables WHERE database = 'default' FORMAT TabSeparated" - ).strip() - if not tables_str: - continue - for table in tables_str.split('\n'): - table = table.strip() - if table: - instance.query(f"DROP TABLE IF EXISTS default.`{table}` SYNC") - except Exception as e: - logging.warning(f"drop_tables_after_test: cleanup failed on {instance_name}: {e}") - - -def create_s3_table(node, s3_table): - node.query(f"CREATE TABLE {s3_table} (id UInt64, year UInt16) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') PARTITION BY year") - - -def create_tables_and_insert_data(node, mt_table, s3_table, replica_name): - node.query(f"DROP TABLE IF EXISTS {mt_table} SYNC") - # enable_block_number_column and enable_block_offset_column are needed for patch parts support - node.query(f"CREATE TABLE {mt_table} (id UInt64, year UInt16) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', '{replica_name}') PARTITION BY year ORDER BY tuple() SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1") - node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020), (3, 2020), (4, 2021)") - - create_s3_table(node, s3_table) - - -def create_sharded_tables_and_insert_data(node, mt_table, s3_table, replica_name): - """Create sharded ReplicatedMergeTree table with {shard} macro in ZooKeeper path.""" - node.query(f"CREATE TABLE {mt_table} (id UInt64, year UInt16) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{mt_table}', '{replica_name}') PARTITION BY year ORDER BY tuple()") - node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2020), (3, 2020), (4, 2021)") - - create_s3_table(node, s3_table) - - -def test_restart_nodes_during_export(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - node2 = cluster.instances["replica2"] - watcher_node = cluster.instances["watcher_node"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"disaster_mt_table_{postfix}" - s3_table = f"disaster_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - create_tables_and_insert_data(node2, mt_table, s3_table, "replica2") - create_s3_table(watcher_node, s3_table) - - # Block S3/MinIO requests to keep exports alive via retry mechanism - # This allows ZooKeeper operations to proceed quickly - minio_ip = cluster.minio_ip - minio_port = cluster.minio_port - - with PartitionManager() as pm: - # Block responses from MinIO (source_port matches MinIO service) - pm_rule_reject_responses_node1 = { - "instance": node, - "destination": node.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_responses_node1) - - pm_rule_reject_responses_node2 = { - "instance": node2, - "destination": node2.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_responses_node2) - - # Block requests to MinIO (destination: MinIO, destination_port: minio_port) - pm_rule_reject_requests_node1 = { - "instance": node, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_requests_node1) - - pm_rule_reject_requests_node2 = { - "instance": node2, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_requests_node2) - - export_queries = f""" - ALTER TABLE {mt_table} - EXPORT PARTITION ID '2020' TO TABLE {s3_table}; - ALTER TABLE {mt_table} - EXPORT PARTITION ID '2021' TO TABLE {s3_table}; - """ - - node.query(export_queries) - - # wait for the exports to start - wait_for_export_to_start(node, mt_table, s3_table, "2020") - wait_for_export_to_start(node, mt_table, s3_table, "2021") - - node.stop_clickhouse(kill=True) - node2.stop_clickhouse(kill=True) - - assert watcher_node.query(f"SELECT count() FROM {s3_table} where year = 2020") == '0\n', "Partition 2020 was written to S3 during network delay crash" - - assert watcher_node.query(f"SELECT count() FROM {s3_table} where year = 2021") == '0\n', "Partition 2021 was written to S3 during network delay crash" - - # start the nodes, they should finish the export - node.start_clickhouse() - node2.start_clickhouse() - - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") - wait_for_export_status(node, mt_table, s3_table, "2021", "COMPLETED") - - assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") != f'0\n', "Export of partition 2020 did not resume after crash" - - assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2021") != f'0\n', "Export of partition 2021 did not resume after crash" - - -def test_kill_export(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - node2 = cluster.instances["replica2"] - watcher_node = cluster.instances["watcher_node"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"kill_export_mt_table_{postfix}" - s3_table = f"kill_export_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - create_tables_and_insert_data(node2, mt_table, s3_table, "replica2") - - # Block S3/MinIO requests to keep exports alive via retry mechanism - # This allows ZooKeeper operations (KILL) to proceed quickly - minio_ip = cluster.minio_ip - minio_port = cluster.minio_port - - with PartitionManager() as pm: - # Block responses from MinIO (source_port matches MinIO service) - pm_rule_reject_responses = { - "instance": node, - "destination": node.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_responses) - - # Block requests to MinIO (destination: MinIO, destination_port: minio_port) - pm_rule_reject_requests = { - "instance": node, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_requests) - - # Block responses from MinIO for node2 - pm_rule_reject_responses_node2 = { - "instance": node2, - "destination": node2.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_responses_node2) - - # Block requests to MinIO from node2 - pm_rule_reject_requests_node2 = { - "instance": node2, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_requests_node2) - - export_queries = f""" - ALTER TABLE {mt_table} - EXPORT PARTITION ID '2020' TO TABLE {s3_table}; - ALTER TABLE {mt_table} - EXPORT PARTITION ID '2021' TO TABLE {s3_table}; - """ - - node.query(export_queries) - - # Kill only 2020 while S3 is blocked - retry mechanism keeps exports alive - # ZooKeeper operations (KILL) proceed quickly since only S3 is blocked - node.query(f"KILL EXPORT PARTITION WHERE partition_id = '2020' and source_table = '{mt_table}' and destination_table = '{s3_table}'") - - # sleep for a while to let the kill to be processed - time.sleep(2) - - # wait for 2021 to finish - wait_for_export_status(node, mt_table, s3_table, "2021", "COMPLETED") - - # checking for the commit file because maybe the data file was too fast? - assert node.query(f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/commit_2020_*', format=LineAsString)") == '0\n', "Partition 2020 was written to S3, it was not killed as expected" - assert node.query(f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/commit_2021_*', format=LineAsString)") != f'0\n', "Partition 2021 was not written to S3, but it should have been" - - # check system.replicated_partition_exports for the export, status should be KILLED - assert node.query(f"SELECT status FROM system.replicated_partition_exports WHERE partition_id = '2020' and source_table = '{mt_table}' and destination_table = '{s3_table}'") == 'KILLED\n', "Partition 2020 was not killed as expected" - assert node.query(f"SELECT status FROM system.replicated_partition_exports WHERE partition_id = '2021' and source_table = '{mt_table}' and destination_table = '{s3_table}'") == 'COMPLETED\n', "Partition 2021 was not completed, this is unexpected" - - # check the data did not land on s3 - assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") == '0\n', "Partition 2020 was written to S3, it was not killed as expected" - - -def test_kill_export_resilient_to_status_handling_failure(cluster): - """KILL EXPORT PARTITION must eventually take effect even when the first - attempt to handle the ZK status-change event throws (simulated via a ONCE - failpoint). The re-queue + reschedule mechanism retries after ~5 s and - the second attempt succeeds because the ONCE failpoint has already fired.""" - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"kill_resilient_mt_{postfix}" - s3_table = f"kill_resilient_s3_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - minio_ip = cluster.minio_ip - minio_port = cluster.minio_port - - with PartitionManager() as pm: - pm.add_rule({ - "instance": node, - "destination": node.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - }) - - pm.add_rule({ - "instance": node, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - }) - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - ) - - node.query("SYSTEM ENABLE FAILPOINT export_partition_status_change_throw") - - node.query( - f"KILL EXPORT PARTITION WHERE partition_id = '2020'" - f" AND source_table = '{mt_table}' AND destination_table = '{s3_table}'") - - # sleep for a while to let the kill to be processed - time.sleep(5) - - # The ONCE failpoint makes the first handleStatusChanges() throw. - # The catch re-queues the key and scheduleAfter(5000) arms a retry. - # Wait up to 15 s (5 s retry delay + margin) for the kill to propagate. - wait_for_export_status(node, mt_table, s3_table, "2020", "KILLED", timeout=15) - - assert ( - node.query( - f"SELECT status FROM system.replicated_partition_exports" - f" WHERE partition_id = '2020'" - f" AND source_table = '{mt_table}'" - f" AND destination_table = '{s3_table}'" - ).strip() == "KILLED" - ), "Export was not killed — status change was lost after the injected failure" - - -def test_drop_source_table_during_export(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - # node2 = cluster.instances["replica2"] - watcher_node = cluster.instances["watcher_node"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"drop_source_table_during_export_mt_table_{postfix}" - s3_table = f"drop_source_table_during_export_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - # create_tables_and_insert_data(node2, mt_table, s3_table, "replica2") - create_s3_table(watcher_node, s3_table) - - # Block S3/MinIO requests to keep exports alive via retry mechanism - # This allows ZooKeeper operations (KILL) to proceed quickly - minio_ip = cluster.minio_ip - minio_port = cluster.minio_port - - with PartitionManager() as pm: - # Block responses from MinIO (source_port matches MinIO service) - pm_rule_reject_responses = { - "instance": node, - "destination": node.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_responses) - - # Block requests to MinIO (destination: MinIO, destination_port: minio_port) - pm_rule_reject_requests = { - "instance": node, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_requests) - - export_queries = f""" - ALTER TABLE {mt_table} - EXPORT PARTITION ID '2020' TO TABLE {s3_table} SETTINGS s3_retry_attempts = 500; - ALTER TABLE {mt_table} - EXPORT PARTITION ID '2021' TO TABLE {s3_table} SETTINGS s3_retry_attempts = 500; - """ - - node.query(export_queries) - - wait_for_export_status(node, mt_table, s3_table, "2020", "PENDING") - wait_for_export_status(node, mt_table, s3_table, "2021", "PENDING") - - # This should kill the background operations and drop the table - node.query(f"DROP TABLE {mt_table}") - - # Sleep some time to let the export finish (assuming it was not properly cancelled) - time.sleep(10) - - assert node.query(f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/commit_*', format=LineAsString)") == '0\n', "Background operations completed even with the table dropped" - - -def test_concurrent_exports_to_different_targets(cluster): - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"concurrent_diff_targets_mt_table_{postfix}" - s3_table_a = f"concurrent_diff_targets_s3_a_{postfix}" - s3_table_b = f"concurrent_diff_targets_s3_b_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table_a, "replica1") - create_s3_table(node, s3_table_b) - - # Launch two exports of the same partition to two different S3 tables concurrently - with PartitionManager() as pm: - pm.add_network_delay(node, delay_ms=1000) - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table_a}" - ) - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table_b}" - ) - - wait_for_export_status(node, mt_table, s3_table_a, "2020", "COMPLETED") - wait_for_export_status(node, mt_table, s3_table_b, "2020", "COMPLETED") - - # Both targets should receive the same data independently - assert node.query(f"SELECT count() FROM {s3_table_a} WHERE year = 2020") == '3\n', "First target did not receive expected rows" - assert node.query(f"SELECT count() FROM {s3_table_b} WHERE year = 2020") == '3\n', "Second target did not receive expected rows" - - # And both should have a commit marker - assert node.query( - f"SELECT count() FROM s3(s3_conn, filename='{s3_table_a}/commit_2020_*', format=LineAsString)" - ) != '0\n', "Commit file missing for first target" - assert node.query( - f"SELECT count() FROM s3(s3_conn, filename='{s3_table_b}/commit_2020_*', format=LineAsString)" - ) != '0\n', "Commit file missing for second target" - - -def test_failure_is_logged_in_system_table(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"failure_is_logged_in_system_table_mt_table_{postfix}" - s3_table = f"failure_is_logged_in_system_table_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - # Block traffic to/from MinIO to force upload errors and retries, following existing S3 tests style - minio_ip = cluster.minio_ip - minio_port = cluster.minio_port - - with PartitionManager() as pm: - # Block responses from MinIO (source_port matches MinIO service) - pm_rule_reject_responses = { - "instance": node, - "destination": node.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_responses) - - # Also block requests to MinIO (destination: MinIO, destination_port: 9001) with REJECT to fail fast - pm_rule_reject_requests = { - "instance": node, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_requests) - - # Blocked MinIO produces transient (retryable) S3 errors. There is no retry - # budget anymore, so the task keeps retrying and is only torn down once the - # absolute task timeout fires (transitioning to KILLED). Use a small timeout - # so the test does not wait for the default (a day). - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - f" SETTINGS export_merge_tree_partition_task_timeout_seconds = 5;" - ) - - # Wait for the timeout to kill the stuck task. The KILL is a Keeper operation - # (MinIO being blocked does not affect it); the status mirror needs roughly one - # manifest-updater poll cycle (~30s) plus watch propagation on top of the 5s - # timeout, so allow a generous budget. - wait_for_export_status(node, mt_table, s3_table, "2020", "KILLED", timeout=90) - - # Network restored; verify the export is marked as KILLED in the system table - # Also verify we captured at least one exception and no commit file exists - status = node.query( - f""" - SELECT status FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '2020' - """ - ) - - assert status.strip() == "KILLED", f"Expected KILLED status, got: {status!r}" - - exception_count = node.query( - f""" - SELECT any(exception_count) FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '2020' - """ - ) - assert int(exception_count.strip()) > 0, "Expected non-zero exception_count in system.replicated_partition_exports" - - # No commit should have been produced for this partition - assert node.query( - f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/commit_2020_*', format=LineAsString)" - ) == '0\n', "Commit file exists despite forced S3 failures" - - -def test_inject_short_living_failures(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"inject_short_living_failures_mt_table_{postfix}" - s3_table = f"inject_short_living_failures_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - # Block traffic to/from MinIO to force upload errors and retries, following existing S3 tests style - minio_ip = cluster.minio_ip - minio_port = cluster.minio_port - - with PartitionManager() as pm: - # Block responses from MinIO (source_port matches MinIO service) - pm_rule_reject_responses = { - "instance": node, - "destination": node.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_responses) - - # Also block requests to MinIO (destination: MinIO, destination_port: 9001) with REJECT to fail fast - pm_rule_reject_requests = { - "instance": node, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_requests) - - # Transient (retryable) failures never fail the task on a budget; it keeps - # retrying until the network is restored and the export completes. - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table};" - ) - - # wait for at least one exception to occur, but not enough to finish the export. - # Use the helper default (>= one manifest-updater poll cycle): system.replicated_partition_exports - # is served from the in-memory mirror, and while the task stays PENDING the mirror only - # picks up new exception leaves on the next poll tick (~30s) — see helper docstring. - wait_for_exception_count(node, mt_table, s3_table, "2020", min_exception_count=1) - - # wait for the export to finish - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") - - # Assert the export succeeded - assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") == '3\n', "Export did not succeed" - assert node.query(f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/commit_2020_*', format=LineAsString)") == '1\n', "Export did not succeed" - - # check system.replicated_partition_exports for the export - assert node.query( - f""" - SELECT status FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '2020' - """ - ) == "COMPLETED\n", "Export should be marked as COMPLETED" - - exception_count = node.query( - f""" - SELECT exception_count FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '2020' - """ - ) - assert int(exception_count.strip()) >= 1, "Expected at least one exception" - - -def test_export_partition_retry_backoff(cluster): - """Verify the per-replica in-memory exponential back-off between failed part exports. - - The back-off is local in-memory state (no ZooKeeper retry_count / next_retry_time - anymore), so it is not directly observable; instead we observe its effect. With a - large back-off, a part that keeps failing (object storage blocked) is parked for the - back-off window after its first failure and must NOT be retried on every ~5s - scheduler tick. We assert that exception_count stays low across a window that spans - several ticks. Once the network is restored and the back-off elapses, the export - completes (there is no retry budget to exhaust).""" - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"retry_backoff_mt_table_{postfix}" - s3_table = f"retry_backoff_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - # Large back-off so a single failed attempt parks the part well beyond the - # ~5s scheduler tick. Kept moderate so the export can still complete promptly - # once the network is restored. - initial_backoff_seconds = 30 - max_backoff_seconds = 30 - - minio_ip = cluster.minio_ip - minio_port = cluster.minio_port - - with PartitionManager() as pm: - # Block responses from MinIO (source_port matches MinIO service) - pm.add_rule({ - "instance": node, - "destination": node.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - }) - # Also block requests to MinIO to fail fast - pm.add_rule({ - "instance": node, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - }) - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_partition_retry_initial_backoff_seconds = {initial_backoff_seconds}, " - f"export_merge_tree_partition_retry_max_backoff_seconds = {max_backoff_seconds}" - ) - - # Wait until the first failure is recorded. - count_after_first = wait_for_exception_count( - node, mt_table, s3_table, "2020", min_exception_count=1, timeout=60 - ) - - # While the part is backing off (~30s) it must not be retried again. Observe - # across a window that spans several scheduler ticks: without back-off the - # ~5s tick would add roughly five more failures, so a small increase proves - # the back-off is pacing retries. - time.sleep(25) - count_during_backoff = int(node.query( - f"SELECT exception_count FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}'" - f" AND destination_table = '{s3_table}'" - f" AND partition_id = '2020'" - ).strip()) - assert count_during_backoff - count_after_first <= 2, ( - f"exception_count jumped during the back-off window: " - f"{count_after_first} -> {count_during_backoff}; back-off was not applied" - ) - - # Network restored; once the back-off elapses the export should complete because - # there is no retry budget to exhaust. - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED", timeout=120) - assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") == "3\n", "Export did not succeed" - - -def test_export_partition_file_already_exists_policy(cluster): - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"export_partition_file_already_exists_policy_mt_table_{postfix}" - s3_table = f"export_partition_file_already_exists_policy_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - # stop merges so part names remain stable. it is important for the test. - node.query(f"SYSTEM STOP MERGES {mt_table}") - - # Export all parts - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}", - ) - - # check system.replicated_partition_exports for the export - assert node.query( - f""" - SELECT status FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '2020' - """ - ) == "COMPLETED\n", "Export should be marked as COMPLETED" - - # wait for the exports to finish - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") - - # plain object storage destinations surface the commit marker file path via - # system.replicated_partition_exports.committed_marker_file - committed_marker_file = node.query( - f""" - SELECT committed_marker_file FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '2020' - """ - ).strip() - # `committed_marker_file` is the absolute key in the bucket (same convention as - # `destination_file_paths`); it may carry the s3_conn URL's in-bucket prefix on - # top of the table's `filename` argument, so use a "contains" check that does - # not depend on knowing that prefix. - assert f"{s3_table}/commit_2020_" in committed_marker_file, \ - f"Expected committed_marker_file under {s3_table}/, got: {committed_marker_file!r}" - # Path relative to the `s3_conn` URL, derived from the absolute key without - # assuming a particular URL prefix. - marker_relative_path = committed_marker_file[committed_marker_file.index(f"{s3_table}/"):] - assert node.query( - f"SELECT count() FROM s3(s3_conn, filename='{marker_relative_path}', format=LineAsString)" - ) == '1\n', f"Commit marker file does not exist at {committed_marker_file!r}" - - # try to export the partition - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} SETTINGS export_merge_tree_partition_force_export=1" - ) - - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") - - assert node.query( - f""" - SELECT count() FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '2020' - AND status = 'COMPLETED' - """ - ) == '1\n', "Expected the export to be marked as COMPLETED" - - # overwrite policy - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} SETTINGS export_merge_tree_partition_force_export=1, export_merge_tree_part_file_already_exists_policy='overwrite'" - ) - - # wait for the export to finish - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") - - # check system.replicated_partition_exports for the export - # ideally we would make sure the transaction id is different, but I do not have the time to do that now - assert node.query( - f""" - SELECT count() FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '2020' - AND status = 'COMPLETED' - """ - ) == '1\n', "Expected the export to be marked as COMPLETED" - - # last but not least, let's try with the error policy. FILE_ALREADY_EXISTS is a - # non-retryable error (retrying always hits the same existing file), so the task - # fails fast without needing a retry budget. - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} SETTINGS export_merge_tree_partition_force_export=1, export_merge_tree_part_file_already_exists_policy='error'", - ) - - # wait for the export to finish - wait_for_export_status(node, mt_table, s3_table, "2020", "FAILED") - - # check system.replicated_partition_exports for the export - assert node.query( - f""" - SELECT count() FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '2020' - AND status = 'FAILED' - """ - ) == '1\n', "Expected the export to be marked as FAILED" - - -def test_export_partition_feature_is_disabled(cluster): - replica_with_export_disabled = cluster.instances["replica_with_export_disabled"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"export_partition_feature_is_disabled_mt_table_{postfix}" - s3_table = f"export_partition_feature_is_disabled_s3_table_{postfix}" - - create_tables_and_insert_data(replica_with_export_disabled, mt_table, s3_table, "replica1") - - error = replica_with_export_disabled.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table};") - assert "experimental" in error, "Expected error about disabled feature" - - # make sure kill operation also throws - error = replica_with_export_disabled.query_and_get_error(f"KILL EXPORT PARTITION WHERE partition_id = '2020' and source_table = '{mt_table}' and destination_table = '{s3_table}'") - assert "experimental" in error, "Expected error about disabled feature" - - -def test_export_partition_permissions(cluster): - """Test that export partition validates permissions correctly: - - User needs ALTER permission on source table - - User needs INSERT permission on destination table - """ - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"permissions_mt_table_{postfix}" - s3_table = f"permissions_s3_table_{postfix}" - - # Create tables as default user - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - # Create test users with specific permissions - node.query("CREATE USER IF NOT EXISTS user_no_alter IDENTIFIED WITH no_password") - node.query("CREATE USER IF NOT EXISTS user_no_insert IDENTIFIED WITH no_password") - node.query("CREATE USER IF NOT EXISTS user_with_permissions IDENTIFIED WITH no_password") - - # Grant basic access to all users - node.query(f"GRANT SELECT ON {mt_table} TO user_no_alter") - node.query(f"GRANT SELECT ON {s3_table} TO user_no_alter") - - # user_no_insert has ALTER on source but no INSERT on destination - node.query(f"GRANT ALTER ON {mt_table} TO user_no_insert") - node.query(f"GRANT SELECT ON {s3_table} TO user_no_insert") - - # user_with_permissions has both ALTER and INSERT - node.query(f"GRANT ALTER ON {mt_table} TO user_with_permissions") - node.query(f"GRANT INSERT ON {s3_table} TO user_with_permissions") - - # Test 1: User without ALTER permission should fail - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}", - user="user_no_alter" - ) - - assert "ACCESS_DENIED" in error or "Not enough privileges" in error, \ - f"Expected ACCESS_DENIED error for user without ALTER, got: {error}" - - # Test 2: User with ALTER but without INSERT permission should fail - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}", - user="user_no_insert" - ) - - assert "ACCESS_DENIED" in error or "Not enough privileges" in error, \ - f"Expected ACCESS_DENIED error for user without INSERT, got: {error}" - - # Test 3: User with both ALTER and INSERT should succeed - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}", - user="user_with_permissions" - ) - - # Wait for export to complete - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") - - # Verify the export succeeded - result = node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") - assert result.strip() == "3", f"Expected 3 rows exported, got: {result}" - - # Verify system table shows COMPLETED status - status = node.query( - f""" - SELECT status FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '2020' - """ - ) - assert status.strip() == "COMPLETED", f"Expected COMPLETED status, got: {status}" - - -# assert multiple exports within a single query are executed. They all share the same query id -# and previously the transaction id was the query id, which would cause problems -def test_multiple_exports_within_a_single_query(cluster): - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multiple_exports_within_a_single_query_mt_table_{postfix}" - s3_table = f"multiple_exports_within_a_single_query_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}, EXPORT PARTITION ID '2021' TO TABLE {s3_table};") - - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") - wait_for_export_status(node, mt_table, s3_table, "2021", "COMPLETED") - - # assert the exports have been executed - assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020") == '3\n', "Export did not succeed" - assert node.query(f"SELECT count() FROM {s3_table} WHERE year = 2021") == '1\n', "Export did not succeed" - - # check system.replicated_partition_exports for the exports - assert node.query( - f""" - SELECT status FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '2020' - """ - ) == "COMPLETED\n", "Export should be marked as COMPLETED" - - assert node.query( - f""" - SELECT status FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '2021' - """ - ) == "COMPLETED\n", "Export should be marked as COMPLETED" - - -def test_pending_mutations_throw_before_export_partition(cluster): - """Test that pending mutations before export partition throw an error.""" - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"pending_mutations_throw_partition_mt_table_{postfix}" - s3_table = f"pending_mutations_throw_partition_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - node.query(f"SYSTEM STOP MERGES {mt_table}") - - node.query(f"ALTER TABLE {mt_table} UPDATE id = id + 100 WHERE year = 2020") - - mutations = node.query(f"SELECT count() FROM system.mutations WHERE table = '{mt_table}' AND is_done = 0") - assert mutations.strip() != '0', "Mutation should be pending" - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_throw_on_pending_mutations=true" - ) - - assert "PENDING_MUTATIONS_NOT_ALLOWED" in error, f"Expected error about pending mutations, got: {error}" - - -def test_pending_mutations_skip_before_export_partition(cluster): - """Test that pending mutations before export partition are skipped with throw_on_pending_mutations=false.""" - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"pending_mutations_skip_partition_mt_table_{postfix}" - s3_table = f"pending_mutations_skip_partition_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - node.query(f"SYSTEM STOP MERGES {mt_table}") - - node.query(f"ALTER TABLE {mt_table} UPDATE id = id + 100 WHERE year = 2020") - - mutations = node.query(f"SELECT count() FROM system.mutations WHERE table = '{mt_table}' AND is_done = 0") - assert mutations.strip() != '0', "Mutation should be pending" - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_throw_on_pending_mutations=false" - ) - - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") - - result = node.query(f"SELECT id FROM {s3_table} WHERE year = 2020 ORDER BY id") - assert "101" not in result and "102" not in result and "103" not in result, \ - "Export should contain original data before mutation" - assert "1\n2\n3" in result, "Export should contain original data" - - -def test_pending_patch_parts_throw_before_export_partition(cluster): - """Test that pending patch parts before export partition throw an error with default settings.""" - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"pending_patches_throw_partition_mt_table_{postfix}" - s3_table = f"pending_patches_throw_partition_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - node.query(f"SYSTEM STOP MERGES {mt_table}") - - node.query(f"UPDATE {mt_table} SET id = id + 100 WHERE year = 2020") - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - ) - - node.query(f"DROP TABLE {mt_table}") - - assert "PENDING_MUTATIONS_NOT_ALLOWED" in error or "pending patch parts" in error.lower(), \ - f"Expected error about pending patch parts, got: {error}" - - -def test_pending_patch_parts_skip_before_export_partition(cluster): - """Test that pending patch parts before export partition are skipped with throw_on_pending_patch_parts=false.""" - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"pending_patches_skip_partition_mt_table_{postfix}" - s3_table = f"pending_patches_skip_partition_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - node.query(f"SYSTEM STOP MERGES {mt_table}") - - node.query(f"UPDATE {mt_table} SET id = id + 100 WHERE year = 2020") - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_throw_on_pending_patch_parts=false" - ) - - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") - - result = node.query(f"SELECT id FROM {s3_table} WHERE year = 2020 ORDER BY id") - assert "1\n2\n3" in result, "Export should contain original data before patch" - - node.query(f"DROP TABLE {mt_table}") - - -def test_mutations_after_export_partition_started(cluster): - """Test that mutations applied after export partition starts don't affect the exported data.""" - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"mutations_after_export_partition_mt_table_{postfix}" - s3_table = f"mutations_after_export_partition_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - # Block traffic to MinIO to delay export - minio_ip = cluster.minio_ip - minio_port = cluster.minio_port - - with PartitionManager() as pm: - pm_rule_reject_responses = { - "instance": node, - "destination": node.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_responses) - - pm_rule_reject_requests = { - "instance": node, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_requests) - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_throw_on_pending_mutations=true" - ) - - # Wait for export to start - wait_for_export_to_start(node, mt_table, s3_table, "2020") - - node.query(f"ALTER TABLE {mt_table} UPDATE id = id + 100 WHERE year = 2020") - - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") - - result = node.query(f"SELECT id FROM {s3_table} WHERE year = 2020 ORDER BY id") - assert "1\n2\n3" in result, "Export should contain original data before mutation" - assert "101" not in result, "Export should not contain mutated data" - - -def test_patch_parts_after_export_partition_started(cluster): - """Test that patch parts created after export partition starts don't affect the exported data.""" - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"patches_after_export_partition_mt_table_{postfix}" - s3_table = f"patches_after_export_partition_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - # Block traffic to MinIO to delay export - minio_ip = cluster.minio_ip - minio_port = cluster.minio_port - - with PartitionManager() as pm: - pm_rule_reject_responses = { - "instance": node, - "destination": node.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_responses) - - pm_rule_reject_requests = { - "instance": node, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - } - pm.add_rule(pm_rule_reject_requests) - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - ) - - # Wait for export to start - wait_for_export_to_start(node, mt_table, s3_table, "2020") - - node.query(f"UPDATE {mt_table} SET id = id + 100 WHERE year = 2020") - - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED") - - result = node.query(f"SELECT id FROM {s3_table} WHERE year = 2020 ORDER BY id") - assert "1\n2\n3" in result, "Export should contain original data before patch" - assert "101" not in result, "Export should not contain patched data" - - node.query(f"DROP TABLE {mt_table}") - - -def test_mutation_in_partition_clause(cluster): - """Test that mutations limited to specific partitions using IN PARTITION clause - allow exports of unaffected partitions to succeed.""" - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"mutation_in_partition_clause_mt_table_{postfix}" - s3_table = f"mutation_in_partition_clause_s3_table_{postfix}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - node.query(f"SYSTEM STOP MERGES {mt_table}") - - # Issue a mutation that uses IN PARTITION to limit it to partition 2020 - node.query(f"ALTER TABLE {mt_table} UPDATE id = id + 100 IN PARTITION '2020' WHERE year = 2020") - - # Verify mutation is pending for 2020 - mutations = node.query( - f"SELECT count() FROM system.mutations WHERE table = '{mt_table}' AND is_done = 0" - ) - assert mutations.strip() != '0', "Mutation should be pending" - - # Export of 2020 should fail (it has pending mutations) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_throw_on_pending_mutations=true" - ) - assert "PENDING_MUTATIONS_NOT_ALLOWED" in error, f"Expected error about pending mutations for partition 2020, got: {error}" - - # Export of 2021 should succeed (no mutations affecting it) - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2021' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_throw_on_pending_mutations=true" - ) - - wait_for_export_status(node, mt_table, s3_table, "2021", "COMPLETED") - - result = node.query(f"SELECT id FROM {s3_table} WHERE year = 2021 ORDER BY id") - assert "4" in result, "Export of partition 2021 should contain original data" - - -def test_export_partition_with_mixed_computed_columns(cluster): - """Test export partition with ALIAS, MATERIALIZED, and EPHEMERAL columns.""" - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"mixed_computed_mt_table_{postfix}" - s3_table = f"mixed_computed_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} ( - id UInt32, - value UInt32, - tag_input String EPHEMERAL, - doubled UInt64 ALIAS value * 2, - tripled UInt64 MATERIALIZED value * 3, - tag String DEFAULT upper(tag_input) - ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY id - ORDER BY id - SETTINGS index_granularity = 1 - """) - - # Create S3 destination table with regular columns (no EPHEMERAL) - node.query(f""" - CREATE TABLE {s3_table} ( - id UInt32, - value UInt32, - doubled UInt64, - tripled UInt64, - tag String - ) ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY id - """) - - node.query(f"INSERT INTO {mt_table} (id, value, tag_input) VALUES (1, 5, 'test'), (1, 10, 'prod')") - - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '1' TO TABLE {s3_table}") - - wait_for_export_status(node, mt_table, s3_table, "1", "COMPLETED") - - # Verify source data (ALIAS computed, EPHEMERAL not stored) - source_result = node.query(f"SELECT id, value, doubled, tripled, tag FROM {mt_table} ORDER BY value") - expected = "1\t5\t10\t15\tTEST\n1\t10\t20\t30\tPROD\n" - assert source_result == expected, f"Source table data mismatch. Expected:\n{expected}\nGot:\n{source_result}" - - dest_result = node.query(f"SELECT id, value, doubled, tripled, tag FROM {s3_table} ORDER BY value") - assert dest_result == expected, f"Exported data mismatch. Expected:\n{expected}\nGot:\n{dest_result}" - - status = node.query(f""" - SELECT status FROM system.replicated_partition_exports - WHERE source_table = '{mt_table}' - AND destination_table = '{s3_table}' - AND partition_id = '1' - """) - assert status.strip() == "COMPLETED", f"Expected COMPLETED status, got: {status}" - - -def test_sharded_export_partition_with_filename_pattern(cluster): - """Test that export partition with filename pattern prevents collisions in sharded setup.""" - shard1_r1 = cluster.instances["shard1_replica1"] - shard2_r1 = cluster.instances["shard2_replica1"] - watcher_node = cluster.instances["watcher_node"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"sharded_mt_table_{postfix}" - s3_table = f"sharded_s3_table_{postfix}" - - # Create sharded tables on all shards with same partition data (same part names) - # Each shard uses different ZooKeeper path via {shard} macro - create_sharded_tables_and_insert_data(shard1_r1, mt_table, s3_table, "replica1") - create_sharded_tables_and_insert_data(shard2_r1, mt_table, s3_table, "replica1") - create_s3_table(watcher_node, s3_table) - - # Export partition from both shards with filename pattern including shard - # This should prevent filename collisions - shard1_r1.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_filename_pattern = '{{part_name}}_{{shard}}_{{replica}}_{{checksum}}'" - ) - shard2_r1.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_filename_pattern = '{{part_name}}_{{shard}}_{{replica}}_{{checksum}}'" - ) - - # Wait for exports to complete - wait_for_export_status(shard1_r1, mt_table, s3_table, "2020", "COMPLETED") - wait_for_export_status(shard2_r1, mt_table, s3_table, "2020", "COMPLETED") - - total_count = watcher_node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip() - assert total_count == "6", f"Expected 6 total rows (3 from each shard), got {total_count}" - - # Verify filenames contain shard information (check via S3 directly) - # Get all files from S3 - query from watcher_node since S3 is shared - files_shard1 = watcher_node.query( - f"SELECT _file FROM s3(s3_conn, filename='{s3_table}/**', format='One') WHERE _file LIKE '%shard1%' LIMIT 1" - ).strip() - files_shard2 = watcher_node.query( - f"SELECT _file FROM s3(s3_conn, filename='{s3_table}/**', format='One') WHERE _file LIKE '%shard2%' LIMIT 1" - ).strip() - - # Both shards should have files with their shard names - assert "shard1" in files_shard1 or files_shard1 == "", f"Expected shard1 in filenames, got: {files_shard1}" - assert "shard2" in files_shard2 or files_shard2 == "", f"Expected shard2 in filenames, got: {files_shard2}" - - -def test_export_partition_from_replicated_database_uses_db_shard_replica_macros(cluster): - """Test that {shard} and {replica} in the filename pattern are expanded from the - DatabaseReplicated identity, NOT from server config macros. - - replica1 has no / entries in its server config section. - Without the fix buildDestinationFilename() leaves macro_info.shard/replica unset, so - Macros::expand() falls through to the config-macros lookup and throws NO_ELEMENTS_IN_CONFIG. - With the fix the DatabaseReplicated shard_name / replica_name are injected into macro_info - before the expand call, and the pattern resolves correctly. - """ - - # The remote disk test suite sets the shard and replica macros in https://github.com/Altinity/ClickHouse/blob/bbabcaa96e8b7fe8f70ecd0bd4f76fb0f76f2166/tests/integration/helpers/cluster.py#L4356 - # When expanding the macros, the configured ones are preferred over the ones from the DatabaseReplicated definition. - # Therefore, this test fails. It is easier to skip it than to fix it. - skip_if_remote_database_disk_enabled(cluster) - - node = cluster.instances["replica1"] - watcher_node = cluster.instances["watcher_node"] - - postfix = str(uuid.uuid4()).replace("-", "_") - db_name = f"repdb_{postfix}" - table_name = "mt_table" - s3_table = f"s3_dbreplicated_{postfix}" - - # These values exist only in the DatabaseReplicated definition – they are NOT - # present anywhere in replica1's server config . - db_shard = "db_shard_x" - db_replica = "db_replica_y" - - node.query( - f"CREATE DATABASE {db_name} " - f"ENGINE = Replicated('/clickhouse/databases/{db_name}', '{db_shard}', '{db_replica}')") - - node.query(f""" - CREATE TABLE {db_name}.{table_name} - (id UInt64, year UInt16) - ENGINE = ReplicatedMergeTree() - PARTITION BY year ORDER BY tuple()""") - - node.query(f"INSERT INTO {db_name}.{table_name} VALUES (1, 2020), (2, 2020), (3, 2020)") - # Stop merges so part names stay stable during the test. - node.query(f"SYSTEM STOP MERGES {db_name}.{table_name}") - - node.query( - f"CREATE TABLE {s3_table} (id UInt64, year UInt16) " - f"ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') " - f"PARTITION BY year") - - watcher_node.query( - f"CREATE TABLE {s3_table} (id UInt64, year UInt16) " - f"ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') " - f"PARTITION BY year") - - # Export with {shard} and {replica} in the pattern. - # Before the fix: Macros::expand throws NO_ELEMENTS_IN_CONFIG because replica1 has - # no / server config macros. - # After the fix: DatabaseReplicated's shard_name/replica_name are wired into - # macro_info before the expand call, so this succeeds and produces the right names. - node.query( - f"ALTER TABLE {db_name}.{table_name} EXPORT PARTITION ID '2020' TO TABLE {s3_table} " - f"SETTINGS export_merge_tree_part_filename_pattern = " - f"'{{part_name}}_{{shard}}_{{replica}}_{{checksum}}'") - - # A FAILED status here almost certainly means the macro expansion threw - # NO_ELEMENTS_IN_CONFIG (i.e. the fix is missing or broken). - wait_for_export_status(node, table_name, s3_table, "2020", "COMPLETED") - - # Data should have landed in S3. - count = watcher_node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip() - assert count == "3", f"Expected 3 exported rows, got {count}" - - # The exported filename must contain the exact shard and replica names from the - # DatabaseReplicated definition, proving the fix injected them (not server config macros). - filename = watcher_node.query( - f"SELECT _file FROM s3(s3_conn, filename='{s3_table}/**/*.parquet', format='One') LIMIT 1" - ).strip() - - assert db_shard in filename, ( - f"Expected filename to contain DatabaseReplicated shard '{db_shard}', got: {filename!r}. " - "Suggests {shard} was not expanded from the DatabaseReplicated identity.") - - assert db_replica in filename, ( - f"Expected filename to contain DatabaseReplicated replica '{db_replica}', got: {filename!r}. " - "Suggests {replica} was not expanded from the DatabaseReplicated identity.") - - -def test_sharded_export_partition_default_pattern(cluster): - shard1_r1 = cluster.instances["shard1_replica1"] - shard2_r1 = cluster.instances["shard2_replica1"] - watcher_node = cluster.instances["watcher_node"] - - mt_table = "sharded_mt_table_default" - s3_table = "sharded_s3_table_default" - - # Create sharded tables with different ZooKeeper paths per shard - create_sharded_tables_and_insert_data(shard1_r1, mt_table, s3_table, "replica1") - create_sharded_tables_and_insert_data(shard2_r1, mt_table, s3_table, "replica1") - create_s3_table(watcher_node, s3_table) - - # Export with default pattern ({part_name}_{checksum}) - may cause collisions if parts have same name and the same checksum - shard1_r1.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - ) - shard2_r1.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - ) - - wait_for_export_status(shard1_r1, mt_table, s3_table, "2020", "COMPLETED") - wait_for_export_status(shard2_r1, mt_table, s3_table, "2020", "COMPLETED") - - # Both exports should complete (even if there are collisions, the overwrite policy handles it) - # S3 tables are shared, so query from watcher_node - total_count = watcher_node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip() - - # only one file with 3 rows should be present - assert int(total_count) == 3, f"Expected 3 rows, got {total_count}" - - -def test_export_partition_scheduler_skipped_when_moves_stopped(cluster): - node = cluster.instances["replica1"] - - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"sched_skip_mt_{uid}" - s3_table = f"sched_skip_s3_{uid}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - node.query(f"SYSTEM STOP MOVES {mt_table}") - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - ) - - wait_for_export_to_start(node, mt_table, s3_table, "2020") - - # Wait for several scheduler cycles (each fires every 5 s). - # If the guard is missing the scheduler would run and data would land in S3. - time.sleep(10) - - status = node.query( - f"SELECT status FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}' AND destination_table = '{s3_table}'" - f" AND partition_id = '2020'" - ).strip() - - assert status == "PENDING", ( - f"Expected PENDING while moves are stopped, got '{status}'" - ) - - row_count = int(node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip()) - assert row_count == 0, ( - f"Expected 0 rows in S3 while scheduler is skipped, got {row_count}" - ) - - node.query(f"SYSTEM START MOVES {mt_table}") - - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED", timeout=60) - - row_count = int(node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip()) - assert row_count == 3, f"Expected 3 rows in S3 after export completed, got {row_count}" - - -def test_export_partition_resumes_after_stop_moves(cluster): - node = cluster.instances["replica1"] - - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"stop_moves_before_mt_{uid}" - s3_table = f"stop_moves_before_s3_{uid}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - node.query(f"SYSTEM STOP MOVES {mt_table}") - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - ) - - wait_for_export_to_start(node, mt_table, s3_table, "2020") - - # Give the scheduler enough time to attempt (and cancel) the part task at - # least once, exercising the lock-release code path. - time.sleep(5) - - status = node.query( - f"SELECT status FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}' AND destination_table = '{s3_table}'" - f" AND partition_id = '2020'" - ).strip() - assert status == "PENDING", f"Expected PENDING while moves are stopped, got '{status}'" - - row_count = int(node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip()) - assert row_count == 0, f"Expected 0 rows in S3 while moves are stopped, got {row_count}" - - node.query(f"SYSTEM START MOVES {mt_table}") - - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED", timeout=60) - - row_count = int(node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip()) - assert row_count == 3, f"Expected 3 rows in S3 after export completed, got {row_count}" - - -def test_export_partition_resumes_after_stop_moves_during_export(cluster): - skip_if_remote_database_disk_enabled(cluster) - - node = cluster.instances["replica1"] - - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"stop_moves_during_mt_{uid}" - s3_table = f"stop_moves_during_s3_{uid}" - - create_tables_and_insert_data(node, mt_table, s3_table, "replica1") - - minio_ip = cluster.minio_ip - minio_port = cluster.minio_port - - with PartitionManager() as pm: - pm.add_rule({ - "instance": node, - "destination": node.ip_address, - "protocol": "tcp", - "source_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - }) - pm.add_rule({ - "instance": node, - "destination": minio_ip, - "protocol": "tcp", - "destination_port": minio_port, - "action": "REJECT --reject-with tcp-reset", - }) - - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - ) - - wait_for_export_to_start(node, mt_table, s3_table, "2020") - - # Let the tasks start executing and failing against the blocked S3. - time.sleep(2) - - node.query(f"SYSTEM STOP MOVES {mt_table}") - - # Give the cancel callback time to fire and the lock-release path to run. - time.sleep(3) - - status = node.query( - f"SELECT status FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}' AND destination_table = '{s3_table}'" - f" AND partition_id = '2020'" - ).strip() - - assert status == "PENDING", ( - f"Expected PENDING while moves are stopped and S3 is blocked, got '{status}'" - ) - - node.query(f"SYSTEM START MOVES {mt_table}") - - # MinIO is now unblocked; the next scheduler cycle should succeed. - wait_for_export_status(node, mt_table, s3_table, "2020", "COMPLETED", timeout=60) - - row_count = int(node.query(f"SELECT count() FROM {s3_table} WHERE year = 2020").strip()) - assert row_count == 3, f"Expected 3 rows in S3 after export completed, got {row_count}" - - -def test_export_partition_all(cluster): - """Happy path for `ALTER TABLE ... EXPORT PARTITION ALL TO TABLE ...`. - - Schedules one export task per active partition in a single ALTER, then - verifies every partition lands in the destination S3 table. - """ - node = cluster.instances["replica1"] - - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"export_all_mt_{uid}" - s3_table = f"export_all_s3_{uid}" - - node.query( - f"CREATE TABLE {mt_table} (id UInt64, year UInt16)" - f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1')" - f" PARTITION BY year ORDER BY tuple()" - ) - node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2021), (3, 2022)") - create_s3_table(node, s3_table) - - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - - for partition_id in ("2020", "2021", "2022"): - wait_for_export_status(node, mt_table, s3_table, partition_id, "COMPLETED", timeout=60) - - row_count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert row_count == 3, f"Expected 3 rows in S3 after EXPORT PARTITION ALL, got {row_count}" - - -def test_export_partition_partition_column_castable_type_mismatch(cluster): - """A lossy partition-column cast (year String -> UInt16) is rejected synchronously - when export_merge_tree_part_allow_lossy_cast is off, scheduling nothing.""" - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"pkey_cast_mismatch_partition_mt_{postfix}" - s3_table = f"pkey_cast_mismatch_partition_s3_{postfix}" - - # Source: year String; destination: year UInt16. PARTITION BY year on - # both sides — same AST text — to defeat the AST equivalence check. - node.query( - f"CREATE TABLE {mt_table} (id UInt64, year String) " - f"ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') " - f"PARTITION BY year " - f"ORDER BY tuple()" - ) - node.query( - f"CREATE TABLE {s3_table} (id UInt64, year UInt16) " - f"ENGINE = S3(s3_conn, filename='{s3_table}', " - f"format=Parquet, partition_strategy='hive') " - f"PARTITION BY year" - ) - - node.query( - f"INSERT INTO {mt_table} VALUES (1, '2020'), (2, '2020'), (3, '2020')" - ) - - # With a String partition column the partition_id is the SipHash of the - # value rather than the textual representation — look it up so we can - # reference the partition explicitly in EXPORT PARTITION ID and in - # subsequent system.replicated_partition_exports queries. - partition_id = node.query( - f"SELECT partition_id FROM system.parts " - f"WHERE database = currentDatabase() AND table = '{mt_table}' " - f" AND active " - f"ORDER BY name LIMIT 1" - ).strip() - assert partition_id, ( - "Expected one active part on the source table after INSERT; " - "system.parts returned nothing." - ) - - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' " - f"TO TABLE {s3_table}" - ) - assert "INCOMPATIBLE_COLUMNS" in error, ( - f"Expected INCOMPATIBLE_COLUMNS for a lossy partition-column cast, " - f"got: {error!r}" - ) - assert "requires a lossy cast" in error and "'year'" in error, ( - f"Expected the error message to report the lossy cast on column " - f"'year', got: {error!r}" - ) - - # Nothing scheduled: no row in system.replicated_partition_exports. - rows_in_system_view = node.query( - f"SELECT count() FROM system.replicated_partition_exports " - f"WHERE source_table = '{mt_table}' " - f" AND destination_table = '{s3_table}' " - f" AND partition_id = '{partition_id}'" - ).strip() - assert rows_in_system_view == "0", ( - f"Expected no row in system.replicated_partition_exports after a " - f"synchronously-rejected export, got {rows_in_system_view}." - ) - - # Nothing written: no parquet file under any year=*/ partition prefix. - files_in_s3 = node.query( - f"SELECT count() FROM s3(s3_conn, " - f"filename='{s3_table}/year=*/*.parquet', format='One')" - ).strip() - assert files_in_s3 == "0", ( - f"Expected no Parquet files in S3 after a synchronously-rejected " - f"export, found {files_in_s3}." - ) - - -def test_export_partition_all_failure_modes(cluster): - """Cover the three values of `export_merge_tree_partition_all_on_error`. - - Set up an already-fully-exported source table, then re-run EXPORT PARTITION ALL - with each failure mode and assert the documented behavior. - """ - node = cluster.instances["replica1"] - - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"export_all_modes_mt_{uid}" - s3_table = f"export_all_modes_s3_{uid}" - empty_mt = f"export_all_empty_mt_{uid}" - - node.query( - f"CREATE TABLE {mt_table} (id UInt64, year UInt16)" - f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1')" - f" PARTITION BY year ORDER BY tuple()" - ) - node.query(f"INSERT INTO {mt_table} VALUES (1, 2020), (2, 2021), (3, 2022)") - create_s3_table(node, s3_table) - - # First run: schedule + wait for all partitions to complete. - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - for partition_id in ("2020", "2021", "2022"): - wait_for_export_status(node, mt_table, s3_table, partition_id, "COMPLETED", timeout=60) - - # Empty table: throws BAD_ARGUMENTS (no active partitions). - node.query( - f"CREATE TABLE {empty_mt} (id UInt64, year UInt16)" - f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{empty_mt}', 'replica1')" - f" PARTITION BY year ORDER BY tuple()" - ) - error = node.query_and_get_error( - f"ALTER TABLE {empty_mt} EXPORT PARTITION ALL TO TABLE {s3_table}" - ) - assert "no active partitions to export" in error, ( - f"Expected 'no active partitions' error, got: {error}" - ) - - # throw_first (default): re-run aborts on the first conflicting partition. - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}" - f" SETTINGS export_merge_tree_partition_all_on_error = 'throw_first'" - ) - assert "EXPORT_PARTITION_ALREADY_EXPORTED" in error, ( - f"Expected EXPORT_PARTITION_ALREADY_EXPORTED in error, got: {error}" - ) - - # collect: aggregated PARTITION_EXPORT_FAILED message lists every conflicting partition. - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}" - f" SETTINGS export_merge_tree_partition_all_on_error = 'collect'" - ) - assert "PARTITION_EXPORT_FAILED" in error, ( - f"Expected PARTITION_EXPORT_FAILED in error, got: {error}" - ) - for partition_id in ("2020", "2021", "2022"): - assert partition_id in error, ( - f"Expected aggregated error to mention partition {partition_id}, got: {error}" - ) - - # skip_conflicts: succeeds silently because every partition conflicts and is skipped. - node.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}" - f" SETTINGS export_merge_tree_partition_all_on_error = 'skip_conflicts'" - ) - - -# ---- Partition-key compatibility gate (unified with the Iceberg gate) -------------------------- -# -# Plain (hive) object storage writes every row of a part to the single directory computed from the -# destination PARTITION BY, so each source partition must map to exactly one destination partition. -# The gate accepts equivalent or finer source keys (e.g. a source that adds partition columns on top -# of the destination's) and rejects source partitions that would span several destination partitions -# or that do not cover the destination partition column. Hive destinations partition by bare columns -# only, so these cases exercise the column-subset and single-value paths. - - -def _run_subset_accept(node, source_key): - """Export a source partitioned by *source_key* (a superset of the destination key ``year``) into a - hive destination partitioned by ``year``, then verify the full dataset, the hive directory layout, - and a round-trip back into MergeTree.""" - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"subset_mt_{uid}" - s3_table = f"subset_s3_{uid}" - roundtrip = f"subset_roundtrip_{uid}" - - node.query( - f"CREATE TABLE {mt_table} (id UInt64, year UInt16, country String)" - f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1')" - f" PARTITION BY {source_key} ORDER BY tuple()" - ) - node.query( - f"INSERT INTO {mt_table} VALUES (1, 2020, 'US'), (2, 2020, 'FR'), (3, 2021, 'US')" - ) - node.query( - f"CREATE TABLE {s3_table} (id UInt64, year UInt16, country String)" - f" ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive')" - f" PARTITION BY year" - ) - - partition_ids = node.query( - f"SELECT DISTINCT partition_id FROM system.parts" - f" WHERE database = currentDatabase() AND table = '{mt_table}' AND active" - ).strip().split("\n") - assert len(partition_ids) == 3, f"expected 3 source partitions, got {partition_ids}" - - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - for pid in partition_ids: - wait_for_export_status(node, mt_table, s3_table, pid, "COMPLETED", timeout=90) - - src = node.query(f"SELECT id, year, country FROM {mt_table} ORDER BY id") - dst = node.query(f"SELECT id, year, country FROM {s3_table} ORDER BY id") - assert dst == src, f"destination rows differ from source:\nsrc={src!r}\ndst={dst!r}" - - # The destination partitions by year only: rows land in the year= hive directory. - rows_2020 = node.query( - f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/year=2020/*.parquet', format='Parquet')" - ).strip() - rows_2021 = node.query( - f"SELECT count() FROM s3(s3_conn, filename='{s3_table}/year=2021/*.parquet', format='Parquet')" - ).strip() - assert rows_2020 == "2", f"expected 2 rows under year=2020, got {rows_2020}" - assert rows_2021 == "1", f"expected 1 row under year=2021, got {rows_2021}" - - node.query( - f"CREATE TABLE {roundtrip} (id UInt64, year UInt16, country String)" - f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{roundtrip}', 'replica1')" - f" PARTITION BY {source_key} ORDER BY tuple()" - ) - node.query(f"INSERT INTO {roundtrip} SELECT * FROM {s3_table}") - rt = node.query(f"SELECT id, year, country FROM {roundtrip} ORDER BY id") - assert rt == src, f"round-trip rows differ from source:\nsrc={src!r}\nrt={rt!r}" - - -def test_export_partition_multicolumn_subset_accepted(cluster): - """Source partitions by (year, country); destination by year only - a coarser key that is covered - by the source key, so every source partition has a single year and maps to exactly one destination - partition. Accepted (this was rejected as a partition-key mismatch before the plain gate was - unified with the Iceberg one).""" - node = cluster.instances["replica1"] - _run_subset_accept(node, "(year, country)") - - -def test_export_partition_subset_reversed_order_accepted(cluster): - """The subset match is order-independent: a source keyed by (country, year) still covers a - destination keyed by year.""" - node = cluster.instances["replica1"] - _run_subset_accept(node, "(country, year)") - - -def test_export_partition_coarser_source_rejected(cluster): - """Source partitions monthly (toYYYYMM(dt)); destination by the raw date. A single source part - holding two different days would map to two destination partitions, so the gate rejects the - export synchronously with BAD_ARGUMENTS and schedules nothing.""" - node = cluster.instances["replica1"] - - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"coarser_mt_{uid}" - s3_table = f"coarser_s3_{uid}" - - node.query( - f"CREATE TABLE {mt_table} (id UInt64, dt Date)" - f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1')" - f" PARTITION BY toYYYYMM(dt) ORDER BY tuple()" - ) - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05'), (2, '2024-03-20')") - node.query( - f"CREATE TABLE {s3_table} (id UInt64, dt Date)" - f" ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive')" - f" PARTITION BY dt" - ) - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error, f"expected BAD_ARGUMENTS, got: {error!r}" - - scheduled = node.query( - f"SELECT count() FROM system.replicated_partition_exports" - f" WHERE source_table = '{mt_table}' AND destination_table = '{s3_table}'" - ).strip() - assert scheduled == "0", f"expected nothing scheduled after a synchronous reject, got {scheduled}" - - -def test_export_partition_dest_column_not_in_source_key_rejected(cluster): - """Destination partitions by a column that is not part of the source partition key; the gate - rejects the export synchronously with BAD_ARGUMENTS naming the uncovered column.""" - node = cluster.instances["replica1"] - - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"nocover_mt_{uid}" - s3_table = f"nocover_s3_{uid}" - - node.query( - f"CREATE TABLE {mt_table} (id UInt64, year UInt16, country String)" - f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1')" - f" PARTITION BY year ORDER BY tuple()" - ) - node.query(f"INSERT INTO {mt_table} VALUES (1, 2020, 'US'), (2, 2020, 'FR')") - node.query( - f"CREATE TABLE {s3_table} (id UInt64, year UInt16, country String)" - f" ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive')" - f" PARTITION BY country" - ) - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error, f"expected BAD_ARGUMENTS, got: {error!r}" - assert "country" in error, f"expected the error to name column 'country', got: {error!r}" - - -def test_export_partition_column_timezone_rendered_in_destination_zone(cluster): - """A hive partition value lives as text in the object path and is read back in the destination - column's time zone, so the export has to spell it the way the destination would. Spelling it in the - source's zone names a different instant and the row reads back shifted by the offset between the - two zones. INSERT SELECT into an identical table is the reference behavior.""" - node = cluster.instances["replica1"] - - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"tz_mt_{uid}" - s3_export = f"tz_export_s3_{uid}" - s3_insert = f"tz_insert_s3_{uid}" - - node.query( - f"CREATE TABLE {mt_table} (id UInt64, ts DateTime('UTC'))" - f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1')" - f" PARTITION BY toDate(ts) ORDER BY tuple()" - ) - node.query(f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 15:00:00')") - for table in (s3_export, s3_insert): - node.query( - f"CREATE TABLE {table} (id UInt64, ts DateTime('Asia/Tokyo'))" - f" ENGINE = S3(s3_conn, filename='{table}', format=Parquet, partition_strategy='hive')" - f" PARTITION BY ts" - ) - - pid = first_partition_id(node, mt_table) - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_export}") - wait_for_export_status(node, mt_table, s3_export, pid, "COMPLETED", timeout=90) - - node.query(f"INSERT INTO {s3_insert} SELECT * FROM {mt_table}") - - source_instant = node.query(f"SELECT toUnixTimestamp(ts) FROM {mt_table}").strip() - exported_instant = node.query(f"SELECT toUnixTimestamp(ts) FROM {s3_export}").strip() - inserted_instant = node.query(f"SELECT toUnixTimestamp(ts) FROM {s3_insert}").strip() - assert exported_instant == source_instant, ( - f"the exported row moved in time: source {source_instant}, destination {exported_instant}" - ) - assert inserted_instant == source_instant, ( - f"INSERT SELECT must not move it either: source {source_instant}," - f" destination {inserted_instant}" - ) - - # 2024-03-05 15:00:00 UTC is 2024-03-06 00:00:00 in Tokyo. - exported_directory = node.query( - f"SELECT DISTINCT extract(_path, 'ts=[^/]*') FROM {s3_export}" - ).strip() - inserted_directory = node.query( - f"SELECT DISTINCT extract(_path, 'ts=[^/]*') FROM {s3_insert}" - ).strip() - assert exported_directory == "ts=2024-03-06 00:00:00", ( - f"unexpected hive directory: {exported_directory!r}" - ) - assert inserted_directory == exported_directory, ( - f"export and INSERT SELECT disagree on the partition directory:" - f" {exported_directory!r} vs {inserted_directory!r}" - ) - - -def create_wildcard_destination(node, table, columns, partition_key): - """A wildcard destination, the only partition strategy that accepts an expression as its - partition key: the hive strategy allows storage columns only.""" - node.query( - f"CREATE TABLE {table} ({columns})" - f" ENGINE = S3(s3_conn, filename='{table}/{{_partition_id}}/{{_file}}.parquet'," - f" format=Parquet, partition_strategy='wildcard')" - f" PARTITION BY {partition_key}" - ) - - -def test_export_partition_dest_argument_order_rejected(cluster): - """The destination key intDiv(x, 100) has to be validated as written. This source part holds - x in [201, 350], which covers the destination partitions 2 and 3, so the export must be rejected. - Reading the arguments in the reverse order would validate intDiv(100, x) instead, which is 0 at - both endpoints and would silently write both destination partitions into one directory.""" - node = cluster.instances["replica1"] - - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"argorder_mt_{uid}" - s3_table = f"argorder_s3_{uid}" - - node.query( - f"CREATE TABLE {mt_table} (id UInt64, x UInt64)" - f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1')" - f" PARTITION BY intDiv(x, 1000) ORDER BY tuple()" - ) - node.query(f"INSERT INTO {mt_table} VALUES (1, 201), (2, 350)") - create_wildcard_destination(node, s3_table, "id UInt64, x UInt64", "intDiv(x, 100)") - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error, f"expected BAD_ARGUMENTS, got: {error!r}" - - -def test_export_partition_dest_finer_expression_single_partition_accepted(cluster): - """The same shape as the rejected case, with x in [100, 150]: the whole source partition maps to - the single destination partition 1, so it is accepted and every row lands in one directory. The - swapped-argument reading would refuse this one, since intDiv(100, 100) != intDiv(100, 150).""" - node = cluster.instances["replica1"] - - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"argorder_ok_mt_{uid}" - s3_table = f"argorder_ok_s3_{uid}" - - node.query( - f"CREATE TABLE {mt_table} (id UInt64, x UInt64)" - f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1')" - f" PARTITION BY intDiv(x, 1000) ORDER BY tuple()" - ) - node.query(f"INSERT INTO {mt_table} VALUES (1, 100), (2, 150)") - create_wildcard_destination(node, s3_table, "id UInt64, x UInt64", "intDiv(x, 100)") - - pid = first_partition_id(node, mt_table) - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_table}") - wait_for_export_status(node, mt_table, s3_table, pid, "COMPLETED", timeout=90) - - # A wildcard destination cannot be read as a table, so read the objects it wrote. - exported = f"s3(s3_conn, filename='{s3_table}/**/*.parquet', format='Parquet', structure='id UInt64, x UInt64')" - src = node.query(f"SELECT id, x FROM {mt_table} ORDER BY id") - dst = node.query(f"SELECT id, x FROM {exported} ORDER BY id") - assert dst == src, f"destination rows differ from source:\nsrc={src!r}\ndst={dst!r}" - - directories = node.query( - f"SELECT DISTINCT extract(_path, '{s3_table}/[^/]*') FROM {exported}" - ).strip() - assert directories == f"{s3_table}/1", f"unexpected destination directories: {directories!r}" - - -def test_export_partition_dest_nested_expression_accepted(cluster): - """A destination key that wraps the source key in a coarser transform - toYYYYMM(toDate(ts)) over - a source keyed by toDate(ts) - is a function of the source key, so every source partition sits - inside one destination partition whatever the data is.""" - node = cluster.instances["replica1"] - - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"nested_mt_{uid}" - s3_table = f"nested_s3_{uid}" - - node.query( - f"CREATE TABLE {mt_table} (id UInt64, ts DateTime)" - f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1')" - f" PARTITION BY toDate(ts) ORDER BY tuple()" - ) - node.query( - f"INSERT INTO {mt_table} VALUES (1, '2024-03-05 01:00:00'), (2, '2024-03-05 20:00:00')" - ) - create_wildcard_destination(node, s3_table, "id UInt64, ts DateTime", "toYYYYMM(toDate(ts))") - - pid = first_partition_id(node, mt_table) - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_table}") - wait_for_export_status(node, mt_table, s3_table, pid, "COMPLETED", timeout=90) - - exported = f"s3(s3_conn, filename='{s3_table}/**/*.parquet', format='Parquet', structure='id UInt64, ts DateTime')" - src = node.query(f"SELECT id, ts FROM {mt_table} ORDER BY id") - dst = node.query(f"SELECT id, ts FROM {exported} ORDER BY id") - assert dst == src, f"destination rows differ from source:\nsrc={src!r}\ndst={dst!r}" - - directories = node.query( - f"SELECT DISTINCT extract(_path, '{s3_table}/[^/]*') FROM {exported}" - ).strip() - assert directories == f"{s3_table}/202403", ( - f"unexpected destination directories: {directories!r}" - ) - - -def test_export_partition_dest_term_over_two_columns_rejected(cluster): - """A destination expression over two columns is only single-valued when the source key pins both. - This source pins b but only intDiv(a, 100), so a spans [10, 90] within one source partition and - intDiv(a + b, 100) takes both 0 and 1 there. Per-column min/max cannot bound such an expression, - so it is rejected; a source keyed by (a, b) would be accepted, since it pins both columns.""" - node = cluster.instances["replica1"] - - uid = str(uuid.uuid4()).replace("-", "_") - mt_table = f"twocol_mt_{uid}" - s3_table = f"twocol_s3_{uid}" - - node.query( - f"CREATE TABLE {mt_table} (id UInt64, a UInt64, b UInt64)" - f" ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1')" - f" PARTITION BY (intDiv(a, 100), b) ORDER BY tuple()" - ) - node.query(f"INSERT INTO {mt_table} VALUES (1, 10, 20), (2, 90, 20)") - create_wildcard_destination( - node, s3_table, "id UInt64, a UInt64, b UInt64", "intDiv(a + b, 100)" - ) - - pid = first_partition_id(node, mt_table) - error = node.query_and_get_error( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{pid}' TO TABLE {s3_table}" - ) - assert "BAD_ARGUMENTS" in error, f"expected BAD_ARGUMENTS, got: {error!r}" -class RejectedPartitionExportCase(NamedTuple): - src_columns: str - src_partition_by: str - dst_columns: str - dst_partition_by: str - insert_values: str - error_substrings: tuple = () - - -REJECTED_PARTITION_EXPORT_CASES = [ - pytest.param( - RejectedPartitionExportCase( - src_columns="a Int32, b Int32", - src_partition_by="a", - dst_columns="b Int32, a Int32", - dst_partition_by="a", - insert_values="(1, 1), (1, 2)", - error_substrings=("partition key column",), - ), - id="same_partition_key_different_column_order_single_column", - ), - pytest.param( - RejectedPartitionExportCase( - src_columns="a Int32, b Int32, c Int32, val String", - src_partition_by="(a, b, c)", - dst_columns="c Int32, b Int32, a Int32, val String", - dst_partition_by="(a, b, c)", - insert_values="(1, 1, 1, 'x'), (1, 1, 1, 'y')", - error_substrings=("partition key column",), - ), - id="same_partition_key_different_column_order_multi_column", - ), - pytest.param( - RejectedPartitionExportCase( - src_columns="a Int32, b Int32, c Int32, val String", - src_partition_by="(a, b)", - dst_columns="a Int32, b Int32, c Int32, val String", - dst_partition_by="(a, b, c)", - insert_values="(1, 2, 3, 'x')", - error_substrings=( - "column 'c', which is not part of the source MergeTree partition key", - ), - ), - id="multi_column_partition_key_more_in_destination", - ), -] - - -@pytest.mark.parametrize("case", REJECTED_PARTITION_EXPORT_CASES) -def test_export_partition_partition_key_mismatch_variants_are_rejected(cluster, case): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"rejected_mt_table_{postfix}" - s3_table = f"rejected_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} ({case.src_columns}) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY {case.src_partition_by} - ORDER BY tuple() - """) - - node.query(f""" - CREATE TABLE {s3_table} ({case.dst_columns}) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY {case.dst_partition_by} - """) - - node.query(f"INSERT INTO {mt_table} VALUES {case.insert_values}") - - partition_id = node.query( - f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - error = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error, f"Expected BAD_ARGUMENTS, got: {error}" - for substring in case.error_substrings: - assert substring in error, f"Expected {substring!r} in error, got: {error}" - - error_all = node.query_and_get_error(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - assert "BAD_ARGUMENTS" in error_all, f"Expected BAD_ARGUMENTS, got: {error_all}" - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 0, f"Expected 0 rows in destination after rejected export, got {count}" - - -@pytest.mark.parametrize( - "dst_partition_by", - ["(a, b, c)", "(c, b, a)", "(a, b)"], - ids=["same", "reordered", "coarser"], -) -def test_export_partition_multi_column_partition_key_success(cluster, dst_partition_by): - """The source key pins every column the destination partitions by, so the destination may - also name them in another order or leave some out: each destination expression is still - single-valued over a source partition.""" - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_pkey_ok_mt_table_{postfix}" - s3_table = f"multi_pkey_ok_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY (a, b, c) - ORDER BY tuple() - """) - - node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY {dst_partition_by} - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (1, 2, 3, 'y')") - - partition_id = node.query( - f"SELECT partition_id FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY name LIMIT 1" - ).strip() - - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ID '{partition_id}' TO TABLE {s3_table}") - wait_for_export_status(node, mt_table, s3_table, partition_id, "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 2, f"Expected 2 rows in destination after export, got {count}" - - result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() - assert result == "1\t2\t3\tx\n1\t2\t3\ty", f"Unexpected exported data:\n{result}" - - -def test_export_partition_multi_column_partition_key_success_all(cluster): - skip_if_remote_database_disk_enabled(cluster) - node = cluster.instances["replica1"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"multi_pkey_ok_all_mt_table_{postfix}" - s3_table = f"multi_pkey_ok_all_s3_table_{postfix}" - - node.query(f""" - CREATE TABLE {mt_table} (a Int32, b Int32, c Int32, val String) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{mt_table}', 'replica1') - PARTITION BY (a, b, c) - ORDER BY tuple() - """) - - node.query(f""" - CREATE TABLE {s3_table} (a Int32, b Int32, c Int32, val String) - ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, partition_strategy='hive') - PARTITION BY (a, b, c) - """) - - node.query(f"INSERT INTO {mt_table} VALUES (1, 2, 3, 'x'), (4, 5, 6, 'y')") - - partition_ids = node.query( - f"SELECT DISTINCT partition_id FROM system.parts WHERE database = currentDatabase() " - f"AND table = '{mt_table}' AND active ORDER BY partition_id" - ).strip().split("\n") - - node.query(f"ALTER TABLE {mt_table} EXPORT PARTITION ALL TO TABLE {s3_table}") - - for pid in partition_ids: - wait_for_export_status(node, mt_table, s3_table, pid, "COMPLETED") - - count = int(node.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 2, f"Expected 2 rows in destination after export, got {count}" - - result = node.query(f"SELECT a, b, c, val FROM {s3_table} ORDER BY val").strip() - assert result == "1\t2\t3\tx\n4\t5\t6\ty", f"Unexpected exported data:\n{result}" - - -@pytest.mark.parametrize("schema_match_mode", EXTRA_SOURCE_COLUMN_MODES) -def test_export_partition_schema_match_mode_honored_by_non_initiating_replica(cluster, schema_match_mode): - replica1 = cluster.instances["replica1"] - replica2 = cluster.instances["replica2"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"schema_mode_cross_replica_mt_{postfix}" - s3_table = f"schema_mode_cross_replica_s3_{postfix}" - - make_rmt(node=replica1, name=mt_table, columns="id UInt64, year UInt16, extra String", - partition_by="year", replica_name="replica1") - make_rmt(node=replica2, name=mt_table, columns="id UInt64, year UInt16, extra String", - partition_by="year", replica_name="replica2") - replica1.query(f"INSERT INTO {mt_table} VALUES (1, 2020, 'foo'), (2, 2020, 'bar'), (3, 2020, 'baz')") - replica2.query(f"SYSTEM SYNC REPLICA {mt_table}") - - create_s3_table(node=replica1, s3_table=s3_table) - create_s3_table(node=replica2, s3_table=s3_table) - - replica1.query(f"SYSTEM STOP MOVES {mt_table}") - - replica1.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - f" SETTINGS export_merge_tree_part_schema_match_mode = '{schema_match_mode}'," - f" export_merge_tree_part_ignore_extra_source_columns = 1" - ) - - wait_for_export_status(node=replica1, source_table=mt_table, dest_table=s3_table, - partition_id="2020", expected_status="COMPLETED", timeout=60) - - count = int(replica1.query(f"SELECT count() FROM {s3_table}").strip()) - assert count == 3, f"Expected 3 rows in destination table after export, got {count}" - - result = replica1.query(f"SELECT id, year FROM {s3_table} ORDER BY id").strip() - assert result == "1\t2020\n2\t2020\n3\t2020", f"Unexpected data:\n{result}" - - replica1.query(f"SYSTEM START MOVES {mt_table}") - - -def test_export_partition_match_by_name_honored_by_non_initiating_replica(cluster): - replica1 = cluster.instances["replica1"] - replica2 = cluster.instances["replica2"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"match_by_name_cross_replica_mt_{postfix}" - s3_table = f"match_by_name_cross_replica_s3_{postfix}" - - source_columns = "id UInt64, year UInt16, omitted String, payload String" - make_rmt( - node=replica1, - name=mt_table, - columns=source_columns, - partition_by="year", - replica_name="replica1", - ) - make_rmt( - node=replica2, - name=mt_table, - columns=source_columns, - partition_by="year", - replica_name="replica2", - ) - replica1.query( - f"INSERT INTO {mt_table} VALUES " - f"(1, 2020, 'left', 'first'), (2, 2020, 'right', 'second')" - ) - replica2.query(f"SYSTEM SYNC REPLICA {mt_table}") - - for replica in (replica1, replica2): - replica.query( - f"CREATE TABLE {s3_table} (payload String, year UInt16, id UInt64) " - f"ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, " - f"partition_strategy='hive') PARTITION BY year" - ) - - replica1.query(f"SYSTEM STOP MOVES {mt_table}") - - replica1.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - f" SETTINGS export_merge_tree_part_schema_match_mode = 'NAME'," - f" export_merge_tree_part_ignore_extra_source_columns = 1" - ) - - wait_for_export_status( - node=replica1, - source_table=mt_table, - dest_table=s3_table, - partition_id="2020", - expected_status="COMPLETED", - timeout=60, - ) - - result = replica1.query( - f"SELECT payload, year, id FROM {s3_table} ORDER BY id" - ).strip() - assert result == "first\t2020\t1\nsecond\t2020\t2", f"Unexpected data:\n{result}" - - replica1.query(f"SYSTEM START MOVES {mt_table}") - - -def test_export_partition_match_by_name_with_equal_column_count_reordered(cluster): - """Test that match_by_name matches columns by name even with an equal source/destination column count.""" - replica1 = cluster.instances["replica1"] - replica2 = cluster.instances["replica2"] - - postfix = str(uuid.uuid4()).replace("-", "_") - mt_table = f"match_by_name_equal_count_mt_{postfix}" - s3_table = f"match_by_name_equal_count_s3_{postfix}" - - source_columns = "id UInt64, year UInt16, payload String" - make_rmt(node=replica1, name=mt_table, columns=source_columns, - partition_by="year", replica_name="replica1") - make_rmt(node=replica2, name=mt_table, columns=source_columns, - partition_by="year", replica_name="replica2") - replica1.query(f"INSERT INTO {mt_table} VALUES (1, 2020, 'foo'), (2, 2020, 'bar')") - replica2.query(f"SYSTEM SYNC REPLICA {mt_table}") - - for replica in (replica1, replica2): - replica.query( - f"CREATE TABLE {s3_table} (payload String, id UInt64, year UInt16) " - f"ENGINE = S3(s3_conn, filename='{s3_table}', format=Parquet, " - f"partition_strategy='hive') PARTITION BY year" - ) - - replica1.query(f"SYSTEM STOP MOVES {mt_table}") - - replica1.query( - f"ALTER TABLE {mt_table} EXPORT PARTITION ID '2020' TO TABLE {s3_table}" - f" SETTINGS export_merge_tree_part_schema_match_mode = 'NAME'" - ) - - wait_for_export_status(node=replica1, source_table=mt_table, dest_table=s3_table, - partition_id="2020", expected_status="COMPLETED", timeout=60) - - result = replica1.query(f"SELECT id, year, payload FROM {s3_table} ORDER BY id").strip() - assert result == "1\t2020\tfoo\n2\t2020\tbar", f"Unexpected data:\n{result}" - - replica1.query(f"SYSTEM START MOVES {mt_table}") diff --git a/tests/queries/0_stateless/03604_export_merge_tree_partition.reference b/tests/queries/0_stateless/03604_export_merge_tree_partition.reference deleted file mode 100644 index d48023362b99..000000000000 --- a/tests/queries/0_stateless/03604_export_merge_tree_partition.reference +++ /dev/null @@ -1,31 +0,0 @@ -Select from source table -1 2020 -2 2020 -3 2020 -4 2021 -5 2021 -6 2022 -7 2022 -Select from destination table -1 2020 -2 2020 -3 2020 -4 2021 -5 2021 -Export partition 2022 -Select from destination table again -1 2020 -2 2020 -3 2020 -4 2021 -5 2021 -6 2022 -7 2022 ----- Data in roundtrip ReplicatedMergeTree table (should match s3_table) -1 2020 -2 2020 -3 2020 -4 2021 -5 2021 -6 2022 -7 2022 diff --git a/tests/queries/0_stateless/03604_export_merge_tree_partition.sh b/tests/queries/0_stateless/03604_export_merge_tree_partition.sh deleted file mode 100755 index a2fc0947dc3b..000000000000 --- a/tests/queries/0_stateless/03604_export_merge_tree_partition.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest, replica, no-parallel, no-replicated-database - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -# shellcheck source=./export_part.lib -. "$CURDIR"/export_part.lib - -rmt_table="rmt_table_${RANDOM}" -s3_table="s3_table_${RANDOM}" -rmt_table_roundtrip="rmt_table_roundtrip_${RANDOM}" - -query() { - $CLICKHOUSE_CLIENT --query "$1" -} - -query "DROP TABLE IF EXISTS $rmt_table, $s3_table, $rmt_table_roundtrip" - -query "CREATE TABLE $rmt_table (id UInt64, year UInt16) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/$rmt_table', 'replica1') PARTITION BY year ORDER BY tuple()" -query "CREATE TABLE $s3_table (id UInt64, year UInt16) ENGINE = S3(s3_conn, filename='$s3_table', format=Parquet, partition_strategy='hive') PARTITION BY year" - -query "INSERT INTO $rmt_table VALUES (1, 2020), (2, 2020), (4, 2021)" - -query "INSERT INTO $rmt_table VALUES (3, 2020), (5, 2021)" - -query "INSERT INTO $rmt_table VALUES (6, 2022), (7, 2022)" - -# sync replicas -query "SYSTEM SYNC REPLICA $rmt_table" - -query "ALTER TABLE $rmt_table EXPORT PARTITION ID '2020' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" - -query "ALTER TABLE $rmt_table EXPORT PARTITION ID '2021' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" - -# wait until 2 partition exports have completed -wait_for_partition_exports 2 - -echo "Select from source table" -query "SELECT * FROM $rmt_table ORDER BY id" - -echo "Select from destination table" -query "SELECT * FROM $s3_table ORDER BY id" - -echo "Export partition 2022" -query "ALTER TABLE $rmt_table EXPORT PARTITION ID '2022' TO TABLE $s3_table SETTINGS allow_experimental_export_merge_tree_part = 1" - -# wait until 3 partition exports have completed (2 from the previous exports) -wait_for_partition_exports 3 - -echo "Select from destination table again" -query "SELECT * FROM $s3_table ORDER BY id" - -query "CREATE TABLE $rmt_table_roundtrip ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/$rmt_table_roundtrip', 'replica1') PARTITION BY year ORDER BY tuple() AS SELECT * FROM $s3_table" - -echo "---- Data in roundtrip ReplicatedMergeTree table (should match s3_table)" -query "SELECT * FROM $rmt_table_roundtrip ORDER BY id" - -query "DROP TABLE IF EXISTS $rmt_table, $s3_table, $rmt_table_roundtrip" \ No newline at end of file diff --git a/tests/queries/0_stateless/03745_system_background_schedule_pool.sql b/tests/queries/0_stateless/03745_system_background_schedule_pool.sql index 7824732f22d4..870ba8e3b68a 100644 --- a/tests/queries/0_stateless/03745_system_background_schedule_pool.sql +++ b/tests/queries/0_stateless/03745_system_background_schedule_pool.sql @@ -15,7 +15,9 @@ DROP TABLE test_table_03745; DROP TABLE IF EXISTS test_merge_tree_03745; CREATE TABLE test_merge_tree_03745 (x UInt64, y String) ENGINE = MergeTree() ORDER BY x SETTINGS refresh_statistics_interval = '0'; INSERT INTO test_merge_tree_03745 VALUES (1, 'a'), (2, 'b'); -SELECT pool, database, table, table_uuid != toUUIDOrDefault(0) AS has_uuid, log_name FROM system.background_schedule_pool WHERE database = currentDatabase() ORDER BY ALL; +-- Exclude the experimental, config-gated EXPORT PARTITION scheduler task so this test stays +-- deterministic regardless of whether `allow_experimental_export_merge_tree_partition` is enabled. +SELECT pool, database, table, table_uuid != toUUIDOrDefault(0) AS has_uuid, log_name FROM system.background_schedule_pool WHERE database = currentDatabase() AND log_name NOT LIKE '%partition_export_task%' ORDER BY ALL; DROP TABLE test_merge_tree_03745; -- Test 3: Distributed table (distributed pool) diff --git a/tests/queries/0_stateless/05027_export_partition_merge_tree.reference b/tests/queries/0_stateless/05027_export_partition_merge_tree.reference new file mode 100644 index 000000000000..c233b89e2101 --- /dev/null +++ b/tests/queries/0_stateless/05027_export_partition_merge_tree.reference @@ -0,0 +1,31 @@ +Export partition 2020 +Export partition 2021 +Select from destination table (2020, 2021) +1 2020 +2 2020 +3 2020 +4 2021 +5 2021 +Re-exporting 2020 without force is rejected +EXPORT_PARTITION_ALREADY_EXPORTED +Export remaining partitions with EXPORT PARTITION ALL (skip existing) +Select from destination table (all partitions) +1 2020 +2 2020 +3 2020 +4 2021 +5 2021 +6 2022 +7 2022 +Roundtrip: create a table from the exported S3 data +1 2020 +2 2020 +3 2020 +4 2021 +5 2021 +6 2022 +7 2022 +system.partition_exports statuses +2020 COMPLETED 2 0 +2021 COMPLETED 2 0 +2022 COMPLETED 1 0 diff --git a/tests/queries/0_stateless/05027_export_partition_merge_tree.sh b/tests/queries/0_stateless/05027_export_partition_merge_tree.sh new file mode 100755 index 000000000000..4b03fb08957f --- /dev/null +++ b/tests/queries/0_stateless/05027_export_partition_merge_tree.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-shared-merge-tree, no-cas-storage +# no-fasttest: requires S3 / MinIO. +# no-shared-merge-tree: this test exercises EXPORT PARTITION on a plain (non-replicated) MergeTree. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# shellcheck source=./export_partition.lib +. "$CUR_DIR"/export_partition.lib + +run_partition_export_roundtrip_test plain diff --git a/tests/queries/0_stateless/05028_export_partition_replicated_merge_tree.reference b/tests/queries/0_stateless/05028_export_partition_replicated_merge_tree.reference new file mode 100644 index 000000000000..c233b89e2101 --- /dev/null +++ b/tests/queries/0_stateless/05028_export_partition_replicated_merge_tree.reference @@ -0,0 +1,31 @@ +Export partition 2020 +Export partition 2021 +Select from destination table (2020, 2021) +1 2020 +2 2020 +3 2020 +4 2021 +5 2021 +Re-exporting 2020 without force is rejected +EXPORT_PARTITION_ALREADY_EXPORTED +Export remaining partitions with EXPORT PARTITION ALL (skip existing) +Select from destination table (all partitions) +1 2020 +2 2020 +3 2020 +4 2021 +5 2021 +6 2022 +7 2022 +Roundtrip: create a table from the exported S3 data +1 2020 +2 2020 +3 2020 +4 2021 +5 2021 +6 2022 +7 2022 +system.partition_exports statuses +2020 COMPLETED 2 0 +2021 COMPLETED 2 0 +2022 COMPLETED 1 0 diff --git a/tests/queries/0_stateless/05028_export_partition_replicated_merge_tree.sh b/tests/queries/0_stateless/05028_export_partition_replicated_merge_tree.sh new file mode 100755 index 000000000000..5fcf98818d09 --- /dev/null +++ b/tests/queries/0_stateless/05028_export_partition_replicated_merge_tree.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, replica, no-parallel, no-replicated-database +# no-fasttest: requires S3 / MinIO. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# shellcheck source=./export_partition.lib +. "$CUR_DIR"/export_partition.lib + +run_partition_export_roundtrip_test replicated diff --git a/tests/queries/0_stateless/05029_export_partition_key_collision_merge_tree.reference b/tests/queries/0_stateless/05029_export_partition_key_collision_merge_tree.reference new file mode 100644 index 000000000000..f2309b3349d7 --- /dev/null +++ b/tests/queries/0_stateless/05029_export_partition_key_collision_merge_tree.reference @@ -0,0 +1,12 @@ +Export partition 2020 to the first destination +The same partition can be exported to the second destination +Both exports are tracked independently +{db} x.y COMPLETED +{db}.x y COMPLETED +Both destinations received the partition +1 2020 +2 2020 +1 2020 +2 2020 +Re-exporting to the first destination is still rejected +EXPORT_PARTITION_ALREADY_EXPORTED diff --git a/tests/queries/0_stateless/05029_export_partition_key_collision_merge_tree.sh b/tests/queries/0_stateless/05029_export_partition_key_collision_merge_tree.sh new file mode 100755 index 000000000000..02e67312fd89 --- /dev/null +++ b/tests/queries/0_stateless/05029_export_partition_key_collision_merge_tree.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-shared-merge-tree, no-cas-storage +# no-fasttest: requires S3 / MinIO. +# no-shared-merge-tree: this test exercises EXPORT PARTITION on a plain (non-replicated) MergeTree. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# shellcheck source=./export_partition.lib +. "$CUR_DIR"/export_partition.lib + +run_partition_export_dotted_destination_test plain diff --git a/tests/queries/0_stateless/05030_export_partition_key_collision_replicated_merge_tree.reference b/tests/queries/0_stateless/05030_export_partition_key_collision_replicated_merge_tree.reference new file mode 100644 index 000000000000..f2309b3349d7 --- /dev/null +++ b/tests/queries/0_stateless/05030_export_partition_key_collision_replicated_merge_tree.reference @@ -0,0 +1,12 @@ +Export partition 2020 to the first destination +The same partition can be exported to the second destination +Both exports are tracked independently +{db} x.y COMPLETED +{db}.x y COMPLETED +Both destinations received the partition +1 2020 +2 2020 +1 2020 +2 2020 +Re-exporting to the first destination is still rejected +EXPORT_PARTITION_ALREADY_EXPORTED diff --git a/tests/queries/0_stateless/05030_export_partition_key_collision_replicated_merge_tree.sh b/tests/queries/0_stateless/05030_export_partition_key_collision_replicated_merge_tree.sh new file mode 100755 index 000000000000..d2ac219d632a --- /dev/null +++ b/tests/queries/0_stateless/05030_export_partition_key_collision_replicated_merge_tree.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, replica, no-parallel, no-replicated-database +# no-fasttest: requires S3 / MinIO. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# shellcheck source=./export_partition.lib +. "$CUR_DIR"/export_partition.lib + +run_partition_export_dotted_destination_test replicated diff --git a/tests/queries/0_stateless/export_part.lib b/tests/queries/0_stateless/export_part.lib index f7564af7a123..453bef86410d 100644 --- a/tests/queries/0_stateless/export_part.lib +++ b/tests/queries/0_stateless/export_part.lib @@ -24,33 +24,4 @@ function wait_for_exports() return 2 } -# EXPORT PARTITION runs in the background, so wait for it instead of sleeping. -# $1 is how many partition exports should have completed by this point, counted from the -# start of the test (the table keeps one row per export operation). -function wait_for_partition_exports() -{ - local expected=$1 - local database=${2:-$CLICKHOUSE_DATABASE} - local timeout=${3:-120} - - while [[ $timeout -gt 0 ]] - do - res=$(${CLICKHOUSE_CLIENT} --query="SELECT countIf(status = 'COMPLETED') FROM system.replicated_partition_exports WHERE source_database = '$database'") - [[ $res -ge $expected ]] && return 0 - - if [[ $(${CLICKHOUSE_CLIENT} --query="SELECT countIf(status IN ('FAILED', 'KILLED')) FROM system.replicated_partition_exports WHERE source_database = '$database'") -gt 0 ]]; then - echo "Partition export failed!" >&2 - ${CLICKHOUSE_CLIENT} --query="SELECT partition_id, status, exception_count, last_exception_per_replica FROM system.replicated_partition_exports WHERE source_database = '$database'" - return 2 - fi - - sleep 1 - timeout=$((timeout - 1)) - done - - echo "Timed out while waiting for $expected partition export(s) to complete!" >&2 - ${CLICKHOUSE_CLIENT} --query="SELECT partition_id, status, parts_count, parts_to_do FROM system.replicated_partition_exports WHERE source_database = '$database'" - return 2 -} - # vi: ft=bash diff --git a/tests/queries/0_stateless/export_partition.lib b/tests/queries/0_stateless/export_partition.lib new file mode 100644 index 000000000000..82d66d783c2b --- /dev/null +++ b/tests/queries/0_stateless/export_partition.lib @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +# Shared bodies for the `ALTER TABLE ... EXPORT PARTITION` tests. +# +# Every body is invoked once per source engine, so a plain `MergeTree` and a `ReplicatedMergeTree` +# run exactly the same scenario. The wrappers share a body's reference file, so the output must not +# depend on the engine. + +# $1 is the engine: "plain" or "replicated". $2 is the table name, used for the ZooKeeper path. +function partition_export_source_engine() +{ + if [ "$1" = "replicated" ] + then + echo "ReplicatedMergeTree('/clickhouse/tables/{database}/$2', 'r1')" + else + echo "MergeTree" + fi +} + +# Poll `system.partition_exports` until the given export reaches the expected status (or timeout). +# $1 source table, $2 destination database, $3 destination table, $4 partition id, $5 expected status. +function wait_for_partition_export_status() +{ + local source_table="$1" + local destination_database="$2" + local destination_table="$3" + local partition_id="$4" + local expected="$5" + + local status + local i=0 + while [ "$i" -lt 120 ] + do + status=$(${CLICKHOUSE_CLIENT} --query "SELECT status FROM system.partition_exports WHERE source_table = '$source_table' AND destination_database = '$destination_database' AND destination_table = '$destination_table' AND partition_id = '$partition_id'") + if [ "$status" = "$expected" ] + then + return 0 + fi + sleep 0.5 + i=$((i + 1)) + done + + echo "TIMEOUT waiting for the export of $partition_id to $destination_database.$destination_table to reach $expected (last: '$status')" + return 1 +} + +# Export partitions one by one and then all the remaining ones, read the result back and roundtrip it +# into a new table. $1 is the engine. +function run_partition_export_roundtrip_test() +{ + local engine="$1" + + local mt_table="mt_table_${CLICKHOUSE_DATABASE}" + local s3_table="s3_table_${CLICKHOUSE_DATABASE}" + local mt_roundtrip="mt_roundtrip_${CLICKHOUSE_DATABASE}" + + local source_engine + source_engine=$(partition_export_source_engine "$engine" "$mt_table") + local roundtrip_engine + roundtrip_engine=$(partition_export_source_engine "$engine" "$mt_roundtrip") + + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS $mt_table" + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS $s3_table" + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS $mt_roundtrip" + + ${CLICKHOUSE_CLIENT} --query "CREATE TABLE $mt_table (id UInt64, year UInt16) ENGINE = $source_engine PARTITION BY year ORDER BY tuple()" + ${CLICKHOUSE_CLIENT} --query "CREATE TABLE $s3_table (id UInt64, year UInt16) ENGINE = S3(s3_conn, filename='$s3_table', format=Parquet, partition_strategy='hive') PARTITION BY year" + + # Stop merges so the number of parts per partition stays stable for the assertions below. + ${CLICKHOUSE_CLIENT} --query "SYSTEM STOP MERGES $mt_table" + + ${CLICKHOUSE_CLIENT} --query "INSERT INTO $mt_table VALUES (1, 2020), (2, 2020), (4, 2021)" + ${CLICKHOUSE_CLIENT} --query "INSERT INTO $mt_table VALUES (3, 2020), (5, 2021)" + ${CLICKHOUSE_CLIENT} --query "INSERT INTO $mt_table VALUES (6, 2022), (7, 2022)" + + echo "Export partition 2020" + ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $mt_table EXPORT PARTITION ID '2020' TO TABLE $s3_table" + wait_for_partition_export_status "$mt_table" "$CLICKHOUSE_DATABASE" "$s3_table" "2020" "COMPLETED" + + echo "Export partition 2021" + ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $mt_table EXPORT PARTITION ID '2021' TO TABLE $s3_table" + wait_for_partition_export_status "$mt_table" "$CLICKHOUSE_DATABASE" "$s3_table" "2021" "COMPLETED" + + echo "Select from destination table (2020, 2021)" + ${CLICKHOUSE_CLIENT} --query "SELECT * FROM $s3_table ORDER BY id" + + echo "Re-exporting 2020 without force is rejected" + ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $mt_table EXPORT PARTITION ID '2020' TO TABLE $s3_table" 2>&1 | grep -o "EXPORT_PARTITION_ALREADY_EXPORTED" | head -1 + + echo "Export remaining partitions with EXPORT PARTITION ALL (skip existing)" + ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $mt_table EXPORT PARTITION ALL TO TABLE $s3_table SETTINGS export_merge_tree_partition_all_on_error = 'skip_conflicts'" + wait_for_partition_export_status "$mt_table" "$CLICKHOUSE_DATABASE" "$s3_table" "2022" "COMPLETED" + + echo "Select from destination table (all partitions)" + ${CLICKHOUSE_CLIENT} --query "SELECT * FROM $s3_table ORDER BY id" + + echo "Roundtrip: create a table from the exported S3 data" + ${CLICKHOUSE_CLIENT} --query "CREATE TABLE $mt_roundtrip ENGINE = $roundtrip_engine PARTITION BY year ORDER BY tuple() AS SELECT * FROM $s3_table" + ${CLICKHOUSE_CLIENT} --query "SELECT * FROM $mt_roundtrip ORDER BY id" + + echo "system.partition_exports statuses" + ${CLICKHOUSE_CLIENT} --query "SELECT partition_id, status, parts_count, parts_to_do FROM system.partition_exports WHERE source_table = '$mt_table' AND destination_table = '$s3_table' ORDER BY partition_id" + + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS $mt_table" + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS $s3_table" + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS $mt_roundtrip" +} + +# Dots are legal in identifiers, so the destinations `.x`.`y` and ``.`x.y` flatten to the +# same qualified name. The export registry must still track them as two independent tasks. +# $1 is the engine. +function run_partition_export_dotted_destination_test() +{ + local engine="$1" + + local mt_table="mt_table_${CLICKHOUSE_DATABASE}" + local dotted_database="${CLICKHOUSE_DATABASE}.x" + + local source_engine + source_engine=$(partition_export_source_engine "$engine" "$mt_table") + + ${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS \`$dotted_database\`" + ${CLICKHOUSE_CLIENT} --query "CREATE DATABASE \`$dotted_database\`" + + ${CLICKHOUSE_CLIENT} --query "CREATE TABLE $mt_table (id UInt64, year UInt16) ENGINE = $source_engine PARTITION BY year ORDER BY tuple()" + ${CLICKHOUSE_CLIENT} --query "CREATE TABLE \`$dotted_database\`.y (id UInt64, year UInt16) ENGINE = S3(s3_conn, filename='first_${CLICKHOUSE_DATABASE}', format=Parquet, partition_strategy='hive') PARTITION BY year" + ${CLICKHOUSE_CLIENT} --query "CREATE TABLE $CLICKHOUSE_DATABASE.\`x.y\` (id UInt64, year UInt16) ENGINE = S3(s3_conn, filename='second_${CLICKHOUSE_DATABASE}', format=Parquet, partition_strategy='hive') PARTITION BY year" + + ${CLICKHOUSE_CLIENT} --query "INSERT INTO $mt_table VALUES (1, 2020), (2, 2020)" + + echo "Export partition 2020 to the first destination" + ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $mt_table EXPORT PARTITION ID '2020' TO TABLE \`$dotted_database\`.y" + wait_for_partition_export_status "$mt_table" "$dotted_database" "y" "2020" "COMPLETED" + + echo "The same partition can be exported to the second destination" + ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $mt_table EXPORT PARTITION ID '2020' TO TABLE $CLICKHOUSE_DATABASE.\`x.y\`" + wait_for_partition_export_status "$mt_table" "$CLICKHOUSE_DATABASE" "x.y" "2020" "COMPLETED" + + echo "Both exports are tracked independently" + ${CLICKHOUSE_CLIENT} --query "SELECT replaceOne(destination_database, '$CLICKHOUSE_DATABASE', '{db}'), destination_table, status FROM system.partition_exports WHERE source_table = '$mt_table' ORDER BY 1, 2" + + echo "Both destinations received the partition" + ${CLICKHOUSE_CLIENT} --query "SELECT * FROM \`$dotted_database\`.y ORDER BY id" + ${CLICKHOUSE_CLIENT} --query "SELECT * FROM $CLICKHOUSE_DATABASE.\`x.y\` ORDER BY id" + + echo "Re-exporting to the first destination is still rejected" + ${CLICKHOUSE_CLIENT} --query "ALTER TABLE $mt_table EXPORT PARTITION ID '2020' TO TABLE \`$dotted_database\`.y" 2>&1 | grep -o "EXPORT_PARTITION_ALREADY_EXPORTED" | head -1 + + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS $CLICKHOUSE_DATABASE.\`x.y\`" + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS $mt_table" + ${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS \`$dotted_database\`" +} + +# vi: ft=bash