From e06e518ba7e388776deb106677c3c166b2efa016 Mon Sep 17 00:00:00 2001 From: irfanghat Date: Wed, 11 Nov 2026 06:33:45 +0000 Subject: [PATCH 1/2] task: Update dataset paths --- Dockerfile => compose/v4.1.x/Dockerfile | 0 compose/v4.1.x/docker-compose.yaml | 2 +- compose/v4.3.x/Dockerfile | 53 ++++++++++++++++++ compose/v4.3.x/docker-compose.yaml | 2 +- .../datasets => datasets}/flights.parquet | Bin .../iot_intrusion_data.csv | 0 {compose/datasets => datasets}/people.csv | 0 {compose/datasets => datasets}/people.json | 0 {compose/datasets => datasets}/people.txt | 0 .../people_no_header.csv | 0 {compose/datasets => datasets}/types.orc | Bin 11 files changed, 55 insertions(+), 2 deletions(-) rename Dockerfile => compose/v4.1.x/Dockerfile (100%) create mode 100644 compose/v4.3.x/Dockerfile rename {compose/datasets => datasets}/flights.parquet (100%) rename {compose/datasets => datasets}/iot_intrusion_data.csv (100%) rename {compose/datasets => datasets}/people.csv (100%) rename {compose/datasets => datasets}/people.json (100%) rename {compose/datasets => datasets}/people.txt (100%) rename {compose/datasets => datasets}/people_no_header.csv (100%) rename {compose/datasets => datasets}/types.orc (100%) diff --git a/Dockerfile b/compose/v4.1.x/Dockerfile similarity index 100% rename from Dockerfile rename to compose/v4.1.x/Dockerfile diff --git a/compose/v4.1.x/docker-compose.yaml b/compose/v4.1.x/docker-compose.yaml index a10cdef..067570e 100644 --- a/compose/v4.1.x/docker-compose.yaml +++ b/compose/v4.1.x/docker-compose.yaml @@ -21,7 +21,7 @@ services: - "4040:4040" - "15002:15002" volumes: - - ../datasets:/opt/spark/work-dir/datasets + - ../../datasets:/opt/spark/work-dir/datasets deploy: resources: limits: diff --git a/compose/v4.3.x/Dockerfile b/compose/v4.3.x/Dockerfile new file mode 100644 index 0000000..10e7382 --- /dev/null +++ b/compose/v4.3.x/Dockerfile @@ -0,0 +1,53 @@ +FROM debian:12-slim + +WORKDIR /workspace + +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + pkg-config \ + libgtest-dev \ + libgmock-dev \ + protobuf-compiler \ + libprotobuf-dev \ + libgrpc-dev \ + libgrpc++-dev \ + protobuf-compiler-grpc \ + uuid-dev \ + libabsl-dev \ + ca-certificates \ + lsb-release \ + wget \ + netcat-traditional \ + gnupg \ + && rm -rf /var/lib/apt/lists/* + +RUN DISTRO=$(lsb_release --id --short | tr 'A-Z' 'a-z') && \ + CODENAME=$(lsb_release --codename --short) && \ + ARROW_PKG="apache-arrow-apt-source-latest-${CODENAME}.deb" && \ + ARROW_URL="https://packages.apache.org/artifactory/arrow/${DISTRO}/${ARROW_PKG}" && \ + echo "Downloading from: ${ARROW_URL}" && \ + wget -q "${ARROW_URL}" -O "${ARROW_PKG}" && \ + apt-get update && \ + apt-get install -y -V "./${ARROW_PKG}" && \ + apt-get update && \ + apt-get install -y -V \ + libarrow-dev \ + libarrow-glib-dev \ + libarrow-dataset-dev \ + libarrow-dataset-glib-dev \ + libarrow-acero-dev \ + libarrow-flight-dev \ + libarrow-flight-glib-dev \ + libarrow-flight-sql-dev \ + libarrow-flight-sql-glib-dev \ + libgandiva-dev \ + libgandiva-glib-dev \ + libparquet-dev \ + libparquet-glib-dev && \ + rm -f "./${ARROW_PKG}" && \ + rm -rf /var/lib/apt/lists/* + +COPY . . + +RUN mkdir -p build \ No newline at end of file diff --git a/compose/v4.3.x/docker-compose.yaml b/compose/v4.3.x/docker-compose.yaml index 14b8112..f1201c0 100644 --- a/compose/v4.3.x/docker-compose.yaml +++ b/compose/v4.3.x/docker-compose.yaml @@ -21,7 +21,7 @@ services: - "4040:4040" - "15002:15002" volumes: - - ../datasets:/opt/spark/work-dir/datasets + - ../../datasets:/opt/spark/work-dir/datasets deploy: resources: limits: diff --git a/compose/datasets/flights.parquet b/datasets/flights.parquet similarity index 100% rename from compose/datasets/flights.parquet rename to datasets/flights.parquet diff --git a/compose/datasets/iot_intrusion_data.csv b/datasets/iot_intrusion_data.csv similarity index 100% rename from compose/datasets/iot_intrusion_data.csv rename to datasets/iot_intrusion_data.csv diff --git a/compose/datasets/people.csv b/datasets/people.csv similarity index 100% rename from compose/datasets/people.csv rename to datasets/people.csv diff --git a/compose/datasets/people.json b/datasets/people.json similarity index 100% rename from compose/datasets/people.json rename to datasets/people.json diff --git a/compose/datasets/people.txt b/datasets/people.txt similarity index 100% rename from compose/datasets/people.txt rename to datasets/people.txt diff --git a/compose/datasets/people_no_header.csv b/datasets/people_no_header.csv similarity index 100% rename from compose/datasets/people_no_header.csv rename to datasets/people_no_header.csv diff --git a/compose/datasets/types.orc b/datasets/types.orc similarity index 100% rename from compose/datasets/types.orc rename to datasets/types.orc From c87c1ee7b5958bc7c7ca3a8014b8b20b32518165 Mon Sep 17 00:00:00 2001 From: irfanghat Date: Thu, 12 Nov 2026 05:55:14 +0000 Subject: [PATCH 2/2] fix: Update proto definitions & wait for compatible Delta Lake version --- compose/v4.3.x/docker-compose.yaml | 2 +- src/spark/connect/base.proto | 2 +- src/spark/connect/catalog.proto | 26 ++++++-------- src/spark/connect/commands.proto | 8 ++++- src/spark/connect/common.proto | 2 +- src/spark/connect/expressions.proto | 30 +++++++++++++++- src/spark/connect/ml.proto | 2 +- src/spark/connect/ml_common.proto | 2 +- src/spark/connect/pipelines.proto | 53 ++++++++++++++++++++++++++++- src/spark/connect/relations.proto | 47 ++++++++++++++++++++++++- src/spark/connect/types.proto | 21 +++++++++++- 11 files changed, 170 insertions(+), 25 deletions(-) diff --git a/compose/v4.3.x/docker-compose.yaml b/compose/v4.3.x/docker-compose.yaml index f1201c0..ffd7973 100644 --- a/compose/v4.3.x/docker-compose.yaml +++ b/compose/v4.3.x/docker-compose.yaml @@ -3,7 +3,7 @@ services: image: "apache/spark:4.3.0-scala2.13-java25-python3-ubuntu" command: > /opt/spark/sbin/start-connect-server.sh - --packages "org.apache.spark:spark-connect_2.13:4.0.0,io.delta:delta-spark_2.13:4.0.0,io.graphframes:graphframes-spark4_2.13:0.10.0,io.graphframes:graphframes-connect-spark4_2.13:0.10.0" + --packages "org.apache.spark:spark-connect_2.13:4.0.0,io.delta:delta-spark_2.13:4.1.0,io.graphframes:graphframes-spark4_2.13:0.10.0,io.graphframes:graphframes-connect-spark4_2.13:0.10.0" --conf "spark.connect.extensions.relation.classes=org.apache.spark.sql.graphframes.GraphFramesConnect" --conf "spark.driver.extraJavaOptions=-Divy.cache.dir=/tmp -Divy.home=/tmp -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError" --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" diff --git a/src/spark/connect/base.proto b/src/spark/connect/base.proto index c724712..9f9e537 100644 --- a/src/spark/connect/base.proto +++ b/src/spark/connect/base.proto @@ -1364,4 +1364,4 @@ service SparkConnectService { // Get status information of different types. rpc GetStatus(GetStatusRequest) returns (GetStatusResponse) {} -} +} \ No newline at end of file diff --git a/src/spark/connect/catalog.proto b/src/spark/connect/catalog.proto index 3d3a97c..dfc1bbc 100644 --- a/src/spark/connect/catalog.proto +++ b/src/spark/connect/catalog.proto @@ -55,17 +55,16 @@ message Catalog { CurrentCatalog current_catalog = 24; SetCurrentCatalog set_current_catalog = 25; ListCatalogs list_catalogs = 26; - ListCachedTables list_cached_tables = 27; - DropTable drop_table = 28; - DropView drop_view = 29; - CreateDatabase create_database = 30; - DropDatabase drop_database = 31; - ListPartitions list_partitions = 32; - ListViews list_views = 33; - GetTableProperties get_table_properties = 34; - GetCreateTableString get_create_table_string = 35; - TruncateTable truncate_table = 36; - AnalyzeTable analyze_table = 37; + DropTable drop_table = 27; + DropView drop_view = 28; + CreateDatabase create_database = 29; + DropDatabase drop_database = 30; + ListPartitions list_partitions = 31; + ListViews list_views = 32; + GetTableProperties get_table_properties = 33; + GetCreateTableString get_create_table_string = 34; + TruncateTable truncate_table = 35; + AnalyzeTable analyze_table = 36; } } @@ -253,9 +252,6 @@ message ListCatalogs { optional string pattern = 1; } -// See `spark.catalog.listCachedTables` -message ListCachedTables { } - // See `spark.catalog.dropTable` message DropTable { // (Required) @@ -325,4 +321,4 @@ message AnalyzeTable { // (Required) string table_name = 1; bool no_scan = 2; -} +} \ No newline at end of file diff --git a/src/spark/connect/commands.proto b/src/spark/connect/commands.proto index c22e76e..976f137 100644 --- a/src/spark/connect/commands.proto +++ b/src/spark/connect/commands.proto @@ -144,6 +144,9 @@ message WriteOperation { // (Optional) Columns used for clustering the table. repeated string clustering_columns = 10; + // (Optional) Whether schema evolution is enabled for the write. + bool with_schema_evolution = 11; + message SaveTable { // (Required) The table name. string table_name = 1; @@ -211,6 +214,9 @@ message WriteOperationV2 { // (Optional) Columns used for clustering the table. repeated string clustering_columns = 9; + + // (Optional) Whether schema evolution is enabled for the write. + bool with_schema_evolution = 10; } // Starts write stream operation as streaming query. Query ID and Run ID of the streaming @@ -551,4 +557,4 @@ message ExecuteExternalCommand { // (Optional) The options for the runner. map options = 3; -} +} \ No newline at end of file diff --git a/src/spark/connect/common.proto b/src/spark/connect/common.proto index c547053..6bd2ebd 100644 --- a/src/spark/connect/common.proto +++ b/src/spark/connect/common.proto @@ -176,4 +176,4 @@ message Doubles { message Strings { repeated string values = 1; -} +} \ No newline at end of file diff --git a/src/spark/connect/expressions.proto b/src/spark/connect/expressions.proto index f74c5af..44e14fa 100644 --- a/src/spark/connect/expressions.proto +++ b/src/spark/connect/expressions.proto @@ -205,6 +205,11 @@ message Expression { SpecializedArray specialized_array = 25; Time time = 26; + + // Nanosecond-capable timestamp literals (precision 7..9). NTZ and LTZ are distinct + // arms so the literal kind is self-describing. + TimestampNTZNanos timestamp_ntz_nanos = 29; + TimestampLTZNanos timestamp_ltz_nanos = 30; } // Reserved for Geometry and Geography. @@ -289,6 +294,29 @@ message Expression { // The precision of this time, if omitted, uses the default value of MICROS_PRECISION. optional int32 precision = 2; } + + // A TIMESTAMP_NTZ literal with nanosecond-capable precision. The physical value is carried + // as microseconds since the UNIX epoch plus the extra nanoseconds within that microsecond, + // because a single int64 of nanoseconds cannot span the supported year range. + message TimestampNTZNanos { + // Microseconds since the UNIX epoch (without timezone information). + int64 epoch_micros = 1; + // Additional nanoseconds within epoch_micros, in [0, 999]. + int32 nanos_within_micro = 2; + // Number of fractional-second digits (7, 8, or 9). If omitted, defaults to 9 (nanoseconds). + optional int32 precision = 3; + } + + // A TIMESTAMP_LTZ literal with nanosecond-capable precision. See TimestampNTZNanos for the + // rationale behind the two-component physical value. + message TimestampLTZNanos { + // Microseconds since the UNIX epoch. + int64 epoch_micros = 1; + // Additional nanoseconds within epoch_micros, in [0, 999]. + int32 nanos_within_micro = 2; + // Number of fractional-second digits (7, 8, or 9). If omitted, defaults to 9 (nanoseconds). + optional int32 precision = 3; + } } // An unresolved attribute that is not explicitly bound to a specific column, but the column @@ -554,4 +582,4 @@ message SubqueryExpression { // (Optional) Whether this is a single partition. optional bool with_single_partition = 3; } -} +} \ No newline at end of file diff --git a/src/spark/connect/ml.proto b/src/spark/connect/ml.proto index ef5c406..23e9ac8 100644 --- a/src/spark/connect/ml.proto +++ b/src/spark/connect/ml.proto @@ -144,4 +144,4 @@ message MlCommandResult { // (Optional) warning message generated during the ML command execution optional string warning_message = 5; } -} +} \ No newline at end of file diff --git a/src/spark/connect/ml_common.proto b/src/spark/connect/ml_common.proto index 06ca4e5..89c802b 100644 --- a/src/spark/connect/ml_common.proto +++ b/src/spark/connect/ml_common.proto @@ -61,4 +61,4 @@ message ObjectRef { // (Required) The ID is used to lookup the object on the server side. // Note it is different from the 'uid' of a ML object. string id = 1; -} +} \ No newline at end of file diff --git a/src/spark/connect/pipelines.proto b/src/spark/connect/pipelines.proto index 6438583..2228e36 100644 --- a/src/spark/connect/pipelines.proto +++ b/src/spark/connect/pipelines.proto @@ -22,6 +22,7 @@ package spark.connect; import "google/protobuf/any.proto"; import "google/protobuf/timestamp.proto"; import "spark/connect/common.proto"; +import "spark/connect/expressions.proto"; import "spark/connect/relations.proto"; import "spark/connect/types.proto"; @@ -144,6 +145,7 @@ message PipelineCommand { oneof details { WriteRelationFlowDetails relation_flow_details = 7; + AutoCdcFlowDetails auto_cdc_flow_details = 10; google.protobuf.Any extension = 999; } @@ -154,6 +156,55 @@ message PipelineCommand { optional spark.connect.Relation relation = 1; } + // Details for Auto CDC flows. + message AutoCdcFlowDetails { + // The name of the CDC source to stream from. + optional string source = 1; + + // Column(s) that uniquely identify a row in source and target data. + repeated Expression keys = 2; + + // Expression to order the source data. + optional Expression sequence_by = 3; + + // Delete condition for the merged operation. + optional Expression apply_as_deletes = 6; + + // Truncate condition for the merged operation. + optional Expression apply_as_truncates = 7; + + // Columns included in the output table. + repeated Expression column_list = 8; + + // Columns excluded from the output table. + repeated Expression except_column_list = 9; + + // SCD Type for target table. + SCDType stored_as_scd_type = 10; + + // SCD2 only. Columns whose value change opens a new history record. When empty, every + // eligible selected user column is tracked. + repeated Expression track_history_column_list = 11; + + // SCD2 only. Columns excluded from history tracking. Mutually exclusive with + // track_history_column_list. + repeated Expression track_history_except_column_list = 12; + + // Subset of columns to ignore null in updates. + repeated Expression ignore_null_updates_column_list = 14; + + // Subset of columns excluded from ignoring null in updates. + repeated Expression ignore_null_updates_except_column_list = 15; + + } + + // SCD Type for Auto CDC target tables. + enum SCDType { + SCD_TYPE_UNSPECIFIED = 0; + SCD_TYPE_1 = 1; + SCD_TYPE_2 = 2; + } + // If true, define the flow as a one-time flow, such as for backfill. // Set to true changes the flow in two ways: // - The flow is run one time by default. If the pipeline is ran with a full refresh, @@ -340,4 +391,4 @@ message PipelineAnalysisContext { // Reserved field for protocol extensions. repeated google.protobuf.Any extension = 999; -} +} \ No newline at end of file diff --git a/src/spark/connect/relations.proto b/src/spark/connect/relations.proto index 4942142..4716bfb 100644 --- a/src/spark/connect/relations.proto +++ b/src/spark/connect/relations.proto @@ -82,6 +82,8 @@ message Relation { LateralJoin lateral_join = 44; ChunkedCachedLocalRelation chunked_cached_local_relation = 45; RelationChanges relation_changes = 46; + NearestByJoin nearest_by_join = 47; + Zip zip = 48; // NA functions NAFill fill_na = 90; @@ -1209,12 +1211,13 @@ message Parse { // (Optional) DataType representing the schema. If not set, Spark will infer the schema. optional DataType schema = 3; - // Options for the csv/json parser. The map key is case insensitive. + // Options for the csv/json/xml parser. The map key is case insensitive. map options = 4; enum ParseFormat { PARSE_FORMAT_UNSPECIFIED = 0; PARSE_FORMAT_CSV = 1; PARSE_FORMAT_JSON = 2; + PARSE_FORMAT_XML = 3; } } @@ -1275,3 +1278,45 @@ message LateralJoin { // (Required) The join type. Join.JoinType join_type = 4; } + +// Relation of type [[NearestByJoin]]. +// +// For each row on the left side, returns up to `num_results` rows from the right side ranked +// by `ranking_expression`. +message NearestByJoin { + // (Required) Left (query) input relation. + Relation left = 1; + + // (Required) Right (base) input relation. + Relation right = 2; + + // (Required) Scalar expression used to rank candidate rows on the right side. + Expression ranking_expression = 3; + + // (Required) Maximum number of matches per left row. Must be between 1 and 100000. + int32 num_results = 4; + + // The following three fields use `string` (not typed enums) for parity with `AsOfJoin`, + // which models analogous fields the same way. Validation happens server-side at planning time. + + // (Required) The join type. Must be one of: "inner", "leftouter". + string join_type = 5; + + // (Required) Search algorithm contract. Must be one of: "approx", "exact". + string mode = 6; + + // (Required) Ranking direction. Must be one of: "distance", "similarity". + string direction = 7; +} + +// Relation of type [[Zip]]. +// +// Combines the columns of two DataFrames side-by-side. Both DataFrames must produce the same +// canonicalized plan after stripping outer Project chains. +message Zip { + // (Required) Left input relation. + Relation left = 1; + + // (Required) Right input relation. + Relation right = 2; +} \ No newline at end of file diff --git a/src/spark/connect/types.proto b/src/spark/connect/types.proto index caaa234..24cc8a5 100644 --- a/src/spark/connect/types.proto +++ b/src/spark/connect/types.proto @@ -76,6 +76,11 @@ message DataType { Unparsed unparsed = 24; Time time = 28; + + // Nanosecond-capable timestamp types (precision 7..9). NTZ and LTZ are distinct kinds + // even though their physical value is identical, mirroring timestamp vs timestamp_ntz. + TimestampNTZNanos timestamp_ntz_nanos = 29; + TimestampLTZNanos timestamp_ltz_nanos = 30; } message Boolean { @@ -136,6 +141,20 @@ message DataType { uint32 type_variation_reference = 2; } + // Timestamp without time zone with nanosecond-capable fractional-second precision. + message TimestampNTZNanos { + // Number of fractional-second digits. Valid values are 7, 8, and 9. If omitted, defaults to 9. + optional int32 precision = 1; + uint32 type_variation_reference = 2; + } + + // Timestamp with local time zone with nanosecond-capable fractional-second precision. + message TimestampLTZNanos { + // Number of fractional-second digits. Valid values are 7, 8, and 9. If omitted, defaults to 9. + optional int32 precision = 1; + uint32 type_variation_reference = 2; + } + message CalendarInterval { uint32 type_variation_reference = 1; } @@ -224,4 +243,4 @@ message DataType { // (Required) The unparsed data type string string data_type_string = 1; } -} +} \ No newline at end of file