Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# Spark Connect C++

[![CodeQL](https://github.com/irfanghat/spark-connect-cpp/actions/workflows/github-code-scanning/codeql/badge.svg?branch=branch-4.1-stable)](https://github.com/irfanghat/spark-connect-cpp/actions/workflows/github-code-scanning/codeql)

[![Build, Test & Coverage (Spark v4.1.x)](https://github.com/irfanghat/spark-connect-cpp/actions/workflows/build_and_test_v4.1.x.yml/badge.svg?branch=branch-4.1-stable)](https://github.com/irfanghat/spark-connect-cpp/actions/workflows/build_and_test_v4.1.x.yml)

[![Build, Test & Coverage (Spark v4.3.x)](https://github.com/irfanghat/spark-connect-cpp/actions/workflows/build_and_test_v4.3.x.yml/badge.svg?branch=branch-4.3-stable)](https://github.com/irfanghat/spark-connect-cpp/actions/workflows/build_and_test_v4.3.x.yml)

A **C++ client for Apache Spark** via **Spark Connect**, enabling C++ applications to execute Spark SQL workloads against remote Spark clusters without requiring JVM dependencies.

Spark Connect introduced a **decoupled client–server architecture** for Apache Spark, where client applications construct logical query plans that are executed remotely by a Spark server. This project delivers an **idiomatic, high-performance C++ interface** for building and submitting Spark queries, with efficient **Apache Arrow–based columnar data exchange** over gRPC.
Expand Down
2 changes: 1 addition & 1 deletion compose/v4.3.x/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ services:
image: "apache/spark:4.3.0-scala2.13-java25-python3-ubuntu"
command: >
/opt/spark/sbin/start-connect-server.sh
--packages "org.apache.spark:spark-connect_2.13:4.0.0,io.delta:delta-spark_2.13:4.0.0,io.graphframes:graphframes-spark4_2.13:0.10.0,io.graphframes:graphframes-connect-spark4_2.13:0.10.0"
--packages "org.apache.spark:spark-connect_2.13:4.0.0,io.delta:delta-spark_2.13:4.1.0,io.graphframes:graphframes-spark4_2.13:0.10.0,io.graphframes:graphframes-connect-spark4_2.13:0.10.0"
--conf "spark.connect.extensions.relation.classes=org.apache.spark.sql.graphframes.GraphFramesConnect"
--conf "spark.driver.extraJavaOptions=-Divy.cache.dir=/tmp -Divy.home=/tmp -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError"
--conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension"
Expand Down
2 changes: 1 addition & 1 deletion src/spark/connect/base.proto
Original file line number Diff line number Diff line change
Expand Up @@ -1364,4 +1364,4 @@ service SparkConnectService {

// Get status information of different types.
rpc GetStatus(GetStatusRequest) returns (GetStatusResponse) {}
}
}
26 changes: 11 additions & 15 deletions src/spark/connect/catalog.proto
Original file line number Diff line number Diff line change
Expand Up @@ -55,17 +55,16 @@ message Catalog {
CurrentCatalog current_catalog = 24;
SetCurrentCatalog set_current_catalog = 25;
ListCatalogs list_catalogs = 26;
ListCachedTables list_cached_tables = 27;
DropTable drop_table = 28;
DropView drop_view = 29;
CreateDatabase create_database = 30;
DropDatabase drop_database = 31;
ListPartitions list_partitions = 32;
ListViews list_views = 33;
GetTableProperties get_table_properties = 34;
GetCreateTableString get_create_table_string = 35;
TruncateTable truncate_table = 36;
AnalyzeTable analyze_table = 37;
DropTable drop_table = 27;
DropView drop_view = 28;
CreateDatabase create_database = 29;
DropDatabase drop_database = 30;
ListPartitions list_partitions = 31;
ListViews list_views = 32;
GetTableProperties get_table_properties = 33;
GetCreateTableString get_create_table_string = 34;
TruncateTable truncate_table = 35;
AnalyzeTable analyze_table = 36;
}
}

Expand Down Expand Up @@ -253,9 +252,6 @@ message ListCatalogs {
optional string pattern = 1;
}

// See `spark.catalog.listCachedTables`
message ListCachedTables { }

// See `spark.catalog.dropTable`
message DropTable {
// (Required)
Expand Down Expand Up @@ -325,4 +321,4 @@ message AnalyzeTable {
// (Required)
string table_name = 1;
bool no_scan = 2;
}
}
8 changes: 7 additions & 1 deletion src/spark/connect/commands.proto
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,9 @@ message WriteOperation {
// (Optional) Columns used for clustering the table.
repeated string clustering_columns = 10;

// (Optional) Whether schema evolution is enabled for the write.
bool with_schema_evolution = 11;

message SaveTable {
// (Required) The table name.
string table_name = 1;
Expand Down Expand Up @@ -211,6 +214,9 @@ message WriteOperationV2 {

// (Optional) Columns used for clustering the table.
repeated string clustering_columns = 9;

// (Optional) Whether schema evolution is enabled for the write.
bool with_schema_evolution = 10;
}

// Starts write stream operation as streaming query. Query ID and Run ID of the streaming
Expand Down Expand Up @@ -551,4 +557,4 @@ message ExecuteExternalCommand {

// (Optional) The options for the runner.
map<string, string> options = 3;
}
}
2 changes: 1 addition & 1 deletion src/spark/connect/common.proto
Original file line number Diff line number Diff line change
Expand Up @@ -176,4 +176,4 @@ message Doubles {

message Strings {
repeated string values = 1;
}
}
30 changes: 29 additions & 1 deletion src/spark/connect/expressions.proto
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,11 @@ message Expression {

SpecializedArray specialized_array = 25;
Time time = 26;

// Nanosecond-capable timestamp literals (precision 7..9). NTZ and LTZ are distinct
// arms so the literal kind is self-describing.
TimestampNTZNanos timestamp_ntz_nanos = 29;
TimestampLTZNanos timestamp_ltz_nanos = 30;
}

// Reserved for Geometry and Geography.
Expand Down Expand Up @@ -289,6 +294,29 @@ message Expression {
// The precision of this time, if omitted, uses the default value of MICROS_PRECISION.
optional int32 precision = 2;
}

// A TIMESTAMP_NTZ literal with nanosecond-capable precision. The physical value is carried
// as microseconds since the UNIX epoch plus the extra nanoseconds within that microsecond,
// because a single int64 of nanoseconds cannot span the supported year range.
message TimestampNTZNanos {
// Microseconds since the UNIX epoch (without timezone information).
int64 epoch_micros = 1;
// Additional nanoseconds within epoch_micros, in [0, 999].
int32 nanos_within_micro = 2;
// Number of fractional-second digits (7, 8, or 9). If omitted, defaults to 9 (nanoseconds).
optional int32 precision = 3;
}

// A TIMESTAMP_LTZ literal with nanosecond-capable precision. See TimestampNTZNanos for the
// rationale behind the two-component physical value.
message TimestampLTZNanos {
// Microseconds since the UNIX epoch.
int64 epoch_micros = 1;
// Additional nanoseconds within epoch_micros, in [0, 999].
int32 nanos_within_micro = 2;
// Number of fractional-second digits (7, 8, or 9). If omitted, defaults to 9 (nanoseconds).
optional int32 precision = 3;
}
}

// An unresolved attribute that is not explicitly bound to a specific column, but the column
Expand Down Expand Up @@ -554,4 +582,4 @@ message SubqueryExpression {
// (Optional) Whether this is a single partition.
optional bool with_single_partition = 3;
}
}
}
2 changes: 1 addition & 1 deletion src/spark/connect/ml.proto
Original file line number Diff line number Diff line change
Expand Up @@ -144,4 +144,4 @@ message MlCommandResult {
// (Optional) warning message generated during the ML command execution
optional string warning_message = 5;
}
}
}
2 changes: 1 addition & 1 deletion src/spark/connect/ml_common.proto
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,4 @@ message ObjectRef {
// (Required) The ID is used to lookup the object on the server side.
// Note it is different from the 'uid' of a ML object.
string id = 1;
}
}
53 changes: 52 additions & 1 deletion src/spark/connect/pipelines.proto
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ package spark.connect;
import "google/protobuf/any.proto";
import "google/protobuf/timestamp.proto";
import "spark/connect/common.proto";
import "spark/connect/expressions.proto";
import "spark/connect/relations.proto";
import "spark/connect/types.proto";

Expand Down Expand Up @@ -144,6 +145,7 @@ message PipelineCommand {

oneof details {
WriteRelationFlowDetails relation_flow_details = 7;
AutoCdcFlowDetails auto_cdc_flow_details = 10;
google.protobuf.Any extension = 999;
}

Expand All @@ -154,6 +156,55 @@ message PipelineCommand {
optional spark.connect.Relation relation = 1;
}

// Details for Auto CDC flows.
message AutoCdcFlowDetails {
// The name of the CDC source to stream from.
optional string source = 1;

// Column(s) that uniquely identify a row in source and target data.
repeated Expression keys = 2;

// Expression to order the source data.
optional Expression sequence_by = 3;

// Delete condition for the merged operation.
optional Expression apply_as_deletes = 6;

// Truncate condition for the merged operation.
optional Expression apply_as_truncates = 7;

// Columns included in the output table.
repeated Expression column_list = 8;

// Columns excluded from the output table.
repeated Expression except_column_list = 9;

// SCD Type for target table.
SCDType stored_as_scd_type = 10;

// SCD2 only. Columns whose value change opens a new history record. When empty, every
// eligible selected user column is tracked.
repeated Expression track_history_column_list = 11;

// SCD2 only. Columns excluded from history tracking. Mutually exclusive with
// track_history_column_list.
repeated Expression track_history_except_column_list = 12;

// Subset of columns to ignore null in updates.
repeated Expression ignore_null_updates_column_list = 14;

// Subset of columns excluded from ignoring null in updates.
repeated Expression ignore_null_updates_except_column_list = 15;

}

// SCD Type for Auto CDC target tables.
enum SCDType {
SCD_TYPE_UNSPECIFIED = 0;
SCD_TYPE_1 = 1;
SCD_TYPE_2 = 2;
}

// If true, define the flow as a one-time flow, such as for backfill.
// Set to true changes the flow in two ways:
// - The flow is run one time by default. If the pipeline is ran with a full refresh,
Expand Down Expand Up @@ -340,4 +391,4 @@ message PipelineAnalysisContext {

// Reserved field for protocol extensions.
repeated google.protobuf.Any extension = 999;
}
}
47 changes: 46 additions & 1 deletion src/spark/connect/relations.proto
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ message Relation {
LateralJoin lateral_join = 44;
ChunkedCachedLocalRelation chunked_cached_local_relation = 45;
RelationChanges relation_changes = 46;
NearestByJoin nearest_by_join = 47;
Zip zip = 48;

// NA functions
NAFill fill_na = 90;
Expand Down Expand Up @@ -1209,12 +1211,13 @@ message Parse {
// (Optional) DataType representing the schema. If not set, Spark will infer the schema.
optional DataType schema = 3;

// Options for the csv/json parser. The map key is case insensitive.
// Options for the csv/json/xml parser. The map key is case insensitive.
map<string, string> options = 4;
enum ParseFormat {
PARSE_FORMAT_UNSPECIFIED = 0;
PARSE_FORMAT_CSV = 1;
PARSE_FORMAT_JSON = 2;
PARSE_FORMAT_XML = 3;
}
}

Expand Down Expand Up @@ -1275,3 +1278,45 @@ message LateralJoin {
// (Required) The join type.
Join.JoinType join_type = 4;
}

// Relation of type [[NearestByJoin]].
//
// For each row on the left side, returns up to `num_results` rows from the right side ranked
// by `ranking_expression`.
message NearestByJoin {
// (Required) Left (query) input relation.
Relation left = 1;

// (Required) Right (base) input relation.
Relation right = 2;

// (Required) Scalar expression used to rank candidate rows on the right side.
Expression ranking_expression = 3;

// (Required) Maximum number of matches per left row. Must be between 1 and 100000.
int32 num_results = 4;

// The following three fields use `string` (not typed enums) for parity with `AsOfJoin`,
// which models analogous fields the same way. Validation happens server-side at planning time.

// (Required) The join type. Must be one of: "inner", "leftouter".
string join_type = 5;

// (Required) Search algorithm contract. Must be one of: "approx", "exact".
string mode = 6;

// (Required) Ranking direction. Must be one of: "distance", "similarity".
string direction = 7;
}

// Relation of type [[Zip]].
//
// Combines the columns of two DataFrames side-by-side. Both DataFrames must produce the same
// canonicalized plan after stripping outer Project chains.
message Zip {
// (Required) Left input relation.
Relation left = 1;

// (Required) Right input relation.
Relation right = 2;
}
21 changes: 20 additions & 1 deletion src/spark/connect/types.proto
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,11 @@ message DataType {
Unparsed unparsed = 24;

Time time = 28;

// Nanosecond-capable timestamp types (precision 7..9). NTZ and LTZ are distinct kinds
// even though their physical value is identical, mirroring timestamp vs timestamp_ntz.
TimestampNTZNanos timestamp_ntz_nanos = 29;
TimestampLTZNanos timestamp_ltz_nanos = 30;
}

message Boolean {
Expand Down Expand Up @@ -136,6 +141,20 @@ message DataType {
uint32 type_variation_reference = 2;
}

// Timestamp without time zone with nanosecond-capable fractional-second precision.
message TimestampNTZNanos {
// Number of fractional-second digits. Valid values are 7, 8, and 9. If omitted, defaults to 9.
optional int32 precision = 1;
uint32 type_variation_reference = 2;
}

// Timestamp with local time zone with nanosecond-capable fractional-second precision.
message TimestampLTZNanos {
// Number of fractional-second digits. Valid values are 7, 8, and 9. If omitted, defaults to 9.
optional int32 precision = 1;
uint32 type_variation_reference = 2;
}

message CalendarInterval {
uint32 type_variation_reference = 1;
}
Expand Down Expand Up @@ -224,4 +243,4 @@ message DataType {
// (Required) The unparsed data type string
string data_type_string = 1;
}
}
}
Loading