From 05d5599ded6bf03014d2e6ecf0d2661bdb6e78ba Mon Sep 17 00:00:00 2001 From: irfanghat Date: Tue, 10 Nov 2026 16:29:11 +0000 Subject: [PATCH 1/2] fix: Remove duplicate branches entries --- .github/workflows/build_and_test_v4.1.x.yml | 3 --- .github/workflows/build_and_test_v4.3.x.yml | 3 --- 2 files changed, 6 deletions(-) diff --git a/.github/workflows/build_and_test_v4.1.x.yml b/.github/workflows/build_and_test_v4.1.x.yml index 7f62da5..9decdf8 100644 --- a/.github/workflows/build_and_test_v4.1.x.yml +++ b/.github/workflows/build_and_test_v4.1.x.yml @@ -11,9 +11,6 @@ on: branches: - branch-4.1-dev - branch-4.1-stable - branches: - - branch-4.1-dev - - branch-4.1-stable workflow_dispatch: jobs: diff --git a/.github/workflows/build_and_test_v4.3.x.yml b/.github/workflows/build_and_test_v4.3.x.yml index 77692a3..d5d625d 100644 --- a/.github/workflows/build_and_test_v4.3.x.yml +++ b/.github/workflows/build_and_test_v4.3.x.yml @@ -11,9 +11,6 @@ on: branches: - branch-4.3-dev - branch-4.3-stable - branches: - - branch-4.3-dev - - branch-4.3-stable workflow_dispatch: jobs: From 8f1bbb6ef41e3bbd5a32450a137a731f8f5a7b7c Mon Sep 17 00:00:00 2001 From: irfanghat Date: Tue, 10 Nov 2026 17:03:06 +0000 Subject: [PATCH 2/2] task: Update docker configurations --- .../datasets}/flights.parquet | Bin .../datasets}/iot_intrusion_data.csv | 0 {datasets => compose/datasets}/people.csv | 0 {datasets => compose/datasets}/people.json | 0 {datasets => compose/datasets}/people.txt | 0 .../datasets}/people_no_header.csv | 0 {datasets => compose/datasets}/types.orc | Bin .../v4.1.x/docker-compose.yaml | 2 +- compose/v4.3.x/docker-compose.yaml | 59 ++++++++++++++++++ docs/SETUP.md | 7 ++- src/dataframe.h | 6 -- 11 files changed, 65 insertions(+), 9 deletions(-) rename {datasets => compose/datasets}/flights.parquet (100%) rename {datasets => compose/datasets}/iot_intrusion_data.csv (100%) rename {datasets => compose/datasets}/people.csv (100%) rename {datasets => compose/datasets}/people.json (100%) rename {datasets => compose/datasets}/people.txt (100%) rename {datasets => compose/datasets}/people_no_header.csv (100%) rename {datasets => compose/datasets}/types.orc (100%) rename docker-compose.yaml => compose/v4.1.x/docker-compose.yaml (97%) create mode 100644 compose/v4.3.x/docker-compose.yaml diff --git a/datasets/flights.parquet b/compose/datasets/flights.parquet similarity index 100% rename from datasets/flights.parquet rename to compose/datasets/flights.parquet diff --git a/datasets/iot_intrusion_data.csv b/compose/datasets/iot_intrusion_data.csv similarity index 100% rename from datasets/iot_intrusion_data.csv rename to compose/datasets/iot_intrusion_data.csv diff --git a/datasets/people.csv b/compose/datasets/people.csv similarity index 100% rename from datasets/people.csv rename to compose/datasets/people.csv diff --git a/datasets/people.json b/compose/datasets/people.json similarity index 100% rename from datasets/people.json rename to compose/datasets/people.json diff --git a/datasets/people.txt b/compose/datasets/people.txt similarity index 100% rename from datasets/people.txt rename to compose/datasets/people.txt diff --git a/datasets/people_no_header.csv b/compose/datasets/people_no_header.csv similarity index 100% rename from datasets/people_no_header.csv rename to compose/datasets/people_no_header.csv diff --git a/datasets/types.orc b/compose/datasets/types.orc similarity index 100% rename from datasets/types.orc rename to compose/datasets/types.orc diff --git a/docker-compose.yaml b/compose/v4.1.x/docker-compose.yaml similarity index 97% rename from docker-compose.yaml rename to compose/v4.1.x/docker-compose.yaml index 046ce2e..a10cdef 100644 --- a/docker-compose.yaml +++ b/compose/v4.1.x/docker-compose.yaml @@ -21,7 +21,7 @@ services: - "4040:4040" - "15002:15002" volumes: - - ./datasets:/opt/spark/work-dir/datasets + - ../datasets:/opt/spark/work-dir/datasets deploy: resources: limits: diff --git a/compose/v4.3.x/docker-compose.yaml b/compose/v4.3.x/docker-compose.yaml new file mode 100644 index 0000000..14b8112 --- /dev/null +++ b/compose/v4.3.x/docker-compose.yaml @@ -0,0 +1,59 @@ +services: + spark: + image: "apache/spark:4.3.0-scala2.13-java25-python3-ubuntu" + command: > + /opt/spark/sbin/start-connect-server.sh + --packages "org.apache.spark:spark-connect_2.13:4.0.0,io.delta:delta-spark_2.13:4.0.0,io.graphframes:graphframes-spark4_2.13:0.10.0,io.graphframes:graphframes-connect-spark4_2.13:0.10.0" + --conf "spark.connect.extensions.relation.classes=org.apache.spark.sql.graphframes.GraphFramesConnect" + --conf "spark.driver.extraJavaOptions=-Divy.cache.dir=/tmp -Divy.home=/tmp -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError" + --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" + --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" + --conf "spark.driver.memory=4g" + --conf "spark.executor.memory=4g" + --conf "spark.memory.fraction=0.8" + --conf "spark.memory.storageFraction=0.3" + --conf "spark.sql.shuffle.partitions=8" + --conf "spark.default.parallelism=8" + --conf "spark.driver.maxResultSize=2g" + environment: + - SPARK_NO_DAEMONIZE=true + ports: + - "4040:4040" + - "15002:15002" + volumes: + - ../datasets:/opt/spark/work-dir/datasets + deploy: + resources: + limits: + memory: 8g + reservations: + memory: 4g + networks: + - default + + spark_client: + build: + context: . + depends_on: + - spark + command: + - /bin/sh + - -c + - | + echo "Waiting for Spark Connect Server to start..." + until nc -z spark 15002; do + echo "Spark Connect Server is not ready yet. Retrying in 1 second..." + sleep 1 + done + echo "Spark Connect Server is ready." + mv CMakeUserPresets.template.json CMakeUserPresets.json + cmake --preset local + cmake --build --preset local + ctest --test-dir ./build --output-on-failure --verbose + networks: + - default + +networks: + default: + driver: bridge + name: spark_connect_cpp \ No newline at end of file diff --git a/docs/SETUP.md b/docs/SETUP.md index ea373c7..d220e4c 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -35,8 +35,10 @@ spark-connect-cpp/ ├── src/ ├── tests/ ├── build/ -├── datasets/ -├── docker-compose.yaml +├── compose/ + ├── datasets/ + ├── v4.x.x/ + ├── docker-compose.yaml ├── spark_connect_cpp.pc.in ├── scripts/install-deps.sh ├── .vscode/ # Should be created manually @@ -99,6 +101,7 @@ cmake --build --preset local ### Run Tests Ensure an instance of Spark is running locally, e.g. via docker: ```bash +cd compose/v4.x.x docker compose up spark --build -d ctest --preset test_local_spark_coverage --verbose --output-on-failure diff --git a/src/dataframe.h b/src/dataframe.h index 9fd2e9c..31317e5 100644 --- a/src/dataframe.h +++ b/src/dataframe.h @@ -171,12 +171,6 @@ class DataFrame * @brief Filters rows using the given SQL condition string. * @param condition SQL expression (e.g., "age > 3") */ - - DataFrame drop(std::string const& col) const; - DataFrame drop(std::vector const& cols) const; - DataFrame drop(std::initializer_list cols) const; - DataFrame drop(spark::sql::functions::Column const& col) const; - DataFrame filter(std::string const& condition) const; /**