Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .github/workflows/build_and_test_v4.1.x.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,6 @@ on:
branches:
- branch-4.1-dev
- branch-4.1-stable
branches:
- branch-4.1-dev
- branch-4.1-stable
workflow_dispatch:

jobs:
Expand Down
3 changes: 0 additions & 3 deletions .github/workflows/build_and_test_v4.3.x.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,6 @@ on:
branches:
- branch-4.3-dev
- branch-4.3-stable
branches:
- branch-4.3-dev
- branch-4.3-stable
workflow_dispatch:

jobs:
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ services:
- "4040:4040"
- "15002:15002"
volumes:
- ./datasets:/opt/spark/work-dir/datasets
- ../datasets:/opt/spark/work-dir/datasets
deploy:
resources:
limits:
Expand Down
59 changes: 59 additions & 0 deletions compose/v4.3.x/docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
services:
spark:
image: "apache/spark:4.3.0-scala2.13-java25-python3-ubuntu"
command: >
/opt/spark/sbin/start-connect-server.sh
--packages "org.apache.spark:spark-connect_2.13:4.0.0,io.delta:delta-spark_2.13:4.0.0,io.graphframes:graphframes-spark4_2.13:0.10.0,io.graphframes:graphframes-connect-spark4_2.13:0.10.0"
--conf "spark.connect.extensions.relation.classes=org.apache.spark.sql.graphframes.GraphFramesConnect"
--conf "spark.driver.extraJavaOptions=-Divy.cache.dir=/tmp -Divy.home=/tmp -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError"
--conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension"
--conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog"
--conf "spark.driver.memory=4g"
--conf "spark.executor.memory=4g"
--conf "spark.memory.fraction=0.8"
--conf "spark.memory.storageFraction=0.3"
--conf "spark.sql.shuffle.partitions=8"
--conf "spark.default.parallelism=8"
--conf "spark.driver.maxResultSize=2g"
environment:
- SPARK_NO_DAEMONIZE=true
ports:
- "4040:4040"
- "15002:15002"
volumes:
- ../datasets:/opt/spark/work-dir/datasets
deploy:
resources:
limits:
memory: 8g
reservations:
memory: 4g
networks:
- default

spark_client:
build:
context: .
depends_on:
- spark
command:
- /bin/sh
- -c
- |
echo "Waiting for Spark Connect Server to start..."
until nc -z spark 15002; do
echo "Spark Connect Server is not ready yet. Retrying in 1 second..."
sleep 1
done
echo "Spark Connect Server is ready."
mv CMakeUserPresets.template.json CMakeUserPresets.json
cmake --preset local
cmake --build --preset local
ctest --test-dir ./build --output-on-failure --verbose
networks:
- default

networks:
default:
driver: bridge
name: spark_connect_cpp
7 changes: 5 additions & 2 deletions docs/SETUP.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,10 @@ spark-connect-cpp/
├── src/
├── tests/
├── build/
├── datasets/
├── docker-compose.yaml
├── compose/
├── datasets/
├── v4.x.x/
├── docker-compose.yaml
├── spark_connect_cpp.pc.in
├── scripts/install-deps.sh
├── .vscode/ # Should be created manually
Expand Down Expand Up @@ -99,6 +101,7 @@ cmake --build --preset local
### Run Tests
Ensure an instance of Spark is running locally, e.g. via docker:
```bash
cd compose/v4.x.x
docker compose up spark --build -d
ctest --preset test_local_spark_coverage --verbose --output-on-failure

Expand Down
6 changes: 0 additions & 6 deletions src/dataframe.h
Original file line number Diff line number Diff line change
Expand Up @@ -171,12 +171,6 @@ class DataFrame
* @brief Filters rows using the given SQL condition string.
* @param condition SQL expression (e.g., "age > 3")
*/

DataFrame drop(std::string const& col) const;
DataFrame drop(std::vector<std::string> const& cols) const;
DataFrame drop(std::initializer_list<std::string> cols) const;
DataFrame drop(spark::sql::functions::Column const& col) const;

DataFrame filter(std::string const& condition) const;

/**
Expand Down
Loading