From 855ee6ac14103e433629cd48d0c37f2d0808dd88 Mon Sep 17 00:00:00 2001 From: irfanghat Date: Tue, 10 Nov 2026 19:22:52 +0000 Subject: [PATCH 1/3] task: Update path to datasets --- tests/spark/dataframe_reader.cpp | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/spark/dataframe_reader.cpp b/tests/spark/dataframe_reader.cpp index 8c44e3a..c4b091f 100644 --- a/tests/spark/dataframe_reader.cpp +++ b/tests/spark/dataframe_reader.cpp @@ -20,11 +20,11 @@ using ::testing::ElementsAre; TEST_F(SparkIntegrationTest, ReadingJson) { - auto df = spark->read().json("datasets/people.json"); + auto df = spark->read().json("compose/datasets/people.json"); // ------------------------------------------------------------------------------------ // Using the fully qualified class name (JSON is usually built-in) // auto df = - // spark->read().format("org.apache.spark.sql.execution.datasources.json.JsonFileFormat").load({"datasets/people.json"}); + // spark->read().format("org.apache.spark.sql.execution.datasources.json.JsonFileFormat").load({"compose/datasets/people.json"}); // ------------------------------------------------------------------------------------ df.show(); } @@ -34,14 +34,14 @@ TEST_F(SparkIntegrationTest, ReadJsonWithOptions) auto df = spark->read() .option("header", "true") .option("inferSchema", "true") - .json("datasets/people.json"); + .json("compose/datasets/people.json"); EXPECT_NO_THROW(df.show(5)); } TEST_F(SparkIntegrationTest, ReadingCsv) { - auto df = spark->read().csv("datasets/people.csv"); + auto df = spark->read().csv("compose/datasets/people.csv"); EXPECT_NO_THROW(df.show(163)); } @@ -50,14 +50,14 @@ TEST_F(SparkIntegrationTest, ReadCsvWithOptions) auto df = spark->read() .option("header", "true") .option("inferSchema", "true") - .csv("datasets/people.csv"); + .csv("compose/datasets/people.csv"); EXPECT_NO_THROW(df.show()); } TEST_F(SparkIntegrationTest, ReadTextFile) { - auto df = spark->read().text("datasets/people.txt"); + auto df = spark->read().text("compose/datasets/people.txt"); EXPECT_NO_THROW(df.show()); } @@ -67,7 +67,7 @@ TEST_F(SparkIntegrationTest, ReadTextFileWithOptions) .option("wholetext", "false") .option("lineSep", "\n") .format("text") - .load({"datasets/people.txt"}); + .load({"compose/datasets/people.txt"}); // ------------------------------------------------------------------------------------------- // Reading text files is a straightforward way to ingest unstructured or @@ -78,7 +78,7 @@ TEST_F(SparkIntegrationTest, ReadTextFileWithOptions) // .option("wholetext", "true") // .option("lineSep", "\n") // .format("text") - // .load({"datasets/people.txt"}); + // .load({"compose/datasets/people.txt"}); // ------------------------------------------------------------------------------------------- EXPECT_NO_THROW(df.show()); @@ -86,7 +86,7 @@ TEST_F(SparkIntegrationTest, ReadTextFileWithOptions) TEST_F(SparkIntegrationTest, ReadParquet) { - auto df = spark->read().parquet("datasets/flights.parquet"); + auto df = spark->read().parquet("compose/datasets/flights.parquet"); EXPECT_NO_THROW(df.show(1000)); } @@ -99,7 +99,7 @@ TEST_F(SparkIntegrationTest, ReadParquetWithOptions) // results in expensive operations "datetimeRebaseMode" - Handles legacy // dates from older Spark versions // ---------------------------------------------------------------------------- - auto df = spark->read().option("mergeSchema", "true").parquet("datasets/flights.parquet"); + auto df = spark->read().option("mergeSchema", "true").parquet("compose/datasets/flights.parquet"); EXPECT_NO_THROW(df.show(10)); } @@ -108,7 +108,7 @@ TEST_F(SparkIntegrationTest, ReadWithDdlSchema) { auto df = spark->read() .schema("name STRING, age INT, salary INT") - .csv("datasets/people_no_header.csv"); + .csv("compose/datasets/people_no_header.csv"); EXPECT_NO_THROW(df.show(4)); } @@ -116,7 +116,7 @@ TEST_F(SparkIntegrationTest, ReadWithDdlSchema) TEST_F(SparkIntegrationTest, SchemaEvaluation) { std::string ddl = "name STRING, age INT, salary INT"; - auto df = spark->read().schema(ddl).csv("datasets/people_no_header.csv"); + auto df = spark->read().schema(ddl).csv("compose/datasets/people_no_header.csv"); auto schema = df.schema(); @@ -138,7 +138,7 @@ TEST_F(SparkIntegrationTest, SchemaEvaluation) TEST_F(SparkIntegrationTest, HandleMissingFile) { - auto df = spark->read().csv("datasets/non_existent.csv"); + auto df = spark->read().csv("compose/datasets/non_existent.csv"); // ----------------------------------------------------------------------------- // Show triggers an ExecutePlan gRPC call which should fail in this @@ -152,13 +152,13 @@ TEST_F(SparkIntegrationTest, OptionEvaluation) // ------------------------------------------------------------------------- // Read without header option (should have column name "value" or "_c0") // ------------------------------------------------------------------------- - auto df1 = spark->read().csv("datasets/people.csv"); + auto df1 = spark->read().csv("compose/datasets/people.csv"); df1.show(2); // ----------------------------------- // Read with header option // ----------------------------------- - auto df2 = spark->read().option("header", "true").csv("datasets/people.csv"); + auto df2 = spark->read().option("header", "true").csv("compose/datasets/people.csv"); df2.show(2); // -------------------------------------- @@ -211,7 +211,7 @@ TEST_F(SparkIntegrationTest, ReaderOptions_VerifySchema) std::map config = { {"header", "true"}, {"inferSchema", "true"}, {"delimiter", ","}}; - auto df = spark->read().options(config).format("csv").load({"datasets/people.csv"}); + auto df = spark->read().options(config).format("csv").load({"compose/datasets/people.csv"}); auto cols = df.columns(); @@ -227,7 +227,7 @@ TEST_F(SparkIntegrationTest, ReaderOptions_InferredTypes) { std::map config = {{"header", "true"}, {"inferSchema", "true"}}; - auto df = spark->read().options(config).format("csv").load({"datasets/people.csv"}); + auto df = spark->read().options(config).format("csv").load({"compose/datasets/people.csv"}); auto schema = df.schema(); // ---------------------------------------------------- @@ -248,7 +248,7 @@ TEST_F(SparkIntegrationTest, ReaderOptions_CorrectValueParsing) { std::map config = { {"header", "true"}, {"delimiter", ","}, {"inferSchema", "true"}}; - auto df = spark->read().options(config).format("csv").load({"datasets/people.csv"}); + auto df = spark->read().options(config).format("csv").load({"compose/datasets/people.csv"}); auto first_row = df.head(); ASSERT_TRUE(first_row.has_value()); @@ -262,7 +262,7 @@ TEST_F(SparkIntegrationTest, ReaderOptions_CorrectValueParsing) TEST_F(SparkIntegrationTest, ReadOrc) { - auto df = spark->read().orc("datasets/types.orc"); + auto df = spark->read().orc("compose/datasets/types.orc"); EXPECT_NO_THROW(df.show(10)); } From 9d8f2076ebac66cbd84b1c30d7f8ae201f5937cf Mon Sep 17 00:00:00 2001 From: irfanghat Date: Wed, 11 Nov 2026 06:33:45 +0000 Subject: [PATCH 2/3] task: Update dataset paths --- Dockerfile => compose/v4.1.x/Dockerfile | 0 compose/v4.1.x/docker-compose.yaml | 2 +- compose/v4.3.x/Dockerfile | 53 ++++++++++++++++++ compose/v4.3.x/docker-compose.yaml | 2 +- .../datasets => datasets}/flights.parquet | Bin .../iot_intrusion_data.csv | 0 {compose/datasets => datasets}/people.csv | 0 {compose/datasets => datasets}/people.json | 0 {compose/datasets => datasets}/people.txt | 0 .../people_no_header.csv | 0 {compose/datasets => datasets}/types.orc | Bin 11 files changed, 55 insertions(+), 2 deletions(-) rename Dockerfile => compose/v4.1.x/Dockerfile (100%) create mode 100644 compose/v4.3.x/Dockerfile rename {compose/datasets => datasets}/flights.parquet (100%) rename {compose/datasets => datasets}/iot_intrusion_data.csv (100%) rename {compose/datasets => datasets}/people.csv (100%) rename {compose/datasets => datasets}/people.json (100%) rename {compose/datasets => datasets}/people.txt (100%) rename {compose/datasets => datasets}/people_no_header.csv (100%) rename {compose/datasets => datasets}/types.orc (100%) diff --git a/Dockerfile b/compose/v4.1.x/Dockerfile similarity index 100% rename from Dockerfile rename to compose/v4.1.x/Dockerfile diff --git a/compose/v4.1.x/docker-compose.yaml b/compose/v4.1.x/docker-compose.yaml index a10cdef..067570e 100644 --- a/compose/v4.1.x/docker-compose.yaml +++ b/compose/v4.1.x/docker-compose.yaml @@ -21,7 +21,7 @@ services: - "4040:4040" - "15002:15002" volumes: - - ../datasets:/opt/spark/work-dir/datasets + - ../../datasets:/opt/spark/work-dir/datasets deploy: resources: limits: diff --git a/compose/v4.3.x/Dockerfile b/compose/v4.3.x/Dockerfile new file mode 100644 index 0000000..10e7382 --- /dev/null +++ b/compose/v4.3.x/Dockerfile @@ -0,0 +1,53 @@ +FROM debian:12-slim + +WORKDIR /workspace + +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + pkg-config \ + libgtest-dev \ + libgmock-dev \ + protobuf-compiler \ + libprotobuf-dev \ + libgrpc-dev \ + libgrpc++-dev \ + protobuf-compiler-grpc \ + uuid-dev \ + libabsl-dev \ + ca-certificates \ + lsb-release \ + wget \ + netcat-traditional \ + gnupg \ + && rm -rf /var/lib/apt/lists/* + +RUN DISTRO=$(lsb_release --id --short | tr 'A-Z' 'a-z') && \ + CODENAME=$(lsb_release --codename --short) && \ + ARROW_PKG="apache-arrow-apt-source-latest-${CODENAME}.deb" && \ + ARROW_URL="https://packages.apache.org/artifactory/arrow/${DISTRO}/${ARROW_PKG}" && \ + echo "Downloading from: ${ARROW_URL}" && \ + wget -q "${ARROW_URL}" -O "${ARROW_PKG}" && \ + apt-get update && \ + apt-get install -y -V "./${ARROW_PKG}" && \ + apt-get update && \ + apt-get install -y -V \ + libarrow-dev \ + libarrow-glib-dev \ + libarrow-dataset-dev \ + libarrow-dataset-glib-dev \ + libarrow-acero-dev \ + libarrow-flight-dev \ + libarrow-flight-glib-dev \ + libarrow-flight-sql-dev \ + libarrow-flight-sql-glib-dev \ + libgandiva-dev \ + libgandiva-glib-dev \ + libparquet-dev \ + libparquet-glib-dev && \ + rm -f "./${ARROW_PKG}" && \ + rm -rf /var/lib/apt/lists/* + +COPY . . + +RUN mkdir -p build \ No newline at end of file diff --git a/compose/v4.3.x/docker-compose.yaml b/compose/v4.3.x/docker-compose.yaml index 14b8112..f1201c0 100644 --- a/compose/v4.3.x/docker-compose.yaml +++ b/compose/v4.3.x/docker-compose.yaml @@ -21,7 +21,7 @@ services: - "4040:4040" - "15002:15002" volumes: - - ../datasets:/opt/spark/work-dir/datasets + - ../../datasets:/opt/spark/work-dir/datasets deploy: resources: limits: diff --git a/compose/datasets/flights.parquet b/datasets/flights.parquet similarity index 100% rename from compose/datasets/flights.parquet rename to datasets/flights.parquet diff --git a/compose/datasets/iot_intrusion_data.csv b/datasets/iot_intrusion_data.csv similarity index 100% rename from compose/datasets/iot_intrusion_data.csv rename to datasets/iot_intrusion_data.csv diff --git a/compose/datasets/people.csv b/datasets/people.csv similarity index 100% rename from compose/datasets/people.csv rename to datasets/people.csv diff --git a/compose/datasets/people.json b/datasets/people.json similarity index 100% rename from compose/datasets/people.json rename to datasets/people.json diff --git a/compose/datasets/people.txt b/datasets/people.txt similarity index 100% rename from compose/datasets/people.txt rename to datasets/people.txt diff --git a/compose/datasets/people_no_header.csv b/datasets/people_no_header.csv similarity index 100% rename from compose/datasets/people_no_header.csv rename to datasets/people_no_header.csv diff --git a/compose/datasets/types.orc b/datasets/types.orc similarity index 100% rename from compose/datasets/types.orc rename to datasets/types.orc From 40191fa88479a01fcfad88a9bb6157d854518b79 Mon Sep 17 00:00:00 2001 From: irfanghat Date: Wed, 11 Nov 2026 07:20:05 +0000 Subject: [PATCH 3/3] task: Resolve PATH_NOT_FOUND issues --- tests/spark/aggregations.cpp | 8 +-- tests/spark/configuration.cpp | 48 ++++++++------ tests/spark/dataframe.cpp | 68 +++++++++++--------- tests/spark/dataframe_columns.cpp | 32 ++++++---- tests/spark/dataframe_reader.cpp | 82 +++++++++++++----------- tests/spark/dataframe_writer.cpp | 75 +++++++++++++--------- tests/spark/expressions.cpp | 5 +- tests/spark/graphframes.cpp | 55 ++++++++++------ tests/spark/risk_management.cpp | 103 +++++++++++++++++++----------- tests/spark/spark_fixture.cpp | 13 ++-- tests/util/env_loader.cpp | 7 +- 11 files changed, 300 insertions(+), 196 deletions(-) diff --git a/tests/spark/aggregations.cpp b/tests/spark/aggregations.cpp index 4a43f34..9344b61 100644 --- a/tests/spark/aggregations.cpp +++ b/tests/spark/aggregations.cpp @@ -1,7 +1,3 @@ -#include -#include -#include - #include "config.h" #include "dataframe.h" #include "functions.h" @@ -10,6 +6,10 @@ #include "spark_fixture.h" #include "types.h" +#include +#include +#include + using namespace spark::sql::functions; using namespace spark::sql::types; using ::testing::ElementsAre; diff --git a/tests/spark/configuration.cpp b/tests/spark/configuration.cpp index 36749b7..b387a16 100644 --- a/tests/spark/configuration.cpp +++ b/tests/spark/configuration.cpp @@ -1,14 +1,12 @@ +#include "dataframe.h" +#include "runtime_config.h" +#include "session.h" + #include #include #include - -#include "runtime_config.h" - #include -#include "dataframe.h" -#include "session.h" - using ::testing::Contains; using ::testing::Pair; @@ -17,7 +15,8 @@ class SparkConfigurationIntegrationTest : public ::testing::Test protected: static SparkSession* spark; - static void SetUpTestSuite() + static void + SetUpTestSuite() { // ------------------------------------------------------------------ // Exercise builder.config() at session creation time @@ -31,11 +30,13 @@ class SparkConfigurationIntegrationTest : public ::testing::Test .appName("SparkConnectCppGTest") .config("spark.sql.shuffle.partitions", int64_t(42)) .config("spark.sql.ansi.enabled", true) - .config("spark.sql.session.timeZone", std::string("America/New_York")) + .config("spark.sql.session.timeZone", + std::string("America/New_York")) .getOrCreate(); } - static void TearDownTestSuite() + static void + TearDownTestSuite() { if (spark) spark->stop(); @@ -44,7 +45,8 @@ class SparkConfigurationIntegrationTest : public ::testing::Test // ----------------------------------------------------------------- // Restore keys touched by tests so each test starts clean // ----------------------------------------------------------------- - void TearDown() override + void + TearDown() override { try { @@ -91,24 +93,30 @@ TEST_F(SparkConfigurationIntegrationTest, BuilderConfigAppliedAtCreation) { EXPECT_EQ(spark->conf().get("spark.sql.shuffle.partitions"), "42"); EXPECT_EQ(spark->conf().get("spark.sql.ansi.enabled"), "true"); - EXPECT_EQ(spark->conf().get("spark.sql.session.timeZone"), "America/New_York"); + EXPECT_EQ(spark->conf().get("spark.sql.session.timeZone"), + "America/New_York"); } -TEST_F(SparkConfigurationIntegrationTest, BuilderConfigFlushesOnSubsequentGetOrCreate) +TEST_F(SparkConfigurationIntegrationTest, + BuilderConfigFlushesOnSubsequentGetOrCreate) { // ------------------------------------------------------------------ // builder.config() on an already-running singleton still flushes — // getOrCreate() applies runtime_configs regardless of whether the // session was just created or already existed. // ------------------------------------------------------------------ - SparkSession::builder().config("spark.sql.shuffle.partitions", int64_t(99)).getOrCreate(); + SparkSession::builder() + .config("spark.sql.shuffle.partitions", int64_t(99)) + .getOrCreate(); EXPECT_EQ(spark->conf().get("spark.sql.shuffle.partitions"), "99"); } TEST_F(SparkConfigurationIntegrationTest, BuilderConfigRawIntOverloading) { - SparkSession::builder().config("spark.sql.shuffle.partitions", 99).getOrCreate(); + SparkSession::builder() + .config("spark.sql.shuffle.partitions", 99) + .getOrCreate(); EXPECT_EQ(spark->conf().get("spark.sql.shuffle.partitions"), "99"); } @@ -136,7 +144,8 @@ TEST_F(SparkConfigurationIntegrationTest, BuilderConfigMapOverload) .getOrCreate(); EXPECT_EQ(spark->conf().get("spark.sql.shuffle.partitions"), "12"); - EXPECT_EQ(spark->conf().get("spark.sql.session.timeZone"), "America/Chicago"); + EXPECT_EQ(spark->conf().get("spark.sql.session.timeZone"), + "America/Chicago"); } // ---------------------------------------------------------------------------------- @@ -155,7 +164,8 @@ TEST_F(SparkConfigurationIntegrationTest, BuilderConfigMapOverload) TEST_F(SparkConfigurationIntegrationTest, SetAndGetString) { spark->conf().set("spark.sql.session.timeZone", "America/New_York"); - EXPECT_EQ(spark->conf().get("spark.sql.session.timeZone"), "America/New_York"); + EXPECT_EQ(spark->conf().get("spark.sql.session.timeZone"), + "America/New_York"); } TEST_F(SparkConfigurationIntegrationTest, SetBoolTrue) @@ -282,7 +292,8 @@ TEST_F(SparkConfigurationIntegrationTest, SetCheckpointDirIsNotModifiable) TEST_F(SparkConfigurationIntegrationTest, SetCheckpointDirThrows) { - EXPECT_THROW(spark->setCheckpointDir("/tmp/cpp-test-checkpoints"), std::runtime_error); + EXPECT_THROW(spark->setCheckpointDir("/tmp/cpp-test-checkpoints"), + std::runtime_error); } // ---------------------------------------------------------------------------------- @@ -290,5 +301,6 @@ TEST_F(SparkConfigurationIntegrationTest, SetCheckpointDirThrows) // ---------------------------------------------------------------------------------- TEST_F(SparkConfigurationIntegrationTest, GetUnknownKeyThrows) { - EXPECT_THROW(spark->conf().get("spark.cpp.test.nonexistent.key"), std::runtime_error); + EXPECT_THROW(spark->conf().get("spark.cpp.test.nonexistent.key"), + std::runtime_error); } diff --git a/tests/spark/dataframe.cpp b/tests/spark/dataframe.cpp index a19d762..e5cf7c8 100644 --- a/tests/spark/dataframe.cpp +++ b/tests/spark/dataframe.cpp @@ -1,16 +1,16 @@ -#include -#include - -#include -#include +#include "dataframe.h" #include "config.h" -#include "dataframe.h" #include "functions.h" #include "session.h" #include "spark_fixture.h" #include "types.h" +#include +#include +#include +#include + using namespace spark::sql::functions; using namespace spark::sql::types; using ::testing::ElementsAre; @@ -186,7 +186,7 @@ TEST_F(SparkIntegrationTest, Head) auto rows = df.head(1); ASSERT_FALSE(rows.empty()); - const auto& row = rows[0]; + auto const& row = rows[0]; std::stringstream ss; ss << row; @@ -250,7 +250,8 @@ TEST_F(SparkIntegrationTest, Filter) // -------------------------------------------- // Complex Logical Operators (AND, OR, NOT) // -------------------------------------------- - auto complex_logic = df.filter("age > 20 AND name LIKE 'J%' OR salary = 130000"); + auto complex_logic = + df.filter("age > 20 AND name LIKE 'J%' OR salary = 130000"); complex_logic.show(); // ---------------------------------------- @@ -461,10 +462,10 @@ TEST_F(SparkIntegrationTest, DataFrameDistinct) auto rows = distinct_df.collect(); std::set unique_rows; - for (const auto& row : rows) + for (auto const& row : rows) { - std::string row_str = - std::get(row["name"]) + std::to_string(row.get_long("id")); + std::string row_str = std::get(row["name"]) + + std::to_string(row.get_long("id")); unique_rows.insert(row_str); } @@ -501,13 +502,15 @@ TEST_F(SparkIntegrationTest, DataFrameDescribe) EXPECT_EQ(std::get(df_stats[0]["weight"]), "3"); EXPECT_EQ(std::get(df_stats[1]["weight"]), "40.73333"); - EXPECT_EQ(std::get(df_stats[2]["weight"]), "3.1722757341273704"); + EXPECT_EQ(std::get(df_stats[2]["weight"]), + "3.1722757341273704"); EXPECT_EQ(std::get(df_stats[3]["weight"]), "37.8"); EXPECT_EQ(std::get(df_stats[4]["weight"]), "44.1"); EXPECT_EQ(std::get(df_stats[0]["height"]), "3"); EXPECT_EQ(std::get(df_stats[1]["height"]), "145.00000"); - EXPECT_EQ(std::get(df_stats[2]["height"]), "4.763402145525822"); + EXPECT_EQ(std::get(df_stats[2]["height"]), + "4.763402145525822"); EXPECT_EQ(std::get(df_stats[3]["height"]), "142.2"); EXPECT_EQ(std::get(df_stats[4]["height"]), "150.5"); } @@ -588,7 +591,7 @@ TEST_F(SparkIntegrationTest, DefaultInnerJoinOnCommonColumns) EXPECT_EQ(rows.size(), 2); std::set names; - for (const auto& row : rows) + for (auto const& row : rows) names.insert(row.get("name")); EXPECT_TRUE(names.count("Alice") == 1); @@ -750,7 +753,8 @@ TEST_F(SparkIntegrationTest, JoinOnExpressionOuter) EXPECT_EQ(rows.size(), 3); } -DataFrame MultiplyColumn(const DataFrame& df, std::string col_name, int factor) +DataFrame +MultiplyColumn(DataFrame const& df, std::string col_name, int factor) { return df.select({col(col_name) * lit(factor)}); } @@ -774,7 +778,8 @@ TEST_F(SparkIntegrationTest, DataFrameTransform) // 50"); // -------------------------------------------------------------------- - auto results = df.transform(MultiplyColumn, "id", 10).filter("(id * 10) > 50"); + auto results = + df.transform(MultiplyColumn, "id", 10).filter("(id * 10) > 50"); EXPECT_NO_THROW(results.show()); @@ -817,22 +822,25 @@ TEST_F(SparkIntegrationTest, ChainedDataFrameTransforms) EXPECT_NO_THROW(raw_df.show()); - auto add_full_name = [](const DataFrame& df) - { - return df.withColumn("full_name", - spark::sql::functions::concat_ws(" ", {col("first"), col("last")})); + auto add_full_name = [](DataFrame const& df) { + return df.withColumn( + "full_name", + spark::sql::functions::concat_ws(" ", {col("first"), col("last")})); }; - auto clean_email = [](const DataFrame& df) - { return df.withColumn("email", lower(col("email"))); }; + auto clean_email = [](DataFrame const& df) { + return df.withColumn("email", lower(col("email"))); + }; - auto add_status = [](const DataFrame& df) - { - return df.withColumn("status", - when(col("age") > lit(18), lit("adult")).otherwise(lit("minor"))); + auto add_status = [](DataFrame const& df) { + return df.withColumn( + "status", + when(col("age") > lit(18), lit("adult")).otherwise(lit("minor"))); }; - auto df = raw_df.transform(add_full_name).transform(clean_email).transform(add_status); + auto df = raw_df.transform(add_full_name) + .transform(clean_email) + .transform(add_status); EXPECT_NO_THROW(df.show()); @@ -854,11 +862,13 @@ TEST_F(SparkIntegrationTest, DataFrameAlias) auto df_as1 = df.alias("df_as1"); auto df_as2 = df.alias("df_as2"); - auto joined_df = df_as1.join_on_expression(df_as2, "df_as1.name = df_as2.name", "inner"); + auto joined_df = + df_as1.join_on_expression(df_as2, "df_as1.name = df_as2.name", "inner"); EXPECT_NO_THROW(joined_df.show()); - EXPECT_NO_THROW(joined_df.select({"df_as1.name", "df_as2.name", "df_as2.age"}).show()); + EXPECT_NO_THROW( + joined_df.select({"df_as1.name", "df_as2.name", "df_as2.age"}).show()); auto rows = joined_df.collect(); diff --git a/tests/spark/dataframe_columns.cpp b/tests/spark/dataframe_columns.cpp index ad1b5cf..2e8146f 100644 --- a/tests/spark/dataframe_columns.cpp +++ b/tests/spark/dataframe_columns.cpp @@ -1,3 +1,6 @@ +#include "dataframe.h" +#include "session.h" + #include #include #include @@ -6,9 +9,6 @@ #include #include -#include "dataframe.h" -#include "session.h" - using ::testing::ElementsAre; using ::testing::HasSubstr; using ::testing::UnorderedElementsAre; @@ -18,7 +18,8 @@ class ColumnTest : public ::testing::Test protected: static SparkSession* spark; - static void SetUpTestSuite() + static void + SetUpTestSuite() { spark = &SparkSession::builder() .master("sc://localhost") @@ -26,7 +27,8 @@ class ColumnTest : public ::testing::Test .getOrCreate(); } - static void TearDownTestSuite() + static void + TearDownTestSuite() { if (spark) spark->stop(); @@ -103,13 +105,16 @@ TEST_F(ColumnTest, SchemaComparison) TEST_F(ColumnTest, SpecialCharactersInNames) { - auto df = spark->sql("SELECT 1 AS `user.id`, 2 AS `count(*)`, 3 AS `col with spaces`"); - EXPECT_THAT(df.columns(), ElementsAre("user.id", "count(*)", "col with spaces")); + auto df = spark->sql( + "SELECT 1 AS `user.id`, 2 AS `count(*)`, 3 AS `col with spaces`"); + EXPECT_THAT(df.columns(), + ElementsAre("user.id", "count(*)", "col with spaces")); } TEST_F(ColumnTest, ColumnsAfterTransformation) { - auto df_selected = spark->sql("SELECT b, a FROM (SELECT 1 AS a, 2 AS b, 3 AS c)"); + auto df_selected = + spark->sql("SELECT b, a FROM (SELECT 1 AS a, 2 AS b, 3 AS c)"); EXPECT_THAT(df_selected.columns(), ElementsAre("b", "a")); } @@ -139,7 +144,8 @@ TEST_F(ColumnTest, ColumnDataTypesValidation) EXPECT_THAT(schema_str, HasSubstr(R"("name":"int_col","type":"integer")")); EXPECT_THAT(schema_str, HasSubstr(R"("name":"str_col","type":"string")")); - EXPECT_THAT(schema_str, HasSubstr(R"("name":"double_col","type":"double")")); + EXPECT_THAT(schema_str, + HasSubstr(R"("name":"double_col","type":"double")")); EXPECT_THAT(schema_str, HasSubstr(R"("name":"date_col","type":"date")")); EXPECT_THAT(schema_str, HasSubstr(R"("name":"bool_col","type":"boolean")")); } @@ -160,13 +166,13 @@ TEST_F(ColumnTest, ComplexTypeDetection) // Check Array // --------------------------- EXPECT_EQ(schema.fields[0].name, "int_array"); - EXPECT_TRUE( - std::holds_alternative(schema.fields[0].data_type->kind)); + EXPECT_TRUE(std::holds_alternative( + schema.fields[0].data_type->kind)); // --------------------------- // Check Map // --------------------------- EXPECT_EQ(schema.fields[1].name, "string_int_map"); - EXPECT_TRUE( - std::holds_alternative(schema.fields[1].data_type->kind)); + EXPECT_TRUE(std::holds_alternative( + schema.fields[1].data_type->kind)); } \ No newline at end of file diff --git a/tests/spark/dataframe_reader.cpp b/tests/spark/dataframe_reader.cpp index c4b091f..ef43501 100644 --- a/tests/spark/dataframe_reader.cpp +++ b/tests/spark/dataframe_reader.cpp @@ -1,9 +1,3 @@ -#include -#include - -#include -#include - #include "config.h" #include "dataframe.h" #include "reader.h" @@ -11,6 +5,12 @@ #include "spark_fixture.h" #include "writer.h" +#include +#include +#include +#include + + // ---------------------------------------------------------------- // The following suite tests reading from various file formats i.e. // json, csv, parquet, orc, avro and text files. @@ -20,11 +20,11 @@ using ::testing::ElementsAre; TEST_F(SparkIntegrationTest, ReadingJson) { - auto df = spark->read().json("compose/datasets/people.json"); + auto df = spark->read().json("datasets/people.json"); // ------------------------------------------------------------------------------------ // Using the fully qualified class name (JSON is usually built-in) // auto df = - // spark->read().format("org.apache.spark.sql.execution.datasources.json.JsonFileFormat").load({"compose/datasets/people.json"}); + // spark->read().format("org.apache.spark.sql.execution.datasources.json.JsonFileFormat").load({"datasets/people.json"}); // ------------------------------------------------------------------------------------ df.show(); } @@ -34,14 +34,14 @@ TEST_F(SparkIntegrationTest, ReadJsonWithOptions) auto df = spark->read() .option("header", "true") .option("inferSchema", "true") - .json("compose/datasets/people.json"); + .json("datasets/people.json"); EXPECT_NO_THROW(df.show(5)); } TEST_F(SparkIntegrationTest, ReadingCsv) { - auto df = spark->read().csv("compose/datasets/people.csv"); + auto df = spark->read().csv("datasets/people.csv"); EXPECT_NO_THROW(df.show(163)); } @@ -50,14 +50,14 @@ TEST_F(SparkIntegrationTest, ReadCsvWithOptions) auto df = spark->read() .option("header", "true") .option("inferSchema", "true") - .csv("compose/datasets/people.csv"); + .csv("datasets/people.csv"); EXPECT_NO_THROW(df.show()); } TEST_F(SparkIntegrationTest, ReadTextFile) { - auto df = spark->read().text("compose/datasets/people.txt"); + auto df = spark->read().text("datasets/people.txt"); EXPECT_NO_THROW(df.show()); } @@ -67,7 +67,7 @@ TEST_F(SparkIntegrationTest, ReadTextFileWithOptions) .option("wholetext", "false") .option("lineSep", "\n") .format("text") - .load({"compose/datasets/people.txt"}); + .load({"datasets/people.txt"}); // ------------------------------------------------------------------------------------------- // Reading text files is a straightforward way to ingest unstructured or @@ -78,7 +78,7 @@ TEST_F(SparkIntegrationTest, ReadTextFileWithOptions) // .option("wholetext", "true") // .option("lineSep", "\n") // .format("text") - // .load({"compose/datasets/people.txt"}); + // .load({"datasets/people.txt"}); // ------------------------------------------------------------------------------------------- EXPECT_NO_THROW(df.show()); @@ -86,7 +86,7 @@ TEST_F(SparkIntegrationTest, ReadTextFileWithOptions) TEST_F(SparkIntegrationTest, ReadParquet) { - auto df = spark->read().parquet("compose/datasets/flights.parquet"); + auto df = spark->read().parquet("datasets/flights.parquet"); EXPECT_NO_THROW(df.show(1000)); } @@ -99,7 +99,9 @@ TEST_F(SparkIntegrationTest, ReadParquetWithOptions) // results in expensive operations "datetimeRebaseMode" - Handles legacy // dates from older Spark versions // ---------------------------------------------------------------------------- - auto df = spark->read().option("mergeSchema", "true").parquet("compose/datasets/flights.parquet"); + auto df = spark->read() + .option("mergeSchema", "true") + .parquet("datasets/flights.parquet"); EXPECT_NO_THROW(df.show(10)); } @@ -108,7 +110,7 @@ TEST_F(SparkIntegrationTest, ReadWithDdlSchema) { auto df = spark->read() .schema("name STRING, age INT, salary INT") - .csv("compose/datasets/people_no_header.csv"); + .csv("datasets/people_no_header.csv"); EXPECT_NO_THROW(df.show(4)); } @@ -116,7 +118,7 @@ TEST_F(SparkIntegrationTest, ReadWithDdlSchema) TEST_F(SparkIntegrationTest, SchemaEvaluation) { std::string ddl = "name STRING, age INT, salary INT"; - auto df = spark->read().schema(ddl).csv("compose/datasets/people_no_header.csv"); + auto df = spark->read().schema(ddl).csv("datasets/people_no_header.csv"); auto schema = df.schema(); @@ -128,17 +130,17 @@ TEST_F(SparkIntegrationTest, SchemaEvaluation) // ------------------------------------------------------- // Check if the variant holds the correct type // ------------------------------------------------------- - EXPECT_TRUE( - std::holds_alternative(schema.fields[0].data_type->kind)); - EXPECT_TRUE( - std::holds_alternative(schema.fields[1].data_type->kind)); - EXPECT_TRUE( - std::holds_alternative(schema.fields[2].data_type->kind)); + EXPECT_TRUE(std::holds_alternative( + schema.fields[0].data_type->kind)); + EXPECT_TRUE(std::holds_alternative( + schema.fields[1].data_type->kind)); + EXPECT_TRUE(std::holds_alternative( + schema.fields[2].data_type->kind)); } TEST_F(SparkIntegrationTest, HandleMissingFile) { - auto df = spark->read().csv("compose/datasets/non_existent.csv"); + auto df = spark->read().csv("datasets/non_existent.csv"); // ----------------------------------------------------------------------------- // Show triggers an ExecutePlan gRPC call which should fail in this @@ -152,13 +154,14 @@ TEST_F(SparkIntegrationTest, OptionEvaluation) // ------------------------------------------------------------------------- // Read without header option (should have column name "value" or "_c0") // ------------------------------------------------------------------------- - auto df1 = spark->read().csv("compose/datasets/people.csv"); + auto df1 = spark->read().csv("datasets/people.csv"); df1.show(2); // ----------------------------------- // Read with header option // ----------------------------------- - auto df2 = spark->read().option("header", "true").csv("compose/datasets/people.csv"); + auto df2 = + spark->read().option("header", "true").csv("datasets/people.csv"); df2.show(2); // -------------------------------------- @@ -211,7 +214,8 @@ TEST_F(SparkIntegrationTest, ReaderOptions_VerifySchema) std::map config = { {"header", "true"}, {"inferSchema", "true"}, {"delimiter", ","}}; - auto df = spark->read().options(config).format("csv").load({"compose/datasets/people.csv"}); + auto df = spark->read().options(config).format("csv").load( + {"datasets/people.csv"}); auto cols = df.columns(); @@ -225,30 +229,34 @@ TEST_F(SparkIntegrationTest, ReaderOptions_VerifySchema) TEST_F(SparkIntegrationTest, ReaderOptions_InferredTypes) { - std::map config = {{"header", "true"}, {"inferSchema", "true"}}; + std::map config = {{"header", "true"}, + {"inferSchema", "true"}}; - auto df = spark->read().options(config).format("csv").load({"compose/datasets/people.csv"}); + auto df = spark->read().options(config).format("csv").load( + {"datasets/people.csv"}); auto schema = df.schema(); // ---------------------------------------------------- // Find the 'age' field and verify it's an Integer // ---------------------------------------------------- auto it = std::find_if(schema.fields.begin(), schema.fields.end(), - [](const auto& f) { return f.name == "age"; }); + [](auto const& f) { return f.name == "age"; }); ASSERT_NE(it, schema.fields.end()) << "Column 'age' not found"; // ------------------------------------------------- // Check if the variant holds IntegerType // ------------------------------------------------- - EXPECT_TRUE(std::holds_alternative(it->data_type->kind)); + EXPECT_TRUE(std::holds_alternative( + it->data_type->kind)); } TEST_F(SparkIntegrationTest, ReaderOptions_CorrectValueParsing) { std::map config = { {"header", "true"}, {"delimiter", ","}, {"inferSchema", "true"}}; - auto df = spark->read().options(config).format("csv").load({"compose/datasets/people.csv"}); + auto df = spark->read().options(config).format("csv").load( + {"datasets/people.csv"}); auto first_row = df.head(); ASSERT_TRUE(first_row.has_value()); @@ -262,7 +270,7 @@ TEST_F(SparkIntegrationTest, ReaderOptions_CorrectValueParsing) TEST_F(SparkIntegrationTest, ReadOrc) { - auto df = spark->read().orc("compose/datasets/types.orc"); + auto df = spark->read().orc("datasets/types.orc"); EXPECT_NO_THROW(df.show(10)); } @@ -298,8 +306,10 @@ TEST_F(SparkIntegrationTest, ReadFromDelta) .format("delta") .save("output/delta_people"); - auto read_df = - spark->read().option("inferSchema", "true").format("delta").load({"output/delta_people"}); + auto read_df = spark->read() + .option("inferSchema", "true") + .format("delta") + .load({"output/delta_people"}); EXPECT_EQ(read_df.count(), df.count()); EXPECT_THAT(read_df.columns(), ElementsAre("id", "name", "age")); diff --git a/tests/spark/dataframe_writer.cpp b/tests/spark/dataframe_writer.cpp index a8170cf..a6b21d0 100644 --- a/tests/spark/dataframe_writer.cpp +++ b/tests/spark/dataframe_writer.cpp @@ -1,16 +1,15 @@ -#include -#include -#include - -#include -#include - #include "config.h" #include "dataframe.h" #include "session.h" #include "spark_fixture.h" #include "writer.h" +#include +#include +#include +#include +#include + using ::testing::ElementsAre; TEST_F(SparkIntegrationTest, ParquetWrite) @@ -40,22 +39,27 @@ TEST_F(SparkIntegrationTest, ParquetReadAndWrite) df.write() .mode("overwrite") - .option("compression", - "snappy") // snappy is faster when compared to gzip. See: - // https://parquet.apache.org/docs/file-format/data-pages/compression/ + .option( + "compression", + "snappy") // snappy is faster when compared to gzip. See: + // https://parquet.apache.org/docs/file-format/data-pages/compression/ .parquet("output/iot_intrusion_data"); - auto iot_data_columns = spark->read().parquet("output/iot_intrusion_data").columns(); - - EXPECT_THAT(iot_data_columns, - ElementsAre("flow_duration", "Header_Length", "Protocol Type", "Duration", "Rate", - "Srate", "Drate", "fin_flag_number", "syn_flag_number", - "rst_flag_number", "psh_flag_number", "ack_flag_number", - "ece_flag_number", "cwr_flag_number", "ack_count", "syn_count", - "fin_count", "urg_count", "rst_count", "HTTP", "HTTPS", "DNS", "Telnet", - "SMTP", "SSH", "IRC", "TCP", "UDP", "DHCP", "ARP", "ICMP", "IPv", "LLC", - "Tot sum", "Min", "Max", "AVG", "Std", "Tot size", "IAT", "Number", - "Magnitue", "Radius", "Covariance", "Variance", "Weight", "label")); + auto iot_data_columns = + spark->read().parquet("output/iot_intrusion_data").columns(); + + EXPECT_THAT( + iot_data_columns, + ElementsAre("flow_duration", "Header_Length", "Protocol Type", + "Duration", "Rate", "Srate", "Drate", "fin_flag_number", + "syn_flag_number", "rst_flag_number", "psh_flag_number", + "ack_flag_number", "ece_flag_number", "cwr_flag_number", + "ack_count", "syn_count", "fin_count", "urg_count", + "rst_count", "HTTP", "HTTPS", "DNS", "Telnet", "SMTP", + "SSH", "IRC", "TCP", "UDP", "DHCP", "ARP", "ICMP", "IPv", + "LLC", "Tot sum", "Min", "Max", "AVG", "Std", "Tot size", + "IAT", "Number", "Magnitue", "Radius", "Covariance", + "Variance", "Weight", "label")); } TEST_F(SparkIntegrationTest, IOTSecurityAnalysisScenario) @@ -71,7 +75,8 @@ TEST_F(SparkIntegrationTest, IOTSecurityAnalysisScenario) // report. // ----------------------------------------------------------------------------------- auto analysis_df = - df.filter("Rate > 0.2").select({"label", "Rate", "Protocol Type", "flow_duration", "IAT"}); + df.filter("Rate > 0.2") + .select({"label", "Rate", "Protocol Type", "flow_duration", "IAT"}); analysis_df.write() .mode("overwrite") @@ -84,13 +89,15 @@ TEST_F(SparkIntegrationTest, IOTSecurityAnalysisScenario) // Validate that filter and projection were preserved // -------------------------------------------------------- auto result_cols = result_df.columns(); - EXPECT_THAT(result_cols, ElementsAre("label", "Rate", "Protocol Type", "flow_duration", "IAT")); + EXPECT_THAT(result_cols, ElementsAre("label", "Rate", "Protocol Type", + "flow_duration", "IAT")); // --------------------------------------------------------------- // Get count of the detected incidents in filtered result // --------------------------------------------------------------- int64_t incident_count = result_df.count(); - std::cout << "Detected " << incident_count << " high-rate traffic incidents." << std::endl; + std::cout << "Detected " << incident_count + << " high-rate traffic incidents." << std::endl; result_df.show(5); @@ -108,7 +115,10 @@ TEST_F(SparkIntegrationTest, CsvWriteWithOptions) .option("delimiter", ";") .csv(csv_path); - auto read_df = spark->read().option("header", "true").option("delimiter", ";").csv(csv_path); + auto read_df = spark->read() + .option("header", "true") + .option("delimiter", ";") + .csv(csv_path); EXPECT_EQ(read_df.count(), 50); EXPECT_THAT(read_df.columns(), ElementsAre("id")); @@ -116,7 +126,8 @@ TEST_F(SparkIntegrationTest, CsvWriteWithOptions) TEST_F(SparkIntegrationTest, WriteToText) { - auto write_df = spark->sql("SELECT 'a' AS alphabets UNION ALL SELECT 'b' AS alphabets"); + auto write_df = + spark->sql("SELECT 'a' AS alphabets UNION ALL SELECT 'b' AS alphabets"); write_df.write().mode("overwrite").text("output/text_write"); @@ -164,7 +175,8 @@ TEST_F(SparkIntegrationTest, MultiColumnPartitioning) // Verify columns exist (Partition columns move to the end in Spark) // ---------------------------------------------------------------------- auto cols = read_df.columns(); - bool has_protocol = std::find(cols.begin(), cols.end(), "Protocol Type") != cols.end(); + bool has_protocol = + std::find(cols.begin(), cols.end(), "Protocol Type") != cols.end(); bool has_label = std::find(cols.begin(), cols.end(), "label") != cols.end(); EXPECT_TRUE(has_protocol); @@ -201,7 +213,8 @@ TEST_F(SparkIntegrationTest, JsonReadWrite) auto read_df = spark->read().format("json").load({"output/test_json"}); EXPECT_EQ(read_df.count(), 1); - EXPECT_THAT(read_df.columns(), ::testing::UnorderedElementsAre("id", "name", "nested")); + EXPECT_THAT(read_df.columns(), + ::testing::UnorderedElementsAre("id", "name", "nested")); } TEST_F(SparkIntegrationTest, PartitionedWriteToParquet) @@ -248,8 +261,10 @@ TEST_F(SparkIntegrationTest, WriteToDelta) .format("delta") .save("output/delta_people"); - auto read_df = - spark->read().option("inferSchema", "true").format("delta").load({"output/delta_people"}); + auto read_df = spark->read() + .option("inferSchema", "true") + .format("delta") + .load({"output/delta_people"}); EXPECT_EQ(read_df.count(), df.count()); EXPECT_THAT(read_df.columns(), ElementsAre("id", "name", "age")); diff --git a/tests/spark/expressions.cpp b/tests/spark/expressions.cpp index a3ace48..5d91be2 100644 --- a/tests/spark/expressions.cpp +++ b/tests/spark/expressions.cpp @@ -4,6 +4,7 @@ #include "session.h" #include "spark_fixture.h" #include "types.h" + #include #include #include @@ -35,8 +36,8 @@ TEST_F(SparkIntegrationTest, ColumnExpressions) // We alias it to "age_plus_one" so we can find it in the resulting Row // ----------------------------------------------------------------------- - auto filtered_df = - df.filter(col("name") == lit("Alice")).select({col("name"), _col.alias("age_plus_one")}); + auto filtered_df = df.filter(col("name") == lit("Alice")) + .select({col("name"), _col.alias("age_plus_one")}); filtered_df.show(); diff --git a/tests/spark/graphframes.cpp b/tests/spark/graphframes.cpp index a407b9a..6bef695 100644 --- a/tests/spark/graphframes.cpp +++ b/tests/spark/graphframes.cpp @@ -1,16 +1,17 @@ -#include -#include - #include "dataframe.h" #include "functions.h" #include "graphframe.h" #include "session.h" +#include +#include + using namespace graphframes; using namespace spark::sql::functions; using ::testing::Contains; -static std::vector gfColumns(DataFrame df) +static std::vector +gfColumns(DataFrame df) { auto rows = df.collect(); if (rows.empty()) @@ -18,12 +19,15 @@ static std::vector gfColumns(DataFrame df) return rows[0].column_names; } -static int64_t gfCount(DataFrame df) +static int64_t +gfCount(DataFrame df) { return static_cast(df.collect().size()); } -template static std::vector gfColumn(DataFrame df, const std::string& col) +template +static std::vector +gfColumn(DataFrame df, std::string const& col) { auto rows = df.collect(); std::vector out; @@ -61,7 +65,8 @@ class SparkGraphFramesIntegrationTest : public ::testing::Test static DataFrame* vertices; static DataFrame* edges; - static void SetUpTestSuite() + static void + SetUpTestSuite() { spark = &SparkSession::builder() .master("sc://localhost") @@ -87,14 +92,16 @@ class SparkGraphFramesIntegrationTest : public ::testing::Test )")); } - static void TearDownTestSuite() + static void + TearDownTestSuite() { delete vertices; delete edges; spark->stop(); } - GraphFrame gf() const + GraphFrame + gf() const { return GraphFrame(*vertices, *edges); } @@ -125,7 +132,8 @@ TEST_F(SparkGraphFramesIntegrationTest, PageRankAllValuesPositive) EXPECT_GT(r, 0.0); } -TEST_F(SparkGraphFramesIntegrationTest, PageRankThrowsWhenBothMaxIterAndTolProvided) +TEST_F(SparkGraphFramesIntegrationTest, + PageRankThrowsWhenBothMaxIterAndTolProvided) { EXPECT_THROW(gf().pageRank(0.15, 5, 0.01), std::invalid_argument); } @@ -155,7 +163,8 @@ TEST_F(SparkGraphFramesIntegrationTest, FindTriangleReturnsThreeRotations) // The single triangle 1 -> 2 -> 3 -> 1 produces 3 rows: // (a=1,b=2,c=3), (a=2,b=3,c=1), (a=3,b=1,c=2) // ---------------------------------------------------------------- - EXPECT_EQ(gfCount(gf().find("(a)-[e1]->(b); (b)-[e2]->(c); (c)-[e3]->(a)")), 3); + EXPECT_EQ(gfCount(gf().find("(a)-[e1]->(b); (b)-[e2]->(c); (c)-[e3]->(a)")), + 3); } TEST_F(SparkGraphFramesIntegrationTest, FindTwoHopReturnsResults) @@ -207,7 +216,8 @@ TEST_F(SparkGraphFramesIntegrationTest, FilterEdgesSQLKeepsFriendEdges) TEST_F(SparkGraphFramesIntegrationTest, FilterEdgesColumnKeepsFriendEdges) { - EXPECT_EQ(gfCount(gf().filterEdges(col("relationship") == lit("friend"))), 2); + EXPECT_EQ(gfCount(gf().filterEdges(col("relationship") == lit("friend"))), + 2); } TEST_F(SparkGraphFramesIntegrationTest, FilterEdgesNoMatchReturnsEmpty) @@ -249,7 +259,8 @@ TEST_F(SparkGraphFramesIntegrationTest, FilterVerticesShow) // -------------------------------------------------------------------------- // dropIsolatedVertices() // -------------------------------------------------------------------------- -TEST_F(SparkGraphFramesIntegrationTest, DropIsolatedVerticesKeepsAllWhenNoneIsolated) +TEST_F(SparkGraphFramesIntegrationTest, + DropIsolatedVerticesKeepsAllWhenNoneIsolated) { EXPECT_EQ(gfCount(gf().dropIsolatedVertices()), 4); } @@ -265,7 +276,8 @@ TEST_F(SparkGraphFramesIntegrationTest, DropIsolatedVerticesRemovesIsolatedNode) (99, 'Ghost', 99) AS people(id, name, age) )"); - EXPECT_EQ(gfCount(GraphFrame(v_with_isolated, *edges).dropIsolatedVertices()), 4); + EXPECT_EQ( + gfCount(GraphFrame(v_with_isolated, *edges).dropIsolatedVertices()), 4); } TEST_F(SparkGraphFramesIntegrationTest, DropIsolatedVerticesShow) @@ -291,7 +303,8 @@ TEST_F(SparkGraphFramesIntegrationTest, BFSNoPathInDirectedGraph) TEST_F(SparkGraphFramesIntegrationTest, BFSWithEdgeFilter) { - EXPECT_GT(gfCount(gf().bfs("id = 1", "id = 2", "relationship = 'friend'")), 0); + EXPECT_GT(gfCount(gf().bfs("id = 1", "id = 2", "relationship = 'friend'")), + 0); } TEST_F(SparkGraphFramesIntegrationTest, BFSWithColumnExpression) @@ -310,7 +323,8 @@ TEST_F(SparkGraphFramesIntegrationTest, BFSShow) // We set use_local_checkpoints=true so no checkpoint directory // needs to be configured on the Spark server. // -------------------------------------------------------------------------- -TEST_F(SparkGraphFramesIntegrationTest, ConnectedComponentsReturnsOneRowPerVertex) +TEST_F(SparkGraphFramesIntegrationTest, + ConnectedComponentsReturnsOneRowPerVertex) { EXPECT_EQ(gfCount(gf().connectedComponents()), 4); } @@ -342,7 +356,8 @@ TEST_F(SparkGraphFramesIntegrationTest, SCCReturnsOneRowPerVertex) TEST_F(SparkGraphFramesIntegrationTest, SCCHasComponentColumn) { - EXPECT_THAT(gfColumns(gf().stronglyConnectedComponents()), Contains("component")); + EXPECT_THAT(gfColumns(gf().stronglyConnectedComponents()), + Contains("component")); } TEST_F(SparkGraphFramesIntegrationTest, SCCProducesMultipleComponents) @@ -351,7 +366,8 @@ TEST_F(SparkGraphFramesIntegrationTest, SCCProducesMultipleComponents) // Triangle 1 - 2 - 3 is one SCC // vertex 4 (leaf) is its own SCC // ------------------------------------------------------------- - auto comps = gfColumn(gf().stronglyConnectedComponents(), "component"); + auto comps = + gfColumn(gf().stronglyConnectedComponents(), "component"); std::set unique(comps.begin(), comps.end()); EXPECT_GT(unique.size(), 1u); } @@ -371,7 +387,8 @@ TEST_F(SparkGraphFramesIntegrationTest, ShortestPathsReturnsOneRowPerVertex) TEST_F(SparkGraphFramesIntegrationTest, ShortestPathsHasDistancesColumn) { - EXPECT_THAT(gfColumns(gf().shortestPaths(std::vector{1})), Contains("distances")); + EXPECT_THAT(gfColumns(gf().shortestPaths(std::vector{1})), + Contains("distances")); } TEST_F(SparkGraphFramesIntegrationTest, ShortestPathsShow) diff --git a/tests/spark/risk_management.cpp b/tests/spark/risk_management.cpp index dc35879..cb8289e 100644 --- a/tests/spark/risk_management.cpp +++ b/tests/spark/risk_management.cpp @@ -1,9 +1,3 @@ -#include -#include - -#include -#include - #include "config.h" #include "dataframe.h" #include "functions.h" @@ -11,6 +5,11 @@ #include "session.h" #include "types.h" +#include +#include +#include +#include + using namespace graphframes; using namespace spark::sql::functions; using namespace spark::sql::types; @@ -54,7 +53,8 @@ TEST_F(SparkIntegrationTest, FraudRing_ConnectedComponentsGroupsRingAndMule) AS t(src, dst, amount) )"); - auto components = GraphFrame(accounts, transfers).connectedComponents().collect(); + auto components = + GraphFrame(accounts, transfers).connectedComponents().collect(); ASSERT_EQ(components.size(), 5u); @@ -97,7 +97,9 @@ TEST_F(SparkIntegrationTest, FraudRing_SCCIsolatesMuleFromRing) AS t(src, dst) )"); - auto components = GraphFrame(accounts, transfers).stronglyConnectedComponents(10).collect(); + auto components = GraphFrame(accounts, transfers) + .stronglyConnectedComponents(10) + .collect(); std::map scc_by_id; for (auto& row : components) @@ -177,7 +179,9 @@ TEST_F(SparkIntegrationTest, Sanctions_BankWithinOneHopOfFlaggedEntity) AS t(src, dst) )"); - auto sp = GraphFrame(banks, wires).shortestPaths(std::vector{9}).collect(); + auto sp = GraphFrame(banks, wires) + .shortestPaths(std::vector{9}) + .collect(); // -------------------------------------------------------------------------------- // Scan a MapData for a given landmark, returning the hop count if @@ -190,21 +194,22 @@ TEST_F(SparkIntegrationTest, Sanctions_BankWithinOneHopOfFlaggedEntity) // same strategy Row::get uses — so the comparison and extraction are // type-agnostic. // -------------------------------------------------------------------------------- - auto hop_count = [](const Row& row, int64_t landmark) -> std::optional - { + auto hop_count = [](Row const& row, + int64_t landmark) -> std::optional { auto map_ptr = row.get>("distances"); if (!map_ptr) return std::nullopt; for (size_t i = 0; i < map_ptr->keys.size(); ++i) { int64_t key = std::visit( - [](auto&& arg) -> int64_t - { + [](auto&& arg) -> int64_t { using T = std::decay_t; - if constexpr (std::is_arithmetic_v && !std::is_same_v) + if constexpr (std::is_arithmetic_v && + !std::is_same_v) return static_cast(arg); - throw std::runtime_error("ShortestPaths: unexpected non-numeric " - "map key"); + throw std::runtime_error( + "ShortestPaths: unexpected non-numeric " + "map key"); }, map_ptr->keys[i]); @@ -212,13 +217,14 @@ TEST_F(SparkIntegrationTest, Sanctions_BankWithinOneHopOfFlaggedEntity) continue; return std::visit( - [](auto&& arg) -> int32_t - { + [](auto&& arg) -> int32_t { using T = std::decay_t; - if constexpr (std::is_arithmetic_v && !std::is_same_v) + if constexpr (std::is_arithmetic_v && + !std::is_same_v) return static_cast(arg); - throw std::runtime_error("ShortestPaths: unexpected non-numeric " - "map value"); + throw std::runtime_error( + "ShortestPaths: unexpected non-numeric " + "map value"); }, map_ptr->values[i]); } @@ -270,7 +276,9 @@ TEST_F(SparkIntegrationTest, Sanctions_IsolatedBankHasNoPathToFlaggedEntity) AS t(src, dst) )"); - auto sp = GraphFrame(banks, wires).shortestPaths(std::vector{9}).collect(); + auto sp = GraphFrame(banks, wires) + .shortestPaths(std::vector{9}) + .collect(); // -------------------------------------------------------------------------------- // Banks 1 and 2 have no directed path to bank 9. @@ -295,7 +303,8 @@ TEST_F(SparkIntegrationTest, Sanctions_IsolatedBankHasNoPathToFlaggedEntity) } } } - EXPECT_FALSE(has_path_to_9) << "Bank id=" << bank_id << " should have no path to bank 9"; + EXPECT_FALSE(has_path_to_9) + << "Bank id=" << bank_id << " should have no path to bank 9"; } } @@ -337,13 +346,15 @@ TEST_F(SparkIntegrationTest, KYC_BFSFindsOwnershipChainWithinDepthLimit) // -------------------------------------------------------------------------------- // HoldCo → SubCo A → OpCo: 2-hop path must exist. // -------------------------------------------------------------------------------- - auto path_to_opco = GraphFrame(entities, ownership).bfs("id = 1", "id = 3").collect(); + auto path_to_opco = + GraphFrame(entities, ownership).bfs("id = 1", "id = 3").collect(); EXPECT_GT(path_to_opco.size(), 0u); // -------------------------------------------------------------------------------- // HoldCo → SubCo A → SubCo B → ShellCo: 3-hop path must exist. // -------------------------------------------------------------------------------- - auto path_to_shell = GraphFrame(entities, ownership).bfs("id = 1", "id = 5").collect(); + auto path_to_shell = + GraphFrame(entities, ownership).bfs("id = 1", "id = 5").collect(); EXPECT_GT(path_to_shell.size(), 0u); } @@ -366,7 +377,8 @@ TEST_F(SparkIntegrationTest, KYC_BFSConfirmsUnrelatedEntityHasNoOwnershipPath) // -------------------------------------------------------------------------------- // TrustA (id=6) has no ownership link to HoldCo: BFS must return empty. // -------------------------------------------------------------------------------- - auto path = GraphFrame(entities, ownership).bfs("id = 1", "id = 6").collect(); + auto path = + GraphFrame(entities, ownership).bfs("id = 1", "id = 6").collect(); EXPECT_EQ(path.size(), 0u); } @@ -391,7 +403,8 @@ TEST_F(SparkIntegrationTest, KYC_DropIsolatedVerticesRemovesOrphanTrust) // dropIsolatedVertices must remove it, reducing the vertex count from 3 // to 2. // -------------------------------------------------------------------------------- - auto active = GraphFrame(entities, ownership).dropIsolatedVertices().collect(); + auto active = + GraphFrame(entities, ownership).dropIsolatedVertices().collect(); EXPECT_EQ(active.size(), 2u); std::set active_ids; @@ -475,7 +488,8 @@ TEST_F(SparkIntegrationTest, HFT_PageRankIdentifiesPriceLeaderVenues) // triangleCount() identifies which traders participated in at least one // closed information loop; those with count > 0 are escalated for review. // --------------------------------------------------------------------------------- -TEST_F(SparkIntegrationTest, InsiderTrading_TriangleCountFlagsInformationSharingRing) +TEST_F(SparkIntegrationTest, + InsiderTrading_TriangleCountFlagsInformationSharingRing) { auto traders = spark->sql(R"( SELECT CAST(id AS INT) AS id, name FROM VALUES @@ -554,7 +568,9 @@ TEST_F(SparkIntegrationTest, CreditRisk_SCCIdentifiesMutualExposureCluster) AS t(src, dst, notional) )"); - auto scc_rows = GraphFrame(counterparties, exposures).stronglyConnectedComponents(10).collect(); + auto scc_rows = GraphFrame(counterparties, exposures) + .stronglyConnectedComponents(10) + .collect(); std::map scc_by_id; for (auto& row : scc_rows) @@ -573,7 +589,8 @@ TEST_F(SparkIntegrationTest, CreditRisk_SCCIdentifiesMutualExposureCluster) EXPECT_NE(scc_by_id[4], scc_by_id[1]); } -TEST_F(SparkIntegrationTest, CreditRisk_BFSTracesContagionPathFromDefaultingEntity) +TEST_F(SparkIntegrationTest, + CreditRisk_BFSTracesContagionPathFromDefaultingEntity) { auto counterparties = spark->sql(R"( SELECT CAST(id AS INT) AS id, name FROM VALUES @@ -599,13 +616,15 @@ TEST_F(SparkIntegrationTest, CreditRisk_BFSTracesContagionPathFromDefaultingEnti // -------------------------------------------------------------------------------- // Contagion path from the defaulting bank to Bank B must exist. // -------------------------------------------------------------------------------- - auto path = GraphFrame(counterparties, exposures).bfs("id = 1", "id = 3").collect(); + auto path = + GraphFrame(counterparties, exposures).bfs("id = 1", "id = 3").collect(); EXPECT_GT(path.size(), 0u); // -------------------------------------------------------------------------------- // Insulated Bank (4) is not reachable from the defaulting entity. // -------------------------------------------------------------------------------- - auto no_path = GraphFrame(counterparties, exposures).bfs("id = 1", "id = 4").collect(); + auto no_path = + GraphFrame(counterparties, exposures).bfs("id = 1", "id = 4").collect(); EXPECT_EQ(no_path.size(), 0u); } @@ -647,7 +666,9 @@ TEST_F(SparkIntegrationTest, AML_MotifFindsTwoHopLayeringChains) AS t(src, dst, amount) )"); - auto chains = GraphFrame(accounts, transfers).find("(a)-[e1]->(b); (b)-[e2]->(c)").collect(); + auto chains = GraphFrame(accounts, transfers) + .find("(a)-[e1]->(b); (b)-[e2]->(c)") + .collect(); // -------------------------------------------------------------------------------- // Two distinct two-hop chains exist. @@ -689,7 +710,8 @@ TEST_F(SparkIntegrationTest, AML_FilterEdgesIsolatesHighValueTransfers) AS t(src, dst, amount) )"); - auto reportable = GraphFrame(accounts, transfers).filterEdges("amount > 50000").collect(); + auto reportable = + GraphFrame(accounts, transfers).filterEdges("amount > 50000").collect(); EXPECT_EQ(reportable.size(), 2u); } @@ -728,8 +750,9 @@ TEST_F(SparkIntegrationTest, WashTrading_SCCDetectsBidirectionalTradePairs) AS t(src, dst, ticker) )"); - auto scc_rows = - GraphFrame(trading_accounts, same_second_trades).stronglyConnectedComponents(5).collect(); + auto scc_rows = GraphFrame(trading_accounts, same_second_trades) + .stronglyConnectedComponents(5) + .collect(); std::map scc_by_id; for (auto& row : scc_rows) @@ -824,7 +847,8 @@ TEST_F(SparkIntegrationTest, Settlement_TripletsEnumerateAllObligations) } } -TEST_F(SparkIntegrationTest, Settlement_FilterEdgesRemovesSmallObligationsBelowNettingThreshold) +TEST_F(SparkIntegrationTest, + Settlement_FilterEdgesRemovesSmallObligationsBelowNettingThreshold) { auto participants = spark->sql(R"( SELECT CAST(id AS INT) AS id, name FROM VALUES @@ -848,8 +872,9 @@ TEST_F(SparkIntegrationTest, Settlement_FilterEdgesRemovesSmallObligationsBelowN // Only obligations at or above the 1 M threshold go to the netting // engine. // -------------------------------------------------------------------------------- - auto netting_eligible = - GraphFrame(participants, obligations).filterEdges("amount >= 1000000").collect(); + auto netting_eligible = GraphFrame(participants, obligations) + .filterEdges("amount >= 1000000") + .collect(); EXPECT_EQ(netting_eligible.size(), 2u); } \ No newline at end of file diff --git a/tests/spark/spark_fixture.cpp b/tests/spark/spark_fixture.cpp index 89fcf47..a26efd4 100644 --- a/tests/spark/spark_fixture.cpp +++ b/tests/spark/spark_fixture.cpp @@ -1,15 +1,20 @@ #include "spark_fixture.h" + #include "util/env_loader.h" SparkSession* SparkIntegrationTest::spark = nullptr; -void SparkIntegrationTest::SetUpTestSuite() +void +SparkIntegrationTest::SetUpTestSuite() { - spark = - &SparkSession::builder().master("localhost").appName("SparkConnectCppGTest").getOrCreate(); + spark = &SparkSession::builder() + .master("localhost") + .appName("SparkConnectCppGTest") + .getOrCreate(); } -void SparkIntegrationTest::TearDownTestSuite() +void +SparkIntegrationTest::TearDownTestSuite() { if (spark) { diff --git a/tests/util/env_loader.cpp b/tests/util/env_loader.cpp index bfba2d3..01eeb7b 100644 --- a/tests/util/env_loader.cpp +++ b/tests/util/env_loader.cpp @@ -1,9 +1,11 @@ #include "env_loader.h" + #include #include #include -static inline std::string trim(const std::string& s) +static inline std::string +trim(std::string const& s) { auto start = s.find_first_not_of(" \t\r\n"); auto end = s.find_last_not_of(" \t\r\n"); @@ -12,7 +14,8 @@ static inline std::string trim(const std::string& s) return s.substr(start, end - start + 1); } -void load_env(const std::string& path) +void +load_env(std::string const& path) { std::ifstream file(path);