Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions docs/API_REFERENCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -665,4 +665,39 @@ auto df_stats = df.describe({"age", "weight", "height"}).show();
| min | 11 | 37.8 | 142.2 |
| max | 13 | 44.1 | 150.5 |
+----------------------+----------------------+----------------------+----------------------+
```

### Summary

```cpp
auto df = spark->sql(
R"(
SELECT *
FROM
VALUES
("Bob", 13, 40.3, 150.5),
("Alice", 12, 37.8, 142.3),
("Tom", 11, 44.1, 142.2)
AS people(name, age, weight, height)
)"
);

df.summary().show();
```

**Output:**

```
+----------------------+----------------------+----------------------+----------------------+----------------------+
| summary | name | age | weight | height |
+----------------------+----------------------+----------------------+----------------------+----------------------+
| count | 3 | 3 | 3 | 3 |
| mean | null | 12.0 | 40.73333 | 145.00000 |
| stddev | null | 1.0 | 3.1722757341273704 | 4.763402145525822 |
| min | Alice | 11 | 37.8 | 142.2 |
| 25% | null | 11 | 37.8 | 142.2 |
| 50% | null | 12 | 40.3 | 142.3 |
| 75% | null | 13 | 44.1 | 150.5 |
| max | Tom | 13 | 44.1 | 150.5 |
+----------------------+----------------------+----------------------+----------------------+----------------------+
```
21 changes: 21 additions & 0 deletions src/dataframe.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -835,4 +835,25 @@ DataFrame DataFrame::describe(const std::vector<std::string> &cols)
}

return DataFrame(stub_, plan, session_id_, user_id_);
}

DataFrame DataFrame::summary(const std::vector<std::string> &statistics)
{
spark::connect::Plan plan;

auto *summary_rel = plan.mutable_root()->mutable_summary();

summary_rel->mutable_input()->CopyFrom(this->plan_.root());

for (const auto &stat : statistics)
{
summary_rel->add_statistics(stat);
}

return DataFrame(stub_, plan, session_id_, user_id_);
}

DataFrame DataFrame::summary()
{
return summary({});
}
33 changes: 33 additions & 0 deletions src/dataframe.h
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,39 @@ class DataFrame
*/
DataFrame describe(const std::vector<std::string> &cols);

/**
* @brief Computes specified statistics for numeric and string columns.
* Available statistics are:
* - count
*
* - mean
*
* - stddev
*
* - min
*
* - max
*
* - arbitrary approximate percentiles specified as a percentage (e.g., 75%)
*
* If no statistics are given, this function computes count, mean, stddev, min,
* approximate quartiles (percentiles at 25%, 50%, and 75%), and max.
*
* @note This function is meant for exploratory data analysis, as we make no
* guarantee about the backward compatibility of the schema of
* the resulting `DataFrame`.
*
* @param statistics Column names to calculate statistics by (default All columns).
* @returns A new `DataFrame` that provides statistics for the given `DataFrame`.
*/
DataFrame summary(const std::vector<std::string> &statistics);

/**
* @brief Overload for empty summary() which computes the default
* set of statistics (count, mean, stddev, min, 25%, 50%, 75%, max).
*/
DataFrame summary();

private:
std::shared_ptr<spark::connect::SparkConnectService::Stub> stub_;
spark::connect::Plan plan_;
Expand Down
53 changes: 46 additions & 7 deletions src/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,43 @@ namespace spark::sql::types
* @brief Strict access. This fails if the type doesn't match exactly.
*/
template <typename T>
const T &get(const std::string &name) const
T get(const std::string &name) const
{
return std::get<T>(values.at(col_index(name)));
const auto &val = values.at(col_index(name));

return std::visit([](auto &&arg) -> T
{
using ArgType = std::decay_t<decltype(arg)>;

// ---------------------------------
// Handle exact Type Match
// ---------------------------------
if constexpr (std::is_same_v<T, ArgType>) {
return arg;
}

// -------------------------------------------------
// Null Handling
// std::monostate to "null"
// -------------------------------------------------
else if constexpr (std::is_same_v<T, std::string> && std::is_same_v<ArgType, std::monostate>) {
return "null";
}

// --------------------------------------------------------------------
// Widening / Numeric Conversion
// This supports get<double> on int32_t columns
// --------------------------------------------------------------------
else if constexpr (std::is_arithmetic_v<T> && std::is_arithmetic_v<ArgType>) {
return static_cast<T>(arg);
}

// -----------------------------------------------
// Failure State
// -----------------------------------------------
else {
throw std::runtime_error("Row::get Type Mismatch: requested type does not match variant state.");
} }, val);
}

/**
Expand All @@ -211,13 +245,18 @@ namespace spark::sql::types
*/
int64_t get_long(const std::string &name) const
{
const auto &val = (*this)[name];
if (std::holds_alternative<std::monostate>(val))
{
throw std::runtime_error("Column " + name + " is null");
}
return std::visit([](auto &&arg) -> int64_t
{
using T = std::decay_t<decltype(arg)>;
if constexpr (std::is_integral_v<T> && !std::is_same_v<T, bool>) {
return static_cast<int64_t>(arg);
}
throw std::runtime_error("Column is not a numeric integral type"); }, (*this)[name]);
using T = std::decay_t<decltype(arg)>;
if constexpr (std::is_integral_v<T> && !std::is_same_v<T, bool>) {
return static_cast<int64_t>(arg);
}
throw std::runtime_error("Column is not a numeric integral type"); }, val);
}

/**
Expand Down
66 changes: 66 additions & 0 deletions tests/dataframe.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,72 @@ TEST_F(SparkIntegrationTest, DataFrameDescribe)

EXPECT_EQ(std::get<std::string>(df_stats[0]["summary"]), "count");
EXPECT_EQ(std::get<std::string>(df_stats[1]["summary"]), "mean");
EXPECT_EQ(std::get<std::string>(df_stats[2]["summary"]), "stddev");
EXPECT_EQ(std::get<std::string>(df_stats[3]["summary"]), "min");
EXPECT_EQ(std::get<std::string>(df_stats[4]["summary"]), "max");

EXPECT_EQ(std::get<std::string>(df_stats[0]["age"]), "3");
EXPECT_EQ(std::get<std::string>(df_stats[1]["age"]), "12.0");
EXPECT_EQ(std::get<std::string>(df_stats[2]["age"]), "1.0");
EXPECT_EQ(std::get<std::string>(df_stats[3]["age"]), "11");
EXPECT_EQ(std::get<std::string>(df_stats[4]["age"]), "13");

EXPECT_EQ(std::get<std::string>(df_stats[0]["weight"]), "3");
EXPECT_EQ(std::get<std::string>(df_stats[1]["weight"]), "40.73333");
EXPECT_EQ(std::get<std::string>(df_stats[2]["weight"]), "3.1722757341273704");
EXPECT_EQ(std::get<std::string>(df_stats[3]["weight"]), "37.8");
EXPECT_EQ(std::get<std::string>(df_stats[4]["weight"]), "44.1");

EXPECT_EQ(std::get<std::string>(df_stats[0]["height"]), "3");
EXPECT_EQ(std::get<std::string>(df_stats[1]["height"]), "145.00000");
EXPECT_EQ(std::get<std::string>(df_stats[2]["height"]), "4.763402145525822");
EXPECT_EQ(std::get<std::string>(df_stats[3]["height"]), "142.2");
EXPECT_EQ(std::get<std::string>(df_stats[4]["height"]), "150.5");
}

TEST_F(SparkIntegrationTest, DataFrameSummary)
{
auto df = spark->sql(
R"(
SELECT *
FROM
VALUES
("Bob", 13, 40.3, 150.5),
("Alice", 12, 37.8, 142.3),
("Tom", 11, 44.1, 142.2)
AS people(name, age, weight, height)
)");

auto df_stats = df.summary().collect();

// ----------------------------------------------------------------------
// ROWS returned:
//
// Row(summary='count', name='3', age='3', weight='3', height='3')
// Row(summary='mean', name=null, age='12.0', weight='40.73333', height='145.00000')
// Row(summary='stddev', name=null, age='1.0', weight='3.1722757341273704', height='4.763402145525822')
// Row(summary='min', name='Alice', age='11', weight='37.8', height='142.2')
// Row(summary='25%', name=null, age='11', weight='37.8', height='142.2')
// Row(summary='50%', name=null, age='12', weight='40.3', height='142.3')
// Row(summary='75%', name=null, age='13', weight='44.1', height='150.5')
// Row(summary='max', name='Tom', age='13', weight='44.1', height='150.5')
// ----------------------------------------------------------------------

EXPECT_EQ(df_stats[0].get<std::string>("summary"), "count");
EXPECT_EQ(df_stats[1].get<std::string>("summary"), "mean");
EXPECT_EQ(df_stats[2].get<std::string>("summary"), "stddev");
EXPECT_EQ(df_stats[3].get<std::string>("summary"), "min");
EXPECT_EQ(df_stats[4].get<std::string>("summary"), "25%");
EXPECT_EQ(df_stats[5].get<std::string>("summary"), "50%");
EXPECT_EQ(df_stats[6].get<std::string>("summary"), "75%");
EXPECT_EQ(df_stats[7].get<std::string>("summary"), "max");

EXPECT_EQ(df_stats[0].get<std::string>("name"), "3");
EXPECT_EQ(df_stats[1].get<std::string>("name"), "null");
EXPECT_EQ(df_stats[2].get<std::string>("name"), "null");
EXPECT_EQ(df_stats[3].get<std::string>("name"), "Alice");
EXPECT_EQ(df_stats[4].get<std::string>("name"), "null");
EXPECT_EQ(df_stats[5].get<std::string>("name"), "null");
EXPECT_EQ(df_stats[6].get<std::string>("name"), "null");
EXPECT_EQ(df_stats[7].get<std::string>("name"), "Tom");
}