Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,13 @@ endif ()

add_custom_target(build_all_plugins)

# Declared here so add_compile_definitions reaches all sibling nes-* targets.
# nes-plugins/CMakeLists.txt re-declares the same option (no-op when cached).
option(NES_ENABLE_MEOS "Enable MEOS plugin (requires libmeos installed on the system)" ON)
if(NES_ENABLE_MEOS)
add_compile_definitions(NES_ENABLE_MEOS)
endif()

# Add target for common lib, which contains a minimal set
# of shared functionality used by all components of nes
file(GLOB NES_DIRECTORIES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "nes-*")
Expand Down
4 changes: 4 additions & 0 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@
tbb
python3
openjdk21
paho-mqtt-c
paho-mqtt-cpp
]) ++ [ follyPkg antlr4Pkg ];

antlr4Jar = pkgs.fetchurl {
Expand Down Expand Up @@ -244,6 +246,7 @@
"-DNES_ENABLES_TESTS=ON"
"-DCMAKE_MODULE_PATH=${libdwarfModule}/share/cmake/Modules"
"-DANTLR4_JAR_LOCATION=${antlr4Jar}"
"-DNES_ENABLE_MEOS=OFF"
];

enableParallelBuilding = true;
Expand Down Expand Up @@ -347,6 +350,7 @@
"-DLLVM_DIR=${commonCmakeEnv.LLVM_DIR}"
"-DANTLR4_JAR_LOCATION=${antlr4Jar}"
"-DCMAKE_MODULE_PATH=${libdwarfModule}/share/cmake/Modules"
"-DNES_ENABLE_MEOS=OFF"
];
shellHook = ''
unset NES_PREBUILT_VCPKG_ROOT
Expand Down
6 changes: 5 additions & 1 deletion nes-physical-operators/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,11 @@ get_source(nes-physical-operators NES_PHYSICAL_OPERATORS_SOURCE_FILES)

# Add Library
add_library(nes-physical-operators ${NES_PHYSICAL_OPERATORS_SOURCE_FILES})
target_link_libraries(nes-physical-operators PUBLIC nes-sources nes-sinks nes-nautilus nes-meos)
if(NES_ENABLE_MEOS)
target_link_libraries(nes-physical-operators PUBLIC nes-sources nes-sinks nes-nautilus nes-meos)
else()
target_link_libraries(nes-physical-operators PUBLIC nes-sources nes-sinks nes-nautilus)
endif()
target_include_directories(nes-physical-operators PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<INSTALL_INTERFACE:include/nebulastream/>)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

if(NES_ENABLE_MEOS)
add_plugin(TemporalSequence AggregationPhysicalFunction nes-physical-operators TemporalSequenceAggregationPhysicalFunction.cpp)
add_plugin(Var AggregationPhysicalFunction nes-physical-operators VarAggregationFunction.cpp)
endif()
2 changes: 2 additions & 0 deletions nes-physical-operators/src/Functions/Meos/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.

if(NES_ENABLE_MEOS)
add_plugin(TemporalIntersects PhysicalFunction nes-physical-operators TemporalIntersectsPhysicalFunction.cpp)
add_plugin(TemporalIntersectsGeometry PhysicalFunction nes-physical-operators TemporalIntersectsGeometryPhysicalFunction.cpp)
add_plugin(TemporalAIntersectsGeometry PhysicalFunction nes-physical-operators TemporalAIntersectsGeometryPhysicalFunction.cpp)
add_plugin(TemporalEContainsGeometry PhysicalFunction nes-physical-operators TemporalEContainsGeometryPhysicalFunction.cpp)
add_plugin(TemporalEDWithinGeometry PhysicalFunction nes-physical-operators TemporalEDWithinGeometryPhysicalFunction.cpp)
add_plugin(TemporalAtStBox PhysicalFunction nes-physical-operators TemporalAtStBoxPhysicalFunction.cpp)
endif()
55 changes: 29 additions & 26 deletions nes-plugins/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,33 +20,36 @@ activate_optional_plugin("Sources/TCPSource" ON)
# Enable the Generator source plugin; required by systests and repl tests
activate_optional_plugin("Sources/GeneratorSource" ON)
activate_optional_plugin("Sinks/VoidSink" ON)
activate_optional_plugin("Sources/MQTTSource" ON)
activate_optional_plugin("Sinks/MQTTSink" ON)

# MEOS is a dependency
activate_optional_plugin("MEOS" ON)
message(STATUS "Enable MEOS Plugin")

# Detect the platform
if (APPLE)
message(STATUS "Building on macOS")
# Set the include and library directories for Homebrew (macOS)
set(MEOS_INCLUDE_DIR "/opt/homebrew/include" CACHE PATH "Path to MEOS include directory")
set(MEOS_PLUGIN_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include" CACHE PATH "Path to MEOS plugin include directory")
include_directories(SYSTEM ${MEOS_INCLUDE_DIR} ${MEOS_PLUGIN_INCLUDE_DIR})
include_directories(SYSTEM ${MEOS_INCLUDE_DIR} ${MEOS_PLUGIN_INCLUDE_DIR})
link_directories(${MEOS_LIBRARY_DIR} )

else()
message(STATUS "Building on Linux")

# Default behavior for Linux
find_package(meos REQUIRED)
if(meos_FOUND)
message(STATUS "MEOS found: include=${meos_INCLUDE_DIR}, library=${meos}")
include_directories(SYSTEM ${meos_INCLUDE_DIR})
option(NES_ENABLE_MQTT "Enable MQTT source and sink plugins (requires PahoMqttCpp)" ON)
activate_optional_plugin("Sources/MQTTSource" ${NES_ENABLE_MQTT})
activate_optional_plugin("Sinks/MQTTSink" ${NES_ENABLE_MQTT})

option(NES_ENABLE_MEOS "Enable MEOS plugin (requires libmeos installed on the system)" ON)
activate_optional_plugin("MEOS" ${NES_ENABLE_MEOS})
if (NES_ENABLE_MEOS)
message(STATUS "Enable MEOS Plugin")

# Detect the platform
if (APPLE)
message(STATUS "Building on macOS")
# Set the include and library directories for Homebrew (macOS)
set(MEOS_INCLUDE_DIR "/opt/homebrew/include" CACHE PATH "Path to MEOS include directory")
set(MEOS_PLUGIN_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include" CACHE PATH "Path to MEOS plugin include directory")
include_directories(SYSTEM ${MEOS_INCLUDE_DIR} ${MEOS_PLUGIN_INCLUDE_DIR})
include_directories(SYSTEM ${MEOS_INCLUDE_DIR} ${MEOS_PLUGIN_INCLUDE_DIR})
link_directories(${MEOS_LIBRARY_DIR} )

else()
message(FATAL_ERROR "MEOS library not found")
message(STATUS "Building on Linux")

# Default behavior for Linux
find_package(meos REQUIRED)
if(meos_FOUND)
message(STATUS "MEOS found: include=${meos_INCLUDE_DIR}, library=${meos}")
include_directories(SYSTEM ${meos_INCLUDE_DIR})
else()
message(FATAL_ERROR "MEOS library not found")
endif()
endif()
endif()

Expand Down
9 changes: 7 additions & 2 deletions nes-plugins/Sinks/MQTTSink/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,10 @@ target_include_directories(mqtt_sink_validation_plugin_library
)

find_package(PahoMqttCpp CONFIG REQUIRED)
target_link_libraries(mqtt_sink_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
target_link_libraries(mqtt_sink_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
if(TARGET PahoMqttCpp::paho-mqttpp3-static)
target_link_libraries(mqtt_sink_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
target_link_libraries(mqtt_sink_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
else()
target_link_libraries(mqtt_sink_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3)
target_link_libraries(mqtt_sink_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3)
endif()
9 changes: 7 additions & 2 deletions nes-plugins/Sources/MQTTSource/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,10 @@ target_include_directories(mqtt_source_validation_plugin_library
)

find_package(PahoMqttCpp CONFIG REQUIRED)
target_link_libraries(mqtt_source_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
target_link_libraries(mqtt_source_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
if(TARGET PahoMqttCpp::paho-mqttpp3-static)
target_link_libraries(mqtt_source_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
target_link_libraries(mqtt_source_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3-static)
else()
target_link_libraries(mqtt_source_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3)
target_link_libraries(mqtt_source_validation_plugin_library PRIVATE PahoMqttCpp::paho-mqttpp3)
endif()
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,10 @@
#include <QueryExecutionConfiguration.hpp>
#include <RewriteRuleRegistry.hpp>
// Special-case lowering for TEMPORAL_SEQUENCE (multi-input) aggregation
#ifdef NES_ENABLE_MEOS
#include <Operators/Windows/Aggregations/Meos/TemporalSequenceAggregationLogicalFunctionV2.hpp>
#include <Aggregation/Function/Meos/TemporalSequenceAggregationPhysicalFunction.hpp>
#endif

namespace NES
{
Expand Down Expand Up @@ -128,6 +130,7 @@ getAggregationPhysicalFunctions(const WindowedAggregationLogicalOperator& logica
const auto name = descriptor->getName();

// Custom lowering path for TEMPORAL_SEQUENCE: needs three field functions (lon, lat, ts)
#ifdef NES_ENABLE_MEOS
if (name == std::string_view("TemporalSequence"))
{
auto tsDescriptor = std::dynamic_pointer_cast<TemporalSequenceAggregationLogicalFunctionV2>(descriptor);
Expand Down Expand Up @@ -159,6 +162,7 @@ getAggregationPhysicalFunctions(const WindowedAggregationLogicalOperator& logica
aggregationPhysicalFunctions.push_back(std::move(phys));
continue;
}
#endif

// Default path: use registry for single-input aggregations
auto aggregationInputFunction = QueryCompilation::FunctionProvider::lowerFunction(descriptor->onField);
Expand Down
5 changes: 5 additions & 0 deletions tools/codegen/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
__pycache__/
*.pyc
meos-idl*.json
.venv/
venv/
125 changes: 125 additions & 0 deletions tools/codegen/PR_COVERAGE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
# Where each open pull request stands against the generator

Every operator a pull request carries is matched against the set the
generator emits from the MEOS-API catalog. A pull request is SUPERSEDED when
the generator emits every operator it carries, PARTIAL when it emits some,
and UNCOVERED when it emits none of them.

The operator lists come from each branch's full diff against main, so a
branch stacked on another reports its whole chain rather than its own
addition. That widens a count, never narrows it: a pull request reported
SUPERSEDED is superseded together with everything beneath it.

Across the 88 open pull requests there are 438 distinct operators, of which the
generator emits 174. The generator emits 463 operators in all, so 289 of them
are carried by no open pull request: the two sets overlap in under half of
either, and the generator is the larger.

## Superseded — the generator emits every operator these carry

| PR | operators | title |
|---|---|---|
| #45 | 59/59 | feat(nebula): Add the ever/always comparison family and signature-driven descriptor builder |
| #46 | 59/59 | feat(nebula): Add 18 int-returning spatial-relation and comparison operators via existing templates |
| #47 | 59/59 | feat(nebula): Add the generalized per-event assembler and scalar operator families |
| #48 | 59/59 | tools(nebula): Add build_local.sh dev-image build script with auto MQTT toggle |
| #49 | 59/59 | feat(nebula): Add the extract marshaler for unary Temporal-to-scalar transforms |
| #50 | 59/59 | feat(nebula): Add the box/span query-literal family via per-event box-literal assembler |
| #51 | 59/59 | feat(nebula): Add windowed extent-to-box aggregates with VARSIZED output |
| #52 | 59/59 | feat(nebula): Add value/time Span extent aggregates with scalar fold |
| #53 | 59/59 | feat(nebula): Add 55 box/temporal position predicates in both argument orders |
| #54 | 59/59 | feat(nebula): Add windowed value-union set-collect aggregates |
| #55 | 59/59 | feat(nebula): Add durable query-literal round-trip for parameterized aggregates |
| #56 | 59/59 | feat(nebula): Add MEOS function library composition over VARSIZED hex-WKB values |
| #57 | 59/59 | feat(nebula): Add TRAJECTORY_WKB windowed aggregate emitting the mini-trip as hex-WKB |
| #58 | 59/59 | feat(nebula): Add the expandable-Temporal* in-process streaming aggregate substrate |
| #59 | 59/59 | feat(nebula): Add value-output windowed aggregates emitting Temporal*-transform results as hex-WKB |
| #60 | 59/59 | feat(nebula): Add tnumber value-output windowed aggregates over the expandable substrate |
| #61 | 59/59 | fix(nebula): Fix three root causes preventing windowed-aggregate query plans from deserializing |
| #62 | 59/59 | feat(nebula): Add tnpoint network-constrained windowed aggregates |
| #63 | 59/59 | fix(harness): Resolve systest tokens for every operator family in the proven callable count |
| #64 | 59/59 | feat(nebula): Add unary temporal-transform value-output windowed aggregates |
| #65 | 59/59 | fix(nebula): Initialize MEOS per worker thread to fix nondeterministic null-timezone segfault |
| #66 | 59/59 | feat(nebula): Add geometry value-output windowed aggregates |
| #67 | 59/59 | feat(nebula): Add cross-vehicle STBox predicates as per-event operators |
| #68 | 59/59 | perf(nebula): Convert extent and union aggregates to incremental MEOS-accumulator slots |
| #69 | 59/59 | feat(nebula): Add cross-vehicle tnumber-vs-tnumber position and topological predicates |
| #70 | 59/59 | feat(nebula): Add cross-vehicle tnpoint and tpose binary scalar predicates |
| #71 | 59/59 | feat(nebula): Add cross-vehicle Temporal*-returning combinators over two per-vehicle trajectories |
| #187 | 59/59 | feat(codegen): recreate aggregation descriptor + wire codegen_aggregations.py (surface reproduces committed byte-for-byte) |
| #188 | 59/59 | fix(nebula): name the multiply operator MUL to derive from canonical mul_tnumber_tnumber |
| #189 | 58/58 | ci(nebula): add the codegen drift-guard for the windowed-aggregation surface |

## Partial — the generator emits some of what these carry

| PR | emitted | carried | title |
|---|---|---|---|
| #192 | 20 | 21 | Generate trgeometry spatial-predicate operators |
| #168 | 54 | 97 | feat(meos): add trgeometry vs geometry spatial predicate NES operators (W148) |
| #111 | 44 | 98 | feat(meos): add TLT_{FLOAT_TFLOAT,TFLOAT_FLOAT,INT_TINT,TINT_INT,TEMPORAL_TEMPORAL} NES operators (W91) |
| #110 | 44 | 98 | feat(meos): add TLE_{FLOAT_TFLOAT,TFLOAT_FLOAT,INT_TINT,TINT_INT,TEMPORAL_TEMPORAL} NES operators (W90) |
| #109 | 44 | 98 | feat(meos): add TGT_{FLOAT_TFLOAT,TFLOAT_FLOAT,INT_TINT,TINT_INT,TEMPORAL_TEMPORAL} NES operators (W89) |
| #108 | 44 | 98 | feat(meos): add TGE_{FLOAT_TFLOAT,TFLOAT_FLOAT,INT_TINT,TINT_INT,TEMPORAL_TEMPORAL} NES operators (W88) |
| #107 | 44 | 98 | feat(meos): add TNE_{BOOL_TBOOL,TBOOL_BOOL,FLOAT_TFLOAT,TFLOAT_FLOAT,INT_TINT,TINT_INT,TEMPORAL_TEMPORAL} NES operators (W87) |
| #106 | 44 | 98 | feat(meos): add TEQ_{BOOL_TBOOL,TBOOL_BOOL,FLOAT_TFLOAT,TFLOAT_FLOAT,INT_TINT,TINT_INT,TEMPORAL_TEMPORAL} NES operators (W86) |
| #105 | 44 | 98 | feat(meos): add TNOT_TBOOL, TAND_{BOOL_TBOOL,TBOOL_BOOL,TBOOL_TBOOL}, TOR_{BOOL_TBOOL,TBOOL_BOOL,TBOOL_TBOOL} NES operators (W85) |
| #104 | 44 | 98 | feat(meos): add EVER/ALWAYS_{EQ,GE,GT,LE,LT,NE}_TEMPORAL_TEMPORAL NES operators (W83-W84) |
| #96 | 43 | 98 | feat(meos): add EVER_EQ/GE/GT/LE/LT/NE_TFLOAT_TFLOAT NES operators (W67) |
| #103 | 40 | 98 | feat(meos): add EVER/ALWAYS_EQ/NE_{TBOOL_BOOL,BOOL_TBOOL} NES operators (W79-W82) |
| #97 | 40 | 98 | feat(meos): add ALWAYS_EQ/GE/GT/LE/LT/NE_TFLOAT_TFLOAT NES operators (W68) |
| #44 | 23 | 58 | Streaming MEOS-parity harness and NebulaStream adapter |
| #98 | 37 | 98 | feat(meos): add EVER_EQ/GE/GT/LE/LT/NE_TINT_TINT NES operators (W69) |
| #99 | 32 | 98 | feat(meos): add ALWAYS_EQ/GE/GT/LE/LT/NE_TINT_TINT NES operators (W70) |
| #194 | 5 | 16 | Generate temporal-to-temporal transform operators via hex-WKB serialization |
| #100 | 29 | 98 | feat(meos): add EVER_EQ/GE/GT/LE/LT/NE_TBIGINT_TBIGINT NES operators (W71) |
| #101 | 27 | 98 | feat(meos): add ALWAYS_EQ/GE/GT/LE/LT/NE_TBIGINT_TBIGINT NES operators (W72) |

## Uncovered — the generator emits none of these, so they are the work it owes

| PR | operators | title |
|---|---|---|
| #42 | 62 | Add the temporal circular buffer nearest-approach distance operators |
| #41 | 62 | Add the temporal pose and network point dwithin operators |
| #40 | 62 | Add the temporal pose and network point nearest-approach distance operators |
| #39 | 62 | Add the temporal network point spatial-relationship operators |
| #38 | 62 | Add the temporal pose–pose spatial-relationship operators |
| #37 | 62 | Add the temporal pose–geometry spatial-relationship operators |
| #36 | 59 | Add the temporal circular buffer dwithin operators |
| #43 | 57 | Convert the base MEOS systests to the runnable DDL format |
| #35 | 53 | Add the temporal circular buffer–circular buffer spatial-relationship operators |
| #34 | 47 | Add the temporal circular buffer–buffer spatial-relationship operators |
| #33 | 38 | Add the temporal circular buffer–geometry spatial-relationship operators |
| #32 | 28 | Add the temporal geo scalar accessor operators |
| #31 | 28 | Add the temporal number average and time-weighted average aggregations |
| #30 | 28 | Add windowed aggregation operators and the aggregation generator |
| #29 | 28 | Add the temporal geo restriction operators |
| #28 | 26 | Add the temporal number nearest-approach distance operators |
| #191 | 25 | Generate tbigint temporal-number comparison operators |
| #27 | 22 | Auto-inject parser glue for the generated operators |
| #26 | 22 | Add the temporal distance operators |
| #25 | 17 | Add the temporal geo–geo spatial-relationship operators |
| #193 | 10 | Generate geo-first tgeo spatial-predicate operators |
| #24 | 8 | Complete the temporal geo–geometry spatial-relationship operators |
| #23 | 6 | Add the temporal geo–geometry spatial-relationship operators |
| #196 | 5 | Generate temporal-vs-geometry transform operators via hex-WKB serialization |
| #195 | 5 | Generate scalar-first tbigint arithmetic operators via hex-WKB serialization |
| #197 | 3 | Generate trgeometry stbox-restriction operators via hex-WKB serialization |
| #175 | 1 | Add the MobilityNebula pin manifest and per-binding generator policy |
| #171 | 1 | fix(ci): make the Nix build work with NES_ENABLE_MEOS=OFF and stock paho-mqtt-cpp |
| #170 | 1 | feat(codegen): IDL-driven NES MEOS-operator generator with idempotent build/grammar/QPC glue |
| #72 | 1 | fix(ci): skip MQTT plugins in the Nix build where PahoMqttCpp is absent |
| #22 | 1 | Add proto extra fields and fix unused-parameter warnings in the aggregations |
| #21 | 1 | Add the MEOS-operator generator for the NebulaStream codegen path |
| #20 | 1 | Parameterize the cross-distance vehicle pair via SQL arguments |
| #19 | 1 | Parameterize the pair-meeting distance threshold via a SQL argument |
| #18 | 1 | Add the BerlinMOD streaming-semantics tier overlay |
| #17 | 1 | Add the pair-meeting and cross-distance aggregations |
| #16 | 1 | Add the temporal length aggregation |
| #15 | 1 | Add the BerlinMOD nine-query three-form parity matrix for NebulaStream |

## Carrying no operator file

| PR | title |
|---|---|
| #190 | Build the libmeos dependency image with all MEOS families via -DALL |

Loading
Loading