diff --git a/pom.xml b/pom.xml index 4d2589463b..76a692eb36 100644 --- a/pom.xml +++ b/pom.xml @@ -387,38 +387,44 @@ org.apache.spark spark-core_2.13 - 4.0.1 + 4.2.0-preview5 provided org.apache.spark spark-sql_2.13 - 4.0.1 + 4.2.0-preview5 provided + + org.apache.hadoop + hadoop-common + 3.4.2 + provided + org.apache.spark spark-mllib_2.13 - 4.0.1 + 4.2.0-preview5 provided org.apache.spark spark-hive_2.13 - 4.0.1 + 4.2.0-preview5 provided + + org.apache.spark + spark-avro_2.13 + 4.2.0-preview5 + org.apache.hadoop hadoop-aws - 3.3.6 + 3.4.3 provided - - org.apache.spark - spark-avro_2.13 - 4.0.1 - org.antlr antlr4-runtime @@ -484,7 +490,7 @@ io.delta delta-spark_2.13 - 4.0.0 + 4.2.0 org.apache.iceberg @@ -514,6 +520,12 @@ mongo-spark-connector_2.13 10.5.0 + + org.apache.logging.log4j + log4j-core + 2.24.3 + compile + com.ibm.icu icu4j diff --git a/src/main/java/org/rumbledb/types/DerivedAtomicItemType.java b/src/main/java/org/rumbledb/types/DerivedAtomicItemType.java index 72b99ad9a3..920e7935b9 100644 --- a/src/main/java/org/rumbledb/types/DerivedAtomicItemType.java +++ b/src/main/java/org/rumbledb/types/DerivedAtomicItemType.java @@ -4,7 +4,7 @@ import java.util.List; import java.util.Set; -import org.apache.commons.collections.ListUtils; +import org.apache.commons.collections4.ListUtils; import org.rumbledb.api.Item; import org.rumbledb.config.RumbleConfiguration; import org.rumbledb.context.DynamicContext; diff --git a/src/main/java/org/rumbledb/types/ObjectItemType.java b/src/main/java/org/rumbledb/types/ObjectItemType.java index ec41e2f889..fb9bc5e093 100644 --- a/src/main/java/org/rumbledb/types/ObjectItemType.java +++ b/src/main/java/org/rumbledb/types/ObjectItemType.java @@ -11,7 +11,7 @@ import java.util.Map; import java.util.Set; -import org.apache.commons.collections.ListUtils; +import org.apache.commons.collections4.ListUtils; import org.rumbledb.api.Item; import org.rumbledb.config.RumbleConfiguration; import org.rumbledb.context.DynamicContext; diff --git a/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-MinMaxScaler.jq b/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-MinMaxScaler.jq index 656f36baaa..23bce27154 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-MinMaxScaler.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-MinMaxScaler.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "label" : 0, "name" : "a", "age" : 20, "weight" : 50, "scaled" : { } }, { "label" : 1, "name" : "b", "age" : 21, "weight" : 55.3, "scaled" : [ 0.2, 0.20703124999999992 ] }, { "label" : 2, "name" : "c", "age" : 22, "weight" : 60.6, "scaled" : [ 0.4, 0.4140625000000001 ] }, { "label" : 3, "name" : "d", "age" : 23, "weight" : 65.9, "scaled" : [ 0.6000000000000001, 0.6210937500000003 ] }, { "label" : 4, "name" : "e", "age" : 24, "weight" : 70.3, "scaled" : [ 0.8, 0.79296875 ] }, { "label" : 5, "name" : "f", "age" : 25, "weight" : 75.6, "scaled" : [ 1, 1 ] })" :) +(:JIQS: ShouldRun; Output="({ "label" : 0, "name" : "a", "age" : 20, "weight" : 50, "scaled" : [ 0, 0 ] }, { "label" : 1, "name" : "b", "age" : 21, "weight" : 55.3, "scaled" : [ 0.2, 0.20703124999999992 ] }, { "label" : 2, "name" : "c", "age" : 22, "weight" : 60.6, "scaled" : [ 0.4, 0.4140625000000001 ] }, { "label" : 3, "name" : "d", "age" : 23, "weight" : 65.9, "scaled" : [ 0.6000000000000001, 0.6210937500000003 ] }, { "label" : 4, "name" : "e", "age" : 24, "weight" : 70.3, "scaled" : [ 0.8, 0.79296875 ] }, { "label" : 5, "name" : "f", "age" : 25, "weight" : 75.6, "scaled" : [ 1, 1 ] })" :) let $data := annotate( json-lines("../../../../queries/rumbleML/sample-ml-data-flat.json"), { "label": "integer", "binaryLabel": "integer", "name": "string", "age": "double", "weight": "double", "booleanCol": "boolean", "nullCol": "null", "stringCol": "string", "stringArrayCol": ["string"], "intArrayCol": ["integer"], "doubleArrayCol": ["double"], "doubleArrayArrayCol": [["double"]] } diff --git a/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-PCA.jq b/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-PCA.jq index 212d57db83..692ec61697 100644 --- a/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-PCA.jq +++ b/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-PCA.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldRun; Output="({ "label" : 0, "name" : "a", "age" : 20, "weight" : 50, "pca" : { } }, { "label" : 1, "name" : "b", "age" : 21, "weight" : 55.3, "pca" : [ 0.2, 0.20703124999999992 ] }, { "label" : 2, "name" : "c", "age" : 22, "weight" : 60.6, "pca" : [ 0.4, 0.4140625000000001 ] }, { "label" : 3, "name" : "d", "age" : 23, "weight" : 65.9, "pca" : [ 0.6000000000000001, 0.6210937500000003 ] }, { "label" : 4, "name" : "e", "age" : 24, "weight" : 70.3, "pca" : [ 0.8, 0.79296875 ] }, { "label" : 5, "name" : "f", "age" : 25, "weight" : 75.6, "pca" : [ 1, 1 ] })" :) +(:JIQS: ShouldRun; Output="({ "label" : 0, "name" : "a", "age" : 20, "weight" : 50, "pca" : [ 0, 0 ] }, { "label" : 1, "name" : "b", "age" : 21, "weight" : 55.3, "pca" : [ 0.2, 0.20703124999999992 ] }, { "label" : 2, "name" : "c", "age" : 22, "weight" : 60.6, "pca" : [ 0.4, 0.4140625000000001 ] }, { "label" : 3, "name" : "d", "age" : 23, "weight" : 65.9, "pca" : [ 0.6000000000000001, 0.6210937500000003 ] }, { "label" : 4, "name" : "e", "age" : 24, "weight" : 70.3, "pca" : [ 0.8, 0.79296875 ] }, { "label" : 5, "name" : "f", "age" : 25, "weight" : 75.6, "pca" : [ 1, 1 ] })" :) let $data := annotate( json-lines("../../../../queries/rumbleML/sample-ml-data-flat.json"), { "label": "integer", "binaryLabel": "integer", "name": "string", "age": "double", "weight": "double", "booleanCol": "boolean", "nullCol": "null", "stringCol": "string", "stringArrayCol": ["string"], "intArrayCol": ["integer"], "doubleArrayCol": ["double"], "doubleArrayArrayCol": [["double"]] }