From 17b4299edea33fe857a2932b3923247d0b435564 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 3 Nov 2025 16:36:04 +0100 Subject: [PATCH 1/3] Simplify code. --- .../java/iq/BackwardsCompatibilityTests.java | 9 +- src/test/java/iq/Bugs.java | 15 +- src/test/java/iq/ComposabilityTests.java | 20 +- src/test/java/iq/DeltaUpdateRuntimeTests.java | 13 +- src/test/java/iq/FrontendTests.java | 36 +- src/test/java/iq/RuntimeTests.java | 13 +- src/test/java/iq/RuntimeTestsNoInlining.java | 11 +- .../java/iq/RuntimeTestsNoParallelism.java | 11 +- .../iq/SequentialClassificationTests.java | 15 +- src/test/java/iq/StaticTypeTests.java | 11 +- src/test/java/iq/XMLTests.java | 97 +++++- src/test/java/iq/XQueryTests.java | 13 +- .../java/iq/base/AnnotationsTestsBase.java | 313 ------------------ 13 files changed, 167 insertions(+), 410 deletions(-) delete mode 100644 src/test/java/iq/base/AnnotationsTestsBase.java diff --git a/src/test/java/iq/BackwardsCompatibilityTests.java b/src/test/java/iq/BackwardsCompatibilityTests.java index 6aa3b1d755..d922e54c16 100644 --- a/src/test/java/iq/BackwardsCompatibilityTests.java +++ b/src/test/java/iq/BackwardsCompatibilityTests.java @@ -20,13 +20,14 @@ package iq; -import iq.base.AnnotationsTestsBase; import org.apache.spark.SparkConf; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.rumbledb.config.RumbleRuntimeConfiguration; +import org.rumbledb.tests.commons.RumbleDBTestCommons; + import scala.util.Properties; import sparksoniq.spark.SparkSessionManager; import utils.FileManager; @@ -36,7 +37,7 @@ import java.util.*; @RunWith(Parameterized.class) -public class BackwardsCompatibilityTests extends AnnotationsTestsBase { +public class BackwardsCompatibilityTests { public static final File runtimeTestsDirectory = new File( System.getProperty("user.dir") @@ -106,8 +107,8 @@ public static void setupSparkSession() { @Test(timeout = 1000000) public final void testRuntimeIterators() throws Throwable { - System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); - AnnotationsTestsBase.testAnnotations( + // System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); + RumbleDBTestCommons.testAnnotations( this.testFile.getAbsolutePath(), getConfiguration(), true, diff --git a/src/test/java/iq/Bugs.java b/src/test/java/iq/Bugs.java index bb05124c36..9cd3953c3d 100644 --- a/src/test/java/iq/Bugs.java +++ b/src/test/java/iq/Bugs.java @@ -20,7 +20,6 @@ package iq; -import iq.base.AnnotationsTestsBase; import scala.Function0; import scala.util.Properties; @@ -29,6 +28,8 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.rumbledb.tests.commons.RumbleDBTestCommons; + import sparksoniq.spark.SparkSessionManager; import utils.FileManager; @@ -38,7 +39,7 @@ import java.util.List; @RunWith(Parameterized.class) -public class Bugs extends AnnotationsTestsBase { +public class Bugs { public static final File runtimeTestsDirectory = new File( System.getProperty("user.dir") @@ -99,13 +100,11 @@ public static void setupSparkSession() { @Test(timeout = 1000000) public void testRuntimeIterators() throws Throwable { - System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); - AnnotationsTestsBase.testAnnotations( + // System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); + RumbleDBTestCommons.testAnnotations( this.testFile.getAbsolutePath(), - getConfiguration(), - true, - getConfiguration().applyUpdates(), - getConfiguration().getResultSizeCap() + RumbleDBTestCommons.getDefaultConfiguration(), + true ); } } diff --git a/src/test/java/iq/ComposabilityTests.java b/src/test/java/iq/ComposabilityTests.java index b0f416a72b..666a38d1ae 100644 --- a/src/test/java/iq/ComposabilityTests.java +++ b/src/test/java/iq/ComposabilityTests.java @@ -21,13 +21,15 @@ package iq; -import iq.base.AnnotationsTestsBase; import org.junit.Test; +import org.rumbledb.tests.commons.RumbleDBTestCommons; + +import java.util.List; import java.io.File; -public class ComposabilityTests extends AnnotationsTestsBase { +public class ComposabilityTests { public static final File composabilityTestsDirectory = new File( System.getProperty("user.dir") + @@ -42,15 +44,13 @@ public class ComposabilityTests extends AnnotationsTestsBase { */ @Test(timeout = 1000000) public void testComposabilityContraints() throws Throwable { - initializeTests(composabilityTestsDirectory); - for (File testFile : this.testFiles) { - System.err.println(counter++ + " : " + testFile); - testAnnotations( + List testFiles = RumbleDBTestCommons.extractTestFilesFromDirectory(composabilityTestsDirectory); + for (File testFile : testFiles) { + // System.err.println(counter++ + " : " + testFile); + RumbleDBTestCommons.testAnnotations( testFile.getAbsolutePath(), - getConfiguration(), - true, - getConfiguration().applyUpdates(), - getConfiguration().getResultSizeCap() + RumbleDBTestCommons.getDefaultConfiguration(), + true ); } } diff --git a/src/test/java/iq/DeltaUpdateRuntimeTests.java b/src/test/java/iq/DeltaUpdateRuntimeTests.java index f09a9b2814..82ff59e0ad 100644 --- a/src/test/java/iq/DeltaUpdateRuntimeTests.java +++ b/src/test/java/iq/DeltaUpdateRuntimeTests.java @@ -20,8 +20,9 @@ package iq; -import iq.base.AnnotationsTestsBase; import org.rumbledb.config.RumbleRuntimeConfiguration; +import org.rumbledb.tests.commons.RumbleDBTestCommons; + import utils.annotations.AnnotationParseException; import utils.annotations.AnnotationProcessor; import org.apache.spark.SparkConf; @@ -41,7 +42,7 @@ import java.util.*; @RunWith(Parameterized.class) -public class DeltaUpdateRuntimeTests extends AnnotationsTestsBase { +public class DeltaUpdateRuntimeTests { public static final File runtimeTestsDirectory = new File( System.getProperty("user.dir") @@ -180,7 +181,7 @@ public static void setupSparkSession() { @Test(timeout = 1000000) public void testRuntimeIterators() throws Throwable { - System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); + // System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); try { AnnotationProcessor.TestAnnotation currentAnnotation = AnnotationProcessor.readAnnotation( new FileReader(this.testFile.getAbsolutePath()) @@ -189,12 +190,10 @@ public void testRuntimeIterators() throws Throwable { e.printStackTrace(); Assert.fail(); } - testAnnotations( + RumbleDBTestCommons.testAnnotations( this.testFile.getAbsolutePath(), getConfiguration(), - true, - getConfiguration().applyUpdates(), - getConfiguration().getResultSizeCap() + true ); } diff --git a/src/test/java/iq/FrontendTests.java b/src/test/java/iq/FrontendTests.java index c357886f56..b8b1f1d2fa 100644 --- a/src/test/java/iq/FrontendTests.java +++ b/src/test/java/iq/FrontendTests.java @@ -21,7 +21,6 @@ package iq; -import iq.base.AnnotationsTestsBase; import org.junit.Assert; import org.junit.Test; import org.rumbledb.compiler.VisitorHelpers; @@ -31,6 +30,7 @@ import org.rumbledb.expressions.module.MainModule; import org.rumbledb.expressions.primary.VariableReferenceExpression; import org.rumbledb.runtime.functions.input.FileSystemUtil; +import org.rumbledb.tests.commons.RumbleDBTestCommons; import org.rumbledb.types.BuiltinTypesCatalogue; import java.io.File; import java.net.URI; @@ -38,7 +38,7 @@ import java.util.List; -public class FrontendTests extends AnnotationsTestsBase { +public class FrontendTests { public static final File grammarTestsDirectory = new File( System.getProperty("user.dir") @@ -64,16 +64,14 @@ public class FrontendTests extends AnnotationsTestsBase { */ @Test(timeout = 1000000) public void testGrammarAndParser() throws Throwable { - initializeTests(grammarTestsDirectory); - for (File testFile : this.testFiles) { - System.err.println(counter++ + " : " + testFile); + List testFiles = RumbleDBTestCommons.extractTestFilesFromDirectory(grammarTestsDirectory); + for (File testFile : testFiles) { + // System.err.println(counter++ + " : " + testFile); // FileReader reader = getReaderForFile(testFile.getAbsolutePath()); - testAnnotations( + RumbleDBTestCommons.testAnnotations( testFile.getAbsolutePath(), - getConfiguration(), - true, - getConfiguration().applyUpdates(), - getConfiguration().getResultSizeCap() + RumbleDBTestCommons.getDefaultConfiguration(), + true ); } @@ -106,25 +104,23 @@ public void testGrammarAndParser() throws Throwable { */ @Test(timeout = 1000000) public void testSematicChecks() throws Throwable { - initializeTests(semanticTestsDirectory); - for (File testFile : this.testFiles) { - System.err.println(counter++ + " : " + testFile); - testAnnotations( + List testFiles = RumbleDBTestCommons.extractTestFilesFromDirectory(semanticTestsDirectory); + for (File testFile : testFiles) { + // System.err.println(counter++ + " : " + testFile); + RumbleDBTestCommons.testAnnotations( testFile.getAbsolutePath(), - getConfiguration(), - true, - getConfiguration().applyUpdates(), - getConfiguration().getResultSizeCap() + RumbleDBTestCommons.getDefaultConfiguration(), + true ); if (Arrays.asList(manualSemanticChecksFiles).contains(testFile.getName())) { URI uri = FileSystemUtil.resolveURIAgainstWorkingDirectory( testFile.getAbsolutePath(), - getConfiguration(), + RumbleDBTestCommons.getDefaultConfiguration(), ExceptionMetadata.EMPTY_METADATA ); MainModule mainModule = VisitorHelpers.parseMainModuleFromLocation( uri, - getConfiguration() + RumbleDBTestCommons.getDefaultConfiguration() ); testVariableTypes(mainModule); diff --git a/src/test/java/iq/RuntimeTests.java b/src/test/java/iq/RuntimeTests.java index 88c8ec7e23..f186c90c8e 100644 --- a/src/test/java/iq/RuntimeTests.java +++ b/src/test/java/iq/RuntimeTests.java @@ -20,7 +20,6 @@ package iq; -import iq.base.AnnotationsTestsBase; import scala.Function0; import scala.util.Properties; @@ -33,13 +32,15 @@ import org.rumbledb.config.RumbleRuntimeConfiguration; import org.rumbledb.context.Name; import org.rumbledb.items.ItemFactory; +import org.rumbledb.tests.commons.RumbleDBTestCommons; + import sparksoniq.spark.SparkSessionManager; import utils.FileManager; import java.io.File; import java.util.*; @RunWith(Parameterized.class) -public class RuntimeTests extends AnnotationsTestsBase { +public class RuntimeTests { public static final File runtimeTestsDirectory = new File( System.getProperty("user.dir") @@ -139,13 +140,11 @@ public static void setupSparkSession() { @Test(timeout = 1000000) public void testRuntimeIterators() throws Throwable { - System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); - testAnnotations( + // System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); + RumbleDBTestCommons.testAnnotations( this.testFile.getAbsolutePath(), getConfiguration(), - true, - getConfiguration().applyUpdates(), - getConfiguration().getResultSizeCap() + true ); } } diff --git a/src/test/java/iq/RuntimeTestsNoInlining.java b/src/test/java/iq/RuntimeTestsNoInlining.java index f97949e411..dd29bdf1a2 100644 --- a/src/test/java/iq/RuntimeTestsNoInlining.java +++ b/src/test/java/iq/RuntimeTestsNoInlining.java @@ -32,8 +32,7 @@ import org.rumbledb.config.RumbleRuntimeConfiguration; import org.rumbledb.context.Name; import org.rumbledb.items.ItemFactory; - -import iq.base.AnnotationsTestsBase; +import org.rumbledb.tests.commons.RumbleDBTestCommons; @RunWith(Parameterized.class) public class RuntimeTestsNoInlining extends RuntimeTests { @@ -79,13 +78,11 @@ public RumbleRuntimeConfiguration getConfiguration() { @Test(timeout = 1000000) public final void testRuntimeIterators() throws Throwable { - System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); - testAnnotations( + // System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); + RumbleDBTestCommons.testAnnotations( this.testFile.getAbsolutePath(), getConfiguration(), - true, - getConfiguration().applyUpdates(), - getConfiguration().getResultSizeCap() + true ); } } diff --git a/src/test/java/iq/RuntimeTestsNoParallelism.java b/src/test/java/iq/RuntimeTestsNoParallelism.java index f6f8bab422..c30470ba88 100644 --- a/src/test/java/iq/RuntimeTestsNoParallelism.java +++ b/src/test/java/iq/RuntimeTestsNoParallelism.java @@ -31,8 +31,7 @@ import org.rumbledb.config.RumbleRuntimeConfiguration; import org.rumbledb.context.Name; import org.rumbledb.items.ItemFactory; - -import iq.base.AnnotationsTestsBase; +import org.rumbledb.tests.commons.RumbleDBTestCommons; @RunWith(Parameterized.class) public class RuntimeTestsNoParallelism extends RuntimeTests { @@ -74,13 +73,11 @@ public RumbleRuntimeConfiguration getConfiguration() { @Test(timeout = 1000000) public final void testRuntimeIterators() throws Throwable { - System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); - testAnnotations( + // System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); + RumbleDBTestCommons.testAnnotations( this.testFile.getAbsolutePath(), getConfiguration(), - true, - getConfiguration().applyUpdates(), - getConfiguration().getResultSizeCap() + true ); } } diff --git a/src/test/java/iq/SequentialClassificationTests.java b/src/test/java/iq/SequentialClassificationTests.java index db4ea04b5a..81e38ac65b 100644 --- a/src/test/java/iq/SequentialClassificationTests.java +++ b/src/test/java/iq/SequentialClassificationTests.java @@ -1,6 +1,5 @@ package iq; -import iq.base.AnnotationsTestsBase; import org.junit.Test; import org.rumbledb.compiler.VisitorHelpers; import org.rumbledb.exceptions.ExceptionMetadata; @@ -26,26 +25,28 @@ import org.rumbledb.expressions.scripting.statement.Statement; import org.rumbledb.expressions.scripting.statement.StatementsAndOptionalExpr; import org.rumbledb.runtime.functions.input.FileSystemUtil; +import org.rumbledb.tests.commons.RumbleDBTestCommons; import java.io.File; import java.io.IOException; import java.net.URI; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -public class SequentialClassificationTests extends AnnotationsTestsBase { +public class SequentialClassificationTests { private MainModule parseAndCompile(String filePath) throws IOException { URI uri = FileSystemUtil.resolveURIAgainstWorkingDirectory( filePath, - getConfiguration(), + RumbleDBTestCommons.getDefaultConfiguration(), ExceptionMetadata.EMPTY_METADATA ); return VisitorHelpers.parseMainModuleFromLocation( uri, - getConfiguration() + RumbleDBTestCommons.getDefaultConfiguration() ); } @@ -292,9 +293,9 @@ public void testNonSequential() throws Throwable { + "/src/test/resources/test_files/sequential/non-sequential" ); - initializeTests(nonsequentialTestsDirectory); - for (File testFile : this.testFiles) { - System.err.println(counter++ + " : " + testFile); + List testFiles = RumbleDBTestCommons.extractTestFilesFromDirectory(nonsequentialTestsDirectory); + for (File testFile : testFiles) { + // System.err.println(counter++ + " : " + testFile); MainModule mainModule = parseAndCompile(testFile.getAbsolutePath()); for (Node descendant : mainModule.getDescendants()) { if (descendant instanceof Expression) { diff --git a/src/test/java/iq/StaticTypeTests.java b/src/test/java/iq/StaticTypeTests.java index 3ac7d33bc2..9de26e5759 100644 --- a/src/test/java/iq/StaticTypeTests.java +++ b/src/test/java/iq/StaticTypeTests.java @@ -1,8 +1,8 @@ package iq; import org.rumbledb.config.RumbleRuntimeConfiguration; +import org.rumbledb.tests.commons.RumbleDBTestCommons; -import iq.base.AnnotationsTestsBase; import scala.Function0; import scala.util.Properties; @@ -20,7 +20,7 @@ import java.util.List; @RunWith(Parameterized.class) -public class StaticTypeTests extends AnnotationsTestsBase { +public class StaticTypeTests { protected static final RumbleRuntimeConfiguration configuration = new RumbleRuntimeConfiguration( new String[] { "--print-iterator-tree", "yes", "--static-typing", "yes" } @@ -85,13 +85,10 @@ public static void setupSparkSession() { @Test(timeout = 1000000) public void testRuntimeIterators() throws Throwable { - System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); - testAnnotations( + RumbleDBTestCommons.testAnnotations( this.testFile.getAbsolutePath(), StaticTypeTests.configuration, - false, - StaticTypeTests.configuration.applyUpdates(), - StaticTypeTests.configuration.getResultSizeCap() + false ); } } diff --git a/src/test/java/iq/XMLTests.java b/src/test/java/iq/XMLTests.java index f23c683f62..ecc9bffcff 100644 --- a/src/test/java/iq/XMLTests.java +++ b/src/test/java/iq/XMLTests.java @@ -22,10 +22,13 @@ import iq.base.AnnotationsTestsBase; import org.apache.spark.SparkConf; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.rumbledb.api.Item; +import org.rumbledb.api.SequenceOfItems; import org.rumbledb.config.RumbleRuntimeConfiguration; import scala.Function0; import scala.util.Properties; @@ -36,6 +39,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; @RunWith(Parameterized.class) public class XMLTests extends AnnotationsTestsBase { @@ -113,12 +117,93 @@ public static void setupSparkSession() { @Test(timeout = 1000000) public final void testRuntimeIterators() throws Throwable { System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); - testAnnotations( - this.testFile.getAbsolutePath(), - getConfiguration(), - true, - getConfiguration().applyUpdates(), - getConfiguration().getResultSizeCap() + testAnnotations(this.testFile.getAbsolutePath(), getConfiguration()); + } + + @Override + protected void checkExpectedOutput( + String expectedOutput, + SequenceOfItems sequence + ) { + String actualOutput; + if (!sequence.availableAsRDD()) { + actualOutput = AnnotationsTestsBase.getIteratorOutput(sequence); + } else { + actualOutput = getRDDResults(sequence); + } + Assert.assertTrue( + "Expected output: " + expectedOutput + "\nActual result: " + actualOutput, + expectedOutput.equals(actualOutput) ); } + + protected String getIteratorOutput(SequenceOfItems sequence) { + sequence.open(); + Item result = null; + if (sequence.hasNext()) { + result = sequence.next(); + } + if (result == null) { + return ""; + } + String singleOutput = result.serialize(); + if (!sequence.hasNext()) { + return singleOutput; + } else { + int itemCount = 1; + StringBuilder sb = new StringBuilder(); + sb.append("("); + sb.append(result.serialize()); + sb.append(", "); + while ( + sequence.hasNext() + && + ((itemCount < getConfiguration().getResultSizeCap() + && getConfiguration().getResultSizeCap() > 0) + || + getConfiguration().getResultSizeCap() == 0) + ) { + sb.append(sequence.next().serialize()); + sb.append(", "); + itemCount++; + } + if (sequence.hasNext() && itemCount == getConfiguration().getResultSizeCap()) { + System.err.println( + "Warning! The output sequence contains a large number of items but its materialization was capped at " + + getConfiguration().getResultSizeCap() + + " items. This value can be configured with the --result-size parameter at startup" + ); + } + // remove last comma + String output = sb.toString(); + output = output.substring(0, output.length() - 2); + output += ")"; + return output; + } + } + + private String getRDDResults(SequenceOfItems sequence) { + List res = sequence.getFirstItemsAsList(getConfiguration().getResultSizeCap()); + List collectedOutput = res.stream().map(item -> item.serialize()).collect(Collectors.toList()); + + if (collectedOutput.isEmpty()) { + return ""; + } + + if (collectedOutput.size() == 1) { + return collectedOutput.get(0); + } + + StringBuilder sb = new StringBuilder(); + sb.append("("); + for (String item : collectedOutput) { + sb.append(item); + sb.append(", "); + } + + String result = sb.toString(); + result = result.substring(0, result.length() - 2); + result += ")"; + return result; + } } diff --git a/src/test/java/iq/XQueryTests.java b/src/test/java/iq/XQueryTests.java index 31af91afd2..208d4704b9 100644 --- a/src/test/java/iq/XQueryTests.java +++ b/src/test/java/iq/XQueryTests.java @@ -20,13 +20,14 @@ package iq; -import iq.base.AnnotationsTestsBase; import org.apache.spark.SparkConf; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.rumbledb.config.RumbleRuntimeConfiguration; +import org.rumbledb.tests.commons.RumbleDBTestCommons; + import scala.Function0; import scala.util.Properties; import sparksoniq.spark.SparkSessionManager; @@ -38,7 +39,7 @@ import java.util.List; @RunWith(Parameterized.class) -public class XQueryTests extends AnnotationsTestsBase { +public class XQueryTests { public static final File runtimeTestsDirectory = new File( System.getProperty("user.dir") @@ -110,13 +111,11 @@ public static void setupSparkSession() { @Test(timeout = 1000000) public final void testRuntimeIterators() throws Throwable { - System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); - testAnnotations( + // System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); + RumbleDBTestCommons.testAnnotations( this.testFile.getAbsolutePath(), getConfiguration(), - true, - getConfiguration().applyUpdates(), - getConfiguration().getResultSizeCap() + true ); } } diff --git a/src/test/java/iq/base/AnnotationsTestsBase.java b/src/test/java/iq/base/AnnotationsTestsBase.java deleted file mode 100644 index aba43f8db9..0000000000 --- a/src/test/java/iq/base/AnnotationsTestsBase.java +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Authors: Stefan Irimescu, Can Berker Cikis - * - */ - -package iq.base; - -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.junit.Assert; -import org.rumbledb.api.Item; -import org.rumbledb.api.Rumble; -import org.rumbledb.api.SequenceOfItems; -import org.rumbledb.config.RumbleRuntimeConfiguration; -import org.rumbledb.context.Name; -import org.rumbledb.exceptions.ExceptionMetadata; -import org.rumbledb.exceptions.ParsingException; -import org.rumbledb.exceptions.SemanticException; -import org.rumbledb.items.ItemFactory; -import org.rumbledb.exceptions.RumbleException; -import org.rumbledb.runtime.functions.input.FileSystemUtil; - -import utils.FileManager; -import utils.annotations.AnnotationParseException; -import utils.annotations.AnnotationProcessor; - -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - -public class AnnotationsTestsBase { - protected static int counter = 0; - protected List testFiles = new ArrayList<>(); - protected static final RumbleRuntimeConfiguration defaultConfiguration = new RumbleRuntimeConfiguration( - new String[] { - "--print-iterator-tree", - "yes", - "--variable:externalUnparsedString", - "unparsed string", - "--materialization-cap", - "200" } - ).setExternalVariableValue( - Name.createVariableInNoNamespace("externalStringItem"), - Collections.singletonList(ItemFactory.getInstance().createStringItem("this is a string")) - ) - .setExternalVariableValue( - Name.createVariableInNoNamespace("externalIntegerItems"), - Arrays.asList( - new Item[] { - ItemFactory.getInstance().createIntItem(1), - ItemFactory.getInstance().createIntItem(2), - ItemFactory.getInstance().createIntItem(3), - ItemFactory.getInstance().createIntItem(4), - ItemFactory.getInstance().createIntItem(5), - } - ) - ); - - public RumbleRuntimeConfiguration getConfiguration() { - return defaultConfiguration; - } - - public void initializeTests(File dir) { - FileManager.loadJiqFiles(dir).forEach(file -> this.testFiles.add(file)); - this.testFiles.sort(Comparator.comparing(File::getName)); - } - - /** - * Tests annotations - */ - public static void testAnnotations( - String path, - RumbleRuntimeConfiguration configuration, - boolean checkOutput, - boolean applyUpdates, - int resultSizeCap - ) - throws IOException { - AnnotationProcessor.TestAnnotation currentAnnotation = null; - try { - currentAnnotation = AnnotationProcessor.readAnnotation(new FileReader(path)); - } catch (AnnotationParseException e) { - e.printStackTrace(); - Assert.fail(); - } - SequenceOfItems sequence = null; - try { - URI uri = FileSystemUtil.resolveURIAgainstWorkingDirectory( - path, - configuration, - ExceptionMetadata.EMPTY_METADATA - ); - Rumble rumble = new Rumble(configuration); - sequence = rumble.runQuery(uri); - } catch (ParsingException exception) { - String errorOutput = exception.getMessage(); - checkErrorCode( - errorOutput, - currentAnnotation.getErrorCode(), - currentAnnotation.getErrorMetadata() - ); - if (currentAnnotation.shouldParse()) { - Assert.fail("Program did not parse when expected to.\nError output: " + errorOutput + "\n"); - return; - } else { - System.out.println(errorOutput); - return; - } - - // SEMANTIC - } catch (SemanticException exception) { - String errorOutput = exception.getMessage(); - checkErrorCode( - errorOutput, - currentAnnotation.getErrorCode(), - currentAnnotation.getErrorMetadata() - ); - try { - if (currentAnnotation.shouldCompile()) { - Assert.fail("Program did not compile when expected to.\nError output: " + errorOutput + "\n"); - return; - } else { - System.out.println(errorOutput); - Assert.assertTrue(true); - return; - } - } catch (Exception ex) { - } - - // RUNTIME - } catch (RumbleException exception) { - String errorOutput = exception.getMessage(); - checkErrorCode( - errorOutput, - currentAnnotation.getErrorCode(), - currentAnnotation.getErrorMetadata() - ); - try { - if (currentAnnotation.shouldRun()) { - Assert.fail("Program did not run when expected to.\nError output: " + errorOutput + "\n"); - return; - } else { - System.out.println(errorOutput); - Assert.assertTrue(true); - return; - } - } catch (Exception ex) { - } - } - - try { - if (!currentAnnotation.shouldCompile()) { - Assert.fail("Program compiled when not expected to.\n"); - return; - } - } catch (Exception ex) { - } - - if (!currentAnnotation.shouldParse()) { - Assert.fail("Program parsed when not expected to.\n"); - return; - } - - // PROGRAM SHOULD RUN - if ( - currentAnnotation instanceof AnnotationProcessor.RunnableTestAnnotation - && - currentAnnotation.shouldRun() - ) { - try { - checkExpectedOutput(currentAnnotation.getOutput(), sequence, checkOutput, applyUpdates, resultSizeCap); - } catch (RumbleException exception) { - String errorOutput = exception.getMessage(); - errorOutput += "\n" + ExceptionUtils.getStackTrace(exception); - Assert.fail("Program did not run when expected to.\nError output: " + errorOutput + "\n"); - } - } else { - // PROGRAM SHOULD CRASH - if ( - currentAnnotation instanceof AnnotationProcessor.UnrunnableTestAnnotation - && - !currentAnnotation.shouldRun() - ) { - try { - checkExpectedOutput( - currentAnnotation.getOutput(), - sequence, - checkOutput, - applyUpdates, - resultSizeCap - ); - } catch (Exception exception) { - String errorOutput = exception.getMessage(); - checkErrorCode( - errorOutput, - currentAnnotation.getErrorCode(), - currentAnnotation.getErrorMetadata() - ); - return; - } - - Assert.fail("Program executed when not expected to"); - } - } - return; - } - - static void checkExpectedOutput( - String expectedOutput, - SequenceOfItems sequence, - boolean checkOutput, - boolean applyUpdates, - int resultSizeCap - ) { - String actualOutput = getIteratorOutput(sequence, resultSizeCap); - if (applyUpdates && sequence.availableAsPUL()) { - sequence.applyPUL(); - } - if (!checkOutput) { - return; - } - Assert.assertTrue( - "Expected output: " + expectedOutput + "\nActual result: " + actualOutput, - expectedOutput.equals(actualOutput) - ); - } - - protected static void checkErrorCode(String errorOutput, String expectedErrorCode, String errorMetadata) { - if (errorOutput != null && expectedErrorCode != null) - Assert.assertTrue( - "Unexpected error code returned; Expected: " - + expectedErrorCode - + - "; Error: " - + errorOutput, - errorOutput.contains(expectedErrorCode) - ); - if (errorOutput != null && errorMetadata != null) - Assert.assertTrue( - "Unexpected metadata returned; Expected: " - + errorMetadata - + - "; Error: " - + errorOutput, - errorOutput.contains(errorMetadata) - ); - } - - public static String getIteratorOutput(SequenceOfItems sequence, int resultSizeCap) { - sequence.open(); - Item result = null; - if (sequence.hasNext()) { - result = sequence.next(); - } - if (result == null) { - return ""; - } - String singleOutput = result.serialize(); - if (!sequence.hasNext()) { - return singleOutput; - } else { - int itemCount = 1; - StringBuilder sb = new StringBuilder(); - sb.append("("); - sb.append(result.serialize()); - sb.append(", "); - while ( - sequence.hasNext() - && - ((itemCount < resultSizeCap - && resultSizeCap > 0) - || - resultSizeCap == 0) - ) { - sb.append(sequence.next().serialize()); - sb.append(", "); - itemCount++; - } - if (sequence.hasNext() && itemCount == resultSizeCap) { - System.err.println( - "Warning! The output sequence contains a large number of items but its materialization was capped at " - + resultSizeCap - + " items. This value can be configured with the --result-size parameter at startup" - ); - } - // remove last comma - String output = sb.toString(); - output = output.substring(0, output.length() - 2); - output += ")"; - return output; - } - } -} From 79ff8c7462182d3d3f91453efd2fa821ac527d1f Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 3 Nov 2025 16:42:54 +0100 Subject: [PATCH 2/3] Fix build. --- .../java/iq/BackwardsCompatibilityTests.java | 4 +- src/test/java/iq/XMLTests.java | 95 +------------------ 2 files changed, 5 insertions(+), 94 deletions(-) diff --git a/src/test/java/iq/BackwardsCompatibilityTests.java b/src/test/java/iq/BackwardsCompatibilityTests.java index d922e54c16..3f9d3c9907 100644 --- a/src/test/java/iq/BackwardsCompatibilityTests.java +++ b/src/test/java/iq/BackwardsCompatibilityTests.java @@ -111,9 +111,7 @@ public final void testRuntimeIterators() throws Throwable { RumbleDBTestCommons.testAnnotations( this.testFile.getAbsolutePath(), getConfiguration(), - true, - getConfiguration().applyUpdates(), - getConfiguration().getResultSizeCap() + true ); } } diff --git a/src/test/java/iq/XMLTests.java b/src/test/java/iq/XMLTests.java index ecc9bffcff..c4db379c8d 100644 --- a/src/test/java/iq/XMLTests.java +++ b/src/test/java/iq/XMLTests.java @@ -20,7 +20,7 @@ package iq; -import iq.base.AnnotationsTestsBase; +import org.rumbledb.tests.commons.RumbleDBTestCommons; import org.apache.spark.SparkConf; import org.junit.Assert; import org.junit.BeforeClass; @@ -42,7 +42,7 @@ import java.util.stream.Collectors; @RunWith(Parameterized.class) -public class XMLTests extends AnnotationsTestsBase { +public class XMLTests { public static final File runtimeTestsDirectory = new File( System.getProperty("user.dir") @@ -116,94 +116,7 @@ public static void setupSparkSession() { @Test(timeout = 1000000) public final void testRuntimeIterators() throws Throwable { - System.err.println(AnnotationsTestsBase.counter++ + " : " + this.testFile); - testAnnotations(this.testFile.getAbsolutePath(), getConfiguration()); - } - - @Override - protected void checkExpectedOutput( - String expectedOutput, - SequenceOfItems sequence - ) { - String actualOutput; - if (!sequence.availableAsRDD()) { - actualOutput = AnnotationsTestsBase.getIteratorOutput(sequence); - } else { - actualOutput = getRDDResults(sequence); - } - Assert.assertTrue( - "Expected output: " + expectedOutput + "\nActual result: " + actualOutput, - expectedOutput.equals(actualOutput) - ); - } - - protected String getIteratorOutput(SequenceOfItems sequence) { - sequence.open(); - Item result = null; - if (sequence.hasNext()) { - result = sequence.next(); - } - if (result == null) { - return ""; - } - String singleOutput = result.serialize(); - if (!sequence.hasNext()) { - return singleOutput; - } else { - int itemCount = 1; - StringBuilder sb = new StringBuilder(); - sb.append("("); - sb.append(result.serialize()); - sb.append(", "); - while ( - sequence.hasNext() - && - ((itemCount < getConfiguration().getResultSizeCap() - && getConfiguration().getResultSizeCap() > 0) - || - getConfiguration().getResultSizeCap() == 0) - ) { - sb.append(sequence.next().serialize()); - sb.append(", "); - itemCount++; - } - if (sequence.hasNext() && itemCount == getConfiguration().getResultSizeCap()) { - System.err.println( - "Warning! The output sequence contains a large number of items but its materialization was capped at " - + getConfiguration().getResultSizeCap() - + " items. This value can be configured with the --result-size parameter at startup" - ); - } - // remove last comma - String output = sb.toString(); - output = output.substring(0, output.length() - 2); - output += ")"; - return output; - } - } - - private String getRDDResults(SequenceOfItems sequence) { - List res = sequence.getFirstItemsAsList(getConfiguration().getResultSizeCap()); - List collectedOutput = res.stream().map(item -> item.serialize()).collect(Collectors.toList()); - - if (collectedOutput.isEmpty()) { - return ""; - } - - if (collectedOutput.size() == 1) { - return collectedOutput.get(0); - } - - StringBuilder sb = new StringBuilder(); - sb.append("("); - for (String item : collectedOutput) { - sb.append(item); - sb.append(", "); - } - - String result = sb.toString(); - result = result.substring(0, result.length() - 2); - result += ")"; - return result; + // System.err.println(RumbleDBTestCommons.counter++ + " : " + this.testFile); + RumbleDBTestCommons.testAnnotations(this.testFile.getAbsolutePath(), getConfiguration(), true); } } From 0708b3925d85e6d443224ea9255550529b72f3bc Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 3 Nov 2025 16:44:44 +0100 Subject: [PATCH 3/3] Add files. --- src/test/java/iq/XMLTests.java | 4 - .../tests/commons/RumbleDBTestCommons.java | 318 ++++++++++++++++++ 2 files changed, 318 insertions(+), 4 deletions(-) create mode 100644 src/test/java/org/rumbledb/tests/commons/RumbleDBTestCommons.java diff --git a/src/test/java/iq/XMLTests.java b/src/test/java/iq/XMLTests.java index c4db379c8d..dc54188036 100644 --- a/src/test/java/iq/XMLTests.java +++ b/src/test/java/iq/XMLTests.java @@ -22,13 +22,10 @@ import org.rumbledb.tests.commons.RumbleDBTestCommons; import org.apache.spark.SparkConf; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.rumbledb.api.Item; -import org.rumbledb.api.SequenceOfItems; import org.rumbledb.config.RumbleRuntimeConfiguration; import scala.Function0; import scala.util.Properties; @@ -39,7 +36,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; @RunWith(Parameterized.class) public class XMLTests { diff --git a/src/test/java/org/rumbledb/tests/commons/RumbleDBTestCommons.java b/src/test/java/org/rumbledb/tests/commons/RumbleDBTestCommons.java new file mode 100644 index 0000000000..3d3458a2b1 --- /dev/null +++ b/src/test/java/org/rumbledb/tests/commons/RumbleDBTestCommons.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Authors: Stefan Irimescu, Can Berker Cikis + * + */ + +package org.rumbledb.tests.commons; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.junit.Assert; +import org.rumbledb.api.Item; +import org.rumbledb.api.Rumble; +import org.rumbledb.api.SequenceOfItems; +import org.rumbledb.config.RumbleRuntimeConfiguration; +import org.rumbledb.context.Name; +import org.rumbledb.exceptions.ExceptionMetadata; +import org.rumbledb.exceptions.ParsingException; +import org.rumbledb.exceptions.SemanticException; +import org.rumbledb.items.ItemFactory; +import org.rumbledb.exceptions.RumbleException; +import org.rumbledb.runtime.functions.input.FileSystemUtil; + +import utils.FileManager; +import utils.annotations.AnnotationParseException; +import utils.annotations.AnnotationProcessor; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class RumbleDBTestCommons { + protected static int counter = 0; + protected static final RumbleRuntimeConfiguration defaultConfiguration = new RumbleRuntimeConfiguration( + new String[] { + "--print-iterator-tree", + "yes", + "--variable:externalUnparsedString", + "unparsed string", + "--materialization-cap", + "200" } + ).setExternalVariableValue( + Name.createVariableInNoNamespace("externalStringItem"), + Collections.singletonList(ItemFactory.getInstance().createStringItem("this is a string")) + ) + .setExternalVariableValue( + Name.createVariableInNoNamespace("externalIntegerItems"), + Arrays.asList( + new Item[] { + ItemFactory.getInstance().createIntItem(1), + ItemFactory.getInstance().createIntItem(2), + ItemFactory.getInstance().createIntItem(3), + ItemFactory.getInstance().createIntItem(4), + ItemFactory.getInstance().createIntItem(5), + } + ) + ); + + public static RumbleRuntimeConfiguration getDefaultConfiguration() { + return defaultConfiguration; + } + + public static List extractTestFilesFromDirectory(File dir) { + List testFiles = new ArrayList<>(); + FileManager.loadJiqFiles(dir).forEach(file -> testFiles.add(file)); + testFiles.sort(Comparator.comparing(File::getName)); + return testFiles; + } + + /** + * Tests annotations + */ + public static void testAnnotations( + String path, + RumbleRuntimeConfiguration configuration, + boolean checkOutput + ) + throws IOException { + AnnotationProcessor.TestAnnotation currentAnnotation = null; + try { + currentAnnotation = AnnotationProcessor.readAnnotation(new FileReader(path)); + } catch (AnnotationParseException e) { + e.printStackTrace(); + Assert.fail(); + } + SequenceOfItems sequence = null; + try { + URI uri = FileSystemUtil.resolveURIAgainstWorkingDirectory( + path, + configuration, + ExceptionMetadata.EMPTY_METADATA + ); + Rumble rumble = new Rumble(configuration); + sequence = rumble.runQuery(uri); + } catch (ParsingException exception) { + String errorOutput = exception.getMessage(); + checkErrorCode( + errorOutput, + currentAnnotation.getErrorCode(), + currentAnnotation.getErrorMetadata() + ); + if (currentAnnotation.shouldParse()) { + Assert.fail("Program did not parse when expected to.\nError output: " + errorOutput + "\n"); + return; + } else { + System.out.println(errorOutput); + return; + } + + // SEMANTIC + } catch (SemanticException exception) { + String errorOutput = exception.getMessage(); + checkErrorCode( + errorOutput, + currentAnnotation.getErrorCode(), + currentAnnotation.getErrorMetadata() + ); + try { + if (currentAnnotation.shouldCompile()) { + Assert.fail("Program did not compile when expected to.\nError output: " + errorOutput + "\n"); + return; + } else { + System.out.println(errorOutput); + Assert.assertTrue(true); + return; + } + } catch (Exception ex) { + } + + // RUNTIME + } catch (RumbleException exception) { + String errorOutput = exception.getMessage(); + checkErrorCode( + errorOutput, + currentAnnotation.getErrorCode(), + currentAnnotation.getErrorMetadata() + ); + try { + if (currentAnnotation.shouldRun()) { + Assert.fail("Program did not run when expected to.\nError output: " + errorOutput + "\n"); + return; + } else { + System.out.println(errorOutput); + Assert.assertTrue(true); + return; + } + } catch (Exception ex) { + } + } + + try { + if (!currentAnnotation.shouldCompile()) { + Assert.fail("Program compiled when not expected to.\n"); + return; + } + } catch (Exception ex) { + } + + if (!currentAnnotation.shouldParse()) { + Assert.fail("Program parsed when not expected to.\n"); + return; + } + + // PROGRAM SHOULD RUN + if ( + currentAnnotation instanceof AnnotationProcessor.RunnableTestAnnotation + && + currentAnnotation.shouldRun() + ) { + try { + checkExpectedOutput( + currentAnnotation.getOutput(), + sequence, + checkOutput, + configuration.applyUpdates(), + configuration.getResultSizeCap() + ); + } catch (RumbleException exception) { + String errorOutput = exception.getMessage(); + errorOutput += "\n" + ExceptionUtils.getStackTrace(exception); + Assert.fail("Program did not run when expected to.\nError output: " + errorOutput + "\n"); + } + } else { + // PROGRAM SHOULD CRASH + if ( + currentAnnotation instanceof AnnotationProcessor.UnrunnableTestAnnotation + && + !currentAnnotation.shouldRun() + ) { + try { + checkExpectedOutput( + currentAnnotation.getOutput(), + sequence, + checkOutput, + configuration.applyUpdates(), + configuration.getResultSizeCap() + ); + } catch (Exception exception) { + String errorOutput = exception.getMessage(); + checkErrorCode( + errorOutput, + currentAnnotation.getErrorCode(), + currentAnnotation.getErrorMetadata() + ); + return; + } + + Assert.fail("Program executed when not expected to"); + } + } + return; + } + + static void checkExpectedOutput( + String expectedOutput, + SequenceOfItems sequence, + boolean checkOutput, + boolean applyUpdates, + int resultSizeCap + ) { + String actualOutput = getIteratorOutput(sequence, resultSizeCap); + if (applyUpdates && sequence.availableAsPUL()) { + sequence.applyPUL(); + } + if (!checkOutput) { + return; + } + Assert.assertTrue( + "Expected output: " + expectedOutput + "\nActual result: " + actualOutput, + expectedOutput.equals(actualOutput) + ); + } + + protected static void checkErrorCode(String errorOutput, String expectedErrorCode, String errorMetadata) { + if (errorOutput != null && expectedErrorCode != null) + Assert.assertTrue( + "Unexpected error code returned; Expected: " + + expectedErrorCode + + + "; Error: " + + errorOutput, + errorOutput.contains(expectedErrorCode) + ); + if (errorOutput != null && errorMetadata != null) + Assert.assertTrue( + "Unexpected metadata returned; Expected: " + + errorMetadata + + + "; Error: " + + errorOutput, + errorOutput.contains(errorMetadata) + ); + } + + public static String getIteratorOutput(SequenceOfItems sequence, int resultSizeCap) { + sequence.open(); + Item result = null; + if (sequence.hasNext()) { + result = sequence.next(); + } + if (result == null) { + return ""; + } + String singleOutput = result.serialize(); + if (!sequence.hasNext()) { + return singleOutput; + } else { + int itemCount = 1; + StringBuilder sb = new StringBuilder(); + sb.append("("); + sb.append(result.serialize()); + sb.append(", "); + while ( + sequence.hasNext() + && + ((itemCount < resultSizeCap + && resultSizeCap > 0) + || + resultSizeCap == 0) + ) { + sb.append(sequence.next().serialize()); + sb.append(", "); + itemCount++; + } + if (sequence.hasNext() && itemCount == resultSizeCap) { + System.err.println( + "Warning! The output sequence contains a large number of items but its materialization was capped at " + + resultSizeCap + + " items. This value can be configured with the --result-size parameter at startup" + ); + } + // remove last comma + String output = sb.toString(); + output = output.substring(0, output.length() - 2); + output += ")"; + return output; + } + } +}