From 1a2406392d712f4fa3828b4bc67c51acbfe07929 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 27 Jul 2026 10:56:50 +0200 Subject: [PATCH 01/12] Fix order by semantics. --- .../flwor/clauses/OrderByClauseIterator.java | 36 ++++++++++++++----- .../udfs/OrderClauseCreateColumnsUDF.java | 16 +++++++++ .../udfs/OrderClauseDetermineTypeUDF.java | 16 +++++++-- 3 files changed, 58 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/rumbledb/runtime/flwor/clauses/OrderByClauseIterator.java b/src/main/java/org/rumbledb/runtime/flwor/clauses/OrderByClauseIterator.java index 976b8f1329..c7c4dc49b9 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/clauses/OrderByClauseIterator.java +++ b/src/main/java/org/rumbledb/runtime/flwor/clauses/OrderByClauseIterator.java @@ -164,14 +164,7 @@ private TreeMap> mapExpressionsToOrderedPairs() { RuntimeIterator iterator = expressionWithIterator.getIterator(); try { Item resultItem = iterator.materializeAtMostOneItemOrNull(tupleContext); - if (resultItem != null && !resultItem.isAtomic()) { - throw new UnexpectedTypeException( - "Keys in an order-by clause must be atomics.", - expressionWithIterator.getIterator().getMetadata() - ); - } - // possibly null for empty sequence. - results.add(resultItem); + results.add(atomizeOrderKey(resultItem, expressionWithIterator)); } catch (MoreThanOneItemException e) { throw new UnexpectedTypeException( "Keys in an order-by clause must be at most one item.", @@ -190,6 +183,33 @@ private TreeMap> mapExpressionsToOrderedPairs() { return keyValuePairs; } + private Item atomizeOrderKey( + Item resultItem, + OrderByClauseAnnotatedChildIterator expressionWithIterator + ) { + if (resultItem == null) { + return null; + } + List atomized = resultItem.atomizedValue(); + if (atomized.size() > 1) { + throw new UnexpectedTypeException( + "Keys in an order-by clause must atomize to at most one item.", + expressionWithIterator.getIterator().getMetadata() + ); + } + if (atomized.isEmpty()) { + return null; + } + Item atomizedItem = atomized.get(0); + if (!atomizedItem.isAtomic()) { + throw new UnexpectedTypeException( + "Keys in an order-by clause must atomize to atomic values.", + expressionWithIterator.getIterator().getMetadata() + ); + } + return atomizedItem; + } + @Override public FlworDataFrame getDataFrame( DynamicContext context diff --git a/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseCreateColumnsUDF.java b/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseCreateColumnsUDF.java index 4f2959adf7..2efe88da4a 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseCreateColumnsUDF.java +++ b/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseCreateColumnsUDF.java @@ -98,6 +98,22 @@ public Row call(Row row) { } while (iterator.hasNext()) { Item nextItem = iterator.next(); + List atomized = nextItem.atomizedValue(); + if (atomized.size() > 1) { + throw new OurBadException( + "Invalid sort key: order by expression must atomize to at most one item." + ); + } + if (atomized.isEmpty()) { + if (expressionWithIterator.getEmptyOrder() == OrderByClauseSortingKey.EMPTY_ORDER.GREATEST) { + this.results.add(emptySequenceOrderIndexLast); + } else { + this.results.add(emptySequenceOrderIndexFirst); + } + this.results.add(null); + continue; + } + nextItem = atomized.get(0); createColumnsForItem(nextItem, expressionIndex); } iterator.close(); diff --git a/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseDetermineTypeUDF.java b/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseDetermineTypeUDF.java index 7292405101..994c0444e6 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseDetermineTypeUDF.java +++ b/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseDetermineTypeUDF.java @@ -82,9 +82,21 @@ private void populateFromExpression(OrderByClauseAnnotatedChildIterator expressi this.result.add(OrderByClauseIterator.StringFlagForEmptySequence); return; } - if (this.nextItem.isArray() || this.nextItem.isObject()) { + List atomized = this.nextItem.atomizedValue(); + if (atomized.size() > 1) { throw new UnexpectedTypeException( - "Order by variable can not contain arrays or objects.", + "Order by variable must atomize to at most one item.", + expressionWithIterator.getIterator().getMetadata() + ); + } + if (atomized.isEmpty()) { + this.result.add(OrderByClauseIterator.StringFlagForEmptySequence); + return; + } + this.nextItem = atomized.get(0); + if (!this.nextItem.isAtomic()) { + throw new UnexpectedTypeException( + "Order by variable must atomize to an atomic value.", expressionWithIterator.getIterator().getMetadata() ); } From 922aa842db93b01952ea262fd1d2f07b83574169 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 27 Jul 2026 11:07:44 +0200 Subject: [PATCH 02/12] Fix collation behavior in order by. --- .../org/rumbledb/compiler/CloneVisitor.java | 3 +- .../compiler/RuntimeIteratorVisitor.java | 3 +- .../rumbledb/compiler/TranslationVisitor.java | 24 +++++++++--- .../compiler/TypeIndependentNodeVisitor.java | 3 +- .../compiler/XQueryTranslationVisitor.java | 24 +++++++++--- .../rumbledb/context/CollationCatalogue.java | 23 ++++++++++- .../org/rumbledb/context/StaticContext.java | 5 ++- .../org/rumbledb/errorcodes/ErrorCode.java | 1 + .../flowr/GroupByVariableDeclaration.java | 15 ++++++++ .../flwor/clauses/GroupByClauseIterator.java | 38 +++++++++++++++++-- .../flwor/clauses/OrderByClauseIterator.java | 11 +++++- .../GroupByClauseSparkIteratorExpression.java | 9 ++++- .../udfs/OrderClauseCreateColumnsUDF.java | 10 ++++- .../udfs/OrderClauseDetermineTypeUDF.java | 9 +++++ .../strings/CompareFunctionIterator.java | 21 +++++----- .../strings/StartsWithFunctionIterator.java | 21 +++++----- .../runtime/misc/ComparisonIterator.java | 20 ++++++++++ 17 files changed, 194 insertions(+), 46 deletions(-) diff --git a/src/main/java/org/rumbledb/compiler/CloneVisitor.java b/src/main/java/org/rumbledb/compiler/CloneVisitor.java index 3e680281d8..f2e891e76d 100644 --- a/src/main/java/org/rumbledb/compiler/CloneVisitor.java +++ b/src/main/java/org/rumbledb/compiler/CloneVisitor.java @@ -312,7 +312,8 @@ public Node visitGroupByClause(GroupByClause clause, Node argument) { variable.getActualSequenceType(), (variable.getExpression() == null) ? variable.getExpression() - : (Expression) visit(variable.getExpression(), argument) + : (Expression) visit(variable.getExpression(), argument), + variable.getCollationURI() ) ); } diff --git a/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java b/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java index 381691f24e..6f971e34a7 100644 --- a/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java +++ b/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java @@ -400,7 +400,8 @@ private RuntimeTupleIterator visitFlowrClause( new GroupByClauseSparkIteratorExpression( groupByExpressionIterator, variableName, - clause.getMetadata() + clause.getMetadata(), + var.getCollationURI() ) ); } diff --git a/src/main/java/org/rumbledb/compiler/TranslationVisitor.java b/src/main/java/org/rumbledb/compiler/TranslationVisitor.java index e784128d01..72fe640e25 100644 --- a/src/main/java/org/rumbledb/compiler/TranslationVisitor.java +++ b/src/main/java/org/rumbledb/compiler/TranslationVisitor.java @@ -1264,9 +1264,9 @@ public Node visitOrderByClause(JsoniqParser.OrderByClauseContext ctx) { public OrderByClauseSortingKey processOrderByExpr(JsoniqParser.OrderByExprContext ctx) { String uri = null; if (ctx.uriLiteral() != null) { - String collation = processURILiteral(ctx.uriLiteral()); + String collation = resolveCollationUri(ctx.uriLiteral()); if (!this.moduleContext.isStaticallyKnownCollation(collation)) { - throw new DefaultCollationException( + throw new UnknownCollationException( "Unknown collation: " + collation, createMetadataFromContext(ctx.uriLiteral()) ); @@ -1294,14 +1294,16 @@ public OrderByClauseSortingKey processOrderByExpr(JsoniqParser.OrderByExprContex } public GroupByVariableDeclaration processGroupByVar(JsoniqParser.GroupByVarContext ctx) { + String collationUri = null; if (ctx.uriLiteral() != null) { - String collation = processURILiteral(ctx.uriLiteral()); + String collation = resolveCollationUri(ctx.uriLiteral()); if (!this.moduleContext.isStaticallyKnownCollation(collation)) { - throw new DefaultCollationException( + throw new UnknownCollationException( "Unknown collation: " + collation, createMetadataFromContext(ctx.uriLiteral()) ); } + collationUri = collation; } SequenceType seq = null; Expression expr = null; @@ -1320,7 +1322,7 @@ public GroupByVariableDeclaration processGroupByVar(JsoniqParser.GroupByVarConte } - return new GroupByVariableDeclaration(var, seq, expr); + return new GroupByVariableDeclaration(var, seq, expr, collationUri); } @Override @@ -4043,7 +4045,7 @@ private void processEmptySequenceOrder(EmptyOrderDeclContext ctx) { } private void processDefaultCollation(DefaultCollationDeclContext ctx) { - String uri = processURILiteral(ctx.uriLiteral()); + String uri = resolveCollationUri(ctx.uriLiteral()); if (!this.moduleContext.isStaticallyKnownCollation(uri)) { throw new DefaultCollationException( "Unknown collation: " + uri, @@ -4053,6 +4055,16 @@ private void processDefaultCollation(DefaultCollationDeclContext ctx) { this.moduleContext.setDefaultCollation(uri); } + private String resolveCollationUri(UriLiteralContext ctx) { + String uriString = processURILiteral(ctx); + URI uri = URILiteralUtils.resolve( + this.moduleContext.getStaticBaseURI(), + uriString, + createMetadataFromContext(ctx) + ); + return uri.toString(); + } + public LibraryModule processModuleImport(JsoniqParser.ModuleImportContext ctx) { String namespace = processURILiteral(ctx.targetNamespace); List locationHints = ctx.locations.stream() diff --git a/src/main/java/org/rumbledb/compiler/TypeIndependentNodeVisitor.java b/src/main/java/org/rumbledb/compiler/TypeIndependentNodeVisitor.java index ec719aef6c..a3c24025fb 100644 --- a/src/main/java/org/rumbledb/compiler/TypeIndependentNodeVisitor.java +++ b/src/main/java/org/rumbledb/compiler/TypeIndependentNodeVisitor.java @@ -136,7 +136,8 @@ public Node visitGroupByClause(GroupByClause clause, Node argument) { variable.getActualSequenceType(), (variable.getExpression() == null) ? variable.getExpression() - : (Expression) visit(variable.getExpression(), argument) + : (Expression) visit(variable.getExpression(), argument), + variable.getCollationURI() ) ); } diff --git a/src/main/java/org/rumbledb/compiler/XQueryTranslationVisitor.java b/src/main/java/org/rumbledb/compiler/XQueryTranslationVisitor.java index b0dd933a72..52ac9ff33d 100644 --- a/src/main/java/org/rumbledb/compiler/XQueryTranslationVisitor.java +++ b/src/main/java/org/rumbledb/compiler/XQueryTranslationVisitor.java @@ -1177,9 +1177,9 @@ public Node visitOrderByClause(XQueryParser.OrderByClauseContext ctx) { public OrderByClauseSortingKey processOrderByExpr(XQueryParser.OrderByExprContext ctx) { String uri = null; if (ctx.uriLiteral() != null) { - String collation = processURILiteral(ctx.uriLiteral()); + String collation = resolveCollationUri(ctx.uriLiteral()); if (!this.moduleContext.isStaticallyKnownCollation(collation)) { - throw new DefaultCollationException( + throw new UnknownCollationException( "Unknown collation: " + collation, createMetadataFromContext(ctx.uriLiteral()) ); @@ -1207,14 +1207,16 @@ public OrderByClauseSortingKey processOrderByExpr(XQueryParser.OrderByExprContex } public GroupByVariableDeclaration processGroupByVar(XQueryParser.GroupByVarContext ctx) { + String collationUri = null; if (ctx.uriLiteral() != null) { - String collation = processURILiteral(ctx.uriLiteral()); + String collation = resolveCollationUri(ctx.uriLiteral()); if (!this.moduleContext.isStaticallyKnownCollation(collation)) { - throw new DefaultCollationException( + throw new UnknownCollationException( "Unknown collation: " + collation, createMetadataFromContext(ctx.uriLiteral()) ); } + collationUri = collation; } SequenceType seq = null; Expression expr = null; @@ -1233,7 +1235,7 @@ public GroupByVariableDeclaration processGroupByVar(XQueryParser.GroupByVarConte } - return new GroupByVariableDeclaration(var, seq, expr); + return new GroupByVariableDeclaration(var, seq, expr, collationUri); } @Override @@ -3735,7 +3737,7 @@ private void processEmptySequenceOrder(EmptyOrderDeclContext ctx) { } private void processDefaultCollation(DefaultCollationDeclContext ctx) { - String uri = processURILiteral(ctx.uriLiteral()); + String uri = resolveCollationUri(ctx.uriLiteral()); if (!this.moduleContext.isStaticallyKnownCollation(uri)) { throw new DefaultCollationException( "Unknown collation: " + uri, @@ -3745,6 +3747,16 @@ private void processDefaultCollation(DefaultCollationDeclContext ctx) { this.moduleContext.setDefaultCollation(uri); } + private String resolveCollationUri(UriLiteralContext ctx) { + String uriString = processURILiteral(ctx); + URI uri = URILiteralUtils.resolve( + this.moduleContext.getStaticBaseURI(), + uriString, + createMetadataFromContext(ctx) + ); + return uri.toString(); + } + public LibraryModule processModuleImport(XQueryParser.ModuleImportContext ctx) { String namespace = processURILiteral(ctx.targetNamespace); List locationHints = ctx.locations.stream() diff --git a/src/main/java/org/rumbledb/context/CollationCatalogue.java b/src/main/java/org/rumbledb/context/CollationCatalogue.java index 9a9500a3f4..035a7ee8d9 100644 --- a/src/main/java/org/rumbledb/context/CollationCatalogue.java +++ b/src/main/java/org/rumbledb/context/CollationCatalogue.java @@ -2,6 +2,7 @@ import java.util.Collections; import java.util.LinkedHashSet; +import java.util.Locale; import java.util.Set; /** @@ -12,6 +13,9 @@ public final class CollationCatalogue { public static final String CODEPOINT_COLLATION = Name.DEFAULT_COLLATION_NS; public static final String FOTS_CASEBLIND_COLLATION = "http://www.w3.org/2010/09/qt-fots-catalog/collation/caseblind"; + public static final String HTML_ASCII_CASE_INSENSITIVE_COLLATION = + "http://www.w3.org/2005/xpath-functions/collation/html-ascii-case-insensitive"; + public static final String UCA_COLLATION_BASE = "http://www.w3.org/2013/collation/UCA"; private static final Set DEFAULT_STATICALLY_KNOWN_COLLATIONS; @@ -19,6 +23,7 @@ public final class CollationCatalogue { Set collations = new LinkedHashSet<>(); collations.add(CODEPOINT_COLLATION); collations.add(FOTS_CASEBLIND_COLLATION); + collations.add(HTML_ASCII_CASE_INSENSITIVE_COLLATION); DEFAULT_STATICALLY_KNOWN_COLLATIONS = Collections.unmodifiableSet(collations); } @@ -30,6 +35,22 @@ public static Set defaultStaticallyKnownCollations() { } public static boolean isDefaultStaticallyKnownCollation(String uri) { - return DEFAULT_STATICALLY_KNOWN_COLLATIONS.contains(uri); + return DEFAULT_STATICALLY_KNOWN_COLLATIONS.contains(uri) + || UCA_COLLATION_BASE.equals(uri) + || uri.startsWith(UCA_COLLATION_BASE + "?"); + } + + public static boolean isCaseInsensitiveCollation(String uri) { + return FOTS_CASEBLIND_COLLATION.equals(uri) + || HTML_ASCII_CASE_INSENSITIVE_COLLATION.equals(uri) + || UCA_COLLATION_BASE.equals(uri) + || uri.startsWith(UCA_COLLATION_BASE + "?"); + } + + public static String normalizeString(String value, String collationUri) { + if (isCaseInsensitiveCollation(collationUri)) { + return value.toLowerCase(Locale.ROOT); + } + return value; } } diff --git a/src/main/java/org/rumbledb/context/StaticContext.java b/src/main/java/org/rumbledb/context/StaticContext.java index 1f8d1d1459..db50be7216 100644 --- a/src/main/java/org/rumbledb/context/StaticContext.java +++ b/src/main/java/org/rumbledb/context/StaticContext.java @@ -661,7 +661,8 @@ public void addStaticallyKnownCollation(String uri) { } public boolean isStaticallyKnownCollation(String uri) { - return getStaticallyKnownCollations().contains(uri); + return getStaticallyKnownCollations().contains(uri) + || CollationCatalogue.isDefaultStaticallyKnownCollation(uri); } public Set getStaticallyKnownCollations() { @@ -677,7 +678,7 @@ public void setDefaultCollation(String uri) { throw new OurBadException("Default collation can only be set in the root static context."); } ensureRootCollationsInitialized(); - if (!this.staticallyKnownCollations.contains(uri)) { + if (!isStaticallyKnownCollation(uri)) { throw new OurBadException("Default collation must be statically known."); } this.defaultCollation = uri; diff --git a/src/main/java/org/rumbledb/errorcodes/ErrorCode.java b/src/main/java/org/rumbledb/errorcodes/ErrorCode.java index cab2e2eb9e..0ef1bfa0b0 100644 --- a/src/main/java/org/rumbledb/errorcodes/ErrorCode.java +++ b/src/main/java/org/rumbledb/errorcodes/ErrorCode.java @@ -202,6 +202,7 @@ public int hashCode() { public static final ErrorCode MoreThanOneBoundarySpaceDeclarationErrorCode = registerBuiltIn("XQST0068"); public static final ErrorCode MoreThanOneEmptyOrderDeclarationErrorCode = registerBuiltIn("XQST0069"); public static final ErrorCode PredefinedPrefixInNamespaceDeclarationErrorCode = registerBuiltIn("XQST0070"); + public static final ErrorCode UnknownCollationInQueryPrologOrClause = registerBuiltIn("XQST0076"); public static final ErrorCode EmptyNamespaceURIForPrefixedBindingErrorCode = registerBuiltIn("XQST0085"); public static final ErrorCode EmptyModuleURIErrorCode = registerBuiltIn("XQST0088"); public static final ErrorCode PositionalVariableNameSameAsForVariable = registerBuiltIn("XQST0089"); diff --git a/src/main/java/org/rumbledb/expressions/flowr/GroupByVariableDeclaration.java b/src/main/java/org/rumbledb/expressions/flowr/GroupByVariableDeclaration.java index 9ddddb002d..7786e2c542 100644 --- a/src/main/java/org/rumbledb/expressions/flowr/GroupByVariableDeclaration.java +++ b/src/main/java/org/rumbledb/expressions/flowr/GroupByVariableDeclaration.java @@ -29,11 +29,21 @@ public class GroupByVariableDeclaration { protected Name variableName; protected Expression expression; protected SequenceType sequenceType; + protected String collationURI; public GroupByVariableDeclaration( Name variableName, SequenceType sequenceType, Expression expression + ) { + this(variableName, sequenceType, expression, null); + } + + public GroupByVariableDeclaration( + Name variableName, + SequenceType sequenceType, + Expression expression, + String collationURI ) { if (variableName == null) { throw new IllegalArgumentException("Flowr var decls cannot be empty"); @@ -41,6 +51,7 @@ public GroupByVariableDeclaration( this.variableName = variableName; this.sequenceType = sequenceType; this.expression = expression; + this.collationURI = collationURI; } public Name getVariableName() { @@ -58,4 +69,8 @@ public SequenceType getSequenceType() { public SequenceType getActualSequenceType() { return this.sequenceType; } + + public String getCollationURI() { + return this.collationURI; + } } diff --git a/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java b/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java index 35dd2c038d..7976146a3f 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java +++ b/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java @@ -49,6 +49,7 @@ import org.rumbledb.runtime.flwor.udfs.GroupClauseArrayMergeAggregateResultsUDF; import org.rumbledb.runtime.flwor.udfs.GroupClauseCreateColumnsUDF; import org.rumbledb.runtime.flwor.udfs.GroupClauseSerializeAggregateResultsUDF; +import org.rumbledb.runtime.misc.CollationSupport; import org.rumbledb.types.TypeMappings; import sparksoniq.jsoniq.tuple.FlworKey; import sparksoniq.jsoniq.tuple.FlworTuple; @@ -187,13 +188,32 @@ private HashMap> mapTuplesToPairs() { ); } if (resultItem != null) { - if (!resultItem.isAtomic()) { + List atomizedResults = resultItem.atomizedValue(); + if (atomizedResults.size() > 1) { throw new UnexpectedTypeException( - "Keys in a group-by clause must be atomics.", + "Keys in a group-by clause must atomize to at most one item.", getMetadata() ); } - newVariableResults = Collections.singletonList(resultItem); + if (atomizedResults.isEmpty()) { + newVariableResults = Collections.emptyList(); + } else { + Item atomizedResult = atomizedResults.get(0); + if (!atomizedResult.isAtomic()) { + throw new UnexpectedTypeException( + "Keys in a group-by clause must atomize to atomic values.", + getMetadata() + ); + } + atomizedResult = CollationSupport.normalizeItemForCollation( + atomizedResult, + expression.getCollationURI() == null + ? getStaticContext().getDefaultCollation() + : expression.getCollationURI(), + getMetadata() + ); + newVariableResults = Collections.singletonList(atomizedResult); + } } else { newVariableResults = Collections.emptyList(); } @@ -225,6 +245,18 @@ private HashMap> mapTuplesToPairs() { getMetadata() ); } + if (atomizedGroupValues.size() == 1) { + atomizedGroupValues.set( + 0, + CollationSupport.normalizeItemForCollation( + atomizedGroupValues.get(0), + expression.getCollationURI() == null + ? getStaticContext().getDefaultCollation() + : expression.getCollationURI(), + getMetadata() + ) + ); + } inputTuple.putValue(groupVariableName, atomizedGroupValues); results.addAll(atomizedGroupValues); } diff --git a/src/main/java/org/rumbledb/runtime/flwor/clauses/OrderByClauseIterator.java b/src/main/java/org/rumbledb/runtime/flwor/clauses/OrderByClauseIterator.java index c7c4dc49b9..61d37bd40f 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/clauses/OrderByClauseIterator.java +++ b/src/main/java/org/rumbledb/runtime/flwor/clauses/OrderByClauseIterator.java @@ -47,6 +47,7 @@ import org.rumbledb.runtime.flwor.expression.OrderByClauseAnnotatedChildIterator; import org.rumbledb.runtime.flwor.udfs.OrderClauseCreateColumnsUDF; import org.rumbledb.runtime.flwor.udfs.OrderClauseDetermineTypeUDF; +import org.rumbledb.runtime.misc.CollationSupport; import org.rumbledb.types.BuiltinTypesCatalogue; import org.rumbledb.types.SequenceType; import org.rumbledb.types.SequenceType.Arity; @@ -207,7 +208,15 @@ private Item atomizeOrderKey( expressionWithIterator.getIterator().getMetadata() ); } - return atomizedItem; + String collationUri = CollationSupport.resolveCollation( + expressionWithIterator.getUri(), + expressionWithIterator.getIterator().getRuntimeStaticContext() + ); + return CollationSupport.normalizeItemForCollation( + atomizedItem, + collationUri, + expressionWithIterator.getIterator().getMetadata() + ); } @Override diff --git a/src/main/java/org/rumbledb/runtime/flwor/expression/GroupByClauseSparkIteratorExpression.java b/src/main/java/org/rumbledb/runtime/flwor/expression/GroupByClauseSparkIteratorExpression.java index cef9b464b8..f7e94953d6 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/expression/GroupByClauseSparkIteratorExpression.java +++ b/src/main/java/org/rumbledb/runtime/flwor/expression/GroupByClauseSparkIteratorExpression.java @@ -36,15 +36,18 @@ public class GroupByClauseSparkIteratorExpression implements Serializable { private final Name variableName; private final RuntimeIterator expression; private final ExceptionMetadata iteratorMetadata; + private final String collationURI; public GroupByClauseSparkIteratorExpression( RuntimeIterator expression, Name variableName, - ExceptionMetadata iteratorMetadata + ExceptionMetadata iteratorMetadata, + String collationURI ) { this.expression = expression; this.variableName = variableName; this.iteratorMetadata = iteratorMetadata; + this.collationURI = collationURI; } public Name getVariableName() { @@ -58,4 +61,8 @@ public ExceptionMetadata getIteratorMetadata() { public RuntimeIterator getExpression() { return this.expression; } + + public String getCollationURI() { + return this.collationURI; + } } diff --git a/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseCreateColumnsUDF.java b/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseCreateColumnsUDF.java index 2efe88da4a..37fccf5db8 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseCreateColumnsUDF.java +++ b/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseCreateColumnsUDF.java @@ -32,6 +32,7 @@ import org.rumbledb.runtime.RuntimeIterator; import org.rumbledb.runtime.flwor.FlworDataFrameColumn; import org.rumbledb.runtime.flwor.expression.OrderByClauseAnnotatedChildIterator; +import org.rumbledb.runtime.misc.CollationSupport; import org.rumbledb.types.BuiltinTypesCatalogue; import java.io.Serial; @@ -113,7 +114,14 @@ public Row call(Row row) { this.results.add(null); continue; } - nextItem = atomized.get(0); + nextItem = CollationSupport.normalizeItemForCollation( + atomized.get(0), + CollationSupport.resolveCollation( + expressionWithIterator.getUri(), + expressionWithIterator.getIterator().getRuntimeStaticContext() + ), + expressionWithIterator.getIterator().getMetadata() + ); createColumnsForItem(nextItem, expressionIndex); } iterator.close(); diff --git a/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseDetermineTypeUDF.java b/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseDetermineTypeUDF.java index 994c0444e6..d6cf451848 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseDetermineTypeUDF.java +++ b/src/main/java/org/rumbledb/runtime/flwor/udfs/OrderClauseDetermineTypeUDF.java @@ -30,6 +30,7 @@ import org.rumbledb.runtime.flwor.FlworDataFrameColumn; import org.rumbledb.runtime.flwor.clauses.OrderByClauseIterator; import org.rumbledb.runtime.flwor.expression.OrderByClauseAnnotatedChildIterator; +import org.rumbledb.runtime.misc.CollationSupport; import java.io.Serial; import java.util.ArrayList; @@ -100,6 +101,14 @@ private void populateFromExpression(OrderByClauseAnnotatedChildIterator expressi expressionWithIterator.getIterator().getMetadata() ); } + this.nextItem = CollationSupport.normalizeItemForCollation( + this.nextItem, + CollationSupport.resolveCollation( + expressionWithIterator.getUri(), + expressionWithIterator.getIterator().getRuntimeStaticContext() + ), + expressionWithIterator.getIterator().getMetadata() + ); this.result.add(this.nextItem.getDynamicType().getName().getLocalName()); } } diff --git a/src/main/java/org/rumbledb/runtime/functions/strings/CompareFunctionIterator.java b/src/main/java/org/rumbledb/runtime/functions/strings/CompareFunctionIterator.java index 9ea4bc7ea1..4309ffd0e6 100644 --- a/src/main/java/org/rumbledb/runtime/functions/strings/CompareFunctionIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/strings/CompareFunctionIterator.java @@ -23,10 +23,10 @@ import org.rumbledb.api.Item; import org.rumbledb.context.DynamicContext; import org.rumbledb.context.RuntimeStaticContext; -import org.rumbledb.exceptions.UnsupportedCollationException; import org.rumbledb.items.ItemFactory; import org.rumbledb.runtime.AtMostOneItemLocalRuntimeIterator; import org.rumbledb.runtime.RuntimeIterator; +import org.rumbledb.runtime.misc.CollationSupport; import java.io.Serial; import java.math.BigInteger; @@ -46,12 +46,9 @@ public CompareFunctionIterator( @Override public Item materializeFirstItemOrNull(DynamicContext context) { - if (this.children.size() == 3) { - String collation = this.children.get(2).materializeFirstItemOrNull(context).getStringValue(); - if (!collation.equals("http://www.w3.org/2005/xpath-functions/collation/codepoint")) { - throw new UnsupportedCollationException("Wrong collation parameter", getMetadata()); - } - } + String collation = this.children.size() == 3 + ? this.children.get(2).materializeFirstItemOrNull(context).getStringValue() + : getRuntimeStaticContext().getDefaultCollation(); Item firstStringItem = this.children.get(0) .materializeFirstItemOrNull(context); Item secondStringItem = this.children.get(1) @@ -60,10 +57,12 @@ public Item materializeFirstItemOrNull(DynamicContext context) { return null; } int result = Integer.signum( - firstStringItem.getStringValue() - .compareTo( - secondStringItem.getStringValue() - ) + CollationSupport.compareStrings( + firstStringItem.getStringValue(), + secondStringItem.getStringValue(), + collation, + getMetadata() + ) ); return ItemFactory.getInstance().createIntegerItem(BigInteger.valueOf(result)); } diff --git a/src/main/java/org/rumbledb/runtime/functions/strings/StartsWithFunctionIterator.java b/src/main/java/org/rumbledb/runtime/functions/strings/StartsWithFunctionIterator.java index d2dcd02b34..5146ae2ef9 100644 --- a/src/main/java/org/rumbledb/runtime/functions/strings/StartsWithFunctionIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/strings/StartsWithFunctionIterator.java @@ -23,10 +23,10 @@ import org.rumbledb.api.Item; import org.rumbledb.context.DynamicContext; import org.rumbledb.context.RuntimeStaticContext; -import org.rumbledb.exceptions.UnsupportedCollationException; import org.rumbledb.items.ItemFactory; import org.rumbledb.runtime.AtMostOneItemLocalRuntimeIterator; import org.rumbledb.runtime.RuntimeIterator; +import org.rumbledb.runtime.misc.CollationSupport; import java.io.Serial; import java.util.List; @@ -45,12 +45,9 @@ public StartsWithFunctionIterator( @Override public Item materializeFirstItemOrNull(DynamicContext context) { - if (this.children.size() == 3) { - String collation = this.children.get(2).materializeFirstItemOrNull(context).getStringValue(); - if (!collation.equals("http://www.w3.org/2005/xpath-functions/collation/codepoint")) { - throw new UnsupportedCollationException("Wrong collation parameter", getMetadata()); - } - } + String collation = this.children.size() == 3 + ? this.children.get(2).materializeFirstItemOrNull(context).getStringValue() + : getRuntimeStaticContext().getDefaultCollation(); Item substringItem = this.children.get(1) .materializeFirstItemOrNull(context); @@ -62,10 +59,12 @@ public Item materializeFirstItemOrNull(DynamicContext context) { if (stringItem == null || stringItem.getStringValue().isEmpty()) { return ItemFactory.getInstance().createBooleanItem(false); } - boolean result = stringItem.getStringValue() - .startsWith( - substringItem.getStringValue() - ); + boolean result = CollationSupport.startsWith( + stringItem.getStringValue(), + substringItem.getStringValue(), + collation, + getMetadata() + ); return ItemFactory.getInstance().createBooleanItem(result); } diff --git a/src/main/java/org/rumbledb/runtime/misc/ComparisonIterator.java b/src/main/java/org/rumbledb/runtime/misc/ComparisonIterator.java index 2c219ae698..c07b2ca3e6 100644 --- a/src/main/java/org/rumbledb/runtime/misc/ComparisonIterator.java +++ b/src/main/java/org/rumbledb/runtime/misc/ComparisonIterator.java @@ -24,6 +24,7 @@ import java.time.*; import org.rumbledb.api.Item; +import org.rumbledb.context.Name; import org.rumbledb.context.DynamicContext; import org.rumbledb.context.RuntimeStaticContext; import org.rumbledb.exceptions.CastException; @@ -156,6 +157,25 @@ public Item materializeFirstItemOrNull(DynamicContext dynamicContext) { throw new IteratorFlowException("Invalid comparison expression", getMetadata()); } + String activeCollation = getRuntimeStaticContext().getDefaultCollation(); + if ( + !Name.DEFAULT_COLLATION_NS.equals(activeCollation) + && CollationSupport.isStringCollationType(this.left) + && CollationSupport.isStringCollationType(this.right) + ) { + int comparison = CollationSupport.compareStrings( + this.left.getStringValue(), + this.right.getStringValue(), + activeCollation, + getMetadata() + ); + return comparisonResultToBooleanItem( + comparison, + this.comparisonOperator, + getMetadata() + ); + } + long comparison = compareItems(this.left, this.right, this.comparisonOperator, getMetadata()); if (comparison == Long.MIN_VALUE) { throw new UnexpectedTypeException( From 5937300311e44bbdeb5b9700ed7835abb10cbf15 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 27 Jul 2026 11:33:09 +0200 Subject: [PATCH 03/12] Change collation catalogue. --- src/main/java/org/rumbledb/context/CollationCatalogue.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/rumbledb/context/CollationCatalogue.java b/src/main/java/org/rumbledb/context/CollationCatalogue.java index 035a7ee8d9..8538ba3b7f 100644 --- a/src/main/java/org/rumbledb/context/CollationCatalogue.java +++ b/src/main/java/org/rumbledb/context/CollationCatalogue.java @@ -48,9 +48,9 @@ public static boolean isCaseInsensitiveCollation(String uri) { } public static String normalizeString(String value, String collationUri) { - if (isCaseInsensitiveCollation(collationUri)) { - return value.toLowerCase(Locale.ROOT); + if (value == null || !isCaseInsensitiveCollation(collationUri)) { + return value; } - return value; + return value.toLowerCase(Locale.ROOT); } } From dff956fded5ebd350f57bb9d21b9993fad002f80 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 27 Jul 2026 11:38:16 +0200 Subject: [PATCH 04/12] Add missing files. --- .../exceptions/UnknownCollationException.java | 33 +++ .../runtime/misc/CollationSupport.java | 239 ++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 src/main/java/org/rumbledb/exceptions/UnknownCollationException.java create mode 100644 src/main/java/org/rumbledb/runtime/misc/CollationSupport.java diff --git a/src/main/java/org/rumbledb/exceptions/UnknownCollationException.java b/src/main/java/org/rumbledb/exceptions/UnknownCollationException.java new file mode 100644 index 0000000000..c162afa684 --- /dev/null +++ b/src/main/java/org/rumbledb/exceptions/UnknownCollationException.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.rumbledb.exceptions; + +import org.rumbledb.errorcodes.ErrorCode; + +import java.io.Serial; + +public class UnknownCollationException extends RumbleException { + + @Serial + private static final long serialVersionUID = 1L; + + public UnknownCollationException(String message, ExceptionMetadata metadata) { + super(message, ErrorCode.UnknownCollationInQueryPrologOrClause, metadata); + } +} diff --git a/src/main/java/org/rumbledb/runtime/misc/CollationSupport.java b/src/main/java/org/rumbledb/runtime/misc/CollationSupport.java new file mode 100644 index 0000000000..ce8c83e111 --- /dev/null +++ b/src/main/java/org/rumbledb/runtime/misc/CollationSupport.java @@ -0,0 +1,239 @@ +package org.rumbledb.runtime.misc; + +import com.ibm.icu.text.Collator; +import com.ibm.icu.text.RuleBasedCollator; +import com.ibm.icu.text.StringSearch; +import com.ibm.icu.util.ULocale; +import org.rumbledb.api.Item; +import org.rumbledb.context.CollationCatalogue; +import org.rumbledb.context.Name; +import org.rumbledb.context.RuntimeStaticContext; +import org.rumbledb.exceptions.ExceptionMetadata; +import org.rumbledb.exceptions.UnsupportedCollationException; +import org.rumbledb.items.ItemFactory; + +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public final class CollationSupport { + + private static final ConcurrentHashMap UCA_COLLATOR_CACHE = new ConcurrentHashMap<>(); + + private CollationSupport() { + } + + public static String resolveCollation(String explicitCollationUri, RuntimeStaticContext staticContext) { + if (explicitCollationUri != null) { + return explicitCollationUri; + } + return staticContext.getDefaultCollation(); + } + + public static void checkCollationSupported(String collationUri, ExceptionMetadata metadata) { + if (CollationCatalogue.isDefaultStaticallyKnownCollation(collationUri)) { + return; + } + throw new UnsupportedCollationException("Wrong collation parameter", metadata); + } + + public static boolean isStringCollationType(Item item) { + return item != null && (item.isString() || item.isAnyURI() || item.isUntypedAtomic()); + } + + public static Item normalizeItemForCollation(Item item, String collationUri, ExceptionMetadata metadata) { + if (item == null) { + return null; + } + checkCollationSupported(collationUri, metadata); + if (!isStringCollationType(item) || Name.DEFAULT_COLLATION_NS.equals(collationUri)) { + return item; + } + if (CollationCatalogue.isUCACollation(collationUri)) { + byte[] sortKeyBytes = getUcaCollator(collationUri, metadata) + .getCollationKey(item.getStringValue()) + .toByteArray(); + return ItemFactory.getInstance().createHexBinaryItem(HexFormat.of().formatHex(sortKeyBytes)); + } + return ItemFactory.getInstance() + .createStringItem( + CollationCatalogue.normalizeString(item.getStringValue(), collationUri) + ); + } + + public static int compareStrings(String left, String right, String collationUri, ExceptionMetadata metadata) { + checkCollationSupported(collationUri, metadata); + if (Name.DEFAULT_COLLATION_NS.equals(collationUri)) { + return left.compareTo(right); + } + if (CollationCatalogue.isUCACollation(collationUri)) { + return getUcaCollator(collationUri, metadata).compare(left, right); + } + return CollationCatalogue.normalizeString(left, collationUri) + .compareTo(CollationCatalogue.normalizeString(right, collationUri)); + } + + public static boolean startsWith(String value, String prefix, String collationUri, ExceptionMetadata metadata) { + checkCollationSupported(collationUri, metadata); + if (Name.DEFAULT_COLLATION_NS.equals(collationUri)) { + return value.startsWith(prefix); + } + if (CollationCatalogue.isUCACollation(collationUri)) { + RuleBasedCollator collator = getUcaCollator(collationUri, metadata); + StringSearch stringSearch = new StringSearch(prefix, value, collator); + return stringSearch.first() == 0; + } + return CollationCatalogue.normalizeString(value, collationUri) + .startsWith(CollationCatalogue.normalizeString(prefix, collationUri)); + } + + private static RuleBasedCollator getUcaCollator(String collationUri, ExceptionMetadata metadata) { + try { + RuleBasedCollator prototype = UCA_COLLATOR_CACHE.computeIfAbsent( + collationUri, + uri -> buildUcaCollator(uri, metadata) + ); + return (RuleBasedCollator) prototype.clone(); + } catch (RuntimeException e) { + if (e instanceof UnsupportedCollationException) { + throw e; + } + throw new UnsupportedCollationException("Wrong collation parameter", metadata); + } + } + + private static RuleBasedCollator buildUcaCollator(String collationUri, ExceptionMetadata metadata) { + UcaParameters parameters = parseUcaParameters(collationUri, metadata); + ULocale locale = parameters.languageTag == null + ? ULocale.ROOT + : ULocale.forLanguageTag(parameters.languageTag); + Collator collator = Collator.getInstance(locale); + if (!(collator instanceof RuleBasedCollator ruleBasedCollator)) { + throw new UnsupportedCollationException("Wrong collation parameter", metadata); + } + + ruleBasedCollator.setStrength(parameters.strength); + ruleBasedCollator.setDecomposition( + parameters.normalization + ? Collator.CANONICAL_DECOMPOSITION + : Collator.NO_DECOMPOSITION + ); + ruleBasedCollator.setCaseLevel(parameters.caseLevel); + ruleBasedCollator.setFrenchCollation(parameters.backwards); + if (parameters.alternateShifted != null) { + ruleBasedCollator.setAlternateHandlingShifted(parameters.alternateShifted); + } + return ruleBasedCollator; + } + + private static UcaParameters parseUcaParameters(String collationUri, ExceptionMetadata metadata) { + UcaParameters parameters = new UcaParameters(); + int queryIndex = collationUri.indexOf('?'); + if (queryIndex < 0 || queryIndex == collationUri.length() - 1) { + return parameters; + } + String query = collationUri.substring(queryIndex + 1); + Map queryParameters = new HashMap<>(); + for (String part : query.split(";")) { + if (part.isEmpty()) { + continue; + } + int separator = part.indexOf('='); + if (separator < 0) { + queryParameters.put(decodeQueryComponent(part), ""); + } else { + queryParameters.put( + decodeQueryComponent(part.substring(0, separator)), + decodeQueryComponent(part.substring(separator + 1)) + ); + } + } + + for (Map.Entry parameter : queryParameters.entrySet()) { + String key = parameter.getKey(); + String value = parameter.getValue(); + switch (key) { + case "lang": + parameters.languageTag = value; + break; + case "strength": + parameters.strength = parseStrength(value, metadata); + break; + case "normalization": + parameters.normalization = parseYesNo(value, key, metadata); + break; + case "backwards": + parameters.backwards = parseYesNo(value, key, metadata); + break; + case "caseLevel": + parameters.caseLevel = parseYesNo(value, key, metadata); + break; + case "alternate": + parameters.alternateShifted = parseAlternate(value, metadata); + break; + case "fallback": + case "version": + break; + default: + if ("no".equals(queryParameters.get("fallback"))) { + throw new UnsupportedCollationException("Wrong collation parameter", metadata); + } + break; + } + } + return parameters; + } + + private static String decodeQueryComponent(String value) { + return URLDecoder.decode(value, StandardCharsets.UTF_8); + } + + private static boolean parseYesNo(String value, String key, ExceptionMetadata metadata) { + if ("yes".equalsIgnoreCase(value)) { + return true; + } + if ("no".equalsIgnoreCase(value)) { + return false; + } + throw new UnsupportedCollationException("Wrong collation parameter", metadata); + } + + private static Boolean parseAlternate(String value, ExceptionMetadata metadata) { + String normalized = value.toLowerCase(Locale.ROOT); + if ("shifted".equals(normalized)) { + return true; + } + if ("non-ignorable".equals(normalized)) { + return false; + } + if ("blanked".equals(normalized)) { + return true; + } + throw new UnsupportedCollationException("Wrong collation parameter", metadata); + } + + private static int parseStrength(String value, ExceptionMetadata metadata) { + String normalized = value.toLowerCase(Locale.ROOT); + return switch (normalized) { + case "1", "primary" -> Collator.PRIMARY; + case "2", "secondary" -> Collator.SECONDARY; + case "3", "tertiary" -> Collator.TERTIARY; + case "4", "quaternary" -> Collator.QUATERNARY; + case "5", "identical" -> Collator.IDENTICAL; + default -> throw new UnsupportedCollationException("Wrong collation parameter", metadata); + }; + } + + private static final class UcaParameters { + private String languageTag; + private int strength = Collator.TERTIARY; + private boolean normalization; + private boolean backwards; + private boolean caseLevel; + private Boolean alternateShifted; + } +} From aa7b069f4725e5a8e6cb7f324565268780b3e396 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 27 Jul 2026 12:04:43 +0200 Subject: [PATCH 05/12] Fix regressions. --- .../rumbledb/compiler/InferTypeVisitor.java | 2 +- .../compiler/RuntimeIteratorVisitor.java | 3 +- .../rumbledb/compiler/TranslationVisitor.java | 4 -- .../compiler/XQueryTranslationVisitor.java | 4 -- .../flwor/clauses/GroupByClauseIterator.java | 42 +++++++++++++++++++ .../GroupByClauseSparkIteratorExpression.java | 10 ++++- .../runtime/misc/CollationSupport.java | 9 +++- 7 files changed, 61 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/rumbledb/compiler/InferTypeVisitor.java b/src/main/java/org/rumbledb/compiler/InferTypeVisitor.java index e05ab4f309..e926bd0f08 100644 --- a/src/main/java/org/rumbledb/compiler/InferTypeVisitor.java +++ b/src/main/java/org/rumbledb/compiler/InferTypeVisitor.java @@ -2645,7 +2645,7 @@ public StaticContext visitGroupByClause(GroupByClause expression, StaticContext inferredType = groupByVarExpr.getStaticSequenceType(); expectedType = inferredType; } else { - inferredType = ((TreatExpression) groupByVarExpr).getMainExpression().getStaticSequenceType(); + inferredType = groupByVarExpr.getStaticSequenceType(); expectedType = declaredType; } checkAndUpdateVariableStaticType( diff --git a/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java b/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java index 6f971e34a7..0a7e84944b 100644 --- a/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java +++ b/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java @@ -401,7 +401,8 @@ private RuntimeTupleIterator visitFlowrClause( groupByExpressionIterator, variableName, clause.getMetadata(), - var.getCollationURI() + var.getCollationURI(), + var.getActualSequenceType() ) ); } diff --git a/src/main/java/org/rumbledb/compiler/TranslationVisitor.java b/src/main/java/org/rumbledb/compiler/TranslationVisitor.java index 72fe640e25..905cdb3e36 100644 --- a/src/main/java/org/rumbledb/compiler/TranslationVisitor.java +++ b/src/main/java/org/rumbledb/compiler/TranslationVisitor.java @@ -1315,10 +1315,6 @@ public GroupByVariableDeclaration processGroupByVar(JsoniqParser.GroupByVarConte if (ctx.ex != null) { expr = (Expression) this.visitExprSingle(ctx.ex); - if (seq != null) { - expr = new TreatExpression(expr, seq, ErrorCode.UnexpectedTypeErrorCode, expr.getMetadata()); - } - } diff --git a/src/main/java/org/rumbledb/compiler/XQueryTranslationVisitor.java b/src/main/java/org/rumbledb/compiler/XQueryTranslationVisitor.java index 52ac9ff33d..5b889c1943 100644 --- a/src/main/java/org/rumbledb/compiler/XQueryTranslationVisitor.java +++ b/src/main/java/org/rumbledb/compiler/XQueryTranslationVisitor.java @@ -1228,10 +1228,6 @@ public GroupByVariableDeclaration processGroupByVar(XQueryParser.GroupByVarConte if (ctx.ex != null) { expr = (Expression) this.visitExprSingle(ctx.ex); - if (seq != null) { - expr = new TreatExpression(expr, seq, ErrorCode.UnexpectedTypeErrorCode, expr.getMetadata()); - } - } diff --git a/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java b/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java index 7976146a3f..5ec8950b50 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java +++ b/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java @@ -50,6 +50,8 @@ import org.rumbledb.runtime.flwor.udfs.GroupClauseCreateColumnsUDF; import org.rumbledb.runtime.flwor.udfs.GroupClauseSerializeAggregateResultsUDF; import org.rumbledb.runtime.misc.CollationSupport; +import org.rumbledb.runtime.typing.InstanceOfIterator; +import org.rumbledb.types.SequenceType; import org.rumbledb.types.TypeMappings; import sparksoniq.jsoniq.tuple.FlworKey; import sparksoniq.jsoniq.tuple.FlworTuple; @@ -217,6 +219,7 @@ private HashMap> mapTuplesToPairs() { } else { newVariableResults = Collections.emptyList(); } + validateGroupingKeySequenceType(expression.getSequenceType(), newVariableResults, tupleContext); // if a new variable is declared inside the group by clause, insert value in tuple inputTuple.putValue(expression.getVariableName(), newVariableResults); @@ -257,6 +260,7 @@ private HashMap> mapTuplesToPairs() { ) ); } + validateGroupingKeySequenceType(expression.getSequenceType(), atomizedGroupValues, tupleContext); inputTuple.putValue(groupVariableName, atomizedGroupValues); results.addAll(atomizedGroupValues); } @@ -272,6 +276,44 @@ private HashMap> mapTuplesToPairs() { return keyValuePairs; } + private void validateGroupingKeySequenceType( + SequenceType declaredType, + List groupingKey, + DynamicContext dynamicContext + ) { + if (declaredType == null) { + return; + } + if (!declaredType.isResolved()) { + declaredType.resolve(dynamicContext, getMetadata()); + } + + boolean validCardinality = switch (declaredType.getArity()) { + case Zero -> groupingKey.isEmpty(); + case One -> groupingKey.size() == 1; + case OneOrZero -> groupingKey.size() <= 1; + case OneOrMore -> !groupingKey.isEmpty(); + case ZeroOrMore -> true; + }; + if (!validCardinality) { + throw new UnexpectedTypeException( + "The grouping key has cardinality " + + groupingKey.size() + + ", but the expected type is " + + declaredType, + getMetadata() + ); + } + for (Item item : groupingKey) { + if (!InstanceOfIterator.doesItemTypeMatchItem(declaredType.getItemType(), item)) { + throw new UnexpectedTypeException( + item.getDynamicType() + " is not expected here. The expected type is " + declaredType, + getMetadata() + ); + } + } + } + /** * Iterate over all tuples to evaluate grouping */ diff --git a/src/main/java/org/rumbledb/runtime/flwor/expression/GroupByClauseSparkIteratorExpression.java b/src/main/java/org/rumbledb/runtime/flwor/expression/GroupByClauseSparkIteratorExpression.java index f7e94953d6..fbfb78aaad 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/expression/GroupByClauseSparkIteratorExpression.java +++ b/src/main/java/org/rumbledb/runtime/flwor/expression/GroupByClauseSparkIteratorExpression.java @@ -24,6 +24,7 @@ import org.rumbledb.context.Name; import org.rumbledb.exceptions.ExceptionMetadata; import org.rumbledb.runtime.RuntimeIterator; +import org.rumbledb.types.SequenceType; import java.io.Serial; import java.io.Serializable; @@ -37,17 +38,20 @@ public class GroupByClauseSparkIteratorExpression implements Serializable { private final RuntimeIterator expression; private final ExceptionMetadata iteratorMetadata; private final String collationURI; + private final SequenceType sequenceType; public GroupByClauseSparkIteratorExpression( RuntimeIterator expression, Name variableName, ExceptionMetadata iteratorMetadata, - String collationURI + String collationURI, + SequenceType sequenceType ) { this.expression = expression; this.variableName = variableName; this.iteratorMetadata = iteratorMetadata; this.collationURI = collationURI; + this.sequenceType = sequenceType; } public Name getVariableName() { @@ -65,4 +69,8 @@ public RuntimeIterator getExpression() { public String getCollationURI() { return this.collationURI; } + + public SequenceType getSequenceType() { + return this.sequenceType; + } } diff --git a/src/main/java/org/rumbledb/runtime/misc/CollationSupport.java b/src/main/java/org/rumbledb/runtime/misc/CollationSupport.java index 4e9bdf3735..2ece61a14e 100644 --- a/src/main/java/org/rumbledb/runtime/misc/CollationSupport.java +++ b/src/main/java/org/rumbledb/runtime/misc/CollationSupport.java @@ -124,7 +124,9 @@ private static RuleBasedCollator buildUcaCollator(String collationUri, Exception : Collator.NO_DECOMPOSITION ); ruleBasedCollator.setCaseLevel(parameters.caseLevel); - ruleBasedCollator.setFrenchCollation(parameters.backwards); + if (parameters.backwards != null) { + ruleBasedCollator.setFrenchCollation(parameters.backwards); + } if (parameters.alternateShifted != null) { ruleBasedCollator.setAlternateHandlingShifted(parameters.alternateShifted); } @@ -178,6 +180,9 @@ private static UcaParameters parseUcaParameters(String collationUri, ExceptionMe break; case "fallback": case "version": + if ("version".equals(key) && "no".equals(queryParameters.get("fallback"))) { + throw new UnsupportedCollationException("Wrong collation parameter", metadata); + } break; default: if ("no".equals(queryParameters.get("fallback"))) { @@ -233,7 +238,7 @@ private static final class UcaParameters { private String languageTag; private int strength = Collator.TERTIARY; private boolean normalization; - private boolean backwards; + private Boolean backwards; private boolean caseLevel; private Boolean alternateShifted; } From 418595f9c14621b5cf30c98f71e61740788858b1 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 27 Jul 2026 12:47:49 +0200 Subject: [PATCH 06/12] Update tests. --- .../iq/MLCoercionWrappedFunctionTests.java | 35 +++++++++++++++++++ .../DataFrames/GroubyClauseTypeCheckError.jq | 2 +- .../GroupbyClauseTypeCheckError.jq | 2 +- .../test_files/runtime/Collation5.jq | 2 +- .../test_files/runtime/Collation7.jq | 2 +- .../test_files/runtime/Collation8.jq | 6 ++++ .../TypeChecking/GroupByClauseType2.jq | 2 +- 7 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 src/test/java/iq/MLCoercionWrappedFunctionTests.java create mode 100644 src/test/resources/test_files/runtime/Collation8.jq diff --git a/src/test/java/iq/MLCoercionWrappedFunctionTests.java b/src/test/java/iq/MLCoercionWrappedFunctionTests.java new file mode 100644 index 0000000000..e6c15fb306 --- /dev/null +++ b/src/test/java/iq/MLCoercionWrappedFunctionTests.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package iq; + +import java.io.File; +import java.util.List; + +public class MLCoercionWrappedFunctionTests extends MLTests { + + private static final File coercionWrappedFunctionTest = new File( + System.getProperty("user.dir") + + + "/src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-CoercionWrappedTransformer1.jq" + ); + + @Override + protected List testFiles() { + return List.of(coercionWrappedFunctionTest); + } +} diff --git a/src/test/resources/test_files/runtime-spark/DataFrames/GroubyClauseTypeCheckError.jq b/src/test/resources/test_files/runtime-spark/DataFrames/GroubyClauseTypeCheckError.jq index 239cd5e503..40a5f3bc0a 100644 --- a/src/test/resources/test_files/runtime-spark/DataFrames/GroubyClauseTypeCheckError.jq +++ b/src/test/resources/test_files/runtime-spark/DataFrames/GroubyClauseTypeCheckError.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldCrash; ErrorCode="XPTY0004"; ErrorMetadata="LINE:3:COLUMN:25:" :) +(:JIQS: ShouldCrash; ErrorCode="XPTY0004"; ErrorMetadata="LINE:3:COLUMN:0:" :) for $j as integer in parallelize((1 to 10)) group by $k as string := $j return $k diff --git a/src/test/resources/test_files/runtime-spark/LocalClauses/GroupbyClauseTypeCheckError.jq b/src/test/resources/test_files/runtime-spark/LocalClauses/GroupbyClauseTypeCheckError.jq index f7faaf2049..a5e9ed3c25 100644 --- a/src/test/resources/test_files/runtime-spark/LocalClauses/GroupbyClauseTypeCheckError.jq +++ b/src/test/resources/test_files/runtime-spark/LocalClauses/GroupbyClauseTypeCheckError.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldCrash; ErrorCode="XPTY0004"; ErrorMetadata="LINE:3:COLUMN:25:" :) +(:JIQS: ShouldCrash; ErrorCode="XPTY0004"; ErrorMetadata="LINE:3:COLUMN:0:" :) for $j as integer in (1 to 10) group by $k as string := $j return $k diff --git a/src/test/resources/test_files/runtime/Collation5.jq b/src/test/resources/test_files/runtime/Collation5.jq index 7eb0576829..37622105bf 100644 --- a/src/test/resources/test_files/runtime/Collation5.jq +++ b/src/test/resources/test_files/runtime/Collation5.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldCrash; ErrorCode="XQST0038"; ErrorMetadata="LINE:5:COLUMN:22:" :) +(:JIQS: ShouldCrash; ErrorCode="XQST0076"; ErrorMetadata="LINE:5:COLUMN:22:" :) declare default collation "http://www.w3.org/2005/xpath-functions/collation/codepoint"; for $i in ("foo", "bar") diff --git a/src/test/resources/test_files/runtime/Collation7.jq b/src/test/resources/test_files/runtime/Collation7.jq index 28606e5275..40d340a6f5 100644 --- a/src/test/resources/test_files/runtime/Collation7.jq +++ b/src/test/resources/test_files/runtime/Collation7.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldCrash; ErrorCode="XQST0038"; ErrorMetadata="LINE:5:COLUMN:28:" :) +(:JIQS: ShouldCrash; ErrorCode="XQST0076"; ErrorMetadata="LINE:5:COLUMN:28:" :) declare default collation "http://www.w3.org/2005/xpath-functions/collation/codepoint"; for $i in ("foo", "bar") diff --git a/src/test/resources/test_files/runtime/Collation8.jq b/src/test/resources/test_files/runtime/Collation8.jq new file mode 100644 index 0000000000..38af78f0bf --- /dev/null +++ b/src/test/resources/test_files/runtime/Collation8.jq @@ -0,0 +1,6 @@ +(:JIQS: ShouldRun; Output="Frog" :) +declare default collation "http://www.w3.org/2005/xpath-functions/collation/codepoint"; + +for $i in ("Frog", "frog") +group by $g := $i collation "http://www.w3.org/2010/09/qt-fots-catalog/collation/caseblind" +return $g diff --git a/src/test/resources/test_files/runtime/TypeChecking/GroupByClauseType2.jq b/src/test/resources/test_files/runtime/TypeChecking/GroupByClauseType2.jq index 0d4a174396..7d26c9cde2 100644 --- a/src/test/resources/test_files/runtime/TypeChecking/GroupByClauseType2.jq +++ b/src/test/resources/test_files/runtime/TypeChecking/GroupByClauseType2.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldCrash; ErrorCode="XPTY0004"; ErrorMetadata="LINE:3:COLUMN:25:" :) +(:JIQS: ShouldCrash; ErrorCode="XPTY0004"; ErrorMetadata="LINE:3:COLUMN:0:" :) for $x in ("foo", "bar") group by $y as string := string-length($x) return $y From 39f89b1565069312c3ca4ebe561a15306cfe0101 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 27 Jul 2026 13:02:48 +0200 Subject: [PATCH 07/12] Fix tests. --- .../flwor/clauses/GroupByClauseIterator.java | 28 +++++++--- .../udfs/GroupClauseCreateColumnsUDF.java | 53 +++++++++++++++++-- 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java b/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java index 5ec8950b50..c38a736d2a 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java +++ b/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java @@ -207,7 +207,7 @@ private HashMap> mapTuplesToPairs() { getMetadata() ); } - atomizedResult = CollationSupport.normalizeItemForCollation( + Item normalizedGroupingKey = CollationSupport.normalizeItemForCollation( atomizedResult, expression.getCollationURI() == null ? getStaticContext().getDefaultCollation() @@ -215,15 +215,16 @@ private HashMap> mapTuplesToPairs() { getMetadata() ); newVariableResults = Collections.singletonList(atomizedResult); + results.add(normalizedGroupingKey); } } else { newVariableResults = Collections.emptyList(); + results.addAll(newVariableResults); } validateGroupingKeySequenceType(expression.getSequenceType(), newVariableResults, tupleContext); // if a new variable is declared inside the group by clause, insert value in tuple inputTuple.putValue(expression.getVariableName(), newVariableResults); - results.addAll(newVariableResults); } else { // if grouping on a variable reference Name groupVariableName = expression.getVariableName(); @@ -248,9 +249,10 @@ private HashMap> mapTuplesToPairs() { getMetadata() ); } + validateGroupingKeySequenceType(expression.getSequenceType(), atomizedGroupValues, tupleContext); + inputTuple.putValue(groupVariableName, atomizedGroupValues); if (atomizedGroupValues.size() == 1) { - atomizedGroupValues.set( - 0, + results.add( CollationSupport.normalizeItemForCollation( atomizedGroupValues.get(0), expression.getCollationURI() == null @@ -259,10 +261,9 @@ private HashMap> mapTuplesToPairs() { getMetadata() ) ); + } else { + results.addAll(atomizedGroupValues); } - validateGroupingKeySequenceType(expression.getSequenceType(), atomizedGroupValues, tupleContext); - inputTuple.putValue(groupVariableName, atomizedGroupValues); - results.addAll(atomizedGroupValues); } } FlworKey key = new FlworKey(results); @@ -508,7 +509,7 @@ public FlworDataFrame getDataFrame( .udf() .register( "createGroupingColumns", - new GroupClauseCreateColumnsUDF(variableAccessNames, context, inputSchema, UDFcolumns, getMetadata()), + new GroupClauseCreateColumnsUDF(this.groupingExpressions, context, inputSchema, UDFcolumns, getMetadata()), DataTypes.createStructType(typedFields) ); @@ -663,6 +664,12 @@ private Dataset tryNativeQuery( DynamicContext context, String input ) { + for (GroupByClauseSparkIteratorExpression expression : this.groupingExpressions) { + if (expression.getSequenceType() != null) { + return null; + } + } + StringBuilder groupByString = new StringBuilder(); String sep = " "; for (Name groupingVar : groupingVariables) { @@ -793,6 +800,11 @@ public boolean isSparkJobNeeded() { @Override public NativeClauseContext generateNativeQuery(NativeClauseContext nativeClauseContext) { + for (GroupByClauseSparkIteratorExpression expression : this.groupingExpressions) { + if (expression.getSequenceType() != null) { + return NativeClauseContext.NoNativeQuery; + } + } List dfColumns = FlworDataFrameUtils.getColumns( (StructType) nativeClauseContext.getSchema(), null, diff --git a/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java b/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java index 87639e9320..47735c974f 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java +++ b/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java @@ -30,6 +30,9 @@ import org.rumbledb.exceptions.ExceptionMetadata; import org.rumbledb.exceptions.UnexpectedTypeException; import org.rumbledb.runtime.flwor.FlworDataFrameColumn; +import org.rumbledb.runtime.flwor.expression.GroupByClauseSparkIteratorExpression; +import org.rumbledb.runtime.typing.InstanceOfIterator; +import org.rumbledb.types.SequenceType; import java.io.Serial; import java.util.ArrayList; @@ -41,6 +44,7 @@ public class GroupClauseCreateColumnsUDF implements UDF1 { private static final long serialVersionUID = 1L; private final DataFrameContext dataFrameContext; private final List groupingVariableNames; + private final List groupingSequenceTypes; private final List results; private final ExceptionMetadata metadata; @@ -58,14 +62,19 @@ public class GroupClauseCreateColumnsUDF implements UDF1 { private static final int dateTimeGroupIndex = 5; public GroupClauseCreateColumnsUDF( - List groupingVariableNames, + List groupingExpressions, DynamicContext context, StructType schema, List columns, ExceptionMetadata metadata ) { this.dataFrameContext = new DataFrameContext(context, columns); - this.groupingVariableNames = groupingVariableNames; + this.groupingVariableNames = new ArrayList<>(); + this.groupingSequenceTypes = new ArrayList<>(); + for (GroupByClauseSparkIteratorExpression expression : groupingExpressions) { + this.groupingVariableNames.add(expression.getVariableName()); + this.groupingSequenceTypes.add(expression.getSequenceType()); + } this.results = new ArrayList<>(); this.metadata = metadata; } @@ -76,7 +85,9 @@ public Row call(Row row) { this.results.clear(); - for (Name groupingVariableName : this.groupingVariableNames) { + for (int i = 0; i < this.groupingVariableNames.size(); i++) { + Name groupingVariableName = this.groupingVariableNames.get(i); + SequenceType declaredType = this.groupingSequenceTypes.get(i); List items = this.dataFrameContext.getContext() .getVariableValues() .getLocalVariableValue( @@ -91,6 +102,8 @@ public Row call(Row row) { ); } + validateGroupingKeySequenceType(declaredType, items); + if (items.isEmpty()) { this.results.add(emptySequenceGroupIndex); this.results.add(null); @@ -106,6 +119,40 @@ public Row call(Row row) { return RowFactory.create(this.results.toArray()); } + private void validateGroupingKeySequenceType(SequenceType declaredType, List groupingKey) { + if (declaredType == null) { + return; + } + if (!declaredType.isResolved()) { + declaredType.resolve(this.dataFrameContext.getContext(), this.metadata); + } + + boolean validCardinality = switch (declaredType.getArity()) { + case Zero -> groupingKey.isEmpty(); + case One -> groupingKey.size() == 1; + case OneOrZero -> groupingKey.size() <= 1; + case OneOrMore -> !groupingKey.isEmpty(); + case ZeroOrMore -> true; + }; + if (!validCardinality) { + throw new UnexpectedTypeException( + "The grouping key has cardinality " + + groupingKey.size() + + ", but the expected type is " + + declaredType, + this.metadata + ); + } + for (Item item : groupingKey) { + if (!InstanceOfIterator.doesItemTypeMatchItem(declaredType.getItemType(), item)) { + throw new UnexpectedTypeException( + item.getDynamicType() + " is not expected here. The expected type is " + declaredType, + this.metadata + ); + } + } + } + private void createColumnsForItem(Item nextItem) { if (nextItem.isNull()) { this.results.add(nullGroupIndex); From 0c0ed50ec8243f0c4999229394a8ffec630b5d6f Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 27 Jul 2026 13:07:31 +0200 Subject: [PATCH 08/12] spotless. --- .../runtime/flwor/clauses/GroupByClauseIterator.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java b/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java index c38a736d2a..90c7ed781d 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java +++ b/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java @@ -509,7 +509,13 @@ public FlworDataFrame getDataFrame( .udf() .register( "createGroupingColumns", - new GroupClauseCreateColumnsUDF(this.groupingExpressions, context, inputSchema, UDFcolumns, getMetadata()), + new GroupClauseCreateColumnsUDF( + this.groupingExpressions, + context, + inputSchema, + UDFcolumns, + getMetadata() + ), DataTypes.createStructType(typedFields) ); From 429f11039c05d3185c159c1273fa768e6e262e5f Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 27 Jul 2026 13:22:05 +0200 Subject: [PATCH 09/12] Fix test. --- .../udfs/GroupClauseCreateColumnsUDF.java | 53 ++++++++++++++----- .../DataFrames/GroupbyClauseError3.jq | 2 +- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java b/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java index 47735c974f..94bdd2010e 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java +++ b/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java @@ -27,6 +27,7 @@ import org.rumbledb.api.Item; import org.rumbledb.context.DynamicContext; import org.rumbledb.context.Name; +import org.rumbledb.exceptions.CannotAtomizeException; import org.rumbledb.exceptions.ExceptionMetadata; import org.rumbledb.exceptions.UnexpectedTypeException; import org.rumbledb.runtime.flwor.FlworDataFrameColumn; @@ -95,16 +96,11 @@ public Row call(Row row) { this.metadata ); - if (items.size() > 1) { - throw new UnexpectedTypeException( - "Can not group on variables with sequences of multiple items.", - this.metadata - ); - } + List atomizedGroupingKey = atomizeGroupingKey(items); - validateGroupingKeySequenceType(declaredType, items); + validateGroupingKeySequenceType(declaredType, atomizedGroupingKey); - if (items.isEmpty()) { + if (atomizedGroupingKey.isEmpty()) { this.results.add(emptySequenceGroupIndex); this.results.add(null); this.results.add(null); @@ -112,13 +108,34 @@ public Row call(Row row) { continue; } - Item nextItem = items.get(0); + Item nextItem = atomizedGroupingKey.get(0); this.createColumnsForItem(nextItem); } return RowFactory.create(this.results.toArray()); } + private List atomizeGroupingKey(List items) { + List atomizedGroupingKey = new ArrayList<>(); + for (Item item : items) { + try { + atomizedGroupingKey.addAll(item.atomizedValue()); + } catch (CannotAtomizeException e) { + throw new UnexpectedTypeException( + "Group by variable can not contain arrays or objects.", + this.metadata + ); + } + } + if (atomizedGroupingKey.size() > 1) { + throw new UnexpectedTypeException( + "Keys in a group-by clause must atomize to at most one item.", + this.metadata + ); + } + return atomizedGroupingKey; + } + private void validateGroupingKeySequenceType(SequenceType declaredType, List groupingKey) { if (declaredType == null) { return; @@ -159,6 +176,7 @@ private void createColumnsForItem(Item nextItem) { this.results.add(null); this.results.add(null); this.results.add(null); + return; } else if (nextItem.isBoolean()) { if (nextItem.getBooleanValue()) { this.results.add(booleanTrueGroupIndex); @@ -168,46 +186,53 @@ private void createColumnsForItem(Item nextItem) { this.results.add(null); this.results.add(null); this.results.add(null); + return; } else if (nextItem.isString() || nextItem.isHexBinary() || nextItem.isBase64Binary()) { this.results.add(stringGroupIndex); this.results.add(nextItem.getStringValue()); this.results.add(null); this.results.add(null); + return; } else if (nextItem.isInteger()) { this.results.add(doubleGroupIndex); this.results.add(null); this.results.add(nextItem.castToDoubleValue()); this.results.add(null); + return; } else if (nextItem.isDecimal()) { this.results.add(doubleGroupIndex); this.results.add(null); this.results.add(nextItem.castToDoubleValue()); this.results.add(null); + return; } else if (nextItem.isDouble()) { this.results.add(doubleGroupIndex); this.results.add(null); this.results.add(nextItem.getDoubleValue()); this.results.add(null); + return; } else if (nextItem.isFloat()) { this.results.add(doubleGroupIndex); this.results.add(null); this.results.add(nextItem.castToDoubleValue()); this.results.add(null); + return; } else if (nextItem.isDuration()) { this.results.add(durationGroupIndex); this.results.add(null); this.results.add(null); this.results.add(nextItem.getEpochMillis()); + return; } else if (nextItem.hasDateTime()) { this.results.add(dateTimeGroupIndex); this.results.add(null); this.results.add(null); this.results.add(nextItem.getEpochMillis()); - } else { - throw new UnexpectedTypeException( - "Group by variable can not contain arrays or objects.", - this.metadata - ); + return; } + throw new UnexpectedTypeException( + "Group by variable can not contain arrays or objects.", + this.metadata + ); } } diff --git a/src/test/resources/test_files/runtime-spark/DataFrames/GroupbyClauseError3.jq b/src/test/resources/test_files/runtime-spark/DataFrames/GroupbyClauseError3.jq index 81b62a384c..8aff0d4313 100644 --- a/src/test/resources/test_files/runtime-spark/DataFrames/GroupbyClauseError3.jq +++ b/src/test/resources/test_files/runtime-spark/DataFrames/GroupbyClauseError3.jq @@ -1,4 +1,4 @@ -(:JIQS: ShouldCrash; ErrorCode="XPTY0004" :) +(:JIQS: ShouldRun; Output="({ "guess" : "Bulgarian", "target" : "Albanian", "country" : "AU", "choices" : [ "Albanian", "Bulgarian", "Russian", "Ukrainian" ], "sample" : "00b85faa8b878a14f8781be334deb137", "date" : "2013-08-19" }, { "guess" : "Bulgarian", "target" : "Albanian", "country" : [ "AU" ], "choices" : [ "Albanian", "Bulgarian", "Russian", "Ukrainian" ], "sample" : "00b85faa8b878a14f8781be334deb137", "date" : "2013-08-19" })" :) for $i in parallelize(( {"guess": "Bulgarian", "target": "Albanian", "country": "AU", "choices": ["Albanian", "Bulgarian", "Russian", "Ukrainian"], "sample": "00b85faa8b878a14f8781be334deb137", "date": "2013-08-19"}, {"guess": "Bulgarian", "target": "Albanian", "country": ["AU"], "choices": ["Albanian", "Bulgarian", "Russian", "Ukrainian"], "sample": "00b85faa8b878a14f8781be334deb137", "date": "2013-08-19"})) From 3fb70e5c71ecc44c9974cbd9717ff9da0e1cdb2e Mon Sep 17 00:00:00 2001 From: Jimmy Cai Date: Mon, 27 Jul 2026 13:42:47 +0200 Subject: [PATCH 10/12] Handle AssertionError in checkErrorCode to provide clearer failure messages in AnnotationTestExecutor --- src/test/java/iq/base/AnnotationTestExecutor.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/java/iq/base/AnnotationTestExecutor.java b/src/test/java/iq/base/AnnotationTestExecutor.java index b14e1962d9..ec3f10bc94 100644 --- a/src/test/java/iq/base/AnnotationTestExecutor.java +++ b/src/test/java/iq/base/AnnotationTestExecutor.java @@ -210,7 +210,11 @@ private static void assertExpectedRuntimeFailureDuringMaterialization( materializeSequence(sequence, applyUpdates, resultSizeCap); Assertions.fail(withTestFile(path, unexpectedSuccessMessage(TestStage.RUNTIME))); } catch (Throwable exception) { - checkErrorCode(errorOutput(exception), annotation.errorCode(), annotation.errorMetadata()); + try { + checkErrorCode(errorOutput(exception), annotation.errorCode(), annotation.errorMetadata()); + } catch (AssertionError assertionError) { + Assertions.fail(withTestFile(path, assertionError.getMessage()), assertionError); + } } } From 35e5277467c5087fefc3bbabb697a45ad6f643bd Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 27 Jul 2026 14:01:19 +0200 Subject: [PATCH 11/12] Fix test. --- .../flwor/clauses/GroupByClauseIterator.java | 20 +++++++++++++++++-- .../udfs/GroupClauseCreateColumnsUDF.java | 4 ++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java b/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java index 90c7ed781d..24dbe9c0c5 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java +++ b/src/main/java/org/rumbledb/runtime/flwor/clauses/GroupByClauseIterator.java @@ -31,6 +31,7 @@ import org.rumbledb.context.DynamicContext; import org.rumbledb.context.Name; import org.rumbledb.context.RuntimeStaticContext; +import org.rumbledb.exceptions.CannotAtomizeException; import org.rumbledb.exceptions.InvalidGroupVariableException; import org.rumbledb.exceptions.IteratorFlowException; import org.rumbledb.exceptions.JobWithinAJobException; @@ -190,7 +191,15 @@ private HashMap> mapTuplesToPairs() { ); } if (resultItem != null) { - List atomizedResults = resultItem.atomizedValue(); + List atomizedResults; + try { + atomizedResults = resultItem.atomizedValue(); + } catch (CannotAtomizeException e) { + throw new UnexpectedTypeException( + "Group by variable must atomize to a supported atomic value.", + getMetadata() + ); + } if (atomizedResults.size() > 1) { throw new UnexpectedTypeException( "Keys in a group-by clause must atomize to at most one item.", @@ -241,7 +250,14 @@ private HashMap> mapTuplesToPairs() { .getLocalVariableValue(groupVariableName, getMetadata()); List atomizedGroupValues = new ArrayList<>(); for (Item groupVariableValue : groupVariableValues) { - atomizedGroupValues.addAll(groupVariableValue.atomizedValue()); + try { + atomizedGroupValues.addAll(groupVariableValue.atomizedValue()); + } catch (CannotAtomizeException e) { + throw new UnexpectedTypeException( + "Group by variable must atomize to a supported atomic value.", + getMetadata() + ); + } } if (atomizedGroupValues.size() > 1) { throw new UnexpectedTypeException( diff --git a/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java b/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java index 94bdd2010e..cc71aad1b0 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java +++ b/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java @@ -122,7 +122,7 @@ private List atomizeGroupingKey(List items) { atomizedGroupingKey.addAll(item.atomizedValue()); } catch (CannotAtomizeException e) { throw new UnexpectedTypeException( - "Group by variable can not contain arrays or objects.", + "Group by variable must atomize to a supported atomic value.", this.metadata ); } @@ -231,7 +231,7 @@ private void createColumnsForItem(Item nextItem) { return; } throw new UnexpectedTypeException( - "Group by variable can not contain arrays or objects.", + "Group by variable must atomize to a supported atomic value.", this.metadata ); } From d84642fb62c039129719a4b2acb3086f188bc2cd Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 27 Jul 2026 14:30:04 +0200 Subject: [PATCH 12/12] Use collations in group by. --- .../org/rumbledb/compiler/RuntimeIteratorVisitor.java | 4 +++- .../flwor/udfs/GroupClauseCreateColumnsUDF.java | 11 ++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java b/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java index 0009d88e7a..a8554a1fe6 100644 --- a/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java +++ b/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java @@ -398,7 +398,9 @@ private RuntimeTupleIterator visitFlowrClause( groupByExpressionIterator, variableName, clause.getMetadata(), - var.getCollationURI(), + var.getCollationURI() == null + ? clause.getStaticContext().getDefaultCollation() + : var.getCollationURI(), var.getActualSequenceType() ) ); diff --git a/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java b/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java index cc71aad1b0..5f13fc4bac 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java +++ b/src/main/java/org/rumbledb/runtime/flwor/udfs/GroupClauseCreateColumnsUDF.java @@ -32,6 +32,7 @@ import org.rumbledb.exceptions.UnexpectedTypeException; import org.rumbledb.runtime.flwor.FlworDataFrameColumn; import org.rumbledb.runtime.flwor.expression.GroupByClauseSparkIteratorExpression; +import org.rumbledb.runtime.misc.CollationSupport; import org.rumbledb.runtime.typing.InstanceOfIterator; import org.rumbledb.types.SequenceType; @@ -45,6 +46,7 @@ public class GroupClauseCreateColumnsUDF implements UDF1 { private static final long serialVersionUID = 1L; private final DataFrameContext dataFrameContext; private final List groupingVariableNames; + private final List groupingCollationURIs; private final List groupingSequenceTypes; private final List results; @@ -71,9 +73,11 @@ public GroupClauseCreateColumnsUDF( ) { this.dataFrameContext = new DataFrameContext(context, columns); this.groupingVariableNames = new ArrayList<>(); + this.groupingCollationURIs = new ArrayList<>(); this.groupingSequenceTypes = new ArrayList<>(); for (GroupByClauseSparkIteratorExpression expression : groupingExpressions) { this.groupingVariableNames.add(expression.getVariableName()); + this.groupingCollationURIs.add(expression.getCollationURI()); this.groupingSequenceTypes.add(expression.getSequenceType()); } this.results = new ArrayList<>(); @@ -88,6 +92,7 @@ public Row call(Row row) { for (int i = 0; i < this.groupingVariableNames.size(); i++) { Name groupingVariableName = this.groupingVariableNames.get(i); + String collationURI = this.groupingCollationURIs.get(i); SequenceType declaredType = this.groupingSequenceTypes.get(i); List items = this.dataFrameContext.getContext() .getVariableValues() @@ -108,7 +113,11 @@ public Row call(Row row) { continue; } - Item nextItem = atomizedGroupingKey.get(0); + Item nextItem = CollationSupport.normalizeItemForCollation( + atomizedGroupingKey.get(0), + collationURI, + this.metadata + ); this.createColumnsForItem(nextItem); }