From 26c2dc99a0de35bb8daa0af15c00e373949aebeb Mon Sep 17 00:00:00 2001 From: Jimmy Cai Date: Fri, 5 Jun 2026 19:02:44 +0200 Subject: [PATCH 1/6] Create PartialFunctionCallIterator to centralize the logic of partial application --- ...iltinPartialApplicationRewriteVisitor.java | 124 ------------ .../compiler/ExecutionModeVisitor.java | 4 +- .../rumbledb/compiler/InferTypeVisitor.java | 8 - .../compiler/RuntimeIteratorVisitor.java | 13 +- .../org/rumbledb/compiler/VisitorHelpers.java | 9 - .../org/rumbledb/context/NamedFunctions.java | 24 ++- .../functions/FunctionItemCallIterator.java | 191 ++---------------- .../PartialFunctionCallIterator.java | 182 +++++++++++++++++ 8 files changed, 225 insertions(+), 330 deletions(-) delete mode 100644 src/main/java/org/rumbledb/compiler/BuiltinPartialApplicationRewriteVisitor.java create mode 100644 src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java diff --git a/src/main/java/org/rumbledb/compiler/BuiltinPartialApplicationRewriteVisitor.java b/src/main/java/org/rumbledb/compiler/BuiltinPartialApplicationRewriteVisitor.java deleted file mode 100644 index 20cccbbbc7..0000000000 --- a/src/main/java/org/rumbledb/compiler/BuiltinPartialApplicationRewriteVisitor.java +++ /dev/null @@ -1,124 +0,0 @@ -package org.rumbledb.compiler; - -import org.rumbledb.context.BuiltinFunction; -import org.rumbledb.context.BuiltinFunctionCatalogue; -import org.rumbledb.context.Name; -import org.rumbledb.expressions.Expression; -import org.rumbledb.expressions.Node; -import org.rumbledb.expressions.postfix.DynamicFunctionCallExpression; -import org.rumbledb.expressions.primary.FunctionCallExpression; -import org.rumbledb.expressions.primary.InlineFunctionExpression; -import org.rumbledb.expressions.primary.NamedFunctionReferenceExpression; -import org.rumbledb.expressions.primary.VariableReferenceExpression; -import org.rumbledb.expressions.scripting.statement.StatementsAndOptionalExpr; -import org.rumbledb.types.SequenceType; - -import java.util.*; -import java.util.stream.Collectors; - -/** - * Rewrites direct partial application of builtins, e.g. {@code fn:max(0, ?)}, - * into - * an equivalent inline function, e.g. {@code function($x) { fn:max(0, $x) }}. - *

- * This keeps builtin partial application on the same code path as ordinary - * inline functions. - */ -public class BuiltinPartialApplicationRewriteVisitor extends CloneVisitor { - - private InlineFunctionExpression rewriteBuiltinPartialApplication( - Name functionName, - BuiltinFunction builtin, - List arguments, - Expression sourceExpression - ) { - List parameterTypes = builtin.getSignature().getParameterTypes(); - Map params = new LinkedHashMap<>(); - - /// We will create a new function call to builtin function, but this time - /// replace ? with real parameters - List fullArguments = new ArrayList<>(arguments.size()); - - for (int i = 0; i < arguments.size(); i++) { - Expression currentArgument = arguments.get(i); - if (currentArgument != null) { - /// It was not a ? - fullArguments.add(currentArgument); - continue; - } - - Name parameterName = Name.createVariableInNoNamespace(String.format("param%s", i)); - params.put(parameterName, parameterTypes.get(i)); - VariableReferenceExpression variableReference = new VariableReferenceExpression( - parameterName, - sourceExpression.getMetadata() - ); - variableReference.setActualType(parameterTypes.get(i)); - fullArguments.add(variableReference); - } - - FunctionCallExpression bodyCall = new FunctionCallExpression( - functionName, - fullArguments, - sourceExpression.getMetadata() - ); - StatementsAndOptionalExpr body = new StatementsAndOptionalExpr( - Collections.emptyList(), - bodyCall, - sourceExpression.getMetadata() - ); - return new InlineFunctionExpression( - Collections.emptyList(), - null, - params, - builtin.getSignature().getReturnType(), - body, - sourceExpression.getMetadata() - ); - } - - @Override - public Node visitFunctionCall(FunctionCallExpression expression, Node argument) { - BuiltinFunction builtin = BuiltinFunctionCatalogue.getBuiltinFunction(expression.getFunctionIdentifier()); - - if (!expression.isPartialApplication() || builtin == null) { - /// In case of non-partial application or non-builtin function, we still need to - /// keep descending - /// Because a partial builtin function might be in the nested level - /// See qt3 test hof-041 - return super.visitFunctionCall(expression, argument); - } - - List arguments = expression.getArguments() - .stream() - .map(expr -> expr != null ? (Expression) visit(expr, argument) : null) - .collect(Collectors.toList()); - return rewriteBuiltinPartialApplication(expression.getFunctionName(), builtin, arguments, expression); - } - - @Override - public Node visitDynamicFunctionCallExpression(DynamicFunctionCallExpression expression, Node argument) { - List arguments = expression.getArguments() - .stream() - .map(expr -> expr != null ? (Expression) visit(expr, argument) : null) - .collect(Collectors.toList()); - Expression rewrittenMainExpression = (Expression) visit(expression.getMainExpression(), argument); - - if (!(rewrittenMainExpression instanceof NamedFunctionReferenceExpression namedFunctionReference)) { - return super.visitDynamicFunctionCallExpression(expression, argument); - } - - BuiltinFunction builtin = BuiltinFunctionCatalogue.getBuiltinFunction(namedFunctionReference.getIdentifier()); - boolean isPartialApplication = arguments.stream().anyMatch(arg -> arg == null); - if (!isPartialApplication || builtin == null) { - return super.visitDynamicFunctionCallExpression(expression, argument); - } - - return rewriteBuiltinPartialApplication( - namedFunctionReference.getIdentifier().getName(), - builtin, - arguments, - expression - ); - } -} diff --git a/src/main/java/org/rumbledb/compiler/ExecutionModeVisitor.java b/src/main/java/org/rumbledb/compiler/ExecutionModeVisitor.java index b2baba266a..db9a830d3d 100644 --- a/src/main/java/org/rumbledb/compiler/ExecutionModeVisitor.java +++ b/src/main/java/org/rumbledb/compiler/ExecutionModeVisitor.java @@ -267,7 +267,9 @@ public StaticContext visitFunctionCall(FunctionCallExpression expression, Static expression.getMetadata() ); } - if (BuiltinFunctionCatalogue.exists(expression.getFunctionIdentifier())) { + if (expression.isPartialApplication()) { + expression.setHighestExecutionMode(ExecutionMode.LOCAL); + } else if (BuiltinFunctionCatalogue.exists(expression.getFunctionIdentifier())) { BuiltinFunction builtinFunction = BuiltinFunctionCatalogue.getBuiltinFunction( expression.getFunctionIdentifier() ); diff --git a/src/main/java/org/rumbledb/compiler/InferTypeVisitor.java b/src/main/java/org/rumbledb/compiler/InferTypeVisitor.java index 9427d465a3..b2332681cd 100644 --- a/src/main/java/org/rumbledb/compiler/InferTypeVisitor.java +++ b/src/main/java/org/rumbledb/compiler/InferTypeVisitor.java @@ -22,7 +22,6 @@ import org.rumbledb.exceptions.OurBadException; import org.rumbledb.exceptions.UnexpectedStaticTypeException; import org.rumbledb.exceptions.UnknownFunctionCallException; -import org.rumbledb.exceptions.UnsupportedFeatureException; import org.rumbledb.expressions.AbstractNodeVisitor; import org.rumbledb.expressions.CommaExpression; import org.rumbledb.expressions.Expression; @@ -766,13 +765,6 @@ public StaticContext visitFunctionCall(FunctionCallExpression expression, Static visitDescendants(expression, argument); if (BuiltinFunctionCatalogue.exists(expression.getFunctionIdentifier())) { - if (expression.isPartialApplication()) { - /// This should never be reached because partial application on built-in functions should have been rewritten before - throw new UnsupportedFeatureException( - "Partial application on built-in functions are not supported.", - expression.getMetadata() - ); - } BuiltinFunction builtinFunction = BuiltinFunctionCatalogue.getBuiltinFunction( expression.getFunctionIdentifier() ); diff --git a/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java b/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java index 9e4674799c..a599bcbc19 100644 --- a/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java +++ b/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java @@ -1323,7 +1323,18 @@ public RuntimeIterator visitFunctionCall(FunctionCallExpression expression, Runt FunctionIdentifier identifier = new FunctionIdentifier(fnName, arity); RuntimeIterator runtimeIterator = null; - if (BuiltinFunctionCatalogue.exists(identifier)) { + if (expression.isPartialApplication()) { + runtimeIterator = new DynamicFunctionCallIterator( + new NamedFunctionRefRuntimeIterator( + identifier, + expression.getStaticContextForRuntime(this.config, this.visitorConfig) + ), + arguments, + expression.getStaticContextForRuntime(this.config, this.visitorConfig) + ); + } + + else if (BuiltinFunctionCatalogue.exists(identifier)) { runtimeIterator = NamedFunctions.getBuiltInFunctionIterator( identifier, arguments, diff --git a/src/main/java/org/rumbledb/compiler/VisitorHelpers.java b/src/main/java/org/rumbledb/compiler/VisitorHelpers.java index aa1ea2f1e7..10460c0901 100644 --- a/src/main/java/org/rumbledb/compiler/VisitorHelpers.java +++ b/src/main/java/org/rumbledb/compiler/VisitorHelpers.java @@ -65,15 +65,6 @@ private static void inferTypes(Module module, RumbleRuntimeConfiguration conf) { private static MainModule applyTypeIndependentOptimizations(MainModule module, RumbleRuntimeConfiguration conf) { MainModule result = module; - if (conf.debug()) { - System.err.println("***************************************"); - System.err.println("Builtin Partial Application Rewrite Visitor"); - System.err.println("***************************************"); - } - result = (MainModule) new BuiltinPartialApplicationRewriteVisitor().visit(result, null); - if (conf.debug()) { - printTree(result, conf); - } // Annotate recursive functions as such if (conf.debug()) { System.err.println("***************************************"); diff --git a/src/main/java/org/rumbledb/context/NamedFunctions.java b/src/main/java/org/rumbledb/context/NamedFunctions.java index a8957937ec..3573c49cd4 100644 --- a/src/main/java/org/rumbledb/context/NamedFunctions.java +++ b/src/main/java/org/rumbledb/context/NamedFunctions.java @@ -29,13 +29,13 @@ import org.rumbledb.exceptions.DuplicateFunctionIdentifierException; import org.rumbledb.exceptions.ExceptionMetadata; import org.rumbledb.exceptions.OurBadException; -import org.rumbledb.exceptions.UnsupportedFeatureException; import org.rumbledb.exceptions.UnknownFunctionCallException; import org.rumbledb.expressions.ExecutionMode; import org.rumbledb.items.FunctionItem; import org.rumbledb.runtime.RuntimeIterator; import org.rumbledb.runtime.functions.BuiltinFunctionItemCallIterator; import org.rumbledb.runtime.functions.FunctionItemCallIterator; +import org.rumbledb.runtime.functions.PartialFunctionCallIterator; import org.rumbledb.runtime.functions.sequences.general.DataFunctionIterator; import org.rumbledb.runtime.typing.AtMostOneItemTypePromotionIterator; import org.rumbledb.runtime.typing.TypePromotionIterator; @@ -104,7 +104,21 @@ public static RuntimeIterator buildFunctionItemCallIterator( List arguments, boolean isTailOptimization ) { - ExceptionMetadata metadata = callerRuntimeContext.getMetadata(); + if (isTailOptimization) { + return new PartialFunctionCallIterator( + functionItem, + arguments, + callerRuntimeContext.withExecutionMode(ExecutionMode.LOCAL), + Name.TAIL_CALL_OPTIMIZATION + ); + } + if (arguments.stream().anyMatch(a -> a == null)) { + return new PartialFunctionCallIterator( + functionItem, + arguments, + callerRuntimeContext.withExecutionMode(ExecutionMode.LOCAL) + ); + } SequenceType sequenceType = functionItem.getSignature().getReturnType(); SequenceType innerSequenceType = functionItem.getBodyIterator().getStaticType(); RuntimeStaticContext outerStaticContext = callerRuntimeContext.withStaticType( @@ -118,12 +132,6 @@ public static RuntimeIterator buildFunctionItemCallIterator( ).withExecutionMode(executionModeForFunctionCall); RuntimeIterator functionCallIterator; if (functionItem.isBuiltinFunction()) { - if (arguments.stream().anyMatch(a -> a == null)) { - throw new UnsupportedFeatureException( - "Partial application of builtin named function references is not supported yet.", - metadata - ); - } functionCallIterator = new BuiltinFunctionItemCallIterator( functionItem, arguments, diff --git a/src/main/java/org/rumbledb/runtime/functions/FunctionItemCallIterator.java b/src/main/java/org/rumbledb/runtime/functions/FunctionItemCallIterator.java index 246ead3fdd..9a9247fcf7 100644 --- a/src/main/java/org/rumbledb/runtime/functions/FunctionItemCallIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/FunctionItemCallIterator.java @@ -20,7 +20,6 @@ package org.rumbledb.runtime.functions; -import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -28,24 +27,13 @@ import org.apache.spark.api.java.JavaRDD; import org.rumbledb.api.Item; import org.rumbledb.context.DynamicContext; -import org.rumbledb.context.FunctionIdentifier; import org.rumbledb.context.Name; import org.rumbledb.context.RuntimeStaticContext; import org.rumbledb.exceptions.IteratorFlowException; -import org.rumbledb.exceptions.OurBadException; -import org.rumbledb.exceptions.UnexpectedTypeException; -import org.rumbledb.expressions.ExecutionMode; -import org.rumbledb.items.FunctionItem; import org.rumbledb.items.structured.JSoundDataFrame; -import org.rumbledb.runtime.ConstantRuntimeIterator; import org.rumbledb.runtime.HybridRuntimeIterator; import org.rumbledb.runtime.RuntimeIterator; -import org.rumbledb.runtime.typing.AtMostOneItemTypePromotionIterator; -import org.rumbledb.runtime.typing.TypePromotionIterator; import org.rumbledb.runtime.update.PendingUpdateList; -import org.rumbledb.types.FunctionSignature; -import org.rumbledb.types.SequenceType; -import org.rumbledb.types.SequenceType.Arity; public class FunctionItemCallIterator extends HybridRuntimeIterator { @@ -55,8 +43,6 @@ public class FunctionItemCallIterator extends HybridRuntimeIterator { private List functionArguments; // calculated fields - private boolean isPartialApplication; - private boolean isTailOptimization; private RuntimeIterator functionBodyIterator; private Item nextResult; private transient DynamicContext dynamicContextForCalls; @@ -70,23 +56,19 @@ public FunctionItemCallIterator( ) { super(null, staticContext); for (RuntimeIterator arg : functionArguments) { - if (arg == null) { - this.isPartialApplication = true; - } else { - this.children.add(arg); - } - } - if (isTailOptimization) { - this.isPartialApplication = true; - this.isTailOptimization = true; + this.children.add(arg); } this.functionItem = functionItem; this.functionArguments = functionArguments; this.functionBodyIterator = null; this.isUpdating = functionItem.getSignature().isUpdating(); - this.validateNumberOfArguments(); - this.wrapArgumentIteratorsWithTypeCheckingIterators(); + FunctionCallArgumentCoercion.validateArity(functionItem, this.functionArguments, getMetadata()); + FunctionCallArgumentCoercion.wrapAccordingToSignature( + functionItem, + this.functionArguments, + staticContext + ); // Prepopulation of the dynamic context (without the parameters) Map> localArgumentValues = new LinkedHashMap<>( @@ -107,155 +89,18 @@ public FunctionItemCallIterator( ); } - private void validateNumberOfArguments() { - if (this.functionItem.getParameterNames().size() != this.functionArguments.size()) { - throw new UnexpectedTypeException( - "Dynamic function " - + this.functionItem.getIdentifier().getName() - + " invoked with incorrect number of arguments. Expected: " - + this.functionItem.getParameterNames().size() - + ", Found: " - + this.functionArguments.size(), - getMetadata() - ); - } - } - - private void wrapArgumentIteratorsWithTypeCheckingIterators() { - if (this.functionItem.getSignature().getParameterTypes() != null) { - for (int i = 0; i < this.functionArguments.size(); i++) { - if ( - this.functionArguments.get(i) != null - && !this.functionItem.getSignature() - .getParameterTypes() - .get(i) - .equals(SequenceType.createSequenceType("item*")) - ) { - SequenceType sequenceType = this.functionItem.getSignature().getParameterTypes().get(i); - ExecutionMode executionMode = this.functionArguments.get(i).getHighestExecutionMode(); - if ( - sequenceType.isEmptySequence() - || sequenceType.getArity().equals(Arity.One) - || sequenceType.getArity().equals(Arity.OneOrZero) - ) { - executionMode = ExecutionMode.LOCAL; - } - RuntimeStaticContext runtimeStaticContext = getRuntimeStaticContext().withStaticType(sequenceType) - .withExecutionMode(executionMode) - .withMetadata(this.functionArguments.get(i).getMetadata()); - if ( - sequenceType.isEmptySequence() - || sequenceType.getArity().equals(Arity.One) - || sequenceType.getArity().equals(Arity.OneOrZero) - ) { - RuntimeIterator typePromotionIterator = new AtMostOneItemTypePromotionIterator( - this.functionArguments.get(i), - sequenceType, - "Invalid argument for " + this.functionItem.getIdentifier().getName() + " function. ", - runtimeStaticContext - ); - this.functionArguments.set(i, typePromotionIterator); - } else { - RuntimeIterator typePromotionIterator = new TypePromotionIterator( - this.functionArguments.get(i), - sequenceType, - "Invalid argument for " + this.functionItem.getIdentifier().getName() + " function. ", - runtimeStaticContext - ); - this.functionArguments.set(i, typePromotionIterator); - } - } - } - } - } - @Override public void openLocal() { - if (this.isPartialApplication) { - this.functionBodyIterator = generatePartiallyAppliedFunction(this.currentDynamicContextForLocalExecution); - } else { - if (this.functionBodyIterator == null) { - this.functionBodyIterator = this.functionItem.getBodyIterator().deepCopy(); - } - this.populateDynamicContextWithArguments( - this.currentDynamicContextForLocalExecution - ); + if (this.functionBodyIterator == null) { + this.functionBodyIterator = this.functionItem.getBodyIterator().deepCopy(); } + this.populateDynamicContextWithArguments( + this.currentDynamicContextForLocalExecution + ); this.functionBodyIterator.open(this.dynamicContextForCalls); setNextResult(); } - /** - * Partial application generates a new function: - * - Supplied parameters are set as NonLocalVariables - * - Argument placeholders form the parameters - * - * @return FunctionRuntimeIterator that contains the newly generated FunctionItem - */ - private RuntimeIterator generatePartiallyAppliedFunction(DynamicContext context) { - Name argName; - RuntimeIterator argIterator; - - Map> localArgumentValues = new LinkedHashMap<>( - this.functionItem.getLocalVariablesInClosure() - ); - Map> RDDArgumentValues = new LinkedHashMap<>( - this.functionItem.getRDDVariablesInClosure() - ); - Map DFArgumentValues = new LinkedHashMap<>( - this.functionItem.getDFVariablesInClosure() - ); - - List partialApplicationParamNames = new ArrayList<>(); - List partialApplicationParamTypes = new ArrayList<>(); - - for (int i = 0; i < this.functionArguments.size(); i++) { - argName = this.functionItem.getParameterNames().get(i); - argIterator = this.functionArguments.get(i); - - if (argIterator == null) { // == ArgumentPlaceholder - partialApplicationParamNames.add(argName); - partialApplicationParamTypes.add(this.functionItem.getSignature().getParameterTypes().get(i)); - } else { - if (argIterator.isDataFrame()) { - DFArgumentValues.put(argName, argIterator.getDataFrame(context)); - } else if (argIterator.isRDDOrDataFrame()) { - RDDArgumentValues.put(argName, argIterator.getRDD(context)); - } else { - localArgumentValues.put(argName, argIterator.materialize(context)); - } - } - } - - Name functionItemName = this.functionItem.getIdentifier().getName(); - if (this.isTailOptimization) { - functionItemName = Name.TAIL_CALL_OPTIMIZATION; - } - FunctionItem partiallyAppliedFunction = new FunctionItem( - new FunctionIdentifier( - functionItemName, - partialApplicationParamNames.size() - ), - partialApplicationParamNames, - new FunctionSignature( - partialApplicationParamTypes, - this.functionItem.getSignature().getReturnType(), - this.functionItem.getSignature().isUpdating() - ), - this.functionItem.getModuleDynamicContext(), - this.functionItem.getBodyIterator(), - localArgumentValues, - RDDArgumentValues, - DFArgumentValues - ); - return new ConstantRuntimeIterator( - partiallyAppliedFunction, - this.staticContext.withStaticType( - SequenceType.createSequenceType("function(*)") - ).withExecutionMode(ExecutionMode.LOCAL).withMetadata(getMetadata()) - ); - } - private void populateDynamicContextWithArguments(DynamicContext context) { Name argName; RuntimeIterator argIterator; @@ -328,12 +173,6 @@ public void setNextResult() { @Override public JavaRDD getRDDAux(DynamicContext dynamicContext) { - if (this.isPartialApplication) { - throw new OurBadException( - "Unexpected program state reached. Partially applied function calls must be evaluated locally." - ); - } - this.populateDynamicContextWithArguments(dynamicContext); this.functionBodyIterator = this.functionItem.getBodyIterator(); return this.functionBodyIterator.getRDD(this.dynamicContextForCalls); @@ -346,12 +185,6 @@ protected boolean implementsDataFrames() { @Override public JSoundDataFrame getDataFrame(DynamicContext dynamicContext) { - if (this.isPartialApplication) { - throw new OurBadException( - "Unexpected program state reached. Partially applied function calls must be evaluated locally." - ); - } - populateDynamicContextWithArguments(dynamicContext); this.functionBodyIterator = this.functionItem.getBodyIterator(); return this.functionBodyIterator.getDataFrame(this.dynamicContextForCalls); diff --git a/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java b/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java new file mode 100644 index 0000000000..1de31edb20 --- /dev/null +++ b/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.rumbledb.runtime.functions; + +import org.apache.spark.api.java.JavaRDD; +import org.rumbledb.api.Item; +import org.rumbledb.context.DynamicContext; +import org.rumbledb.context.FunctionIdentifier; +import org.rumbledb.context.Name; +import org.rumbledb.context.RuntimeStaticContext; +import org.rumbledb.expressions.ExecutionMode; +import org.rumbledb.items.FunctionItem; +import org.rumbledb.items.structured.JSoundDataFrame; +import org.rumbledb.runtime.AtMostOneItemLocalRuntimeIterator; +import org.rumbledb.runtime.RuntimeIterator; +import org.rumbledb.runtime.primary.VariableReferenceIterator; +import org.rumbledb.types.FunctionSignature; +import org.rumbledb.types.SequenceType; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Creates a partially-applied function item by capturing supplied arguments in the closure and + * exposing each placeholder as a parameter of the returned function item. + */ +public class PartialFunctionCallIterator extends AtMostOneItemLocalRuntimeIterator { + + private static final long serialVersionUID = 1L; + private static final Name PARTIAL_FUNCTION_TARGET_NAME = Name.createVariableInNoNamespace( + "$4deff1a0-0d7f-43d1-bf96-09934cc6a539" + ); + + private final Item functionItem; + private final List functionArguments; + private final Name functionNameOverride; + + public PartialFunctionCallIterator( + Item functionItem, + List functionArguments, + RuntimeStaticContext staticContext + ) { + this(functionItem, functionArguments, staticContext, null); + } + + public PartialFunctionCallIterator( + Item functionItem, + List functionArguments, + RuntimeStaticContext staticContext, + Name functionNameOverride + ) { + super(null, staticContext); + for (RuntimeIterator arg : functionArguments) { + if (arg != null) { + this.children.add(arg); + } + } + this.functionItem = functionItem; + this.functionArguments = functionArguments; + this.functionNameOverride = functionNameOverride; + + FunctionCallArgumentCoercion.validateArity(functionItem, this.functionArguments, getMetadata()); + FunctionCallArgumentCoercion.wrapAccordingToSignature( + functionItem, + this.functionArguments, + staticContext + ); + } + + @Override + public Item materializeFirstItemOrNull(DynamicContext context) { + Map> localArgumentValues = new LinkedHashMap<>( + this.functionItem.getLocalVariablesInClosure() + ); + Map> rddArgumentValues = new LinkedHashMap<>( + this.functionItem.getRDDVariablesInClosure() + ); + Map dfArgumentValues = new LinkedHashMap<>( + this.functionItem.getDFVariablesInClosure() + ); + + List partialApplicationParamNames = new ArrayList<>(); + List partialApplicationParamTypes = new ArrayList<>(); + + for (int i = 0; i < this.functionArguments.size(); i++) { + Name parameterName = this.functionItem.getParameterNames().get(i); + RuntimeIterator argumentIterator = this.functionArguments.get(i); + + if (argumentIterator == null) { + partialApplicationParamNames.add(parameterName); + partialApplicationParamTypes.add(this.functionItem.getSignature().getParameterTypes().get(i)); + } else if (argumentIterator.isDataFrame()) { + dfArgumentValues.put(parameterName, argumentIterator.getDataFrame(context)); + } else if (argumentIterator.isRDDOrDataFrame()) { + rddArgumentValues.put(parameterName, argumentIterator.getRDD(context)); + } else { + localArgumentValues.put(parameterName, argumentIterator.materialize(context)); + } + } + + RuntimeIterator functionBodyIterator = this.functionItem.getBodyIterator(); + boolean isBuiltinFunctionItem = this.functionItem.isBuiltinFunction(); + if (isBuiltinFunctionItem) { + localArgumentValues.put(PARTIAL_FUNCTION_TARGET_NAME, List.of(this.functionItem)); + functionBodyIterator = createPartialBuiltinBodyIterator(); + } + + return new FunctionItem( + new FunctionIdentifier( + this.functionNameOverride == null + ? this.functionItem.getIdentifier().getName() + : this.functionNameOverride, + partialApplicationParamNames.size() + ), + partialApplicationParamNames, + new FunctionSignature( + partialApplicationParamTypes, + this.functionItem.getSignature().getReturnType(), + this.functionItem.getSignature().isUpdating() + ), + this.functionItem.getModuleDynamicContext(), + functionBodyIterator, + localArgumentValues, + rddArgumentValues, + dfArgumentValues, + false + ); + } + + private RuntimeIterator createPartialBuiltinBodyIterator() { + RuntimeStaticContext targetStaticContext = getRuntimeStaticContext() + .withStaticType(SequenceType.createSequenceType("function(*)")) + .withExecutionMode(ExecutionMode.LOCAL); + + RuntimeIterator targetIterator = new VariableReferenceIterator( + PARTIAL_FUNCTION_TARGET_NAME, + targetStaticContext + ); + + List callArguments = new ArrayList<>(); + for (int i = 0; i < this.functionArguments.size(); i++) { + RuntimeStaticContext argumentStaticContext; + if (this.functionArguments.get(i) == null) { + argumentStaticContext = getRuntimeStaticContext() + .withStaticType(this.functionItem.getSignature().getParameterTypes().get(i)) + .withExecutionMode(ExecutionMode.LOCAL); + } else { + argumentStaticContext = this.functionArguments.get(i).getRuntimeStaticContext(); + } + callArguments.add( + new VariableReferenceIterator( + this.functionItem.getParameterNames().get(i), + argumentStaticContext + ) + ); + } + + return new DynamicFunctionCallIterator( + targetIterator, + callArguments, + getRuntimeStaticContext() + .withStaticType(this.functionItem.getSignature().getReturnType()) + .withExecutionMode(ExecutionMode.LOCAL) + ); + } +} From fbc2d5c0341361c115778e342a43f3217b664ba9 Mon Sep 17 00:00:00 2001 From: Jimmy Cai Date: Wed, 17 Jun 2026 00:15:57 +0200 Subject: [PATCH 2/6] Return null function name in case of partial function --- .../runtime/functions/PartialFunctionCallIterator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java b/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java index 1de31edb20..4b722b4308 100644 --- a/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java @@ -123,9 +123,7 @@ public Item materializeFirstItemOrNull(DynamicContext context) { return new FunctionItem( new FunctionIdentifier( - this.functionNameOverride == null - ? this.functionItem.getIdentifier().getName() - : this.functionNameOverride, + this.functionNameOverride, partialApplicationParamNames.size() ), partialApplicationParamNames, From f82f2bd9a4f981826a1f32fee908842a1a151cbd Mon Sep 17 00:00:00 2001 From: Jimmy Cai Date: Wed, 17 Jun 2026 00:23:41 +0200 Subject: [PATCH 3/6] Remove function-name debug output --- .../runtime/functions/typing/FunctionNameFunctionIterator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/rumbledb/runtime/functions/typing/FunctionNameFunctionIterator.java b/src/main/java/org/rumbledb/runtime/functions/typing/FunctionNameFunctionIterator.java index b7fdbd85a3..797c9db91c 100644 --- a/src/main/java/org/rumbledb/runtime/functions/typing/FunctionNameFunctionIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/typing/FunctionNameFunctionIterator.java @@ -37,7 +37,6 @@ public Item materializeFirstItemOrNull(DynamicContext context) { getMetadata() ); } - System.err.println("Item is of type function"); Item functionItem = functionIterator.materializeFirstItemOrNull(context); if (functionItem == null || !(functionItem instanceof FunctionItem)) { throw new OurBadException("Expected argument to be of type function and not be null"); From 7c531f34b95bb7bfe5a7728ad24495ef6f609ebf Mon Sep 17 00:00:00 2001 From: Jimmy Cai Date: Thu, 18 Jun 2026 12:12:23 +0200 Subject: [PATCH 4/6] Added generic PartiallyAppliedFunctionItem and solve the runtime mode for partial applied builtin functions --- .../org/rumbledb/context/NamedFunctions.java | 99 ++++++++++++ .../items/PartiallyAppliedFunctionItem.java | 145 ++++++++++++++++++ .../CapturedFunctionArgumentIterator.java | 98 ++++++++++++ .../DynamicFunctionCallIterator.java | 22 +-- .../PartialFunctionCallIterator.java | 93 +++-------- .../PartialBuiltinFunctionApplication1.jq | 7 + .../PartialBuiltinFunctionApplication2.jq | 4 + .../PartialBuiltinFunctionApplicationRDD1.jq | 3 + .../PartialBuiltinFunctionApplicationRDD2.jq | 3 + 9 files changed, 382 insertions(+), 92 deletions(-) create mode 100644 src/main/java/org/rumbledb/items/PartiallyAppliedFunctionItem.java create mode 100644 src/main/java/org/rumbledb/runtime/functions/CapturedFunctionArgumentIterator.java create mode 100644 src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplication1.jq create mode 100644 src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplication2.jq create mode 100644 src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplicationRDD1.jq create mode 100644 src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplicationRDD2.jq diff --git a/src/main/java/org/rumbledb/context/NamedFunctions.java b/src/main/java/org/rumbledb/context/NamedFunctions.java index 3573c49cd4..3606123cce 100644 --- a/src/main/java/org/rumbledb/context/NamedFunctions.java +++ b/src/main/java/org/rumbledb/context/NamedFunctions.java @@ -32,8 +32,16 @@ import org.rumbledb.exceptions.UnknownFunctionCallException; import org.rumbledb.expressions.ExecutionMode; import org.rumbledb.items.FunctionItem; +import org.rumbledb.items.PartiallyAppliedFunctionItem; +import org.rumbledb.items.PartiallyAppliedFunctionItem.ArgumentBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.DataFrameBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.LocalBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.PlaceholderBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.RDDBinding; import org.rumbledb.runtime.RuntimeIterator; import org.rumbledb.runtime.functions.BuiltinFunctionItemCallIterator; +import org.rumbledb.runtime.functions.CapturedFunctionArgumentIterator; +import org.rumbledb.runtime.functions.FunctionCallArgumentCoercion; import org.rumbledb.runtime.functions.FunctionItemCallIterator; import org.rumbledb.runtime.functions.PartialFunctionCallIterator; import org.rumbledb.runtime.functions.sequences.general.DataFunctionIterator; @@ -44,6 +52,7 @@ import java.io.Serializable; import java.lang.reflect.Constructor; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -104,6 +113,32 @@ public static RuntimeIterator buildFunctionItemCallIterator( List arguments, boolean isTailOptimization ) { + if (functionItem instanceof PartiallyAppliedFunctionItem partiallyAppliedFunction) { + FunctionCallArgumentCoercion.validateArity(functionItem, arguments, callerRuntimeContext.getMetadata()); + FunctionCallArgumentCoercion.wrapAccordingToSignature( + functionItem, + arguments, + callerRuntimeContext + ); + List expandedArguments = expandPartialArguments( + partiallyAppliedFunction, + arguments, + callerRuntimeContext + ); + Item targetFunction = partiallyAppliedFunction.getTargetFunction(); + ExecutionMode targetExecutionMode = resolveFunctionItemExecutionMode( + targetFunction, + expandedArguments, + callerRuntimeContext + ); + return buildFunctionItemCallIterator( + targetFunction, + callerRuntimeContext, + targetExecutionMode, + expandedArguments, + false + ); + } if (isTailOptimization) { return new PartialFunctionCallIterator( functionItem, @@ -177,6 +212,70 @@ public static RuntimeIterator buildFunctionItemCallIterator( } } + public static ExecutionMode resolveFunctionItemExecutionMode( + Item functionItem, + List arguments, + RuntimeStaticContext callerRuntimeContext + ) { + if (functionItem instanceof PartiallyAppliedFunctionItem partiallyAppliedFunction) { + List expandedArguments = expandPartialArguments( + partiallyAppliedFunction, + arguments, + callerRuntimeContext + ); + return resolveFunctionItemExecutionMode( + partiallyAppliedFunction.getTargetFunction(), + expandedArguments, + callerRuntimeContext + ); + } + if (functionItem.isBuiltinFunction()) { + BuiltinFunction builtin = BuiltinFunctionCatalogue.getBuiltinFunction(functionItem.getIdentifier()); + ExecutionMode firstArgumentMode = arguments.isEmpty() || arguments.get(0) == null + ? ExecutionMode.LOCAL + : arguments.get(0).getHighestExecutionMode(); + return BuiltinFunctionExecutionModes.resolve( + builtin, + firstArgumentMode, + callerRuntimeContext.getConfiguration() + ); + } + return functionItem.getBodyIterator().getHighestExecutionMode(); + } + + private static List expandPartialArguments( + PartiallyAppliedFunctionItem functionItem, + List suppliedArguments, + RuntimeStaticContext callerRuntimeContext + ) { + List result = new ArrayList<>(); + int suppliedIndex = 0; + for (ArgumentBinding binding : functionItem.getArgumentBindings()) { + if (binding instanceof PlaceholderBinding) { + result.add(suppliedArguments.get(suppliedIndex++)); + continue; + } + ExecutionMode executionMode; + if (binding instanceof DataFrameBinding) { + executionMode = ExecutionMode.DATAFRAME; + } else if (binding instanceof RDDBinding) { + executionMode = ExecutionMode.RDD; + } else if (binding instanceof LocalBinding) { + executionMode = ExecutionMode.LOCAL; + } else { + throw new OurBadException("Unsupported partial-function argument binding."); + } + result.add( + new CapturedFunctionArgumentIterator( + binding, + callerRuntimeContext.withStaticType(binding.sequenceType()) + .withExecutionMode(executionMode) + ) + ); + } + return result; + } + public void addUserDefinedFunction(Item function, ExceptionMetadata meta) { if (!function.isFunction()) { throw new OurBadException("Only a function item can be added as a user-defined function."); diff --git a/src/main/java/org/rumbledb/items/PartiallyAppliedFunctionItem.java b/src/main/java/org/rumbledb/items/PartiallyAppliedFunctionItem.java new file mode 100644 index 0000000000..6fff374ab9 --- /dev/null +++ b/src/main/java/org/rumbledb/items/PartiallyAppliedFunctionItem.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.rumbledb.items; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import org.apache.spark.api.java.JavaRDD; +import org.rumbledb.api.Item; +import org.rumbledb.context.DynamicContext; +import org.rumbledb.context.FunctionIdentifier; +import org.rumbledb.context.Name; +import org.rumbledb.items.structured.JSoundDataFrame; +import org.rumbledb.types.FunctionSignature; +import org.rumbledb.types.SequenceType; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; + +/** + * Function item produced by partial application. + * + * It stores the original target and an argument binding for every target parameter. Invocation expands + * these bindings and delegates to the ordinary function-item dispatch path. + */ +public class PartiallyAppliedFunctionItem extends FunctionItem { + + private static final long serialVersionUID = 1L; + + public sealed interface ArgumentBinding extends Serializable + permits PlaceholderBinding, LocalBinding, RDDBinding, DataFrameBinding { + + SequenceType sequenceType(); + } + + public record PlaceholderBinding(SequenceType sequenceType) implements ArgumentBinding { + + private static final long serialVersionUID = 1L; + } + + public record LocalBinding(SequenceType sequenceType, List value) implements ArgumentBinding { + + private static final long serialVersionUID = 1L; + } + + public record RDDBinding(SequenceType sequenceType, JavaRDD value) implements ArgumentBinding { + + private static final long serialVersionUID = 1L; + } + + public record DataFrameBinding(SequenceType sequenceType, JSoundDataFrame value) implements ArgumentBinding { + + private static final long serialVersionUID = 1L; + } + + private Item targetFunction; + private List argumentBindings; + + public PartiallyAppliedFunctionItem() { + super(); + } + + public PartiallyAppliedFunctionItem( + FunctionIdentifier identifier, + List parameterNames, + FunctionSignature signature, + DynamicContext dynamicModuleContext, + Item targetFunction, + List argumentBindings + ) { + super( + identifier, + parameterNames, + signature, + dynamicModuleContext, + targetFunction.getBodyIterator(), + new HashMap<>(), + new HashMap<>(), + new HashMap<>(), + false + ); + this.targetFunction = targetFunction; + this.argumentBindings = argumentBindings; + } + + public Item getTargetFunction() { + return this.targetFunction; + } + + public List getArgumentBindings() { + return this.argumentBindings; + } + + @Override + public void write(Kryo kryo, Output output) { + super.write(kryo, output); + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + ObjectOutputStream objects = new ObjectOutputStream(bytes); + objects.writeObject(this.targetFunction); + objects.writeObject(this.argumentBindings); + objects.flush(); + byte[] serialized = bytes.toByteArray(); + output.writeInt(serialized.length); + output.writeBytes(serialized); + } catch (IOException e) { + throw new IllegalStateException("Could not serialize partially applied function metadata.", e); + } + } + + @SuppressWarnings("unchecked") + @Override + public void read(Kryo kryo, Input input) { + super.read(kryo, input); + try { + int length = input.readInt(); + byte[] serialized = input.readBytes(length); + ObjectInputStream objects = new ObjectInputStream(new ByteArrayInputStream(serialized)); + this.targetFunction = (Item) objects.readObject(); + this.argumentBindings = (List) objects.readObject(); + } catch (IOException | ClassNotFoundException e) { + throw new IllegalStateException("Could not deserialize partially applied function metadata.", e); + } + } +} diff --git a/src/main/java/org/rumbledb/runtime/functions/CapturedFunctionArgumentIterator.java b/src/main/java/org/rumbledb/runtime/functions/CapturedFunctionArgumentIterator.java new file mode 100644 index 0000000000..be8c46d71c --- /dev/null +++ b/src/main/java/org/rumbledb/runtime/functions/CapturedFunctionArgumentIterator.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.rumbledb.runtime.functions; + +import org.apache.spark.api.java.JavaRDD; +import org.rumbledb.api.Item; +import org.rumbledb.context.DynamicContext; +import org.rumbledb.context.RuntimeStaticContext; +import org.rumbledb.exceptions.IteratorFlowException; +import org.rumbledb.items.PartiallyAppliedFunctionItem.ArgumentBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.DataFrameBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.LocalBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.RDDBinding; +import org.rumbledb.items.structured.JSoundDataFrame; +import org.rumbledb.runtime.HybridRuntimeIterator; + +import java.util.List; + +/** + * Context-independent iterator over a value captured by partial application. + */ +public class CapturedFunctionArgumentIterator extends HybridRuntimeIterator { + + private static final long serialVersionUID = 1L; + + private final ArgumentBinding binding; + private int index; + + public CapturedFunctionArgumentIterator(ArgumentBinding binding, RuntimeStaticContext staticContext) { + super(null, staticContext); + this.binding = binding; + } + + @Override + protected void openLocal() { + this.index = 0; + this.hasNext = !getLocalValue().isEmpty(); + } + + @Override + protected boolean hasNextLocal() { + return this.hasNext; + } + + @Override + protected Item nextLocal() { + if (!this.hasNext) { + throw new IteratorFlowException(FLOW_EXCEPTION_MESSAGE, getMetadata()); + } + Item result = getLocalValue().get(this.index++); + this.hasNext = this.index < getLocalValue().size(); + return result; + } + + @Override + protected void resetLocal() { + this.index = 0; + this.hasNext = !getLocalValue().isEmpty(); + } + + @Override + protected void closeLocal() { + this.index = 0; + } + + @Override + public JavaRDD getRDDAux(DynamicContext context) { + return ((RDDBinding) this.binding).value(); + } + + @Override + protected boolean implementsDataFrames() { + return true; + } + + @Override + public JSoundDataFrame getDataFrame(DynamicContext context) { + return ((DataFrameBinding) this.binding).value(); + } + + private List getLocalValue() { + return ((LocalBinding) this.binding).value(); + } +} diff --git a/src/main/java/org/rumbledb/runtime/functions/DynamicFunctionCallIterator.java b/src/main/java/org/rumbledb/runtime/functions/DynamicFunctionCallIterator.java index 9132baba41..e9cadb656c 100644 --- a/src/main/java/org/rumbledb/runtime/functions/DynamicFunctionCallIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/DynamicFunctionCallIterator.java @@ -24,9 +24,6 @@ import org.apache.spark.api.java.JavaRDD; import org.rumbledb.api.Item; -import org.rumbledb.context.BuiltinFunction; -import org.rumbledb.context.BuiltinFunctionCatalogue; -import org.rumbledb.context.BuiltinFunctionExecutionModes; import org.rumbledb.context.DynamicContext; import org.rumbledb.context.NamedFunctions; import org.rumbledb.context.RuntimeStaticContext; @@ -250,20 +247,11 @@ private ExecutionMode getCalleeExecutionModeForFunctionItemCall() { if (this.isPartialApplication) { return ExecutionMode.LOCAL; } - if (this.functionItem.isBuiltinFunction()) { - BuiltinFunction builtin = - BuiltinFunctionCatalogue.getBuiltinFunction(this.functionItem.getIdentifier()); - // assume that the passed builtin function is valid - ExecutionMode firstArgumentMode = ExecutionMode.LOCAL; - for (RuntimeIterator arg : this.functionArguments) { - if (arg != null) { - firstArgumentMode = arg.getHighestExecutionMode(); - break; - } - } - return BuiltinFunctionExecutionModes.resolve(builtin, firstArgumentMode, getConfiguration()); - } - return this.functionItem.getBodyIterator().getHighestExecutionMode(); + return NamedFunctions.resolveFunctionItemExecutionMode( + this.functionItem, + this.functionArguments, + this.staticContext + ); } @Override diff --git a/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java b/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java index 4b722b4308..47b590843a 100644 --- a/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java @@ -16,25 +16,24 @@ */ package org.rumbledb.runtime.functions; -import org.apache.spark.api.java.JavaRDD; import org.rumbledb.api.Item; import org.rumbledb.context.DynamicContext; import org.rumbledb.context.FunctionIdentifier; import org.rumbledb.context.Name; import org.rumbledb.context.RuntimeStaticContext; -import org.rumbledb.expressions.ExecutionMode; -import org.rumbledb.items.FunctionItem; -import org.rumbledb.items.structured.JSoundDataFrame; +import org.rumbledb.items.PartiallyAppliedFunctionItem; +import org.rumbledb.items.PartiallyAppliedFunctionItem.ArgumentBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.DataFrameBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.LocalBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.PlaceholderBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.RDDBinding; import org.rumbledb.runtime.AtMostOneItemLocalRuntimeIterator; import org.rumbledb.runtime.RuntimeIterator; -import org.rumbledb.runtime.primary.VariableReferenceIterator; import org.rumbledb.types.FunctionSignature; import org.rumbledb.types.SequenceType; import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; /** * Creates a partially-applied function item by capturing supplied arguments in the closure and @@ -43,10 +42,6 @@ public class PartialFunctionCallIterator extends AtMostOneItemLocalRuntimeIterator { private static final long serialVersionUID = 1L; - private static final Name PARTIAL_FUNCTION_TARGET_NAME = Name.createVariableInNoNamespace( - "$4deff1a0-0d7f-43d1-bf96-09934cc6a539" - ); - private final Item functionItem; private final List functionArguments; private final Name functionNameOverride; @@ -85,43 +80,31 @@ public PartialFunctionCallIterator( @Override public Item materializeFirstItemOrNull(DynamicContext context) { - Map> localArgumentValues = new LinkedHashMap<>( - this.functionItem.getLocalVariablesInClosure() - ); - Map> rddArgumentValues = new LinkedHashMap<>( - this.functionItem.getRDDVariablesInClosure() - ); - Map dfArgumentValues = new LinkedHashMap<>( - this.functionItem.getDFVariablesInClosure() - ); - List partialApplicationParamNames = new ArrayList<>(); List partialApplicationParamTypes = new ArrayList<>(); + List argumentBindings = new ArrayList<>(); for (int i = 0; i < this.functionArguments.size(); i++) { Name parameterName = this.functionItem.getParameterNames().get(i); RuntimeIterator argumentIterator = this.functionArguments.get(i); + SequenceType parameterType = this.functionItem.getSignature().getParameterTypes().get(i); if (argumentIterator == null) { partialApplicationParamNames.add(parameterName); - partialApplicationParamTypes.add(this.functionItem.getSignature().getParameterTypes().get(i)); + partialApplicationParamTypes.add(parameterType); + argumentBindings.add(new PlaceholderBinding(parameterType)); } else if (argumentIterator.isDataFrame()) { - dfArgumentValues.put(parameterName, argumentIterator.getDataFrame(context)); + argumentBindings.add( + new DataFrameBinding(parameterType, argumentIterator.getDataFrame(context)) + ); } else if (argumentIterator.isRDDOrDataFrame()) { - rddArgumentValues.put(parameterName, argumentIterator.getRDD(context)); + argumentBindings.add(new RDDBinding(parameterType, argumentIterator.getRDD(context))); } else { - localArgumentValues.put(parameterName, argumentIterator.materialize(context)); + argumentBindings.add(new LocalBinding(parameterType, argumentIterator.materialize(context))); } } - RuntimeIterator functionBodyIterator = this.functionItem.getBodyIterator(); - boolean isBuiltinFunctionItem = this.functionItem.isBuiltinFunction(); - if (isBuiltinFunctionItem) { - localArgumentValues.put(PARTIAL_FUNCTION_TARGET_NAME, List.of(this.functionItem)); - functionBodyIterator = createPartialBuiltinBodyIterator(); - } - - return new FunctionItem( + return new PartiallyAppliedFunctionItem( new FunctionIdentifier( this.functionNameOverride, partialApplicationParamNames.size() @@ -133,48 +116,8 @@ public Item materializeFirstItemOrNull(DynamicContext context) { this.functionItem.getSignature().isUpdating() ), this.functionItem.getModuleDynamicContext(), - functionBodyIterator, - localArgumentValues, - rddArgumentValues, - dfArgumentValues, - false - ); - } - - private RuntimeIterator createPartialBuiltinBodyIterator() { - RuntimeStaticContext targetStaticContext = getRuntimeStaticContext() - .withStaticType(SequenceType.createSequenceType("function(*)")) - .withExecutionMode(ExecutionMode.LOCAL); - - RuntimeIterator targetIterator = new VariableReferenceIterator( - PARTIAL_FUNCTION_TARGET_NAME, - targetStaticContext - ); - - List callArguments = new ArrayList<>(); - for (int i = 0; i < this.functionArguments.size(); i++) { - RuntimeStaticContext argumentStaticContext; - if (this.functionArguments.get(i) == null) { - argumentStaticContext = getRuntimeStaticContext() - .withStaticType(this.functionItem.getSignature().getParameterTypes().get(i)) - .withExecutionMode(ExecutionMode.LOCAL); - } else { - argumentStaticContext = this.functionArguments.get(i).getRuntimeStaticContext(); - } - callArguments.add( - new VariableReferenceIterator( - this.functionItem.getParameterNames().get(i), - argumentStaticContext - ) - ); - } - - return new DynamicFunctionCallIterator( - targetIterator, - callArguments, - getRuntimeStaticContext() - .withStaticType(this.functionItem.getSignature().getReturnType()) - .withExecutionMode(ExecutionMode.LOCAL) + this.functionItem, + argumentBindings ); } } diff --git a/src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplication1.jq b/src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplication1.jq new file mode 100644 index 0000000000..e4eb53bc33 --- /dev/null +++ b/src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplication1.jq @@ -0,0 +1,7 @@ +(:JIQS: ShouldRun; Output="(3, false)" :) +let $partial := max#1(?) +return +( + $partial((1, 2, 3)), + exists(function-name($partial)) +) diff --git a/src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplication2.jq b/src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplication2.jq new file mode 100644 index 0000000000..a7a7a9576e --- /dev/null +++ b/src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplication2.jq @@ -0,0 +1,4 @@ +(:JIQS: ShouldRun; Output="(2, 3, 4)" :) +let $partial := tail(?) +let $partial-again := $partial(?) +return $partial-again(1 to 4) diff --git a/src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplicationRDD1.jq b/src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplicationRDD1.jq new file mode 100644 index 0000000000..7ba7b2e853 --- /dev/null +++ b/src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplicationRDD1.jq @@ -0,0 +1,3 @@ +(:JIQS: ShouldRun; Output="(2, 3, 4, 5)" :) +let $partial := tail(?) +return $partial(parallelize(1 to 5)) diff --git a/src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplicationRDD2.jq b/src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplicationRDD2.jq new file mode 100644 index 0000000000..c59d7b948c --- /dev/null +++ b/src/test/resources/test_files/runtime/FunctionPartialApplication/PartialBuiltinFunctionApplicationRDD2.jq @@ -0,0 +1,3 @@ +(:JIQS: ShouldRun; Output="(1, 9, 2, 3, 4)" :) +let $partial := insert-before(parallelize(1 to 4), ?, 9) +return $partial(2) From e6b9bd327c7824cd2afc939512b5e83bb9448c52 Mon Sep 17 00:00:00 2001 From: Jimmy Cai Date: Thu, 18 Jun 2026 12:30:08 +0200 Subject: [PATCH 5/6] Refactor PartiallyAppliedFunctionItem and NamedFunctions for improved function resolution and argument binding validation; add unit tests for immutability and serialization. --- .../org/rumbledb/context/NamedFunctions.java | 108 ++++++++----- .../items/PartiallyAppliedFunctionItem.java | 38 ++++- .../CapturedFunctionArgumentIterator.java | 56 +++++-- .../DynamicFunctionCallIterator.java | 37 ++--- .../PartialFunctionCallIterator.java | 4 +- .../PartiallyAppliedFunctionItemTest.java | 145 ++++++++++++++++++ 6 files changed, 310 insertions(+), 78 deletions(-) create mode 100644 src/test/java/org/rumbledb/items/PartiallyAppliedFunctionItemTest.java diff --git a/src/main/java/org/rumbledb/context/NamedFunctions.java b/src/main/java/org/rumbledb/context/NamedFunctions.java index 3606123cce..9579837c16 100644 --- a/src/main/java/org/rumbledb/context/NamedFunctions.java +++ b/src/main/java/org/rumbledb/context/NamedFunctions.java @@ -37,7 +37,7 @@ import org.rumbledb.items.PartiallyAppliedFunctionItem.DataFrameBinding; import org.rumbledb.items.PartiallyAppliedFunctionItem.LocalBinding; import org.rumbledb.items.PartiallyAppliedFunctionItem.PlaceholderBinding; -import org.rumbledb.items.PartiallyAppliedFunctionItem.RDDBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.RddBinding; import org.rumbledb.runtime.RuntimeIterator; import org.rumbledb.runtime.functions.BuiltinFunctionItemCallIterator; import org.rumbledb.runtime.functions.CapturedFunctionArgumentIterator; @@ -61,6 +61,12 @@ public class NamedFunctions implements Serializable, KryoSerializable { private static final long serialVersionUID = 1L; + public record ResolvedFunctionCall( + Item functionItem, + List arguments, + ExecutionMode executionMode) { + } + // two maps for User defined function are needed as execution mode is known at // static analysis phase // but functions items are fully known at runtimeIterator generation @@ -113,32 +119,41 @@ public static RuntimeIterator buildFunctionItemCallIterator( List arguments, boolean isTailOptimization ) { - if (functionItem instanceof PartiallyAppliedFunctionItem partiallyAppliedFunction) { - FunctionCallArgumentCoercion.validateArity(functionItem, arguments, callerRuntimeContext.getMetadata()); - FunctionCallArgumentCoercion.wrapAccordingToSignature( - functionItem, - arguments, + if (functionItem instanceof PartiallyAppliedFunctionItem) { + return buildResolvedFunctionItemCallIterator( + resolveFunctionItemCall(functionItem, arguments, callerRuntimeContext), callerRuntimeContext ); - List expandedArguments = expandPartialArguments( - partiallyAppliedFunction, - arguments, - callerRuntimeContext - ); - Item targetFunction = partiallyAppliedFunction.getTargetFunction(); - ExecutionMode targetExecutionMode = resolveFunctionItemExecutionMode( - targetFunction, - expandedArguments, - callerRuntimeContext - ); - return buildFunctionItemCallIterator( - targetFunction, - callerRuntimeContext, - targetExecutionMode, - expandedArguments, - false - ); } + return buildDirectFunctionItemCallIterator( + functionItem, + callerRuntimeContext, + executionModeForFunctionCall, + arguments, + isTailOptimization + ); + } + + public static RuntimeIterator buildResolvedFunctionItemCallIterator( + ResolvedFunctionCall resolvedCall, + RuntimeStaticContext callerRuntimeContext + ) { + return buildDirectFunctionItemCallIterator( + resolvedCall.functionItem(), + callerRuntimeContext, + resolvedCall.executionMode(), + new ArrayList<>(resolvedCall.arguments()), + false + ); + } + + private static RuntimeIterator buildDirectFunctionItemCallIterator( + Item functionItem, + RuntimeStaticContext callerRuntimeContext, + ExecutionMode executionModeForFunctionCall, + List arguments, + boolean isTailOptimization + ) { if (isTailOptimization) { return new PartialFunctionCallIterator( functionItem, @@ -212,23 +227,42 @@ public static RuntimeIterator buildFunctionItemCallIterator( } } - public static ExecutionMode resolveFunctionItemExecutionMode( + public static ResolvedFunctionCall resolveFunctionItemCall( Item functionItem, List arguments, RuntimeStaticContext callerRuntimeContext ) { - if (functionItem instanceof PartiallyAppliedFunctionItem partiallyAppliedFunction) { - List expandedArguments = expandPartialArguments( - partiallyAppliedFunction, - arguments, + Item resolvedFunction = functionItem; + List resolvedArguments = new ArrayList<>(arguments); + while (resolvedFunction instanceof PartiallyAppliedFunctionItem partiallyAppliedFunction) { + FunctionCallArgumentCoercion.validateArity( + resolvedFunction, + resolvedArguments, + callerRuntimeContext.getMetadata() + ); + FunctionCallArgumentCoercion.wrapAccordingToSignature( + resolvedFunction, + resolvedArguments, callerRuntimeContext ); - return resolveFunctionItemExecutionMode( - partiallyAppliedFunction.getTargetFunction(), - expandedArguments, + resolvedArguments = expandPartialArguments( + partiallyAppliedFunction, + resolvedArguments, callerRuntimeContext ); + resolvedFunction = partiallyAppliedFunction.getTargetFunction(); } + ExecutionMode executionMode = resolvedArguments.stream().anyMatch(argument -> argument == null) + ? ExecutionMode.LOCAL + : resolveDirectFunctionItemExecutionMode(resolvedFunction, resolvedArguments, callerRuntimeContext); + return new ResolvedFunctionCall(resolvedFunction, resolvedArguments, executionMode); + } + + private static ExecutionMode resolveDirectFunctionItemExecutionMode( + Item functionItem, + List arguments, + RuntimeStaticContext callerRuntimeContext + ) { if (functionItem.isBuiltinFunction()) { BuiltinFunction builtin = BuiltinFunctionCatalogue.getBuiltinFunction(functionItem.getIdentifier()); ExecutionMode firstArgumentMode = arguments.isEmpty() || arguments.get(0) == null @@ -258,7 +292,7 @@ private static List expandPartialArguments( ExecutionMode executionMode; if (binding instanceof DataFrameBinding) { executionMode = ExecutionMode.DATAFRAME; - } else if (binding instanceof RDDBinding) { + } else if (binding instanceof RddBinding) { executionMode = ExecutionMode.RDD; } else if (binding instanceof LocalBinding) { executionMode = ExecutionMode.LOCAL; @@ -266,10 +300,10 @@ private static List expandPartialArguments( throw new OurBadException("Unsupported partial-function argument binding."); } result.add( - new CapturedFunctionArgumentIterator( - binding, - callerRuntimeContext.withStaticType(binding.sequenceType()) - .withExecutionMode(executionMode) + CapturedFunctionArgumentIterator.create( + binding, + callerRuntimeContext.withStaticType(binding.sequenceType()) + .withExecutionMode(executionMode) ) ); } diff --git a/src/main/java/org/rumbledb/items/PartiallyAppliedFunctionItem.java b/src/main/java/org/rumbledb/items/PartiallyAppliedFunctionItem.java index 6fff374ab9..0e19cca7a0 100644 --- a/src/main/java/org/rumbledb/items/PartiallyAppliedFunctionItem.java +++ b/src/main/java/org/rumbledb/items/PartiallyAppliedFunctionItem.java @@ -24,6 +24,7 @@ import org.rumbledb.context.DynamicContext; import org.rumbledb.context.FunctionIdentifier; import org.rumbledb.context.Name; +import org.rumbledb.exceptions.OurBadException; import org.rumbledb.items.structured.JSoundDataFrame; import org.rumbledb.types.FunctionSignature; import org.rumbledb.types.SequenceType; @@ -48,7 +49,7 @@ public class PartiallyAppliedFunctionItem extends FunctionItem { private static final long serialVersionUID = 1L; public sealed interface ArgumentBinding extends Serializable - permits PlaceholderBinding, LocalBinding, RDDBinding, DataFrameBinding { + permits PlaceholderBinding, LocalBinding, RddBinding, DataFrameBinding { SequenceType sequenceType(); } @@ -61,9 +62,13 @@ public record PlaceholderBinding(SequenceType sequenceType) implements ArgumentB public record LocalBinding(SequenceType sequenceType, List value) implements ArgumentBinding { private static final long serialVersionUID = 1L; + + public LocalBinding { + value = List.copyOf(value); + } } - public record RDDBinding(SequenceType sequenceType, JavaRDD value) implements ArgumentBinding { + public record RddBinding(SequenceType sequenceType, JavaRDD value) implements ArgumentBinding { private static final long serialVersionUID = 1L; } @@ -93,14 +98,22 @@ public PartiallyAppliedFunctionItem( parameterNames, signature, dynamicModuleContext, - targetFunction.getBodyIterator(), + getTargetBody(targetFunction), new HashMap<>(), new HashMap<>(), new HashMap<>(), false ); + validateBindings(identifier, targetFunction, argumentBindings); this.targetFunction = targetFunction; - this.argumentBindings = argumentBindings; + this.argumentBindings = List.copyOf(argumentBindings); + } + + private static org.rumbledb.runtime.RuntimeIterator getTargetBody(Item targetFunction) { + if (targetFunction == null || !targetFunction.isFunction()) { + throw new OurBadException("A partially applied function must have a function target."); + } + return targetFunction.getBodyIterator(); } public Item getTargetFunction() { @@ -111,6 +124,20 @@ public List getArgumentBindings() { return this.argumentBindings; } + private static void validateBindings( + FunctionIdentifier identifier, + Item targetFunction, + List argumentBindings + ) { + if (argumentBindings.size() != targetFunction.getIdentifier().getArity()) { + throw new OurBadException("Partial-function bindings do not match the target arity."); + } + long placeholderCount = argumentBindings.stream().filter(PlaceholderBinding.class::isInstance).count(); + if (placeholderCount != identifier.getArity()) { + throw new OurBadException("Partial-function placeholders do not match the partial signature."); + } + } + @Override public void write(Kryo kryo, Output output) { super.write(kryo, output); @@ -137,7 +164,8 @@ public void read(Kryo kryo, Input input) { byte[] serialized = input.readBytes(length); ObjectInputStream objects = new ObjectInputStream(new ByteArrayInputStream(serialized)); this.targetFunction = (Item) objects.readObject(); - this.argumentBindings = (List) objects.readObject(); + this.argumentBindings = List.copyOf((List) objects.readObject()); + validateBindings(getIdentifier(), this.targetFunction, this.argumentBindings); } catch (IOException | ClassNotFoundException e) { throw new IllegalStateException("Could not deserialize partially applied function metadata.", e); } diff --git a/src/main/java/org/rumbledb/runtime/functions/CapturedFunctionArgumentIterator.java b/src/main/java/org/rumbledb/runtime/functions/CapturedFunctionArgumentIterator.java index be8c46d71c..1e474cf442 100644 --- a/src/main/java/org/rumbledb/runtime/functions/CapturedFunctionArgumentIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/CapturedFunctionArgumentIterator.java @@ -21,10 +21,11 @@ import org.rumbledb.context.DynamicContext; import org.rumbledb.context.RuntimeStaticContext; import org.rumbledb.exceptions.IteratorFlowException; +import org.rumbledb.exceptions.OurBadException; import org.rumbledb.items.PartiallyAppliedFunctionItem.ArgumentBinding; import org.rumbledb.items.PartiallyAppliedFunctionItem.DataFrameBinding; import org.rumbledb.items.PartiallyAppliedFunctionItem.LocalBinding; -import org.rumbledb.items.PartiallyAppliedFunctionItem.RDDBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.RddBinding; import org.rumbledb.items.structured.JSoundDataFrame; import org.rumbledb.runtime.HybridRuntimeIterator; @@ -37,18 +38,43 @@ public class CapturedFunctionArgumentIterator extends HybridRuntimeIterator { private static final long serialVersionUID = 1L; - private final ArgumentBinding binding; + private final List localValue; + private final JavaRDD rddValue; + private final JSoundDataFrame dataFrameValue; private int index; - public CapturedFunctionArgumentIterator(ArgumentBinding binding, RuntimeStaticContext staticContext) { + private CapturedFunctionArgumentIterator( + List localValue, + JavaRDD rddValue, + JSoundDataFrame dataFrameValue, + RuntimeStaticContext staticContext + ) { super(null, staticContext); - this.binding = binding; + this.localValue = localValue; + this.rddValue = rddValue; + this.dataFrameValue = dataFrameValue; + } + + public static CapturedFunctionArgumentIterator create( + ArgumentBinding binding, + RuntimeStaticContext staticContext + ) { + if (binding instanceof LocalBinding local) { + return new CapturedFunctionArgumentIterator(local.value(), null, null, staticContext); + } + if (binding instanceof RddBinding rdd) { + return new CapturedFunctionArgumentIterator(null, rdd.value(), null, staticContext); + } + if (binding instanceof DataFrameBinding dataFrame) { + return new CapturedFunctionArgumentIterator(null, null, dataFrame.value(), staticContext); + } + throw new OurBadException("A placeholder cannot be converted to a captured argument."); } @Override protected void openLocal() { this.index = 0; - this.hasNext = !getLocalValue().isEmpty(); + this.hasNext = !this.localValue.isEmpty(); } @Override @@ -61,15 +87,14 @@ protected Item nextLocal() { if (!this.hasNext) { throw new IteratorFlowException(FLOW_EXCEPTION_MESSAGE, getMetadata()); } - Item result = getLocalValue().get(this.index++); - this.hasNext = this.index < getLocalValue().size(); + Item result = this.localValue.get(this.index++); + this.hasNext = this.index < this.localValue.size(); return result; } @Override protected void resetLocal() { - this.index = 0; - this.hasNext = !getLocalValue().isEmpty(); + openLocal(); } @Override @@ -79,20 +104,19 @@ protected void closeLocal() { @Override public JavaRDD getRDDAux(DynamicContext context) { - return ((RDDBinding) this.binding).value(); + if (this.rddValue != null) { + return this.rddValue; + } + return dataFrameToRDDOfItems(this.dataFrameValue, getMetadata()); } @Override protected boolean implementsDataFrames() { - return true; + return this.dataFrameValue != null; } @Override public JSoundDataFrame getDataFrame(DynamicContext context) { - return ((DataFrameBinding) this.binding).value(); - } - - private List getLocalValue() { - return ((LocalBinding) this.binding).value(); + return this.dataFrameValue; } } diff --git a/src/main/java/org/rumbledb/runtime/functions/DynamicFunctionCallIterator.java b/src/main/java/org/rumbledb/runtime/functions/DynamicFunctionCallIterator.java index e9cadb656c..52dedac80c 100644 --- a/src/main/java/org/rumbledb/runtime/functions/DynamicFunctionCallIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/DynamicFunctionCallIterator.java @@ -223,9 +223,24 @@ private void setFunctionItemAndIteratorWithCurrentContext(DynamicContext context getMetadata() ); } - ExecutionMode calleeExecutionMode = getCalleeExecutionModeForFunctionItemCall(); + if (this.isPartialApplication) { + this.functionCallIterator = NamedFunctions.buildFunctionItemCallIterator( + this.functionItem, + this.staticContext, + ExecutionMode.LOCAL, + this.functionArguments, + false + ); + return; + } + + NamedFunctions.ResolvedFunctionCall resolvedCall = NamedFunctions.resolveFunctionItemCall( + this.functionItem, + this.functionArguments, + this.staticContext + ); if ( - calleeExecutionMode.equals(ExecutionMode.LOCAL) + resolvedCall.executionMode().equals(ExecutionMode.LOCAL) && this.getHighestExecutionMode().equals(ExecutionMode.DATAFRAME) ) { throw new OurBadException( @@ -234,22 +249,8 @@ private void setFunctionItemAndIteratorWithCurrentContext(DynamicContext context getMetadata() ); } - this.functionCallIterator = NamedFunctions.buildFunctionItemCallIterator( - this.functionItem, - this.staticContext, - this.isPartialApplication ? ExecutionMode.LOCAL : calleeExecutionMode, - this.functionArguments, - false - ); - } - - private ExecutionMode getCalleeExecutionModeForFunctionItemCall() { - if (this.isPartialApplication) { - return ExecutionMode.LOCAL; - } - return NamedFunctions.resolveFunctionItemExecutionMode( - this.functionItem, - this.functionArguments, + this.functionCallIterator = NamedFunctions.buildResolvedFunctionItemCallIterator( + resolvedCall, this.staticContext ); } diff --git a/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java b/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java index 47b590843a..899db041b4 100644 --- a/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/PartialFunctionCallIterator.java @@ -26,7 +26,7 @@ import org.rumbledb.items.PartiallyAppliedFunctionItem.DataFrameBinding; import org.rumbledb.items.PartiallyAppliedFunctionItem.LocalBinding; import org.rumbledb.items.PartiallyAppliedFunctionItem.PlaceholderBinding; -import org.rumbledb.items.PartiallyAppliedFunctionItem.RDDBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.RddBinding; import org.rumbledb.runtime.AtMostOneItemLocalRuntimeIterator; import org.rumbledb.runtime.RuntimeIterator; import org.rumbledb.types.FunctionSignature; @@ -98,7 +98,7 @@ public Item materializeFirstItemOrNull(DynamicContext context) { new DataFrameBinding(parameterType, argumentIterator.getDataFrame(context)) ); } else if (argumentIterator.isRDDOrDataFrame()) { - argumentBindings.add(new RDDBinding(parameterType, argumentIterator.getRDD(context))); + argumentBindings.add(new RddBinding(parameterType, argumentIterator.getRDD(context))); } else { argumentBindings.add(new LocalBinding(parameterType, argumentIterator.materialize(context))); } diff --git a/src/test/java/org/rumbledb/items/PartiallyAppliedFunctionItemTest.java b/src/test/java/org/rumbledb/items/PartiallyAppliedFunctionItemTest.java new file mode 100644 index 0000000000..6e8ce7d644 --- /dev/null +++ b/src/test/java/org/rumbledb/items/PartiallyAppliedFunctionItemTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.rumbledb.items; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import org.junit.Assert; +import org.junit.Test; +import org.rumbledb.api.Item; +import org.rumbledb.config.RumbleRuntimeConfiguration; +import org.rumbledb.context.BuiltinFunction; +import org.rumbledb.context.BuiltinFunctionCatalogue; +import org.rumbledb.context.DynamicContext; +import org.rumbledb.context.FunctionIdentifier; +import org.rumbledb.context.Name; +import org.rumbledb.exceptions.ExceptionMetadata; +import org.rumbledb.items.PartiallyAppliedFunctionItem.ArgumentBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.LocalBinding; +import org.rumbledb.items.PartiallyAppliedFunctionItem.PlaceholderBinding; +import org.rumbledb.types.FunctionSignature; +import org.rumbledb.types.SequenceType; + +import java.util.ArrayList; +import java.util.List; + +public class PartiallyAppliedFunctionItemTest { + + @Test + public void localBindingsAndBindingListAreImmutable() { + List capturedValue = new ArrayList<>(); + capturedValue.add(ItemFactory.getInstance().createIntItem(1)); + LocalBinding binding = new LocalBinding(parameterType(), capturedValue); + capturedValue.add(ItemFactory.getInstance().createIntItem(2)); + Assert.assertEquals(1, binding.value().size()); + + List bindings = new ArrayList<>(); + bindings.add(binding); + PartiallyAppliedFunctionItem partial = createPartial(builtinTarget(), bindings, 0); + bindings.clear(); + Assert.assertEquals(1, partial.getArgumentBindings().size()); + } + + @Test + public void deepCopyPreservesNestedPartialFunctionType() { + PartiallyAppliedFunctionItem inner = createPartial( + builtinTarget(), + List.of(new PlaceholderBinding(parameterType())), + 1 + ); + PartiallyAppliedFunctionItem outer = createPartial( + inner, + List.of(new PlaceholderBinding(parameterType())), + 1 + ); + + FunctionItem copy = outer.deepCopy(); + Assert.assertTrue(copy instanceof PartiallyAppliedFunctionItem); + Item copiedTarget = ((PartiallyAppliedFunctionItem) copy).getTargetFunction(); + Assert.assertTrue(copiedTarget instanceof PartiallyAppliedFunctionItem); + } + + @Test + public void kryoRoundTripPreservesLocalBindings() { + PartiallyAppliedFunctionItem partial = createPartial( + builtinTarget(), + List.of( + new LocalBinding( + parameterType(), + List.of(ItemFactory.getInstance().createIntItem(7)) + ) + ), + 0 + ); + Kryo kryo = new Kryo(); + kryo.setRegistrationRequired(false); + Output output = new Output(4096, -1); + kryo.writeClassAndObject(output, partial); + Input input = new Input(output.toBytes()); + + Object deserialized = kryo.readClassAndObject(input); + Assert.assertTrue(deserialized instanceof PartiallyAppliedFunctionItem); + ArgumentBinding binding = ((PartiallyAppliedFunctionItem) deserialized).getArgumentBindings().get(0); + Assert.assertTrue(binding instanceof LocalBinding); + Assert.assertEquals(7, ((LocalBinding) binding).value().get(0).getIntValue()); + } + + private static PartiallyAppliedFunctionItem createPartial( + Item target, + List bindings, + int arity + ) { + List parameterNames = arity == 0 + ? List.of() + : List.of(Name.createVariableInNoNamespace("$p0")); + List parameterTypes = arity == 0 ? List.of() : List.of(parameterType()); + return new PartiallyAppliedFunctionItem( + new FunctionIdentifier(null, arity), + parameterNames, + new FunctionSignature(parameterTypes, target.getSignature().getReturnType(), false), + target.getModuleDynamicContext(), + target, + bindings + ); + } + + private static FunctionItem builtinTarget() { + RumbleRuntimeConfiguration configuration = new RumbleRuntimeConfiguration(new String[] {}); + DynamicContext context = new DynamicContext(configuration); + FunctionIdentifier identifier = new FunctionIdentifier( + Name.createVariableInDefaultFunctionNamespace("max"), + 1 + ); + BuiltinFunction builtin = BuiltinFunctionCatalogue.getBuiltinFunction(identifier); + return FunctionItemFactory.createBuiltinNamedReference( + identifier, + context, + configuration, + ExceptionMetadata.EMPTY_METADATA, + builtin + ); + } + + private static SequenceType parameterType() { + FunctionIdentifier identifier = new FunctionIdentifier( + Name.createVariableInDefaultFunctionNamespace("max"), + 1 + ); + return BuiltinFunctionCatalogue.getBuiltinFunction(identifier).getSignature().getParameterTypes().get(0); + } +} From b28b29ebd73716b7ccdc62630e34f1fc185618d5 Mon Sep 17 00:00:00 2001 From: Jimmy Cai Date: Thu, 18 Jun 2026 12:42:23 +0200 Subject: [PATCH 6/6] Update implementsDataFrames method to always return true for data frame support --- .../runtime/functions/CapturedFunctionArgumentIterator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/rumbledb/runtime/functions/CapturedFunctionArgumentIterator.java b/src/main/java/org/rumbledb/runtime/functions/CapturedFunctionArgumentIterator.java index 1e474cf442..447748f4c2 100644 --- a/src/main/java/org/rumbledb/runtime/functions/CapturedFunctionArgumentIterator.java +++ b/src/main/java/org/rumbledb/runtime/functions/CapturedFunctionArgumentIterator.java @@ -112,7 +112,7 @@ public JavaRDD getRDDAux(DynamicContext context) { @Override protected boolean implementsDataFrames() { - return this.dataFrameValue != null; + return true; } @Override