diff --git a/src/main/java/evaluation/CaseCollector.java b/src/main/java/evaluation/CaseCollector.java index cdb2530..6c30698 100644 --- a/src/main/java/evaluation/CaseCollector.java +++ b/src/main/java/evaluation/CaseCollector.java @@ -30,7 +30,10 @@ public class CaseCollector { "moduleImport", "olson-timezone", "serialization", - "staticTyping"); + "staticTyping", + "schemaImport", + "schemaValidation", + "schema-location-hint"); private static final Set SUPPORTED_SPECS = Set.of("XQ10+", "XQ30+", "XQ31", "XQ31+"); private Path testsRepositoryDirectoryPath; diff --git a/src/main/java/evaluation/Environment.java b/src/main/java/evaluation/Environment.java index 68249cb..0271dfa 100644 --- a/src/main/java/evaluation/Environment.java +++ b/src/main/java/evaluation/Environment.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -20,9 +21,10 @@ public class Environment { private final Map runtimeResourceLookup = new HashMap<>(); private final Map paramLookup = new HashMap<>(); private final Map externalParamLookup = new HashMap<>(); - private final Map roleLookup = new HashMap<>(); + private final Map roleLookup = new LinkedHashMap<>(); private final Map importResourceLookup = new HashMap<>(); private final Map> moduleLocationHints = new HashMap<>(); + private final Map> schemaLocationHints = new LinkedHashMap<>(); private final Map namespaceLookup = new HashMap<>(); @@ -52,6 +54,9 @@ private Environment(Environment environment) { environment.moduleLocationHints.forEach((namespace, locations) -> { this.moduleLocationHints.put(namespace, new ArrayList<>(locations)); }); + environment.schemaLocationHints.forEach((namespace, locations) -> { + this.schemaLocationHints.put(namespace, new ArrayList<>(locations)); + }); this.namespaceLookup.putAll(environment.namespaceLookup); this.decimalFormatDeclarations.addAll(environment.decimalFormatDeclarations); this.staticBaseUriUndefined = environment.staticBaseUriUndefined; @@ -192,11 +197,12 @@ private void initSources(XdmNode environmentNode, Path envPath) { String file = envPath.resolve(source.attribute("file")).toUri().toString(); String uri = source.attribute("uri"); String role = source.attribute("role"); + String validation = source.attribute("validation"); if (uri != null && !file.equals(uri)) { runtimeResourceLookup.put(uri, file); } if (role != null) { - roleLookup.put(role, file); + roleLookup.put(role, new SourceBinding(file, validation)); } } } @@ -204,9 +210,14 @@ private void initSources(XdmNode environmentNode, Path envPath) { private void addImportResources(ImportResources imports) { // The compiler currently supports one physical location per logical URI. imports.logicalToPhysical().forEach(importResourceLookup::putIfAbsent); - imports.moduleLocationHints().forEach((namespace, locations) -> { - List knownLocations = - this.moduleLocationHints.computeIfAbsent(namespace, ignored -> new ArrayList<>()); + mergeLocationHints(this.moduleLocationHints, imports.moduleLocationHints()); + mergeLocationHints(this.schemaLocationHints, imports.schemaLocationHints()); + } + + private void mergeLocationHints( + Map> target, Map> additionalLocationHints) { + additionalLocationHints.forEach((namespace, locations) -> { + List knownLocations = target.computeIfAbsent(namespace, ignored -> new ArrayList<>()); for (String location : locations) { if (!knownLocations.contains(location)) { knownLocations.add(location); @@ -218,6 +229,7 @@ private void addImportResources(ImportResources imports) { private static ImportResources collectImportResources(XdmNode node, Path basePath) { Map imports = new HashMap<>(); Map> moduleLocationHints = new HashMap<>(); + Map> schemaLocationHints = new LinkedHashMap<>(); for (String elementName : List.of("module", "schema")) { for (XdmNode resource : node.select(Steps.descendant(elementName)).asList()) { String uri = resource.attribute("uri"); @@ -227,13 +239,18 @@ private static ImportResources collectImportResources(XdmNode node, Path basePat .computeIfAbsent(uri, ignored -> new ArrayList<>()) .add(basePath.resolve(file).toUri().toString()); } + if ("schema".equals(elementName) && file != null) { + schemaLocationHints + .computeIfAbsent(uri == null ? "" : uri, ignored -> new ArrayList<>()) + .add(basePath.resolve(file).toUri().toString()); + } URI logicalUri = parseLogicalUri(uri); if (logicalUri != null && file != null) { imports.putIfAbsent(logicalUri, basePath.resolve(file).toUri()); } } } - return new ImportResources(imports, moduleLocationHints); + return new ImportResources(imports, moduleLocationHints, schemaLocationHints); } private static URI parseLogicalUri(String uri) { @@ -261,27 +278,38 @@ public ResourceResolver getResourceResolver() { */ public String applyToQuery(String query) { return EnvironmentQueryRewriter.rewrite( - query, createDeclarations(), externalParamLookup, runtimeResourceLookup, moduleLocationHints); + query, + this.namespaceLookup, + createDeclarations(), + this.externalParamLookup, + this.runtimeResourceLookup, + this.moduleLocationHints, + this.schemaLocationHints, + hasSchemaValidatedSource()); + } + + private boolean hasSchemaValidatedSource() { + return this.roleLookup.values().stream().anyMatch(SourceBinding::requiresSchemaValidation); } private String createDeclarations() { StringBuilder declarations = new StringBuilder(); - declarations.append(createDecimalFormatAndNamespaceProlog()); - for (Map.Entry r : roleLookup.entrySet()) { + declarations.append(createDecimalFormatProlog()); + for (Map.Entry r : roleLookup.entrySet()) { String role = r.getKey(); - String file = r.getValue(); + SourceBinding source = r.getValue(); if (role.equals(".")) { declarations - .append("declare context item := doc(\"") - .append(file) - .append("\"); "); + .append("declare context item := ") + .append(source.documentExpression()) + .append("; "); } else { declarations .append("declare variable ") .append(role) - .append(" := doc(\"") - .append(file) - .append("\"); "); + .append(" := ") + .append(source.documentExpression()) + .append("; "); } } for (Map.Entry param : paramLookup.entrySet()) { @@ -297,30 +325,36 @@ private String createDeclarations() { return declarations.toString(); } - public String createDecimalFormatAndNamespaceProlog() { - if (namespaceLookup.isEmpty() && decimalFormatDeclarations.isEmpty()) { - return ""; - } - + private String createDecimalFormatProlog() { StringBuilder prolog = new StringBuilder(); - - for (Map.Entry namespace : namespaceLookup.entrySet()) { - prolog.append("declare namespace ") - .append(namespace.getKey()) - .append(" = ") - .append(toXQueryStringLiteral(namespace.getValue())) - .append(";\n"); - } - - for (String decimalFormatDeclaration : decimalFormatDeclarations) { + for (String decimalFormatDeclaration : this.decimalFormatDeclarations) { prolog.append(decimalFormatDeclaration).append("\n"); } return prolog.toString(); } - private record ImportResources(Map logicalToPhysical, Map> moduleLocationHints) { + private record ImportResources( + Map logicalToPhysical, + Map> moduleLocationHints, + Map> schemaLocationHints) { private boolean isEmpty() { - return this.logicalToPhysical.isEmpty() && this.moduleLocationHints.isEmpty(); + return this.logicalToPhysical.isEmpty() + && this.moduleLocationHints.isEmpty() + && this.schemaLocationHints.isEmpty(); + } + } + + private record SourceBinding(String file, String validation) { + private boolean requiresSchemaValidation() { + return "strict".equals(this.validation) || "lax".equals(this.validation); + } + + private String documentExpression() { + String document = "doc(\"" + file + "\")"; + if (requiresSchemaValidation()) { + return "validate " + validation + " { " + document + " }"; + } + return document; } } } diff --git a/src/main/java/evaluation/conversion/EnvironmentQueryRewriter.java b/src/main/java/evaluation/conversion/EnvironmentQueryRewriter.java index 0288988..1d25ce7 100644 --- a/src/main/java/evaluation/conversion/EnvironmentQueryRewriter.java +++ b/src/main/java/evaluation/conversion/EnvironmentQueryRewriter.java @@ -1,6 +1,7 @@ package evaluation.conversion; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -17,10 +18,13 @@ private EnvironmentQueryRewriter() {} public static String rewrite( String query, + Map environmentNamespaces, String declarations, Map externalParams, Map resources, - Map> moduleLocationHints) { + Map> moduleLocationHints, + Map> schemaLocationHints, + boolean injectEnvironmentSchemaImports) { XQueryParser.ModuleAndThisIsItContext module = XQueryParsing.parseValidModule(query); if (module == null) { return query; @@ -29,7 +33,13 @@ public static String rewrite( ConversionContext context = new ConversionContext(query, module); new ExternalParamVisitor(context, externalParams).visit(module); new ModuleImportVisitor(context, moduleLocationHints).visit(module); - insertDeclarations(context, module, declarations); + new SchemaImportVisitor(context, schemaLocationHints).visit(module); + insertDeclarations( + context, + module, + environmentSchemaImports(injectEnvironmentSchemaImports ? schemaLocationHints : Map.of(), module) + + environmentNamespaceDeclarations(environmentNamespaces, module) + + declarations); String queryWithDeclarations = context.result(); // Parse the intermediate query so resource URIs inside injected parameter values are rewritten too. @@ -44,6 +54,122 @@ public static String rewrite( return resourceContext.result(); } + private static String environmentSchemaImports( + Map> schemaLocationHints, XQueryParser.ModuleAndThisIsItContext module) { + if (schemaLocationHints.isEmpty() || module.module().main == null) { + return ""; + } + + LinkedHashSet importedNamespaces = new LinkedHashSet<>(); + for (XQueryParser.SchemaImportContext schemaImport : + module.module().main.prolog().schemaImport()) { + String namespace = XQueryStringLiteral.parse(schemaImport.nsURI.getText()); + if (namespace != null) { + importedNamespaces.add(namespace); + } + } + + StringBuilder declarations = new StringBuilder(); + for (Map.Entry> schema : schemaLocationHints.entrySet()) { + if (importedNamespaces.contains(schema.getKey())) { + continue; + } + List locations = new ArrayList<>(); + for (String location : schema.getValue()) { + locations.add(XQueryStringLiteral.serialize(location, '"')); + } + if (locations.isEmpty()) { + continue; + } + declarations + .append("import schema ") + .append(XQueryStringLiteral.serialize(schema.getKey(), '"')) + .append(" at ") + .append(String.join(", ", locations)) + .append(";\n"); + } + return declarations.toString(); + } + + private static String environmentNamespaceDeclarations( + Map environmentNamespaces, XQueryParser.ModuleAndThisIsItContext module) { + if (environmentNamespaces.isEmpty()) { + return ""; + } + + Map queryNamespaces = queryNamespaces(module); + StringBuilder declarations = new StringBuilder(); + for (Map.Entry environmentNamespace : environmentNamespaces.entrySet()) { + String prefix = environmentNamespace.getKey(); + String uri = environmentNamespace.getValue(); + String queryUri = queryNamespaces.get(prefix); + if (queryUri == null) { + appendNamespaceDeclaration(declarations, prefix, uri); + } else if (!queryUri.equals(uri)) { + throw new IllegalArgumentException("QT3 environment binds prefix " + + prefix + + " to " + + uri + + ", but the query binds it to " + + queryUri + + "."); + } + } + return declarations.toString(); + } + + private static void appendNamespaceDeclaration(StringBuilder declarations, String prefix, String uri) { + if (prefix.isEmpty()) { + declarations.append("declare default element namespace "); + } else { + declarations.append("declare namespace ").append(prefix).append(" = "); + } + declarations.append(XQueryStringLiteral.serialize(uri, '"')).append(";\n"); + } + + private static Map queryNamespaces(XQueryParser.ModuleAndThisIsItContext module) { + if (module.module().main == null) { + return Map.of(); + } + + Map result = new HashMap<>(); + XQueryParser.PrologContext prolog = module.module().main.prolog(); + for (XQueryParser.NamespaceDeclContext namespaceDeclaration : prolog.namespaceDecl()) { + addNamespace( + result, + namespaceDeclaration.ncName().getText(), + namespaceDeclaration.uriLiteral().getText()); + } + for (XQueryParser.DefaultNamespaceDeclContext namespaceDeclaration : prolog.defaultNamespaceDecl()) { + if (namespaceDeclaration.type.getText().equals("element")) { + addNamespace(result, "", namespaceDeclaration.uri.getText()); + } + } + for (XQueryParser.SchemaImportContext schemaImport : prolog.schemaImport()) { + if (schemaImport.schemaPrefix() == null) { + continue; + } + if (schemaImport.schemaPrefix().ncName() == null) { + addNamespace(result, "", schemaImport.nsURI.getText()); + } else { + addNamespace(result, schemaImport.schemaPrefix().ncName().getText(), schemaImport.nsURI.getText()); + } + } + for (XQueryParser.ModuleImportContext moduleImport : prolog.moduleImport()) { + if (moduleImport.ncName() != null) { + addNamespace(result, moduleImport.ncName().getText(), moduleImport.targetNamespace.getText()); + } + } + return result; + } + + private static void addNamespace(Map namespaces, String prefix, String uriLiteral) { + String uri = XQueryStringLiteral.parse(uriLiteral); + if (uri != null) { + namespaces.put(prefix, uri); + } + } + private static void insertDeclarations( ConversionContext context, XQueryParser.ModuleAndThisIsItContext module, String declarations) { if (declarations.isEmpty() || module.module().main == null) { @@ -155,9 +281,52 @@ public Void visitModuleImport(XQueryParser.ModuleImportContext moduleImport) { } else { replacement.append(source); } - replacement.append(" at ").append(String.join(", ", serializedLocations)); + replacement + .append(" at ") + .append(String.join(", ", serializedLocations)) + .append(";"); this.context.replace(moduleImport, replacement.toString()); return null; } } + + private static final class SchemaImportVisitor extends XQueryParserBaseVisitor { + + private final ConversionContext context; + private final Map> schemaLocationHints; + + private SchemaImportVisitor(ConversionContext context, Map> schemaLocationHints) { + this.context = context; + this.schemaLocationHints = schemaLocationHints; + } + + @Override + public Void visitSchemaImport(XQueryParser.SchemaImportContext schemaImport) { + String source = this.context.text(schemaImport.nsURI); + String namespace = XQueryStringLiteral.parse(source); + List environmentLocations = this.schemaLocationHints.get(namespace); + if (environmentLocations == null || environmentLocations.isEmpty()) { + return null; + } + + List serializedLocations = new ArrayList<>(); + for (String location : environmentLocations) { + serializedLocations.add(XQueryStringLiteral.serialize(location, '"')); + } + + StringBuilder replacement = new StringBuilder("import schema "); + if (schemaImport.schemaPrefix() != null) { + replacement + .append(this.context.text(schemaImport.schemaPrefix())) + .append(" "); + } + replacement + .append(source) + .append(" at ") + .append(String.join(", ", serializedLocations)) + .append(";"); + this.context.replace(schemaImport, replacement.toString()); + return null; + } + } } diff --git a/src/test/java/evaluation/EnvironmentTest.java b/src/test/java/evaluation/EnvironmentTest.java index e32ed83..b34073b 100644 --- a/src/test/java/evaluation/EnvironmentTest.java +++ b/src/test/java/evaluation/EnvironmentTest.java @@ -88,6 +88,55 @@ public void acceptsMalformedLogicalUrisUsedByNegativeTests() throws Exception { ""); } + @Test + public void preservesSourceValidationModesInInjectedDeclarations() throws Exception { + Path strict = Files.writeString(this.directory.resolve("strict.xml"), ""); + Path lax = Files.writeString(this.directory.resolve("lax.xml"), ""); + Environment environment = new Environment( + element( + "" + + "" + + "" + + "", + "environment"), + this.directory); + + assertEquals( + "declare context item := validate strict { doc(\"" + + strict.toUri() + + "\") }; declare variable $other := validate lax { doc(\"" + + lax.toUri() + + "\") }; 1", + environment.applyToQuery("1")); + } + + @Test + public void importsEnvironmentSchemasWithoutBindingTheirNamespaces() throws Exception { + Path schema = Files.writeString(this.directory.resolve("schema.xsd"), ""); + Path noNamespaceSchema = Files.writeString(this.directory.resolve("no-namespace-schema.xsd"), ""); + Environment environment = new Environment( + element( + "" + + "" + + "" + + "" + + "", + "environment"), + this.directory); + + assertEquals( + "import schema \"urn:schema\" at \"" + + schema.toUri() + + "\";\n" + + "import schema \"\" at \"" + + noNamespaceSchema.toUri() + + "\";\n" + + "declare context item := validate strict { doc(\"" + + this.directory.resolve("document.xml").toUri() + + "\") }; 1", + environment.applyToQuery("1")); + } + @Test public void doesNotMutateASharedEnvironment() throws Exception { Path fallback = Files.writeString(this.directory.resolve("fallback.xq"), "fallback"); diff --git a/src/test/java/evaluation/conversion/EnvironmentQueryRewriterTest.java b/src/test/java/evaluation/conversion/EnvironmentQueryRewriterTest.java index 649126f..d18e5df 100644 --- a/src/test/java/evaluation/conversion/EnvironmentQueryRewriterTest.java +++ b/src/test/java/evaluation/conversion/EnvironmentQueryRewriterTest.java @@ -5,6 +5,7 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class EnvironmentQueryRewriterTest { @@ -15,7 +16,14 @@ public void replacesOnlyCompleteStringLiteralValues() { assertEquals( "\"file:///resource.xml\", \"prefix urn:test\", (: \"urn:test\" :) urn:test", EnvironmentQueryRewriter.rewrite( - query, "", Map.of(), Map.of("urn:test", "file:///resource.xml"), Map.of())); + query, + Map.of(), + "", + Map.of(), + Map.of("urn:test", "file:///resource.xml"), + Map.of(), + Map.of(), + false)); } @Test @@ -30,14 +38,18 @@ public void bindsOnlyTheMatchingExternalVariableDeclaration() { + "declare variable $value external := (42);\n" + "declare variable $value-more external;\n" + "\"declare variable $value external;\"", - EnvironmentQueryRewriter.rewrite(query, "", Map.of("value", "42"), Map.of(), Map.of())); + EnvironmentQueryRewriter.rewrite( + query, Map.of(), "", Map.of("value", "42"), Map.of(), Map.of(), Map.of(), false)); } @Test public void preservesAnExistingExternalDefault() { String query = "declare variable $value external := 1; $value"; - assertEquals(query, EnvironmentQueryRewriter.rewrite(query, "", Map.of("value", "42"), Map.of(), Map.of())); + assertEquals( + query, + EnvironmentQueryRewriter.rewrite( + query, Map.of(), "", Map.of("value", "42"), Map.of(), Map.of(), Map.of(), false)); } @Test @@ -56,7 +68,14 @@ public void insertsDeclarationsBetweenLeadingAndAnnotatedPrologDeclarations() { + "declare variable $existing external;\n" + "$existing", EnvironmentQueryRewriter.rewrite( - query, "declare variable $environment := 1;", Map.of(), Map.of(), Map.of())); + query, + Map.of(), + "declare variable $environment := 1;", + Map.of(), + Map.of(), + Map.of(), + Map.of(), + false)); } @Test @@ -68,10 +87,13 @@ public void replacesResourcesInsideInjectedValues() { + "declare variable $external external := (\"file:///resource.xml\"); $external", EnvironmentQueryRewriter.rewrite( query, + Map.of(), "declare variable $environment := \"urn:test\";", Map.of("external", "\"urn:test\""), Map.of("urn:test", "file:///resource.xml"), - Map.of())); + Map.of(), + Map.of(), + false)); } @Test @@ -82,10 +104,13 @@ public void leavesInvalidXQueryUntouched() { query, EnvironmentQueryRewriter.rewrite( query, + Map.of(), "declare variable $environment := 1;", Map.of(), Map.of("urn:test", "file:///resource.xml"), - Map.of())); + Map.of(), + Map.of(), + false)); } @Test @@ -96,9 +121,91 @@ public void injectsEnvironmentModuleLocationsIntoModuleImports() { "import module namespace m=\"urn:module\" at \"file:///module1.xq\", \"file:///module2.xq\"; 1", EnvironmentQueryRewriter.rewrite( query, + Map.of(), + "", + Map.of(), + Map.of(), + Map.of("urn:module", java.util.List.of("file:///module1.xq", "file:///module2.xq")), + Map.of(), + false)); + } + + @Test + public void doesNotDuplicateNamespaceBoundBySchemaImport() { + String query = "import schema namespace atomic=\"urn:atomic\"; \"ABC\""; + + assertEquals( + "import schema namespace atomic=\"urn:atomic\"; " + + "declare context item := doc(\"file:///atomic.xml\"); \"ABC\"", + EnvironmentQueryRewriter.rewrite( + query, + Map.of("atomic", "urn:atomic"), + "declare context item := doc(\"file:///atomic.xml\"); ", + Map.of(), + Map.of(), + Map.of(), + Map.of(), + false)); + } + + @Test + public void replacesExistingSchemaImportLocationsWithoutInjectingAnotherImport() { + String query = "import schema namespace s = \"urn:schema\"; 1"; + + assertEquals( + "import schema namespace s = \"urn:schema\" at \"file:///schema.xsd\"; 1", + EnvironmentQueryRewriter.rewrite( + query, + Map.of(), + "", + Map.of(), + Map.of(), + Map.of(), + Map.of("urn:schema", java.util.List.of("file:///schema.xsd")), + false)); + } + + @Test + public void doesNotInjectEnvironmentSchemaWithoutAValidatedSource() { + assertEquals( + "1", + EnvironmentQueryRewriter.rewrite( + "1", + Map.of(), "", Map.of(), Map.of(), - Map.of("urn:module", java.util.List.of("file:///module1.xq", "file:///module2.xq")))); + Map.of(), + Map.of("urn:schema", java.util.List.of("file:///schema.xsd")), + false)); + } + + @Test + public void doesNotDuplicateNamespaceBoundByModuleImport() { + String query = "import module namespace module=\"urn:module\"; \"ABC\""; + + assertEquals( + query, + EnvironmentQueryRewriter.rewrite( + query, Map.of("module", "urn:module"), "", Map.of(), Map.of(), Map.of(), Map.of(), false)); + } + + @Test + public void rejectsConflictingEnvironmentAndQueryNamespaceBindings() { + IllegalArgumentException exception = assertThrows( + IllegalArgumentException.class, + () -> EnvironmentQueryRewriter.rewrite( + "import schema namespace atomic=\"urn:query\"; \"ABC\"", + Map.of("atomic", "urn:environment"), + "", + Map.of(), + Map.of(), + Map.of(), + Map.of(), + false)); + + assertEquals( + "QT3 environment binds prefix atomic to urn:environment, but the query binds it to urn:query.", + exception.getMessage()); } } diff --git a/src/test/java/iq/base/TestBase.java b/src/test/java/iq/base/TestBase.java index cd480f0..df4f1c6 100644 --- a/src/test/java/iq/base/TestBase.java +++ b/src/test/java/iq/base/TestBase.java @@ -11,6 +11,8 @@ import java.util.stream.Collectors; import org.xmlunit.builder.DiffBuilder; +import org.xmlunit.diff.ComparisonResult; +import org.xmlunit.diff.ComparisonType; import org.xmlunit.diff.Diff; import evaluation.*; @@ -228,10 +230,13 @@ private void checkAssertion(XdmNode assertion, AssertionContext context, Path te + ""; String expectedXml = "" + assertionText(assertion, testSetDirectory) + ""; - Diff diff = DiffBuilder.compare(expectedXml) - .withTest(actualXml) - .ignoreWhitespace() - .build(); + DiffBuilder diffBuilder = + DiffBuilder.compare(expectedXml).withTest(actualXml).ignoreWhitespace(); + if ("true".equals(assertion.attribute("ignore-prefixes"))) { + diffBuilder.withDifferenceEvaluator((comparison, outcome) -> + comparison.getType() == ComparisonType.NAMESPACE_PREFIX ? ComparisonResult.EQUAL : outcome); + } + Diff diff = diffBuilder.build(); assertFalse(diff.hasDifferences(), "Expected vs actual XML are different:\n" + diff.toString()); break;