diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java b/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java
new file mode 100644
index 0000000000..3d3aaa1fd0
--- /dev/null
+++ b/opennlp-api/src/main/java/opennlp/tools/document/Annotation.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.document;
+
+import opennlp.tools.util.Span;
+
+/**
+ * One annotation of a {@link Document}: a typed value anchored to a {@link Span} of the
+ * document's original text, or a span-less value under a
+ * {@link LayerKey.Scope#DOCUMENT document-scoped} key.
+ *
+ *
The span always refers to the text the document was created with, never to a
+ * normalized or otherwise derived form, so any annotation can be highlighted in what the
+ * caller supplied. Whether a span is present is decided by the layer key's scope, not
+ * per annotation: the container rejects a span-less annotation under a positional key
+ * and a spanned annotation under a document-scoped key. Annotations that need to
+ * reference other annotations, for example a dependency arc naming its head token, do
+ * so by the index of the target annotation within its layer, never by object
+ * identity.
+ *
+ * @param span The location of the annotation in the original text, or {@code null} for
+ * a value under a document-scoped key.
+ * @param value The annotation value. Must not be {@code null}.
+ * @param The type of the annotation value.
+ *
+ * @since 3.0.0
+ */
+public record Annotation(Span span, T value) {
+
+ /**
+ * Validates the annotation.
+ *
+ * @throws IllegalArgumentException Thrown if {@code value} is {@code null}.
+ */
+ public Annotation {
+ if (value == null) {
+ throw new IllegalArgumentException("value must not be null");
+ }
+ }
+
+ /**
+ * Creates a span-less annotation for a {@link LayerKey.Scope#DOCUMENT
+ * document-scoped} layer.
+ *
+ * @param value The annotation value. Must not be {@code null}.
+ * @param The type of the annotation value.
+ * @return An {@link Annotation} without a span. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code value} is {@code null}.
+ */
+ public static Annotation of(T value) {
+ return new Annotation<>(null, value);
+ }
+}
diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Document.java b/opennlp-api/src/main/java/opennlp/tools/document/Document.java
new file mode 100644
index 0000000000..4d22d3a18b
--- /dev/null
+++ b/opennlp-api/src/main/java/opennlp/tools/document/Document.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.document;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * An offset-anchored annotation container: the original text of one document plus any
+ * number of typed annotation layers over it.
+ *
+ *
A layer is a list of {@link Annotation annotations} identified by a
+ * {@link LayerKey}. The container itself knows nothing about specific layers; every
+ * analysis capability contributes its results as one more layer without any change to
+ * this interface, which is what keeps new capabilities additive. All spans refer to
+ * {@link #text()} as supplied, never to a derived form. A
+ * {@link LayerKey.Scope#DOCUMENT document-scoped} layer carries whole-document values
+ * without spans, for example a language id.
+ *
+ *
A document is never modified in place: {@link #with(LayerKey, List)} leaves its
+ * receiver untouched and returns a new document. Thread safety is implementation
+ * specific.
+ *
+ *
Three invariants make index-based references sound. A layer preserves its
+ * insertion order, and the container never sorts or reorders it. A layer is immutable
+ * once added: the returned lists reject modification and are detached from the
+ * caller's input list. Providing a layer that already exists is rejected loudly: the
+ * add is once-only, and the exception names the offending key. An annotation that
+ * references another annotation by its index within a layer, for example a dependency
+ * arc naming its head token, therefore stays valid for the lifetime of the
+ * document.
+ *
+ * @since 3.0.0
+ */
+public interface Document {
+
+ /**
+ * Creates an empty {@link Document} over a text. The returned document is immutable
+ * and safe to share between threads: it captures the text's content at construction,
+ * so later changes to a mutable {@code CharSequence} do not reach the document.
+ *
+ * @param text The original document text. Must not be {@code null}.
+ * @return A {@link Document} without any layers. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
+ static Document of(CharSequence text) {
+ return ImmutableDocument.empty(text);
+ }
+
+ /**
+ * @return The original text of the document. Never {@code null}.
+ */
+ CharSequence text();
+
+ /**
+ * Retrieves the annotations of one layer.
+ *
+ * @param layer The layer to read. Must not be {@code null}.
+ * @param The type of the layer's annotation values.
+ * @return The layer's annotations in their layer order, or an empty list when the
+ * layer is absent. Never {@code null}; the list is unmodifiable.
+ * @throws IllegalArgumentException Thrown if {@code layer} is {@code null}.
+ */
+ List> get(LayerKey layer);
+
+ /**
+ * @return The keys of all layers present on the document. Never {@code null}; the set
+ * is unmodifiable.
+ */
+ Set> layers();
+
+ /**
+ * Returns a new document with one layer added.
+ *
+ * @param layer The key of the layer to add. Must not be {@code null} and must not
+ * already be present.
+ * @param annotations The annotations of the layer. Must not be {@code null}, must not
+ * contain {@code null}, and every value must be assignable to the
+ * layer's type. Under a positional key every annotation must carry
+ * a span within the text bounds; under a document-scoped key no
+ * annotation may carry a span.
+ * @param The type of the layer's annotation values.
+ * @return A new {@link Document} sharing this document's text and existing layers.
+ * Never {@code null}.
+ * @throws IllegalArgumentException Thrown if any of the above constraints is violated.
+ */
+ Document with(LayerKey layer, List> annotations);
+
+ /**
+ * How {@link #merge(Document, DuplicateLayerPolicy)} treats a layer key that is
+ * present on both documents.
+ */
+ enum DuplicateLayerPolicy {
+
+ /** Reject any layer key present on both documents. */
+ REJECT,
+
+ /**
+ * Keep one copy of a layer key present on both documents when the two layers are
+ * structurally equal, for example when two parallel branches ran the same
+ * tokenizer. Layers whose contents differ are rejected as with {@link #REJECT}.
+ * Equality is {@link Annotation} equality: spans compare by offsets and type,
+ * never by probability, and values by their own {@code equals}.
+ */
+ KEEP_EQUAL
+ }
+
+ /**
+ * Returns a new document combining this document's layers with another document's
+ * layers over the same text, joining documents grown independently, for example by
+ * pipelines that ran in parallel.
+ *
+ * @param other The document whose layers are added on top of this document's layers.
+ * Must not be {@code null}, must carry the same text content, and must
+ * not provide a layer this document already has.
+ * @return A new {@link Document} carrying the layers of both documents. Never
+ * {@code null}; both source documents are left untouched.
+ * @throws IllegalArgumentException Thrown if {@code other} is {@code null}, if its
+ * text content differs, or if a layer key is present on both documents; the
+ * exception names the offending key.
+ */
+ default Document merge(Document other) {
+ return merge(other, DuplicateLayerPolicy.REJECT);
+ }
+
+ /**
+ * Returns a new document combining this document's layers with another document's
+ * layers over the same text, resolving duplicate layer keys with
+ * {@code duplicateLayers}.
+ *
+ * @param other The document whose layers are added on top of this document's layers.
+ * Must not be {@code null} and must carry the same text content.
+ * @param duplicateLayers How to treat a layer key present on both documents. Must
+ * not be {@code null}.
+ * @return A new {@link Document} carrying the layers of both documents. Never
+ * {@code null}; both source documents are left untouched.
+ * @throws IllegalArgumentException Thrown if either argument is {@code null}, if the
+ * text content differs, or if a layer key is present on both documents and
+ * the policy does not keep it; the exception names the offending key.
+ */
+ default Document merge(Document other, DuplicateLayerPolicy duplicateLayers) {
+ if (other == null) {
+ throw new IllegalArgumentException("other must not be null");
+ }
+ if (duplicateLayers == null) {
+ throw new IllegalArgumentException("duplicateLayers must not be null");
+ }
+ if (!text().toString().contentEquals(other.text())) {
+ throw new IllegalArgumentException(
+ "merge requires both documents to carry the same text");
+ }
+ Document merged = this;
+ for (final LayerKey layer : other.layers()) {
+ if (duplicateLayers == DuplicateLayerPolicy.KEEP_EQUAL
+ && merged.layers().contains(layer)) {
+ if (layersEqual(merged, layer, other)) {
+ continue;
+ }
+ throw new IllegalArgumentException(
+ "layer is present on both documents with differing contents: " + layer);
+ }
+ merged = addLayer(merged, layer, other);
+ }
+ return merged;
+ }
+
+ /**
+ * @return Whether the two documents carry structurally equal contents for the layer.
+ */
+ private static boolean layersEqual(Document first, LayerKey layer, Document second) {
+ return first.get(layer).equals(second.get(layer));
+ }
+
+ /**
+ * Adds one layer of {@code from} to {@code base} through {@link #with(LayerKey, List)},
+ * capturing the key's value type.
+ */
+ private static Document addLayer(Document base, LayerKey layer, Document from) {
+ return base.with(layer, from.get(layer));
+ }
+}
diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java
new file mode 100644
index 0000000000..e12a468b5a
--- /dev/null
+++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnalyzer.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.document;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Runs a fixed sequence of {@link DocumentAnnotator annotators} over a text, producing
+ * one {@link Document} that carries every step's layers.
+ *
+ *
The pipeline is validated at build time: every annotator's required layers must be
+ * provided by an earlier annotator, and no two annotators may provide the same layer, so
+ * a misordered or conflicting pipeline fails when it is assembled rather than midway
+ * through a document. The analyzer holds no per-call state; it is as thread-safe as the
+ * annotators it is built from.
+ *
+ * @since 3.0.0
+ */
+public final class DocumentAnalyzer {
+
+ private final List annotators;
+
+ private DocumentAnalyzer(List annotators) {
+ this.annotators = annotators;
+ }
+
+ /**
+ * @return A new {@link Builder}. Never {@code null}.
+ */
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ /**
+ * Analyzes a text by running every annotator in order.
+ *
+ * @param text The original document text. Must not be {@code null}.
+ * @return The annotated {@link Document}. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
+ public Document analyze(CharSequence text) {
+ Document document = Document.of(text);
+ for (final DocumentAnnotator annotator : annotators) {
+ document = annotator.annotate(document);
+ }
+ return document;
+ }
+
+ /**
+ * Assembles a {@link DocumentAnalyzer} from annotators in execution order.
+ */
+ public static final class Builder {
+
+ private final List annotators = new ArrayList<>();
+
+ private Builder() {
+ }
+
+ /**
+ * Appends an annotator to the pipeline.
+ *
+ * @param annotator The annotator to run after the ones already added. Must not be
+ * {@code null}.
+ * @return This {@link Builder}. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code annotator} is {@code null}.
+ */
+ public Builder add(DocumentAnnotator annotator) {
+ if (annotator == null) {
+ throw new IllegalArgumentException("annotator must not be null");
+ }
+ annotators.add(annotator);
+ return this;
+ }
+
+ /**
+ * Validates the pipeline and builds the analyzer.
+ *
+ * @return A {@link DocumentAnalyzer}. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if the pipeline is empty, an annotator
+ * requires a layer no earlier annotator provides, or two annotators provide
+ * the same layer.
+ */
+ public DocumentAnalyzer build() {
+ if (annotators.isEmpty()) {
+ throw new IllegalArgumentException("a pipeline needs at least one annotator");
+ }
+ final Map, Integer> providers = new HashMap<>();
+ for (int position = 0; position < annotators.size(); position++) {
+ final DocumentAnnotator annotator = annotators.get(position);
+ for (final LayerKey required : annotator.requires()) {
+ if (!providers.containsKey(required)) {
+ throw new IllegalArgumentException("annotator " + annotator
+ + " requires layer " + required + ", which no earlier annotator provides");
+ }
+ }
+ for (final LayerKey provided : annotator.provides()) {
+ final Integer earlier = providers.putIfAbsent(provided, position);
+ if (earlier != null) {
+ throw new IllegalArgumentException("annotators at positions " + earlier
+ + " and " + position + " both provide layer " + provided);
+ }
+ }
+ }
+ return new DocumentAnalyzer(List.copyOf(annotators));
+ }
+ }
+}
diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java
new file mode 100644
index 0000000000..6bc8417e6c
--- /dev/null
+++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotator.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.document;
+
+import java.util.Set;
+
+/**
+ * A pipeline step that reads layers from a {@link Document} and returns a new document
+ * with its own layers added.
+ *
+ *
An annotator declares the layers it {@link #requires()} and {@link #provides()}, so
+ * a {@link DocumentAnalyzer} can validate a pipeline before running it. Annotators are
+ * usually thin adapters over an existing analysis component. Thread safety is
+ * implementation specific.
A required layer must be present on the document, but it may be empty: an empty
+ * required layer is valid input and yields the annotator's provided layers present but
+ * empty, so a pipeline degrades gracefully on documents without content.
+ *
+ * @param document The document to annotate. Must not be {@code null} and must contain
+ * every layer named by {@link #requires()}.
+ * @return A new {@link Document} carrying the layers named by {@link #provides()} in
+ * addition to the input layers. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code document} is {@code null} or lacks
+ * a required layer.
+ */
+ Document annotate(Document document);
+
+ /**
+ * @return The keys of the layers this annotator reads. Never {@code null}.
+ */
+ default Set> requires() {
+ return Set.of();
+ }
+
+ /**
+ * @return The keys of the layers this annotator adds. Never {@code null}.
+ */
+ Set> provides();
+}
diff --git a/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java
new file mode 100644
index 0000000000..fd5f81b858
--- /dev/null
+++ b/opennlp-api/src/main/java/opennlp/tools/document/DocumentAnnotators.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.document;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Support methods shared by {@link DocumentAnnotator} implementations: the
+ * required-layer check and the per-sentence walk over the token layer.
+ *
+ *
These helpers keep the annotators' shared behavior identical across
+ * implementations: an absent required layer is always rejected with the same message
+ * naming the layer, and every per-sentence adapter applies the same sentence-to-token
+ * mapping, including the loud rejection of a token lying outside every sentence.
+ *
+ * @since 3.0.0
+ */
+public final class DocumentAnnotators {
+
+ /**
+ * Receives one sentence's contiguous token run during
+ * {@link #forEachSentence(List, List, SentenceTokenConsumer)}.
+ */
+ @FunctionalInterface
+ public interface SentenceTokenConsumer {
+
+ /**
+ * Consumes one sentence's tokens.
+ *
+ * @param first The position of the sentence's first token in the token layer.
+ * @param words The sentence's token values in layer order. Never {@code null} or
+ * empty; the run covers the token layer positions
+ * {@code [first, first + words.length)}.
+ */
+ void accept(int first, String[] words);
+ }
+
+ /**
+ * Verifies that a document is present and carries every given layer.
+ *
+ * @param document The document to check.
+ * @param layers The required layers, in the order they are to be reported.
+ * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, or if
+ * a layer is absent; the message names the first absent layer.
+ */
+ public static void requireLayers(Document document, LayerKey... layers) {
+ if (document == null) {
+ throw new IllegalArgumentException("document must not be null");
+ }
+ final Set> present = document.layers();
+ for (final LayerKey layer : layers) {
+ if (!present.contains(layer)) {
+ throw new IllegalArgumentException("document lacks the required layer " + layer);
+ }
+ }
+ }
+
+ /**
+ * Walks the token layer sentence by sentence and hands each sentence's contiguous
+ * token run to the consumer.
+ *
+ *
Both layers must be in text order. Each sentence consumes the contiguous run of
+ * tokens whose spans it encloses; a sentence without tokens is skipped. Every token
+ * must belong to a sentence: a token lying outside every sentence is rejected loudly
+ * after the walk, so it can never be silently dropped.
+ *
+ * @param sentences The sentence layer, in text order. Must not be {@code null}.
+ * @param tokens The token layer, in text order. Must not be {@code null}.
+ * @param consumer Receives each token-carrying sentence's run. Must not be
+ * {@code null}.
+ * @throws IllegalArgumentException Thrown if a token lies outside every sentence.
+ */
+ public static void forEachSentence(List> sentences,
+ List> tokens, SentenceTokenConsumer consumer) {
+ int next = 0;
+ for (final Annotation sentence : sentences) {
+ final int first = next;
+ while (next < tokens.size()
+ && tokens.get(next).span().getStart() >= sentence.span().getStart()
+ && tokens.get(next).span().getEnd() <= sentence.span().getEnd()) {
+ next++;
+ }
+ final int count = next - first;
+ if (count == 0) {
+ continue;
+ }
+ final String[] words = new String[count];
+ for (int i = 0; i < count; i++) {
+ words[i] = tokens.get(first + i).value();
+ }
+ consumer.accept(first, words);
+ }
+ if (next != tokens.size()) {
+ throw new IllegalArgumentException("token at " + tokens.get(next).span()
+ + " lies outside every sentence");
+ }
+ }
+
+ private DocumentAnnotators() {
+ // Not instantiated; this class provides static support methods only.
+ }
+}
diff --git a/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java
new file mode 100644
index 0000000000..e175c05f6c
--- /dev/null
+++ b/opennlp-api/src/main/java/opennlp/tools/document/ImmutableDocument.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.document;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import opennlp.tools.util.Span;
+
+/**
+ * The default {@link Document} implementation: an unmodifiable map from layer key to an
+ * unmodifiable annotation list.
+ *
+ *
Instances are immutable: {@link #with(LayerKey, List)} returns a new document that
+ * shares the unchanged layers with its ancestor, copying the map but not the layers, so
+ * documents grown from a common ancestor share their layer lists. The text is captured
+ * as a {@link String} at construction, so a mutable {@link CharSequence} handed to
+ * {@link #empty(CharSequence)} cannot change the document afterwards. That immutability
+ * makes instances safe to share between threads.
+ */
+final class ImmutableDocument implements Document {
+
+ private final String text;
+ private final Map, List>> layers;
+
+ private ImmutableDocument(String text, Map, List>> layers) {
+ this.text = text;
+ this.layers = Collections.unmodifiableMap(layers);
+ }
+
+ /**
+ * Creates a document without any layers.
+ *
+ * @param text The original document text, captured as its content at this moment.
+ * Must not be {@code null}.
+ * @return An empty {@link ImmutableDocument}. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+ */
+ static ImmutableDocument empty(CharSequence text) {
+ if (text == null) {
+ throw new IllegalArgumentException("text must not be null");
+ }
+ return new ImmutableDocument(text.toString(), Collections.emptyMap());
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public CharSequence text() {
+ return text;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ @SuppressWarnings("unchecked")
+ public List> get(LayerKey layer) {
+ if (layer == null) {
+ throw new IllegalArgumentException("layer must not be null");
+ }
+ final List> annotations = layers.get(layer);
+ if (annotations == null) {
+ return List.of();
+ }
+ // This cast is safe because with(LayerKey, List) verified every value against the
+ // key's type when the layer was inserted.
+ return (List>) (List) annotations;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Set> layers() {
+ // The unmodifiable map exposes an unmodifiable key set and caches it, so this
+ // accessor allocates no wrapper per call.
+ return layers.keySet();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Document with(LayerKey layer, List> annotations) {
+ if (layer == null) {
+ throw new IllegalArgumentException("layer must not be null");
+ }
+ if (annotations == null) {
+ throw new IllegalArgumentException("annotations must not be null");
+ }
+ if (layers.containsKey(layer)) {
+ throw new IllegalArgumentException("layer is already present: " + layer);
+ }
+ validate(layer, annotations);
+ final Map, List>> grown = new LinkedHashMap<>(layers);
+ grown.put(layer, List.copyOf(annotations));
+ return new ImmutableDocument(text, grown);
+ }
+
+ /**
+ * {@inheritDoc}
+ * This implementation copies the layer map once, not once per added layer.
+ */
+ @Override
+ public Document merge(Document other, DuplicateLayerPolicy duplicateLayers) {
+ if (other == null) {
+ throw new IllegalArgumentException("other must not be null");
+ }
+ if (duplicateLayers == null) {
+ throw new IllegalArgumentException("duplicateLayers must not be null");
+ }
+ if (!text.contentEquals(other.text())) {
+ throw new IllegalArgumentException(
+ "merge requires both documents to carry the same text");
+ }
+ final Map, List>> combined = new LinkedHashMap<>(layers);
+ for (final LayerKey layer : other.layers()) {
+ if (combined.containsKey(layer)) {
+ if (duplicateLayers == DuplicateLayerPolicy.KEEP_EQUAL
+ && get(layer).equals(other.get(layer))) {
+ continue;
+ }
+ throw new IllegalArgumentException(duplicateLayers == DuplicateLayerPolicy.KEEP_EQUAL
+ ? "layer is present on both documents with differing contents: " + layer
+ : "layer is already present: " + layer);
+ }
+ combined.put(layer, copyValidated(layer, other));
+ }
+ if (combined.size() == layers.size()) {
+ return this;
+ }
+ return new ImmutableDocument(text, combined);
+ }
+
+ /**
+ * {@return a validated immutable copy of one of {@code from}'s layers, capturing the
+ * key's value type}
+ */
+ private List> copyValidated(LayerKey layer, Document from) {
+ final List> annotations = from.get(layer);
+ if (annotations == null) {
+ throw new IllegalArgumentException("annotations must not be null");
+ }
+ validate(layer, annotations);
+ return List.copyOf(annotations);
+ }
+
+ /**
+ * Checks one layer's annotations against the key's contract: no null elements, values
+ * assignable to the key's type, spans present and within the text bounds under a
+ * positional key, absent under a document-scoped key.
+ *
+ * @throws IllegalArgumentException Thrown if any check fails; the message names the
+ * layer.
+ */
+ private void validate(LayerKey layer, List> annotations) {
+ for (final Annotation annotation : annotations) {
+ if (annotation == null) {
+ throw new IllegalArgumentException("annotations must not contain null: " + layer);
+ }
+ if (!layer.type().isInstance(annotation.value())) {
+ throw new IllegalArgumentException("value of type "
+ + annotation.value().getClass().getName() + " does not match layer " + layer);
+ }
+ final Span span = annotation.span();
+ if (layer.scope() == LayerKey.Scope.POSITIONAL) {
+ if (span == null) {
+ throw new IllegalArgumentException(
+ "positional layer " + layer + " requires a span on every annotation");
+ }
+ if (span.getEnd() > text.length()) {
+ throw new IllegalArgumentException("span " + span + " exceeds the text length "
+ + text.length() + " in layer " + layer);
+ }
+ } else if (span != null) {
+ throw new IllegalArgumentException(
+ "document-scoped layer " + layer + " must not carry spans");
+ }
+ }
+ }
+}
diff --git a/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java
new file mode 100644
index 0000000000..e29870a06c
--- /dev/null
+++ b/opennlp-api/src/main/java/opennlp/tools/document/LayerKey.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.document;
+
+import java.util.Objects;
+
+import opennlp.tools.util.StringUtil;
+
+/**
+ * Identifies one annotation layer of a {@link Document} and carries the type of that
+ * layer's annotation values, so reading a layer back is statically typed.
+ *
+ *
The key space is deliberately open: any producer may define new keys in its own
+ * package, and the container never enumerates them. Two keys are equal when their id,
+ * their value type, and their {@link Scope} are equal, so independently created
+ * constants for the same layer interoperate. Standard keys for the toolkit's own
+ * results live in {@link Layers}.
+ *
+ *
A key declares its {@link Scope}: a {@link Scope#POSITIONAL positional} key
+ * guarantees a span on every annotation, and a {@link Scope#DOCUMENT document-scoped}
+ * key carries whole-document values without spans, for example a language id or a
+ * category distribution. The scope is declared per key, never per annotation, so
+ * consumers of a positional layer never null-check a span.
+ *
+ * @param The type of the annotation values stored under this key.
+ *
+ * @since 3.0.0
+ */
+public final class LayerKey {
+
+ /** How the annotations of a layer relate to the document text. */
+ public enum Scope {
+ /** Every annotation of the layer is anchored to a span of the text. */
+ POSITIONAL,
+ /** The layer's values describe the document as a whole and carry no spans. */
+ DOCUMENT
+ }
+
+ private final String id;
+ private final Class type;
+ private final Scope scope;
+
+ private LayerKey(String id, Class type, Scope scope) {
+ this.id = id;
+ this.type = type;
+ this.scope = scope;
+ }
+
+ /**
+ * Creates a {@link LayerKey}.
+ *
+ * @param id The layer identifier, for example {@code opennlp:tokens}. Keys defined by
+ * the toolkit carry the {@code opennlp:} prefix, an extension uses its own
+ * prefix, and a bare id is legal for an application-local layer. Must not
+ * be {@code null} or blank.
+ * @param type The class of the annotation values stored under the key. Must not be
+ * {@code null}.
+ * @param The type of the annotation values.
+ * @return A {@link Scope#POSITIONAL positional} {@link LayerKey}. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code id} is {@code null} or blank, or
+ * {@code type} is {@code null}.
+ */
+ public static LayerKey of(String id, Class type) {
+ return key(id, type, Scope.POSITIONAL);
+ }
+
+ /**
+ * Creates a {@link Scope#DOCUMENT document-scoped} {@link LayerKey} for values that
+ * describe the document as a whole, for example a language id, a category
+ * distribution, or provenance. Annotations under such a key carry no span.
+ *
+ * @param id The layer identifier, following the same prefix rules as
+ * {@link #of(String, Class)}. Must not be {@code null} or blank.
+ * @param type The class of the annotation values stored under the key. Must not be
+ * {@code null}.
+ * @param The type of the annotation values.
+ * @return A document-scoped {@link LayerKey}. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code id} is {@code null} or blank, or
+ * {@code type} is {@code null}.
+ */
+ public static LayerKey document(String id, Class type) {
+ return key(id, type, Scope.DOCUMENT);
+ }
+
+ /**
+ * Validates the components and creates the key.
+ *
+ * @param id The layer identifier.
+ * @param type The value class.
+ * @param scope The declared scope.
+ * @param The type of the annotation values.
+ * @return The {@link LayerKey}. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code id} is {@code null} or blank, or
+ * {@code type} is {@code null}.
+ */
+ private static LayerKey key(String id, Class type, Scope scope) {
+ if (id == null || StringUtil.isBlank(id)) {
+ throw new IllegalArgumentException("id must not be null or blank");
+ }
+ if (type == null) {
+ throw new IllegalArgumentException("type must not be null");
+ }
+ return new LayerKey<>(id, type, scope);
+ }
+
+ /**
+ * @return The layer identifier. Never {@code null}.
+ */
+ public String id() {
+ return id;
+ }
+
+ /**
+ * @return The class of the annotation values stored under this key. Never {@code null}.
+ */
+ public Class type() {
+ return type;
+ }
+
+ /**
+ * @return The declared scope of the layer. Never {@code null}.
+ */
+ public Scope scope() {
+ return scope;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof LayerKey other)) {
+ return false;
+ }
+ return id.equals(other.id) && type.equals(other.type) && scope == other.scope;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, type, scope);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString() {
+ return id + '<' + type.getSimpleName() + '>';
+ }
+}
diff --git a/opennlp-api/src/main/java/opennlp/tools/document/Layers.java b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java
new file mode 100644
index 0000000000..0257afc384
--- /dev/null
+++ b/opennlp-api/src/main/java/opennlp/tools/document/Layers.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.document;
+
+import opennlp.tools.util.StringUtil;
+
+/**
+ * The standard {@link LayerKey layer keys} for the results the toolkit produces itself.
+ *
+ *
This class is a convenience, not a registry: the key space stays open, and any
+ * producer may define further keys in its own package. New capabilities must never
+ * require an addition here to function.
+ *
+ *
Namespace rule: every key the toolkit itself defines carries the
+ * {@code opennlp:} id prefix. An extension defines its keys under its own prefix, and
+ * a bare id without a prefix is legal for an application-local layer, so ids from
+ * independent producers cannot collide. Toolkit keys are created through
+ * {@link #key(String, Class)} and {@link #documentKey(String, Class)}, which apply the
+ * prefix, so no producer spells it.
+ *
+ *
Gold versus predicted: a corpus may carry a hand-annotated version of a layer
+ * beside a produced one. The convention is a {@code gold:} id prefix on the same key
+ * scheme, for example {@code gold:opennlp:tokens} beside {@code opennlp:tokens}.
+ * Because adding a layer is once-only, competing versions of a layer always live under
+ * distinct keys and never replace each other.
+ *
+ *
Placement rule: this class holds only the keys of the core linguistic layers
+ * every pipeline shares (sentences, tokens, tags, entities). A capability-specific
+ * layer's key lives on the annotator that provides it, for example the lemma layer's
+ * key on its adapter, so adding a capability never touches this class.
+ *
+ * @since 3.0.0
+ */
+public final class Layers {
+
+ /** The id prefix of every key the toolkit defines. */
+ private static final String NAMESPACE = "opennlp:";
+
+ /**
+ * Sentence boundaries; each annotation covers one sentence and carries its text.
+ */
+ public static final LayerKey SENTENCES = key("sentences", String.class);
+
+ /**
+ * Token boundaries; each annotation covers one token and carries its text.
+ */
+ public static final LayerKey TOKENS = key("tokens", String.class);
+
+ /**
+ * Part-of-speech tags; one annotation per token, aligned with {@link #TOKENS} by
+ * position, carrying the tag.
+ */
+ public static final LayerKey POS_TAGS = key("pos", String.class);
+
+ /**
+ * Named entities; each annotation covers one mention and carries the entity type as
+ * its value. The annotation's span carries offsets only.
+ */
+ public static final LayerKey ENTITIES = key("entities", String.class);
+
+ /**
+ * Creates a {@link LayerKey.Scope#POSITIONAL positional} key in the toolkit's
+ * {@code opennlp:} namespace, for a layer the toolkit itself produces.
+ *
+ * @param name The layer name without a namespace, for example {@code tokens}. Must
+ * not be {@code null}, blank, or contain {@code ':'}.
+ * @param type The class of the annotation values stored under the key. Must not be
+ * {@code null}.
+ * @param The type of the annotation values.
+ * @return A key whose id is the name under the {@code opennlp:} prefix. Never
+ * {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, blank, or
+ * contains {@code ':'}, or {@code type} is {@code null}.
+ */
+ public static LayerKey key(String name, Class type) {
+ return LayerKey.of(NAMESPACE + validName(name), type);
+ }
+
+ /**
+ * Creates a {@link LayerKey.Scope#DOCUMENT document-scoped} key in the toolkit's
+ * {@code opennlp:} namespace, for a whole-document value the toolkit itself produces.
+ *
+ * @param name The layer name without a namespace, for example {@code language}. Must
+ * not be {@code null}, blank, or contain {@code ':'}.
+ * @param type The class of the annotation values stored under the key. Must not be
+ * {@code null}.
+ * @param The type of the annotation values.
+ * @return A document-scoped key whose id is the name under the {@code opennlp:}
+ * prefix. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, blank, or
+ * contains {@code ':'}, or {@code type} is {@code null}.
+ */
+ public static LayerKey documentKey(String name, Class type) {
+ return LayerKey.document(NAMESPACE + validName(name), type);
+ }
+
+ /**
+ * Validates a namespace-free layer name.
+ *
+ * @param name The name to validate.
+ * @return The validated name.
+ * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, blank, or
+ * contains {@code ':'}.
+ */
+ private static String validName(String name) {
+ if (name == null || StringUtil.isBlank(name)) {
+ throw new IllegalArgumentException("name must not be null or blank");
+ }
+ if (name.indexOf(':') >= 0) {
+ throw new IllegalArgumentException(
+ "name must not contain ':', the namespace is applied by this factory: " + name);
+ }
+ return name;
+ }
+
+ private Layers() {
+ // Not instantiated; this class provides constants and static key factories only.
+ }
+}
diff --git a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
index 98cca59891..82041f7764 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
@@ -268,6 +268,30 @@ public static boolean isEmpty(CharSequence theString) {
return theString.length() == 0;
}
+ /**
+ * Determines whether a {@link CharSequence} is blank: empty, or made up entirely of
+ * code points that {@link #isWhitespace(int)} accepts. Unlike
+ * {@link String#isBlank()}, this follows the toolkit's whitespace definition, which
+ * includes the no-break spaces the JDK predicate leaves out, so a value spelled
+ * entirely from them cannot pass a blank check as content. Unlike
+ * {@link #isUnicodeBlank(CharSequence)}, it resolves through the active
+ * {@link WhitespaceMode} and does not treat {@code null} as blank.
+ *
+ * @param theString The {@link CharSequence} to examine. Must not be {@code null}.
+ * @return {@code true} if {@code theString} is empty or all whitespace.
+ * @throws NullPointerException Thrown if {@code theString} is {@code null}.
+ */
+ public static boolean isBlank(CharSequence theString) {
+ for (int i = 0; i < theString.length(); ) {
+ final int codePoint = Character.codePointAt(theString, i);
+ if (!isWhitespace(codePoint)) {
+ return false;
+ }
+ i += Character.charCount(codePoint);
+ }
+ return true;
+ }
+
/**
* Get the minimum of three values.
*
diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java
new file mode 100644
index 0000000000..bef5db118f
--- /dev/null
+++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentAnnotatorsTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.document;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.util.Span;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Tests the {@link DocumentAnnotators} support methods directly: the required-layer
+ * check's exact rejection messages and the per-sentence walk's slicing, skipping, and
+ * loud rejection of a token outside every sentence. The adapter tests exercise the same
+ * behavior through the annotators; this class pins the helpers as public API on their
+ * own.
+ */
+public class DocumentAnnotatorsTest {
+
+ @Test
+ void testRequireLayersAcceptsPresentLayers() {
+ final Document document = Document.of("the")
+ .with(Layers.SENTENCES, List.of())
+ .with(Layers.TOKENS, List.of());
+ DocumentAnnotators.requireLayers(document, Layers.SENTENCES, Layers.TOKENS);
+ DocumentAnnotators.requireLayers(document);
+ }
+
+ @Test
+ void testRequireLayersRejectsNullDocument() {
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.requireLayers(null, Layers.TOKENS));
+ assertEquals("document must not be null", e.getMessage());
+ }
+
+ /**
+ * Verifies that an absent layer is rejected with the shared message naming the first
+ * absent layer in the order the caller listed them.
+ */
+ @Test
+ void testRequireLayersNamesTheFirstAbsentLayer() {
+ final Document document = Document.of("the").with(Layers.TOKENS, List.of());
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.requireLayers(document,
+ Layers.SENTENCES, Layers.TOKENS, Layers.POS_TAGS));
+ assertEquals("document lacks the required layer opennlp:sentences",
+ e.getMessage());
+ }
+
+ /**
+ * Verifies the walk contract: each sentence receives the contiguous run of tokens its
+ * span encloses with the run's first token layer position, and a sentence without
+ * tokens is skipped rather than reported as an empty run.
+ */
+ @Test
+ void testForEachSentenceSlicesContiguousRuns() {
+ final List> sentences = List.of(
+ new Annotation<>(new Span(0, 9), "Ana runs."),
+ new Annotation<>(new Span(10, 11), "!"),
+ new Annotation<>(new Span(12, 21), "Bob sits."));
+ final List> tokens = List.of(
+ new Annotation<>(new Span(0, 3), "Ana"),
+ new Annotation<>(new Span(4, 9), "runs."),
+ new Annotation<>(new Span(12, 15), "Bob"),
+ new Annotation<>(new Span(16, 21), "sits."));
+
+ final List firsts = new ArrayList<>();
+ final List> runs = new ArrayList<>();
+ DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> {
+ firsts.add(first);
+ runs.add(List.of(words));
+ });
+
+ assertEquals(List.of(0, 2), firsts);
+ assertEquals(List.of(
+ List.of("Ana", "runs."),
+ List.of("Bob", "sits.")), runs);
+ }
+
+ @Test
+ void testForEachSentenceRejectsTokenOutsideEverySentence() {
+ final List> sentences = List.of(
+ new Annotation<>(new Span(0, 9), "Ana runs."));
+ final List> tokens = List.of(
+ new Annotation<>(new Span(0, 3), "Ana"),
+ new Annotation<>(new Span(4, 9), "runs."),
+ new Annotation<>(new Span(10, 13), "Bob"));
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnnotators.forEachSentence(sentences, tokens, (first, words) -> {
+ }));
+ assertEquals("token at [10..13) lies outside every sentence", e.getMessage());
+ }
+
+ @Test
+ void testForEachSentenceOverEmptyLayersConsumesNothing() {
+ final List> runs = new ArrayList<>();
+ DocumentAnnotators.forEachSentence(List.of(), List.of(),
+ (first, words) -> runs.add(List.of(words)));
+ assertTrue(runs.isEmpty());
+ }
+}
diff --git a/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java
new file mode 100644
index 0000000000..27a6a1948f
--- /dev/null
+++ b/opennlp-api/src/test/java/opennlp/tools/document/DocumentContractTest.java
@@ -0,0 +1,644 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.document;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.util.Span;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Pins down the observable contract of the document container at its edges: key
+ * equality, layer ordering, span boundary cases, list immutability, the exact rejection
+ * messages, and the analyzer's build-time validation. Every expected value in this class
+ * is asserted exactly so any behavioral drift is caught, not just gross breakage.
+ */
+public class DocumentContractTest {
+
+ private static final LayerKey WORDS = LayerKey.of("words", String.class);
+
+ /**
+ * Verifies that two independently created keys with the same id and the same type are
+ * equal, hash alike, and therefore address the same layer, while remaining distinct
+ * instances. This is what lets separately compiled producers agree on a layer without
+ * sharing a constant.
+ */
+ @Test
+ void testKeysWithSameIdAndTypeAddressTheSameLayer() {
+ final LayerKey first = LayerKey.of("words", String.class);
+ final LayerKey second = LayerKey.of("words", String.class);
+ assertNotSame(first, second);
+ assertEquals(first, second);
+ assertEquals(first.hashCode(), second.hashCode());
+ assertEquals("words", first.toString());
+
+ final Document document = Document.of("the")
+ .with(first, List.of(new Annotation<>(new Span(0, 3), "the")));
+ // Reading through the other, equal key yields the very layer added above.
+ assertEquals(1, document.get(second).size());
+ assertEquals("the", document.get(second).get(0).value());
+ // A duplicate add through the equal key is rejected like any duplicate.
+ assertThrows(IllegalArgumentException.class, () -> document.with(second, List.of()));
+ }
+
+ /**
+ * Verifies that two keys sharing an id but differing in value type are unequal and
+ * denote two independent layers that can coexist on one document.
+ */
+ @Test
+ void testKeysWithSameIdButDifferentTypesAreDifferentLayers() {
+ final LayerKey asString = LayerKey.of("marks", String.class);
+ final LayerKey asInteger = LayerKey.of("marks", Integer.class);
+ assertNotEquals(asString, asInteger);
+
+ final Document document = Document.of("ab")
+ .with(asString, List.of(new Annotation<>(new Span(0, 1), "a")))
+ .with(asInteger, List.of(new Annotation<>(new Span(1, 2), 7)));
+ assertEquals(Set.of(asString, asInteger), document.layers());
+ assertEquals("a", document.get(asString).get(0).value());
+ assertEquals(7, document.get(asInteger).get(0).value());
+ }
+
+ /**
+ * Verifies the per-key scope contract: a document-scoped layer carries span-less
+ * values and round-trips them, a spanned annotation under a document-scoped key is
+ * rejected with a message naming the layer, a span-less annotation under a
+ * positional key is rejected likewise, and two keys differing only in scope are
+ * unequal and never address the same layer.
+ */
+ @Test
+ void testDocumentScopedLayersCarrySpanlessValues() {
+ final LayerKey language = LayerKey.document("language", String.class);
+ assertEquals(LayerKey.Scope.DOCUMENT, language.scope());
+ assertEquals(LayerKey.Scope.POSITIONAL, WORDS.scope());
+
+ final Document document = Document.of("the dog")
+ .with(language, List.of(Annotation.of("eng")));
+ assertEquals(1, document.get(language).size());
+ assertEquals("eng", document.get(language).get(0).value());
+ assertNull(document.get(language).get(0).span());
+
+ final IllegalArgumentException spanned = assertThrows(IllegalArgumentException.class,
+ () -> Document.of("the").with(language,
+ List.of(new Annotation<>(new Span(0, 3), "eng"))));
+ assertEquals("document-scoped layer language must not carry spans",
+ spanned.getMessage());
+
+ final IllegalArgumentException spanless = assertThrows(IllegalArgumentException.class,
+ () -> Document.of("the").with(WORDS, List.of(Annotation.of("the"))));
+ assertEquals("positional layer words requires a span on every annotation",
+ spanless.getMessage());
+
+ final LayerKey positionalTwin = LayerKey.of("language", String.class);
+ assertNotEquals(language, positionalTwin);
+ final Document both = Document.of("the")
+ .with(language, List.of(Annotation.of("eng")))
+ .with(positionalTwin, List.of(new Annotation<>(new Span(0, 3), "the")));
+ assertEquals(2, both.layers().size());
+ }
+
+ /**
+ * Verifies the toolkit-namespace factories: {@link Layers#key(String, Class)} and
+ * {@link Layers#documentKey(String, Class)} apply the {@code opennlp:} prefix and
+ * yield keys equal to independently spelled ones, and a name that is null, blank, or
+ * already carries a namespace is rejected.
+ */
+ @Test
+ void testToolkitKeysCarryTheNamespacePrefix() {
+ assertEquals(LayerKey.of("opennlp:things", String.class),
+ Layers.key("things", String.class));
+ assertEquals("opennlp:things", Layers.key("things", String.class).id());
+ assertEquals(Layers.TOKENS, Layers.key("tokens", String.class));
+
+ final LayerKey whole = Layers.documentKey("language", String.class);
+ assertEquals("opennlp:language", whole.id());
+ assertEquals(LayerKey.Scope.DOCUMENT, whole.scope());
+
+ assertThrows(IllegalArgumentException.class, () -> Layers.key(" ", String.class));
+ assertThrows(IllegalArgumentException.class, () -> Layers.key(null, String.class));
+ assertThrows(IllegalArgumentException.class, () -> Layers.key("things", null));
+ final IllegalArgumentException nested = assertThrows(IllegalArgumentException.class,
+ () -> Layers.key("opennlp:things", String.class));
+ assertEquals("name must not contain ':', the namespace is applied by this factory: "
+ + "opennlp:things", nested.getMessage());
+ assertThrows(IllegalArgumentException.class,
+ () -> Layers.documentKey("app:x", String.class));
+ }
+
+ /**
+ * Verifies the gold-layer convention: a hand-annotated version of a layer lives
+ * under the {@code gold:} prefixed key beside the produced layer, both are readable
+ * independently, and the once-only rule keeps either from replacing the other.
+ */
+ @Test
+ void testGoldLayerLivesBesideThePredictedLayer() {
+ final LayerKey gold = LayerKey.of("gold:" + Layers.TOKENS.id(), String.class);
+ final Document document = Document.of("the dog")
+ .with(Layers.TOKENS, List.of(new Annotation<>(new Span(0, 3), "the")))
+ .with(gold, List.of(
+ new Annotation<>(new Span(0, 3), "the"),
+ new Annotation<>(new Span(4, 7), "dog")));
+ assertEquals("gold:opennlp:tokens", gold.id());
+ assertEquals(1, document.get(Layers.TOKENS).size());
+ assertEquals(2, document.get(gold).size());
+ assertThrows(IllegalArgumentException.class,
+ () -> document.with(gold, List.of()));
+ }
+
+ /**
+ * Verifies that a layer preserves the insertion order of its annotations: the
+ * container does not sort by span, so a producer that wants span order must supply
+ * span order.
+ */
+ @Test
+ void testLayerPreservesInsertionOrder() {
+ final Document document = Document.of("the dog")
+ .with(WORDS, List.of(
+ new Annotation<>(new Span(4, 7), "dog"),
+ new Annotation<>(new Span(0, 3), "the")));
+ final List> words = document.get(WORDS);
+ assertEquals(2, words.size());
+ assertEquals(new Span(4, 7), words.get(0).span());
+ assertEquals("dog", words.get(0).value());
+ assertEquals(new Span(0, 3), words.get(1).span());
+ assertEquals("the", words.get(1).value());
+ }
+
+ /**
+ * Verifies that several annotations may share one span within a layer, for example
+ * alternative readings of the same region, and all of them are retained in order.
+ */
+ @Test
+ void testAnnotationsWithIdenticalSpansAreAllRetained() {
+ final Document document = Document.of("bank")
+ .with(WORDS, List.of(
+ new Annotation<>(new Span(0, 4), "institution"),
+ new Annotation<>(new Span(0, 4), "riverside")));
+ final List> words = document.get(WORDS);
+ assertEquals(2, words.size());
+ assertEquals("institution", words.get(0).value());
+ assertEquals("riverside", words.get(1).value());
+ assertEquals(words.get(0).span(), words.get(1).span());
+ }
+
+ /**
+ * Verifies that zero-length spans are accepted anywhere within the bounds, including
+ * at the very end of the text where start and end equal the text length.
+ */
+ @Test
+ void testZeroLengthSpansAreAccepted() {
+ final Document document = Document.of("ab")
+ .with(WORDS, List.of(
+ new Annotation<>(new Span(1, 1), "between"),
+ new Annotation<>(new Span(2, 2), "at the end")));
+ final List> words = document.get(WORDS);
+ assertEquals(new Span(1, 1), words.get(0).span());
+ assertEquals(new Span(2, 2), words.get(1).span());
+ assertEquals(0, words.get(0).span().length());
+ }
+
+ /**
+ * Verifies that spans are indexed in {@code char} units, like {@link Span} itself: a
+ * supplementary-plane character counts as two, so a span over it covers the whole
+ * surrogate pair and the char-based text length is what bounds a span.
+ */
+ @Test
+ void testSpansAreCharIndexedOverSupplementaryCharacters() {
+ // U+1F600, a supplementary-plane character, is two chars in the text
+ final String text = "\uD83D\uDE00 ok";
+ final Document document = Document.of(text)
+ .with(WORDS, List.of(
+ new Annotation<>(new Span(0, 2), "emoji"),
+ new Annotation<>(new Span(3, 5), "ok")));
+ final List> words = document.get(WORDS);
+ assertEquals("\uD83D\uDE00",
+ words.get(0).span().getCoveredText(document.text()).toString());
+ assertEquals("ok", words.get(1).span().getCoveredText(document.text()).toString());
+ assertThrows(IllegalArgumentException.class, () -> Document.of("\uD83D\uDE00")
+ .with(WORDS, List.of(new Annotation<>(new Span(0, 3), "past the end"))));
+ }
+
+ /**
+ * Verifies that a span reaching past the end of the text is rejected on insertion
+ * with a message naming the span, the text length, and the layer.
+ */
+ @Test
+ void testSpanBeyondTextLengthIsRejectedWithExactMessage() {
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> Document.of("the").with(WORDS,
+ List.of(new Annotation<>(new Span(0, 4), "the?"))));
+ assertEquals("span [0..4) exceeds the text length 3 in layer words",
+ e.getMessage());
+ }
+
+ /**
+ * Verifies that adding a layer under a key that is already present is rejected with a
+ * message naming the offending layer.
+ */
+ @Test
+ void testDuplicateLayerIsRejectedWithExactMessage() {
+ final Document document = Document.of("the").with(WORDS, List.of());
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> document.with(WORDS, List.of()));
+ assertEquals("layer is already present: words", e.getMessage());
+ }
+
+ /**
+ * Verifies that reading an absent layer yields an empty, unmodifiable list rather
+ * than {@code null}, so callers can iterate without a presence check.
+ */
+ @Test
+ void testAbsentLayerReadsAsUnmodifiableEmptyList() {
+ final List> absent = Document.of("the").get(WORDS);
+ assertTrue(absent.isEmpty());
+ assertThrows(UnsupportedOperationException.class,
+ () -> absent.add(new Annotation<>(new Span(0, 3), "the")));
+ }
+
+ /**
+ * Verifies that the list returned for a present layer is unmodifiable and detached
+ * from the caller's input list: mutating the input after the add does not change the
+ * document.
+ */
+ @Test
+ void testPresentLayerListIsUnmodifiableAndDetachedFromInput() {
+ final List> input = new ArrayList<>();
+ input.add(new Annotation<>(new Span(0, 3), "the"));
+ final Document document = Document.of("the").with(WORDS, input);
+
+ final List> words = document.get(WORDS);
+ assertThrows(UnsupportedOperationException.class, () -> words.remove(0));
+ assertThrows(UnsupportedOperationException.class,
+ () -> words.add(new Annotation<>(new Span(0, 3), "the")));
+
+ input.clear();
+ assertEquals(1, document.get(WORDS).size());
+ }
+
+ /**
+ * Verifies that the layer key set exposed by a document cannot be mutated by callers.
+ */
+ @Test
+ void testLayerKeySetIsUnmodifiable() {
+ final Document document = Document.of("the").with(WORDS, List.of());
+ assertThrows(UnsupportedOperationException.class, () -> document.layers().clear());
+ }
+
+ /**
+ * Verifies that an analyzer whose annotator requires a layer no earlier annotator
+ * provides fails at build time with a message naming the annotator and the missing
+ * layer. The annotator under test overrides {@code toString()} so the whole message
+ * can be asserted exactly.
+ */
+ @Test
+ void testUnsatisfiedRequirementFailsAtBuildTimeWithExactMessage() {
+ final DocumentAnnotator needsTags = new DocumentAnnotator() {
+
+ @Override
+ public Document annotate(Document document) {
+ throw new IllegalStateException("must never run; the pipeline must not build");
+ }
+
+ @Override
+ public Set> requires() {
+ return Set.of(Layers.POS_TAGS);
+ }
+
+ @Override
+ public Set> provides() {
+ return Set.of(WORDS);
+ }
+
+ @Override
+ public String toString() {
+ return "tag-consumer";
+ }
+ };
+ final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder().add(needsTags);
+ final IllegalArgumentException e =
+ assertThrows(IllegalArgumentException.class, builder::build);
+ assertEquals("annotator tag-consumer requires layer opennlp:pos,"
+ + " which no earlier annotator provides", e.getMessage());
+ }
+
+ /**
+ * Verifies that an analyzer whose annotators would provide the same layer twice fails
+ * at build time with a message naming the layer and the positions of both providers,
+ * instead of crashing midway through the first document.
+ */
+ @Test
+ void testDuplicateProviderFailsAtBuildTimeWithExactMessage() {
+ final DocumentAnnotator provider = new DocumentAnnotator() {
+
+ @Override
+ public Document annotate(Document document) {
+ throw new IllegalStateException("must never run; the pipeline must not build");
+ }
+
+ @Override
+ public Set> provides() {
+ return Set.of(WORDS);
+ }
+ };
+ final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder()
+ .add(provider).add(provider);
+ final IllegalArgumentException e =
+ assertThrows(IllegalArgumentException.class, builder::build);
+ assertEquals("annotators at positions 0 and 1 both provide layer words",
+ e.getMessage());
+ }
+
+ /**
+ * Verifies that building an analyzer without any annotator fails with a message
+ * stating that a pipeline needs at least one annotator.
+ */
+ @Test
+ void testEmptyPipelineFailsWithExactMessage() {
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> DocumentAnalyzer.builder().build());
+ assertEquals("a pipeline needs at least one annotator", e.getMessage());
+ }
+
+ /**
+ * Verifies that a caller can add a new layer type without changing the document
+ * container.
+ */
+ @Test
+ void testCustomLayerNeedsNoContainerChange() {
+ record Sentiment(String polarity, double score) {
+ }
+ final LayerKey sentiment = LayerKey.of("sentiment", Sentiment.class);
+ final DocumentAnnotator annotator = new DocumentAnnotator() {
+
+ @Override
+ public Document annotate(Document document) {
+ final Span all = new Span(0, document.text().length());
+ return document.with(sentiment,
+ List.of(new Annotation<>(all, new Sentiment("positive", 0.9d))));
+ }
+
+ @Override
+ public Set> provides() {
+ return Set.of(sentiment);
+ }
+ };
+ final Document document = DocumentAnalyzer.builder().add(annotator).build()
+ .analyze("good dog");
+ assertEquals("positive", document.get(sentiment).get(0).value().polarity());
+ }
+
+ /**
+ * Verifies that the value type travels through {@link LayerKey}: a layer added under
+ * an {@code Integer} key reads back as {@code Annotation}, so its values
+ * participate in arithmetic without a cast, and a mismatched value can never enter
+ * the layer in the first place.
+ */
+ @Test
+ void testValueTypeTravelsThroughTheKey() {
+ final LayerKey counts = LayerKey.of("counts", Integer.class);
+ final Document document = Document.of("ab cd")
+ .with(counts, List.of(
+ new Annotation<>(new Span(0, 2), 2),
+ new Annotation<>(new Span(3, 5), 40)));
+ int sum = 0;
+ for (final Annotation count : document.get(counts)) {
+ sum += count.value();
+ }
+ assertEquals(42, sum);
+
+ // The insertion-time check backs the typed read: a raw-typed caller cannot place a
+ // String under the Integer key.
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ final LayerKey