spans = new ArrayList<>();
+ spans.add(new Span(0, 3));
+ spans.add(null);
+ assertThrows(IllegalArgumentException.class, () -> new TermVector("dog", 2, spans));
+ assertThrows(IllegalArgumentException.class, () -> TermVector.withSpans("dog", spans));
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {0, -1, Integer.MIN_VALUE})
+ void testFrequencyBelowOneIsRejected(int frequency) {
+ assertThrows(IllegalArgumentException.class, () -> TermVector.count("dog", frequency));
+ }
+
+ @Test
+ void testEmptySpanListCannotDeriveAFrequency() {
+ assertThrows(IllegalArgumentException.class,
+ () -> TermVector.withSpans("dog", List.of()));
+ }
+
+ @Test
+ void testPartialSpanListIsRejected() {
+ // Two occurrences but only one recorded span: neither full nor scoring-only.
+ assertThrows(IllegalArgumentException.class,
+ () -> new TermVector("dog", 2, List.of(new Span(0, 3))));
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
new file mode 100644
index 0000000000..4ed12d0d13
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.termvector;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import opennlp.tools.document.Annotation;
+import opennlp.tools.document.Document;
+import opennlp.tools.document.DocumentAnnotator;
+import opennlp.tools.document.LayerKey;
+import opennlp.tools.document.Layers;
+import opennlp.tools.util.Span;
+import opennlp.tools.util.normalizer.AlignedText;
+import opennlp.tools.util.normalizer.CharSequenceNormalizer;
+import opennlp.tools.util.normalizer.OffsetAwareNormalizer;
+
+/**
+ * Rolls the token layer up into a term vector layer for index consumers: one
+ * {@link TermVector} per distinct term, carrying the term string, its occurrence count,
+ * and (in {@link Mode#FULL full mode}) the occurrence offsets.
+ *
+ * Term identity comes from the annotator's inputs, not from logic of its own. Without
+ * a normalizer, the term is the token layer's value as-is, that is, the token's covered
+ * text in the original document. With a plain {@link CharSequenceNormalizer}, the general
+ * path, each token's covered text is normalized on its own to produce the term, so any
+ * normalizer works: case folding, NFC, accent folding, a stemmer-backed normalizer. With
+ * an {@link OffsetAwareNormalizer}, the whole document text is normalized once with its
+ * alignment recorded, each token span is mapped forward to the normalized form, and the
+ * covered normalized text is the term; this path can see across token boundaries but is
+ * limited to alignment-reporting normalizers. On every path, tokens that differ only by
+ * a normalization fold (case, an eszett expansion, collapsed whitespace) group together,
+ * and the occurrence spans emitted in {@link Mode#FULL full mode} are the token layer's
+ * own spans and therefore always point into the original text. A token whose normalized
+ * form is empty, for example one the normalizer deleted entirely, is omitted from the
+ * layer; an empty string is no term, and the token layer still accounts for the
+ * token.
+ *
+ * The layer is {@link LayerKey.Scope#DOCUMENT document-scoped}: each {@link TermVector}
+ * is a whole-document statistic, so the annotations carry no span of their own and the
+ * occurrence offsets live inside the payload. The layer preserves first-occurrence
+ * order: the first token of a term fixes its position in the layer.
+ *
+ * The annotator holds no per-call state; it is as thread-safe as the normalizer it
+ * was built with.
+ *
+ * @since 3.0.0
+ */
+public final class TermVectorAnnotator implements DocumentAnnotator {
+
+ /**
+ * The key of the term vector layer this annotator provides: a document-scoped layer
+ * of {@link TermVector} values, one per distinct term.
+ */
+ public static final LayerKey TERM_VECTORS =
+ Layers.documentKey("term-vectors", TermVector.class);
+
+ /** How much each {@link TermVector} records. */
+ public enum Mode {
+ /**
+ * Counts occurrences and stores every occurrence span in original text coordinates.
+ */
+ FULL,
+ /**
+ * Counts occurrences only; the emitted {@link TermVector term vectors} carry no
+ * spans, so scoring-only consumers do not pay for offset storage.
+ */
+ SCORING_ONLY
+ }
+
+ private final OffsetAwareNormalizer normalizer;
+ private final CharSequenceNormalizer tokenNormalizer;
+ private final Mode mode;
+
+ /**
+ * Initializes a {@link Mode#FULL full mode} annotator that groups tokens by their
+ * covered text as-is.
+ */
+ public TermVectorAnnotator() {
+ this(Mode.FULL);
+ }
+
+ /**
+ * Initializes an annotator that groups tokens by their covered text as-is.
+ *
+ * @param mode How much each {@link TermVector} records. Must not be {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code mode} is {@code null}.
+ */
+ public TermVectorAnnotator(Mode mode) {
+ if (mode == null) {
+ throw new IllegalArgumentException("mode must not be null");
+ }
+ this.normalizer = null;
+ this.tokenNormalizer = null;
+ this.mode = mode;
+ }
+
+ /**
+ * Initializes a {@link Mode#FULL full mode} annotator that groups tokens by their
+ * per-token normalized form. This is the general path: the normalizer is applied to
+ * each token's covered text on its own, so any {@link CharSequenceNormalizer} works,
+ * including the folds that cannot report an alignment (case folding, NFC, accent
+ * folding, stemmer-backed normalizers). The occurrence spans stay the tokens' own
+ * spans in the original text.
+ *
+ * @param normalizer The normalizer that defines term identity, applied to each token's
+ * covered text. Must not be {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code normalizer} is {@code null}.
+ */
+ public TermVectorAnnotator(CharSequenceNormalizer normalizer) {
+ this(normalizer, Mode.FULL);
+ }
+
+ /**
+ * Initializes an annotator that groups tokens by their per-token normalized form. This
+ * is the general path: the normalizer is applied to each token's covered text on its
+ * own, so any {@link CharSequenceNormalizer} works, including the folds that cannot
+ * report an alignment (case folding, NFC, accent folding, stemmer-backed normalizers).
+ * The occurrence spans stay the tokens' own spans in the original text.
+ *
+ * @param normalizer The normalizer that defines term identity, applied to each token's
+ * covered text. Must not be {@code null}.
+ * @param mode How much each {@link TermVector} records. Must not be {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code normalizer} or {@code mode} is
+ * {@code null}.
+ */
+ public TermVectorAnnotator(CharSequenceNormalizer normalizer, Mode mode) {
+ if (normalizer == null) {
+ throw new IllegalArgumentException("normalizer must not be null");
+ }
+ if (mode == null) {
+ throw new IllegalArgumentException("mode must not be null");
+ }
+ this.normalizer = null;
+ this.tokenNormalizer = normalizer;
+ this.mode = mode;
+ }
+
+ /**
+ * Initializes a {@link Mode#FULL full mode} annotator that groups tokens by their
+ * normalized form through a whole-document alignment. Prefer the
+ * {@link #TermVectorAnnotator(CharSequenceNormalizer) plain-normalizer constructor}
+ * as the general path; this one only accepts alignment-reporting normalizers but can
+ * see across token boundaries, for example a whitespace collapse spanning two tokens.
+ *
+ * @param normalizer The normalizer that defines term identity, applied to the whole
+ * document text so token spans can be mapped into the normalized
+ * form through its alignment. Must not be {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code normalizer} is {@code null}.
+ */
+ public TermVectorAnnotator(OffsetAwareNormalizer normalizer) {
+ this(normalizer, Mode.FULL);
+ }
+
+ /**
+ * Initializes an annotator that groups tokens by their normalized form through a
+ * whole-document alignment. Prefer the
+ * {@link #TermVectorAnnotator(CharSequenceNormalizer, Mode) plain-normalizer
+ * constructor} as the general path; this one only accepts alignment-reporting
+ * normalizers but can see across token boundaries, for example a whitespace collapse
+ * spanning two tokens.
+ *
+ * @param normalizer The normalizer that defines term identity, applied to the whole
+ * document text so token spans can be mapped into the normalized
+ * form through its alignment. Must not be {@code null}.
+ * @param mode How much each {@link TermVector} records. Must not be {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code normalizer} or {@code mode} is
+ * {@code null}.
+ */
+ public TermVectorAnnotator(OffsetAwareNormalizer normalizer, Mode mode) {
+ if (normalizer == null) {
+ throw new IllegalArgumentException("normalizer must not be null");
+ }
+ if (mode == null) {
+ throw new IllegalArgumentException("mode must not be null");
+ }
+ this.normalizer = normalizer;
+ this.tokenNormalizer = null;
+ this.mode = mode;
+ }
+
+ /**
+ * Aggregates the token layer into the {@link #TERM_VECTORS} layer. A present-but-empty
+ * token layer yields a present-but-empty term vector layer.
+ *
+ * @param document The document to annotate. Must not be {@code null} and must contain
+ * the {@link Layers#TOKENS} layer.
+ * @return A new {@link Document} with the {@link #TERM_VECTORS} layer added. Never
+ * {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, lacks
+ * the {@link Layers#TOKENS} layer, or already carries the
+ * {@link #TERM_VECTORS} layer.
+ */
+ @Override
+ public Document annotate(Document document) {
+ if (document == null) {
+ throw new IllegalArgumentException("document must not be null");
+ }
+ if (!document.layers().contains(Layers.TOKENS)) {
+ throw new IllegalArgumentException("document lacks the required layer "
+ + Layers.TOKENS);
+ }
+ final List> tokens = document.get(Layers.TOKENS);
+ final AlignedText aligned =
+ normalizer != null ? normalizer.normalizeAligned(document.text()) : null;
+ final String normalized = aligned != null ? aligned.normalizedString() : null;
+ return document.with(TERM_VECTORS, mode == Mode.FULL
+ ? fullVectors(tokens, aligned, normalized)
+ : countVectors(tokens, aligned, normalized));
+ }
+
+ /**
+ * Aggregates the tokens into full term vectors, keeping every occurrence span.
+ *
+ * @param tokens The token layer.
+ * @param aligned The normalized document text with its alignment, or {@code null} when
+ * no normalizer is present.
+ * @param normalized The normalized document text, or {@code null} when no normalizer is
+ * present.
+ * @return One annotation per distinct term, in first-occurrence order.
+ */
+ private List> fullVectors(List> tokens,
+ AlignedText aligned, String normalized) {
+ final Map> spansByTerm = new LinkedHashMap<>();
+ for (final Annotation token : tokens) {
+ final String term = termOf(token, aligned, normalized);
+ if (!term.isEmpty()) {
+ spansByTerm.computeIfAbsent(term, key -> new ArrayList<>()).add(token.span());
+ }
+ }
+ final List> vectors = new ArrayList<>(spansByTerm.size());
+ for (final Map.Entry> entry : spansByTerm.entrySet()) {
+ vectors.add(Annotation.of(TermVector.withSpans(entry.getKey(), entry.getValue())));
+ }
+ return vectors;
+ }
+
+ /**
+ * Aggregates the tokens into scoring-only term vectors, so no offset storage is ever
+ * allocated.
+ *
+ * @param tokens The token layer.
+ * @param aligned The normalized document text with its alignment, or {@code null} when
+ * no normalizer is present.
+ * @param normalized The normalized document text, or {@code null} when no normalizer is
+ * present.
+ * @return One annotation per distinct term, in first-occurrence order.
+ */
+ private List> countVectors(List> tokens,
+ AlignedText aligned, String normalized) {
+ final Map frequencies = new LinkedHashMap<>();
+ for (final Annotation token : tokens) {
+ final String term = termOf(token, aligned, normalized);
+ if (!term.isEmpty()) {
+ frequencies.merge(term, 1, Integer::sum);
+ }
+ }
+ final List> vectors = new ArrayList<>(frequencies.size());
+ for (final Map.Entry entry : frequencies.entrySet()) {
+ vectors.add(Annotation.of(TermVector.count(entry.getKey(), entry.getValue())));
+ }
+ return vectors;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Set> requires() {
+ return Set.of(Layers.TOKENS);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Set> provides() {
+ return Set.of(TERM_VECTORS);
+ }
+
+ /**
+ * Determines the term one token groups under: its covered text as-is, its covered text
+ * normalized on its own when a plain per-token normalizer is present, or the covered
+ * text of its span mapped into the normalized form when an offset-aware normalizer is
+ * present.
+ *
+ * @param token The token annotation.
+ * @param aligned The normalized document text with its alignment, or {@code null}
+ * when no offset-aware normalizer is present.
+ * @param normalized The normalized document text, or {@code null} when no offset-aware
+ * normalizer is present.
+ * @return The term string. Never {@code null}, possibly empty.
+ */
+ private String termOf(Annotation token, AlignedText aligned, String normalized) {
+ if (tokenNormalizer != null) {
+ return tokenNormalizer.normalize(token.value()).toString();
+ }
+ if (aligned == null) {
+ return token.value();
+ }
+ final Span span = aligned.toNormalizedSpan(token.span().getStart(), token.span().getEnd());
+ return normalized.substring(span.getStart(), span.getEnd());
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java
new file mode 100644
index 0000000000..4f7beb4643
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.termvector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import opennlp.tools.document.Annotation;
+import opennlp.tools.tokenize.Tokenizer;
+import opennlp.tools.util.Span;
+
+/**
+ * Deterministic single-space tokenization shared by the term vector tests: splits on
+ * single space characters and keeps all other characters, including sentence-final
+ * periods, attached to their token. Runs of spaces yield no empty tokens, so every
+ * expected span follows directly from the input text.
+ */
+final class SingleSpaceTokens {
+
+ /**
+ * A {@link Tokenizer} view of {@link #spans(String)}. Only the span-producing method
+ * is implemented because the annotator adapter calls no other method.
+ */
+ static final Tokenizer TOKENIZER = new Tokenizer() {
+
+ @Override
+ public String[] tokenize(String s) {
+ throw new UnsupportedOperationException("the adapter only calls tokenizePos");
+ }
+
+ @Override
+ public Span[] tokenizePos(String s) {
+ return spans(s).toArray(new Span[0]);
+ }
+ };
+
+ private SingleSpaceTokens() {
+ }
+
+ /**
+ * Computes the token spans of a text split on single space characters.
+ *
+ * @param text The text to split.
+ * @return One span per token, in text order.
+ */
+ static List spans(String text) {
+ final List spans = new ArrayList<>();
+ int start = -1;
+ for (int i = 0; i <= text.length(); i++) {
+ final boolean boundary = i == text.length() || text.charAt(i) == ' ';
+ if (boundary && start >= 0) {
+ spans.add(new Span(start, i));
+ start = -1;
+ } else if (!boundary && start < 0) {
+ start = i;
+ }
+ }
+ return spans;
+ }
+
+ /**
+ * Builds a token layer from {@link #spans(String)}, each token valued with its covered
+ * text.
+ *
+ * @param text The text to split.
+ * @return One annotation per token, in text order.
+ */
+ static List> tokens(String text) {
+ final List> tokens = new ArrayList<>();
+ for (final Span span : spans(text)) {
+ tokens.add(new Annotation<>(span, text.substring(span.getStart(), span.getEnd())));
+ }
+ return tokens;
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java
new file mode 100644
index 0000000000..33559371f9
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java
@@ -0,0 +1,424 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.termvector;
+
+import java.io.Serial;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.document.Annotation;
+import opennlp.tools.document.Document;
+import opennlp.tools.document.LayerKey;
+import opennlp.tools.document.Layers;
+import opennlp.tools.util.Span;
+import opennlp.tools.util.normalizer.AlignedText;
+import opennlp.tools.util.normalizer.Alignment;
+import opennlp.tools.util.normalizer.CharSequenceNormalizer;
+import opennlp.tools.util.normalizer.OffsetAwareNormalizer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Verifies the {@link TermVectorAnnotator} roll-up: term identity with and without a
+ * normalizer, both recording modes, and the graceful-degradation rules of the document
+ * pipeline. The token layer is built directly here; the wiring through a
+ * {@code DocumentAnalyzer} is covered by {@code TermVectorPipelineTest}.
+ */
+public class TermVectorAnnotatorTest {
+
+ /**
+ * A deterministic {@link OffsetAwareNormalizer} that collapses every run of
+ * whitespace to one space and applies a full case fold, including the German eszett
+ * expansion {@code ß -> ss}. Every edit is recorded in an {@link Alignment}, so the
+ * expected spans below follow directly from the input text.
+ */
+ private static final class WhitespaceCaseFoldNormalizer implements OffsetAwareNormalizer {
+
+ @Serial
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public CharSequence normalize(CharSequence text) {
+ return normalizeAligned(text).normalized();
+ }
+
+ @Override
+ public AlignedText normalizeAligned(CharSequence text) {
+ final String source = text.toString();
+ final StringBuilder out = new StringBuilder(source.length());
+ final Alignment.Builder alignment = new Alignment.Builder(source.length());
+ int i = 0;
+ while (i < source.length()) {
+ final char c = source.charAt(i);
+ if (Character.isWhitespace(c)) {
+ int end = i;
+ while (end < source.length() && Character.isWhitespace(source.charAt(end))) {
+ end++;
+ }
+ out.append(' ');
+ alignment.replace(end - i, 1);
+ i = end;
+ } else if (c == 'ß') {
+ out.append("ss");
+ alignment.replace(1, 2);
+ i++;
+ } else {
+ final String folded = String.valueOf(c).toLowerCase(Locale.ROOT);
+ out.append(folded);
+ if (folded.length() == 1 && folded.charAt(0) == c) {
+ alignment.equal(1);
+ } else {
+ alignment.replace(1, folded.length());
+ }
+ i++;
+ }
+ }
+ return new AlignedText(source, out.toString(), alignment.build(source.length()));
+ }
+ }
+
+ /**
+ * A deterministic {@link OffsetAwareNormalizer} that deletes every digit and copies
+ * every other character unchanged, so a token made up of digits alone normalizes to
+ * the empty string.
+ */
+ private static final class DigitDeletingNormalizer implements OffsetAwareNormalizer {
+
+ @Serial
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public CharSequence normalize(CharSequence text) {
+ return normalizeAligned(text).normalized();
+ }
+
+ @Override
+ public AlignedText normalizeAligned(CharSequence text) {
+ final String source = text.toString();
+ final StringBuilder out = new StringBuilder(source.length());
+ final Alignment.Builder alignment = new Alignment.Builder(source.length());
+ for (int i = 0; i < source.length(); i++) {
+ final char c = source.charAt(i);
+ if (Character.isDigit(c)) {
+ alignment.replace(1, 0);
+ } else {
+ out.append(c);
+ alignment.equal(1);
+ }
+ }
+ return new AlignedText(source, out.toString(), alignment.build(source.length()));
+ }
+ }
+
+ private static final OffsetAwareNormalizer FOLD = new WhitespaceCaseFoldNormalizer();
+
+ private static final OffsetAwareNormalizer DROP_DIGITS = new DigitDeletingNormalizer();
+
+ /**
+ * A plain, alignment-free case folder for the per-token path: it stands in for the
+ * shipped normalizers (case fold, NFC, accent fold) that cannot report offsets and
+ * therefore cannot implement {@link OffsetAwareNormalizer}.
+ */
+ private static final CharSequenceNormalizer PLAIN_LOWER =
+ text -> text.toString().toLowerCase(Locale.ROOT);
+
+ /**
+ * A plain normalizer that deletes every digit, so a token made up of digits alone
+ * normalizes to the empty string on the per-token path.
+ */
+ private static final CharSequenceNormalizer PLAIN_DROP_DIGITS = text -> {
+ final StringBuilder out = new StringBuilder(text.length());
+ for (int i = 0; i < text.length(); i++) {
+ if (!Character.isDigit(text.charAt(i))) {
+ out.append(text.charAt(i));
+ }
+ }
+ return out.toString();
+ };
+
+ private static Document documentWithTokens(String text) {
+ return Document.of(text).with(Layers.TOKENS, SingleSpaceTokens.tokens(text));
+ }
+
+ @Test
+ void testFullModeGroupsByCoveredTextAsIs() {
+ final Document document = new TermVectorAnnotator()
+ .annotate(documentWithTokens("The dog barks. The dog naps."));
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(4, vectors.size());
+ // The layer preserves first-occurrence order.
+ assertEquals(new TermVector("The", 2, List.of(new Span(0, 3), new Span(15, 18))),
+ vectors.get(0).value());
+ assertEquals(new TermVector("dog", 2, List.of(new Span(4, 7), new Span(19, 22))),
+ vectors.get(1).value());
+ assertEquals(new TermVector("barks.", 1, List.of(new Span(8, 14))),
+ vectors.get(2).value());
+ assertEquals(new TermVector("naps.", 1, List.of(new Span(23, 28))),
+ vectors.get(3).value());
+ }
+
+ @Test
+ void testLayerIsDocumentScopedAndCarriesNoAnnotationSpans() {
+ final Document document = new TermVectorAnnotator()
+ .annotate(documentWithTokens("The dog barks."));
+ assertEquals(LayerKey.Scope.DOCUMENT, TermVectorAnnotator.TERM_VECTORS.scope());
+ for (final Annotation vector : document.get(TermVectorAnnotator.TERM_VECTORS)) {
+ assertNull(vector.span());
+ }
+ }
+
+ @Test
+ void testScoringOnlyModeOmitsSpans() {
+ final Document document = new TermVectorAnnotator(TermVectorAnnotator.Mode.SCORING_ONLY)
+ .annotate(documentWithTokens("The dog barks. The dog naps."));
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(4, vectors.size());
+ assertEquals(TermVector.count("The", 2), vectors.get(0).value());
+ assertEquals(TermVector.count("dog", 2), vectors.get(1).value());
+ assertEquals(TermVector.count("barks.", 1), vectors.get(2).value());
+ assertEquals(TermVector.count("naps.", 1), vectors.get(3).value());
+ for (final Annotation vector : vectors) {
+ assertTrue(vector.value().spans().isEmpty());
+ }
+ }
+
+ /**
+ * The offset-fidelity case: whitespace collapse shifts offsets and the eszett case
+ * fold grows the text, yet every emitted occurrence span must land on the original
+ * text. {@code "Groß groß GROSS"} normalizes to {@code "gross gross gross"}, so all
+ * three tokens group under one term while their spans keep pointing at the original
+ * surface forms.
+ */
+ @Test
+ void testNormalizationGroupsFoldedTokensWithOriginalOffsets() {
+ final String text = "Groß groß GROSS";
+ final Document document = new TermVectorAnnotator(FOLD)
+ .annotate(documentWithTokens(text));
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(1, vectors.size());
+ final TermVector vector = vectors.get(0).value();
+ assertEquals("gross", vector.term());
+ assertEquals(3, vector.frequency());
+ assertEquals(List.of(new Span(0, 4), new Span(6, 10), new Span(12, 17)), vector.spans());
+
+ // Every occurrence span covers the original surface form, not the normalized one.
+ final List surfaceForms =
+ vector.spans().stream().map(s -> s.getCoveredText(text).toString()).toList();
+ assertEquals(List.of("Groß", "groß", "GROSS"), surfaceForms);
+
+ // The same spans round-trip through the alignment: a token's normalized span maps
+ // back to exactly its original span, through both edits.
+ final AlignedText aligned = FOLD.normalizeAligned(text);
+ assertEquals("gross gross gross", aligned.normalizedString());
+ assertEquals(new Span(0, 4), aligned.toOriginalSpan(0, 5));
+ assertEquals(new Span(6, 10), aligned.toOriginalSpan(6, 11));
+ assertEquals(new Span(12, 17), aligned.toOriginalSpan(12, 17));
+ }
+
+ @Test
+ void testNormalizationKeepsDistinctTermsApart() {
+ final String text = "Das große Haus ist groß";
+ final Document document = new TermVectorAnnotator(FOLD)
+ .annotate(documentWithTokens(text));
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(5, vectors.size());
+ assertEquals(new TermVector("das", 1, List.of(new Span(0, 3))), vectors.get(0).value());
+ assertEquals(new TermVector("grosse", 1, List.of(new Span(5, 10))), vectors.get(1).value());
+ assertEquals(new TermVector("haus", 1, List.of(new Span(12, 16))), vectors.get(2).value());
+ assertEquals(new TermVector("ist", 1, List.of(new Span(18, 21))), vectors.get(3).value());
+ assertEquals(new TermVector("gross", 1, List.of(new Span(23, 27))), vectors.get(4).value());
+ }
+
+ @Test
+ void testScoringOnlyModeWithNormalizerCountsWithoutOffsets() {
+ final Document document = new TermVectorAnnotator(FOLD,
+ TermVectorAnnotator.Mode.SCORING_ONLY).annotate(documentWithTokens("Groß groß GROSS"));
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(1, vectors.size());
+ assertEquals(TermVector.count("gross", 3), vectors.get(0).value());
+ }
+
+ /**
+ * The general per-token path: a plain {@link CharSequenceNormalizer} defines term
+ * identity by folding each token's covered text, no alignment involved, while every
+ * occurrence span stays the token's own span in the original text. This admits the
+ * folds {@code buildAligned()} rejects (case fold, NFC, accent fold).
+ */
+ @Test
+ void testPlainNormalizerGroupsFoldedTokensWithOriginalSpans() {
+ final String text = "Word word WORD";
+ final Document document = new TermVectorAnnotator(PLAIN_LOWER)
+ .annotate(documentWithTokens(text));
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(1, vectors.size());
+ assertEquals(new TermVector("word", 3,
+ List.of(new Span(0, 4), new Span(5, 9), new Span(10, 14))), vectors.get(0).value());
+
+ // Every occurrence span covers the original surface form, not the folded one.
+ final List surfaceForms = vectors.get(0).value().spans().stream()
+ .map(s -> s.getCoveredText(text).toString()).toList();
+ assertEquals(List.of("Word", "word", "WORD"), surfaceForms);
+ }
+
+ @Test
+ void testPlainNormalizerScoringOnlyModeCountsWithoutOffsets() {
+ final Document document = new TermVectorAnnotator(PLAIN_LOWER,
+ TermVectorAnnotator.Mode.SCORING_ONLY).annotate(documentWithTokens("Word word WORD"));
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(1, vectors.size());
+ assertEquals(TermVector.count("word", 3), vectors.get(0).value());
+ }
+
+ /**
+ * Empty-term omission behaves identically on the per-token path: tokens a plain
+ * normalizer folds to the empty string are left out of the layer, matching
+ * {@link #testTokensNormalizedAwayAreOmitted()}.
+ */
+ @Test
+ void testPlainNormalizerOmitsDeletedTokens() {
+ final Document document = new TermVectorAnnotator(PLAIN_DROP_DIGITS)
+ .annotate(documentWithTokens("dog 42 dog 7"));
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(1, vectors.size());
+ assertEquals(new TermVector("dog", 2, List.of(new Span(0, 3), new Span(7, 10))),
+ vectors.get(0).value());
+ }
+
+ @Test
+ void testNullPlainNormalizerIsRejected() {
+ final CharSequenceNormalizer noNormalizer = null;
+ assertThrows(IllegalArgumentException.class, () -> new TermVectorAnnotator(noNormalizer));
+ assertThrows(IllegalArgumentException.class,
+ () -> new TermVectorAnnotator(noNormalizer, TermVectorAnnotator.Mode.FULL));
+ assertThrows(IllegalArgumentException.class,
+ () -> new TermVectorAnnotator(PLAIN_LOWER, null));
+ }
+
+ /**
+ * A token the normalizer deletes entirely is omitted from the layer: an empty string
+ * is no term, and the token layer still accounts for the token itself.
+ */
+ @Test
+ void testTokensNormalizedAwayAreOmitted() {
+ final Document document = new TermVectorAnnotator(DROP_DIGITS)
+ .annotate(documentWithTokens("dog 42 dog 7"));
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(1, vectors.size());
+ assertEquals(new TermVector("dog", 2, List.of(new Span(0, 3), new Span(7, 10))),
+ vectors.get(0).value());
+ }
+
+ /**
+ * Spans are UTF-16 offsets, so a supplementary-plane character occupies two positions:
+ * {@code "𝕏 x 𝕏"} tokenizes to spans of width two around the surrogate pairs, and both
+ * occurrences group under one term whose spans still cover the original text exactly.
+ */
+ @Test
+ void testSupplementaryPlaneTokensKeepUtf16Offsets() {
+ final String text = "𝕏 x 𝕏";
+ final Document document = new TermVectorAnnotator()
+ .annotate(documentWithTokens(text));
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(2, vectors.size());
+ assertEquals(new TermVector("𝕏", 2, List.of(new Span(0, 2), new Span(5, 7))),
+ vectors.get(0).value());
+ assertEquals(new TermVector("x", 1, List.of(new Span(3, 4))), vectors.get(1).value());
+ for (final Annotation vector : vectors) {
+ for (final Span span : vector.value().spans()) {
+ assertEquals(vector.value().term(), span.getCoveredText(text).toString());
+ }
+ }
+ }
+
+ /**
+ * A document whose every token normalizes to the empty string yields the layer
+ * present but empty, the same graceful degradation as an empty token layer.
+ */
+ @Test
+ void testAllTokensNormalizedAwayYieldPresentButEmptyLayer() {
+ final Document document = new TermVectorAnnotator(DROP_DIGITS)
+ .annotate(documentWithTokens("42 7"));
+ assertTrue(document.layers().contains(TermVectorAnnotator.TERM_VECTORS));
+ assertTrue(document.get(TermVectorAnnotator.TERM_VECTORS).isEmpty());
+ }
+
+ @Test
+ void testEmptyTokenLayerYieldsPresentButEmptyLayer() {
+ final Document document = new TermVectorAnnotator()
+ .annotate(Document.of("").with(Layers.TOKENS, List.of()));
+ assertTrue(document.layers().contains(TermVectorAnnotator.TERM_VECTORS));
+ assertTrue(document.get(TermVectorAnnotator.TERM_VECTORS).isEmpty());
+ }
+
+ @Test
+ void testMissingTokenLayerIsRejected() {
+ final TermVectorAnnotator annotator = new TermVectorAnnotator();
+ final Document bare = Document.of("The dog barks.");
+ assertThrows(IllegalArgumentException.class, () -> annotator.annotate(bare));
+ }
+
+ @Test
+ void testNullDocumentIsRejected() {
+ final TermVectorAnnotator annotator = new TermVectorAnnotator();
+ assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null));
+ }
+
+ @Test
+ void testNullConstructorArgumentsAreRejected() {
+ final TermVectorAnnotator.Mode noMode = null;
+ final OffsetAwareNormalizer noNormalizer = null;
+ assertThrows(IllegalArgumentException.class, () -> new TermVectorAnnotator(noMode));
+ assertThrows(IllegalArgumentException.class, () -> new TermVectorAnnotator(noNormalizer));
+ assertThrows(IllegalArgumentException.class, () -> new TermVectorAnnotator(FOLD, null));
+ assertThrows(IllegalArgumentException.class,
+ () -> new TermVectorAnnotator(null, TermVectorAnnotator.Mode.FULL));
+ }
+
+ @Test
+ void testRequiresAndProvides() {
+ final TermVectorAnnotator annotator = new TermVectorAnnotator();
+ assertEquals(Set.of(Layers.TOKENS), annotator.requires());
+ assertEquals(Set.of(TermVectorAnnotator.TERM_VECTORS), annotator.provides());
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java
new file mode 100644
index 0000000000..2868b021e7
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.termvector;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.document.Annotation;
+import opennlp.tools.document.Document;
+import opennlp.tools.document.DocumentAnalyzer;
+import opennlp.tools.tokenize.TokenizerAnnotator;
+import opennlp.tools.tokenize.WhitespaceTokenizer;
+import opennlp.tools.util.Span;
+import opennlp.tools.util.normalizer.CharSequenceNormalizer;
+import opennlp.tools.util.normalizer.TextNormalizer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Mirrors the normalized-term workflow shown in the term vector section of the manual:
+ * a whitespace tokenizer feeding a {@link TermVectorAnnotator} built with a shipped,
+ * plain {@link CharSequenceNormalizer} case folder. The folder defines term identity per
+ * token, so case variants group under one term, while every occurrence span stays the
+ * token's own span in the original text.
+ */
+public class TermVectorNormalizedExampleTest {
+
+ /**
+ * The documented example: {@code "Word word WORD"} yields one term, {@code "word"},
+ * with three occurrence spans, each the token's exact original span.
+ */
+ @Test
+ void testCaseFoldedTermsKeepOriginalSpans() {
+ final CharSequenceNormalizer folder = TextNormalizer.builder().caseFold().build();
+ final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
+ .add(new TokenizerAnnotator(WhitespaceTokenizer.INSTANCE))
+ .add(new TermVectorAnnotator(folder))
+ .build();
+
+ final Document document = analyzer.analyze("Word word WORD");
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(1, vectors.size());
+ assertEquals(new TermVector("word", 3,
+ List.of(new Span(0, 4), new Span(5, 9), new Span(10, 14))), vectors.get(0).value());
+
+ // The spans point at the original surface forms, not the folded term.
+ final List surfaceForms = vectors.get(0).value().spans().stream()
+ .map(s -> s.getCoveredText(document.text()).toString()).toList();
+ assertEquals(List.of("Word", "word", "WORD"), surfaceForms);
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java
new file mode 100644
index 0000000000..8510cfdac9
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.termvector;
+
+import java.util.List;
+import java.util.Set;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.document.Annotation;
+import opennlp.tools.document.Document;
+import opennlp.tools.document.DocumentAnalyzer;
+import opennlp.tools.document.Layers;
+import opennlp.tools.tokenize.TokenizerAnnotator;
+import opennlp.tools.util.Span;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Wires the {@link TermVectorAnnotator} into a {@link DocumentAnalyzer} behind a
+ * {@link TokenizerAnnotator}: the token layer goes in, the term vector layer comes out,
+ * and nothing else about the document changes. The tokenizer is the shared deterministic
+ * single-space fixture, so every expected span follows directly from the input text.
+ */
+public class TermVectorPipelineTest {
+
+ /**
+ * Runs tokenizer plus term vector roll-up over a text with repeated tokens and reads
+ * the aggregated layer back, span by span, in original text coordinates.
+ */
+ @Test
+ void testTokenizerAndTermVectorPipeline() {
+ final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
+ .add(new TokenizerAnnotator(SingleSpaceTokens.TOKENIZER))
+ .add(new TermVectorAnnotator())
+ .build();
+
+ final Document document = analyzer.analyze("The dog barks. The dog naps.");
+
+ // The document carries exactly the two layers the pipeline provides.
+ assertEquals(Set.of(Layers.TOKENS, TermVectorAnnotator.TERM_VECTORS), document.layers());
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(4, vectors.size());
+ assertEquals(new TermVector("The", 2, List.of(new Span(0, 3), new Span(15, 18))),
+ vectors.get(0).value());
+ assertEquals(new TermVector("dog", 2, List.of(new Span(4, 7), new Span(19, 22))),
+ vectors.get(1).value());
+ assertEquals(new TermVector("barks.", 1, List.of(new Span(8, 14))),
+ vectors.get(2).value());
+ assertEquals(new TermVector("naps.", 1, List.of(new Span(23, 28))),
+ vectors.get(3).value());
+
+ // Every occurrence span indexes into the original text and covers its own term.
+ for (final Annotation vector : vectors) {
+ for (final Span span : vector.value().spans()) {
+ assertEquals(vector.value().term(), span.getCoveredText(document.text()).toString());
+ }
+ }
+ }
+
+ /**
+ * The same pipeline in scoring-only mode: counts survive, offsets are never stored.
+ */
+ @Test
+ void testScoringOnlyPipeline() {
+ final DocumentAnalyzer analyzer = DocumentAnalyzer.builder()
+ .add(new TokenizerAnnotator(SingleSpaceTokens.TOKENIZER))
+ .add(new TermVectorAnnotator(TermVectorAnnotator.Mode.SCORING_ONLY))
+ .build();
+
+ final Document document = analyzer.analyze("The dog barks. The dog naps.");
+
+ final List> vectors =
+ document.get(TermVectorAnnotator.TERM_VECTORS);
+ assertEquals(4, vectors.size());
+ assertEquals(TermVector.count("The", 2), vectors.get(0).value());
+ assertEquals(TermVector.count("dog", 2), vectors.get(1).value());
+ }
+
+ /**
+ * The analyzer validates the pipeline at build time: the term vector annotator
+ * requires the token layer, so a pipeline without a tokenizer fails when it is
+ * assembled.
+ */
+ @Test
+ void testPipelineWithoutTokenizerIsRejectedAtBuildTime() {
+ final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder()
+ .add(new TermVectorAnnotator());
+ assertThrows(IllegalArgumentException.class, builder::build);
+ }
+}
diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml
index 7177fdf0bb..169a34c2fd 100644
--- a/opennlp-docs/src/docbkx/document.xml
+++ b/opennlp-docs/src/docbkx/document.xml
@@ -335,4 +335,60 @@ int firstTokenLength = lengths.get(0).value(); // 3, for "The" at [0..3)]]>
adapters use.
+
+
+ Term vectors
+
+ TermVectorAnnotator rolls the token layer up into a document-scoped
+ term vector layer for index consumers: one TermVector per distinct
+ term, carrying the term string, its occurrence count, and the occurrence spans in
+ original text coordinates. Term identity comes from the annotator's inputs: without
+ a normalizer the term is the token's covered text as-is; with a plain
+ CharSequenceNormalizer, the general path, each token's covered text is
+ normalized on its own to become the term; with an
+ OffsetAwareNormalizer the document text is normalized once and each
+ token's covered normalized text becomes the term. On every path tokens differing
+ only by a normalization fold group together while their spans keep pointing into
+ the original text. A token whose normalized form is empty is omitted from the
+ layer. The layer preserves first-occurrence order.
+ TermVectorPipelineTest#testTokenizerAndTermVectorPipeline asserts the
+ behavior shown here.
+ > vectors = document.get(TermVectorAnnotator.TERM_VECTORS);
+// vectors.get(1).value() is ("dog", 2, [[4..7), [19..22)])]]>
+
+
+
+ The normalized-term workflow passes any shipped
+ CharSequenceNormalizer to the plain-normalizer constructor: the
+ normalizer is applied to each token's covered text to produce the term, so the
+ case, NFC, and accent folds that buildAligned() rejects all work
+ here, and every occurrence span stays the token's own span in the original text.
+ TermVectorNormalizedExampleTest#testCaseFoldedTermsKeepOriginalSpans
+ asserts the behavior shown here.
+ > vectors = document.get(TermVectorAnnotator.TERM_VECTORS);
+// vectors.get(0).value() is ("word", 3, [[0..4), [5..9), [10..14)])]]>
+
+
+
+ Scoring-only consumers that never read offsets can skip storing them:
+ new TermVectorAnnotator(TermVectorAnnotator.Mode.SCORING_ONLY) emits
+ vectors that carry counts but no spans, and
+ TermVector.count("dog", 2) builds the matching expectation in tests.
+
+