From cb8c43b1ff45ddbacce5aa74f27afd25bacfa936 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 26 Jul 2026 18:42:24 -0400 Subject: [PATCH 1/9] OPENNLP-1897: Add a term vector layer rolling tokens up to (term, frequency, offsets) A new opennlp.tools.termvector package aggregates the document token layer into a document-scoped layer of TermVector records for index consumers, without touching the opennlp.tools.document container. TermVector carries the term string, the occurrence count, and the occurrence spans in original text coordinates. It comes in two shapes: full (one span per occurrence) and scoring-only (counts only, no offset storage). TermVectorAnnotator implements DocumentAnnotator: it requires Layers.TOKENS and provides its own opennlp:term-vectors key. Term identity is delegated, never analyzed: without a normalizer the token's covered text groups as-is; with an OffsetAwareNormalizer the document text is normalized once and each token span is mapped through the alignment, so tokens differing only by a normalization fold (case, eszett expansion, collapsed whitespace) group together while every emitted occurrence span still points into the original text. --- .../opennlp/tools/termvector/TermVector.java | 111 +++++++ .../tools/termvector/TermVectorAnnotator.java | 243 ++++++++++++++++ .../termvector/TermVectorAnnotatorTest.java | 273 ++++++++++++++++++ .../termvector/TermVectorPipelineTest.java | 142 +++++++++ .../tools/termvector/TermVectorTest.java | 93 ++++++ 5 files changed, 862 insertions(+) create mode 100644 opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java new file mode 100644 index 0000000000..7dba46c044 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.termvector; + +import java.util.List; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.Span; + +/** + * One entry of a term vector layer: a term, how often it occurs in the document, and + * where. + * + *

The {@link #term()} is the term's identity as the producing annotator determined + * it, typically a normalized form; the {@link #spans()} are the occurrence offsets and + * always point into the document's original text, never into a normalized form, + * so an index consumer can highlight every occurrence in what the caller supplied.

+ * + *

A term vector comes in one of two shapes. A full vector carries one span + * per occurrence, so {@code spans().size() == frequency()}. A scoring-only + * vector carries no spans at all, so consumers that only need term frequencies do not + * pay for offset storage; the empty span list is what distinguishes the shape, never a + * flag. These are the only legal shapes: a non-empty span list must match the frequency + * exactly.

+ * + *

Instances are immutable: the span list is copied on construction and the copy is + * unmodifiable.

+ * + * @param term The term string. Must not be {@code null}. + * @param frequency The number of occurrences in the document. Must be at least one. + * @param spans The occurrence spans in original text coordinates, one per occurrence, + * or an empty list for a scoring-only vector. Must not be {@code null} and, + * when non-empty, must contain exactly {@code frequency} spans. + * + * @since 3.0.0 + */ +@ThreadSafe +public record TermVector(String term, int frequency, List spans) { + + /** + * Validates the term vector and detaches the span list from the caller's input. + * + * @throws IllegalArgumentException Thrown if {@code term} is {@code null}, + * {@code frequency} is below one, {@code spans} is {@code null}, or a + * non-empty {@code spans} list does not hold exactly {@code frequency} spans. + */ + public TermVector { + if (term == null) { + throw new IllegalArgumentException("term must not be null"); + } + if (frequency < 1) { + throw new IllegalArgumentException("frequency must be at least one: " + frequency); + } + if (spans == null) { + throw new IllegalArgumentException("spans must not be null"); + } + if (!spans.isEmpty() && spans.size() != frequency) { + throw new IllegalArgumentException("a full term vector holds one span per " + + "occurrence: frequency is " + frequency + " but spans holds " + spans.size()); + } + spans = List.copyOf(spans); + } + + /** + * Creates a full {@link TermVector} whose frequency is derived from the occurrence + * spans. + * + * @param term The term string. Must not be {@code null}. + * @param spans The occurrence spans in original text coordinates. Must not be + * {@code null} or empty. + * @return A {@link TermVector} with {@code frequency() == spans.size()}. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code term} is {@code null} or + * {@code spans} is {@code null} or empty. + */ + public static TermVector withSpans(String term, List spans) { + if (spans == null || spans.isEmpty()) { + throw new IllegalArgumentException("spans must not be null or empty"); + } + return new TermVector(term, spans.size(), spans); + } + + /** + * Creates a scoring-only {@link TermVector} that carries the occurrence count without + * any offsets. + * + * @param term The term string. Must not be {@code null}. + * @param frequency The number of occurrences in the document. Must be at least one. + * @return A {@link TermVector} whose {@link #spans()} is empty. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code term} is {@code null} or + * {@code frequency} is below one. + */ + public static TermVector count(String term, int frequency) { + return new TermVector(term, frequency, List.of()); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java new file mode 100644 index 0000000000..ff3f2af7b5 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.termvector; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnnotator; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; +import opennlp.tools.util.normalizer.AlignedText; +import opennlp.tools.util.normalizer.OffsetAwareNormalizer; + +/** + * Rolls the token layer up into a term vector layer for index consumers: one + * {@link TermVector} per distinct term, carrying the term string, its occurrence count, + * and (in {@link Mode#FULL full mode}) the occurrence offsets. + * + *

The annotator aggregates; it does not analyze. What makes two tokens the same term + * is decided by the inputs it is given, never by logic of its own. Without a normalizer, + * the term is the token layer's value as-is, that is, the token's covered text in the + * original document. With an {@link OffsetAwareNormalizer}, the whole document text is + * normalized once with its alignment recorded, each token span is mapped forward to the + * normalized form, and the covered normalized text is the term, so tokens that differ + * only by a normalization fold (case, an eszett expansion, collapsed whitespace around + * them) group together. Either way, the occurrence spans emitted in + * {@link Mode#FULL full mode} are the token layer's own spans and therefore always + * point into the original text. A token whose normalized form is empty, for example one + * the normalizer deleted entirely, groups under the empty string rather than being + * dropped.

+ * + *

The layer is {@link LayerKey.Scope#DOCUMENT document-scoped}: each {@link TermVector} + * is a whole-document statistic, so the annotations carry no span of their own and the + * occurrence offsets live inside the payload. The layer preserves first-occurrence + * order: the first token of a term fixes its position in the layer.

+ * + *

The annotator holds no per-call state; it is as thread-safe as the normalizer it + * was built with.

+ * + * @since 3.0.0 + */ +public class TermVectorAnnotator implements DocumentAnnotator { + + /** + * The key of the term vector layer this annotator provides: a document-scoped layer + * of {@link TermVector} values, one per distinct term. + */ + public static final LayerKey TERM_VECTORS = + Layers.documentKey("term-vectors", TermVector.class); + + /** How much each {@link TermVector} records. */ + public enum Mode { + /** + * Counts occurrences and stores every occurrence span in original text coordinates. + */ + FULL, + /** + * Counts occurrences only; the emitted {@link TermVector term vectors} carry no + * spans, so scoring-only consumers do not pay for offset storage. + */ + SCORING_ONLY + } + + private final OffsetAwareNormalizer normalizer; + private final Mode mode; + + /** + * Initializes a {@link Mode#FULL full mode} annotator that groups tokens by their + * covered text as-is. + */ + public TermVectorAnnotator() { + this.normalizer = null; + this.mode = Mode.FULL; + } + + /** + * Initializes an annotator that groups tokens by their covered text as-is. + * + * @param mode How much each {@link TermVector} records. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code mode} is {@code null}. + */ + public TermVectorAnnotator(Mode mode) { + this.normalizer = null; + this.mode = requireMode(mode); + } + + /** + * Initializes a {@link Mode#FULL full mode} annotator that groups tokens by their + * normalized form. + * + * @param normalizer The normalizer that defines term identity, applied to the whole + * document text so token spans can be mapped into the normalized + * form through its alignment. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code normalizer} is {@code null}. + */ + public TermVectorAnnotator(OffsetAwareNormalizer normalizer) { + this.normalizer = requireNormalizer(normalizer); + this.mode = Mode.FULL; + } + + /** + * Initializes an annotator that groups tokens by their normalized form. + * + * @param normalizer The normalizer that defines term identity, applied to the whole + * document text so token spans can be mapped into the normalized + * form through its alignment. Must not be {@code null}. + * @param mode How much each {@link TermVector} records. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code normalizer} or {@code mode} is + * {@code null}. + */ + public TermVectorAnnotator(OffsetAwareNormalizer normalizer, Mode mode) { + this.normalizer = requireNormalizer(normalizer); + this.mode = requireMode(mode); + } + + /** + * Aggregates the token layer into the {@link #TERM_VECTORS} layer. A present-but-empty + * token layer yields a present-but-empty term vector layer. + * + * @param document The document to annotate. Must not be {@code null} and must contain + * the {@link Layers#TOKENS} layer. + * @return A new {@link Document} with the {@link #TERM_VECTORS} layer added. Never + * {@code null}. + * @throws IllegalArgumentException Thrown if {@code document} is {@code null}, lacks + * the {@link Layers#TOKENS} layer, or already carries the + * {@link #TERM_VECTORS} layer. + */ + @Override + public Document annotate(Document document) { + if (document == null) { + throw new IllegalArgumentException("document must not be null"); + } + if (!document.layers().contains(Layers.TOKENS)) { + throw new IllegalArgumentException("document lacks the required layer " + + Layers.TOKENS); + } + final List> tokens = document.get(Layers.TOKENS); + final AlignedText aligned = + normalizer != null ? normalizer.normalizeAligned(document.text()) : null; + final String normalized = aligned != null ? aligned.normalizedString() : null; + + final Map frequencies = new LinkedHashMap<>(); + // Null in SCORING_ONLY mode, so offset storage is never allocated. + final Map> spansByTerm = mode == Mode.FULL + ? new LinkedHashMap<>() : null; + for (final Annotation token : tokens) { + final String term = termOf(token, aligned, normalized); + frequencies.merge(term, 1, Integer::sum); + if (spansByTerm != null) { + spansByTerm.computeIfAbsent(term, key -> new ArrayList<>()).add(token.span()); + } + } + + final List> vectors = new ArrayList<>(frequencies.size()); + for (final Map.Entry entry : frequencies.entrySet()) { + final TermVector vector = spansByTerm != null + ? TermVector.withSpans(entry.getKey(), spansByTerm.get(entry.getKey())) + : TermVector.count(entry.getKey(), entry.getValue()); + vectors.add(Annotation.of(vector)); + } + return document.with(TERM_VECTORS, vectors); + } + + /** {@inheritDoc} */ + @Override + public Set> requires() { + return Set.of(Layers.TOKENS); + } + + /** {@inheritDoc} */ + @Override + public Set> provides() { + return Set.of(TERM_VECTORS); + } + + /** + * Determines the term one token groups under: its covered text as-is, or the covered + * text of its span mapped into the normalized form when a normalizer is present. + * + * @param token The token annotation. + * @param aligned The normalized document text with its alignment, or {@code null} + * when no normalizer is present. + * @param normalized The normalized document text, or {@code null} likewise. + * @return The term string. Never {@code null}, possibly empty. + */ + private static String termOf(Annotation token, AlignedText aligned, + String normalized) { + if (aligned == null) { + return token.value(); + } + final Span span = aligned.toNormalizedSpan(token.span().getStart(), token.span().getEnd()); + return normalized.substring(span.getStart(), span.getEnd()); + } + + /** + * Validates a mode argument. + * + * @param mode The mode to validate. + * @return The validated mode. + * @throws IllegalArgumentException Thrown if {@code mode} is {@code null}. + */ + private static Mode requireMode(Mode mode) { + if (mode == null) { + throw new IllegalArgumentException("mode must not be null"); + } + return mode; + } + + /** + * Validates a normalizer argument. + * + * @param normalizer The normalizer to validate. + * @return The validated normalizer. + * @throws IllegalArgumentException Thrown if {@code normalizer} is {@code null}. + */ + private static OffsetAwareNormalizer requireNormalizer(OffsetAwareNormalizer normalizer) { + if (normalizer == null) { + throw new IllegalArgumentException("normalizer must not be null"); + } + return normalizer; + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java new file mode 100644 index 0000000000..869431d59c --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.termvector; + +import java.io.Serial; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.LayerKey; +import opennlp.tools.document.Layers; +import opennlp.tools.util.Span; +import opennlp.tools.util.normalizer.AlignedText; +import opennlp.tools.util.normalizer.Alignment; +import opennlp.tools.util.normalizer.OffsetAwareNormalizer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verifies the {@link TermVectorAnnotator} roll-up: term identity with and without a + * normalizer, both recording modes, and the graceful-degradation rules of the document + * pipeline. The token layer is built directly here; the wiring through a + * {@code DocumentAnalyzer} is covered by {@code TermVectorPipelineTest}. + */ +public class TermVectorAnnotatorTest { + + /** + * A deterministic {@link OffsetAwareNormalizer} that collapses every run of + * whitespace to one space and applies a full case fold, including the German eszett + * expansion {@code ß -> ss}. Every edit is recorded in an {@link Alignment} built the + * way {@code AlignmentTest} builds them, so the expected spans below follow directly + * from the input text. + */ + private static final class WhitespaceCaseFoldNormalizer implements OffsetAwareNormalizer { + + @Serial + private static final long serialVersionUID = 1L; + + @Override + public CharSequence normalize(CharSequence text) { + return normalizeAligned(text).normalized(); + } + + @Override + public AlignedText normalizeAligned(CharSequence text) { + final String source = text.toString(); + final StringBuilder out = new StringBuilder(source.length()); + final Alignment.Builder alignment = new Alignment.Builder(source.length()); + int i = 0; + while (i < source.length()) { + final char c = source.charAt(i); + if (Character.isWhitespace(c)) { + int end = i; + while (end < source.length() && Character.isWhitespace(source.charAt(end))) { + end++; + } + out.append(' '); + alignment.replace(end - i, 1); + i = end; + } else if (c == 'ß') { + out.append("ss"); + alignment.replace(1, 2); + i++; + } else { + final String folded = String.valueOf(c).toLowerCase(Locale.ROOT); + out.append(folded); + if (folded.length() == 1 && folded.charAt(0) == c) { + alignment.equal(1); + } else { + alignment.replace(1, folded.length()); + } + i++; + } + } + return new AlignedText(source, out.toString(), alignment.build(source.length())); + } + } + + private static final OffsetAwareNormalizer FOLD = new WhitespaceCaseFoldNormalizer(); + + /** + * Splits a text on single space characters into a token layer, skipping the empty + * tokens a collapsed run would produce, so double spaces yield no tokens. + */ + private static List> splitOnSpaces(String text) { + final List> tokens = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= text.length(); i++) { + final boolean boundary = i == text.length() || text.charAt(i) == ' '; + if (boundary && start >= 0) { + tokens.add(new Annotation<>(new Span(start, i), text.substring(start, i))); + start = -1; + } else if (!boundary && start < 0) { + start = i; + } + } + return tokens; + } + + private static Document documentWithTokens(String text) { + return Document.of(text).with(Layers.TOKENS, splitOnSpaces(text)); + } + + @Test + void testFullModeGroupsByCoveredTextAsIs() { + final Document document = new TermVectorAnnotator() + .annotate(documentWithTokens("The dog barks. The dog naps.")); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(4, vectors.size()); + // The layer preserves first-occurrence order. + assertEquals(new TermVector("The", 2, List.of(new Span(0, 3), new Span(15, 18))), + vectors.get(0).value()); + assertEquals(new TermVector("dog", 2, List.of(new Span(4, 7), new Span(19, 22))), + vectors.get(1).value()); + assertEquals(new TermVector("barks.", 1, List.of(new Span(8, 14))), + vectors.get(2).value()); + assertEquals(new TermVector("naps.", 1, List.of(new Span(23, 28))), + vectors.get(3).value()); + } + + @Test + void testLayerIsDocumentScopedAndCarriesNoAnnotationSpans() { + final Document document = new TermVectorAnnotator() + .annotate(documentWithTokens("The dog barks.")); + assertEquals(LayerKey.Scope.DOCUMENT, TermVectorAnnotator.TERM_VECTORS.scope()); + for (final Annotation vector : document.get(TermVectorAnnotator.TERM_VECTORS)) { + assertNull(vector.span()); + } + } + + @Test + void testScoringOnlyModeOmitsSpans() { + final Document document = new TermVectorAnnotator(TermVectorAnnotator.Mode.SCORING_ONLY) + .annotate(documentWithTokens("The dog barks. The dog naps.")); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(4, vectors.size()); + assertEquals(TermVector.count("The", 2), vectors.get(0).value()); + assertEquals(TermVector.count("dog", 2), vectors.get(1).value()); + assertEquals(TermVector.count("barks.", 1), vectors.get(2).value()); + assertEquals(TermVector.count("naps.", 1), vectors.get(3).value()); + for (final Annotation vector : vectors) { + assertTrue(vector.value().spans().isEmpty()); + } + } + + /** + * The offset-fidelity case: whitespace collapse shifts offsets and the eszett case + * fold grows the text, yet every emitted occurrence span must land on the original + * text. {@code "Groß groß GROSS"} normalizes to {@code "gross gross gross"}, so all + * three tokens group under one term while their spans keep pointing at the original + * surface forms. + */ + @Test + void testNormalizationGroupsFoldedTokensWithOriginalOffsets() { + final String text = "Groß groß GROSS"; + final Document document = new TermVectorAnnotator(FOLD) + .annotate(documentWithTokens(text)); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(1, vectors.size()); + final TermVector vector = vectors.get(0).value(); + assertEquals("gross", vector.term()); + assertEquals(3, vector.frequency()); + assertEquals(List.of(new Span(0, 4), new Span(6, 10), new Span(12, 17)), vector.spans()); + + // Every occurrence span covers the original surface form, not the normalized one. + final List surfaceForms = + vector.spans().stream().map(s -> s.getCoveredText(text).toString()).toList(); + assertEquals(List.of("Groß", "groß", "GROSS"), surfaceForms); + + // The same spans round-trip through the alignment: a token's normalized span maps + // back to exactly its original span, through both edits. + final AlignedText aligned = FOLD.normalizeAligned(text); + assertEquals("gross gross gross", aligned.normalizedString()); + assertEquals(new Span(0, 4), aligned.toOriginalSpan(0, 5)); + assertEquals(new Span(6, 10), aligned.toOriginalSpan(6, 11)); + assertEquals(new Span(12, 17), aligned.toOriginalSpan(12, 17)); + } + + @Test + void testNormalizationKeepsDistinctTermsApart() { + final String text = "Das große Haus ist groß"; + final Document document = new TermVectorAnnotator(FOLD) + .annotate(documentWithTokens(text)); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(5, vectors.size()); + assertEquals(new TermVector("das", 1, List.of(new Span(0, 3))), vectors.get(0).value()); + assertEquals(new TermVector("grosse", 1, List.of(new Span(5, 10))), vectors.get(1).value()); + assertEquals(new TermVector("haus", 1, List.of(new Span(12, 16))), vectors.get(2).value()); + assertEquals(new TermVector("ist", 1, List.of(new Span(18, 21))), vectors.get(3).value()); + assertEquals(new TermVector("gross", 1, List.of(new Span(23, 27))), vectors.get(4).value()); + } + + @Test + void testScoringOnlyModeWithNormalizerCountsWithoutOffsets() { + final Document document = new TermVectorAnnotator(FOLD, + TermVectorAnnotator.Mode.SCORING_ONLY).annotate(documentWithTokens("Groß groß GROSS")); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(1, vectors.size()); + assertEquals(TermVector.count("gross", 3), vectors.get(0).value()); + } + + @Test + void testEmptyTokenLayerYieldsPresentButEmptyLayer() { + final Document document = new TermVectorAnnotator() + .annotate(Document.of("").with(Layers.TOKENS, List.of())); + assertTrue(document.layers().contains(TermVectorAnnotator.TERM_VECTORS)); + assertTrue(document.get(TermVectorAnnotator.TERM_VECTORS).isEmpty()); + } + + @Test + void testMissingTokenLayerIsRejected() { + final TermVectorAnnotator annotator = new TermVectorAnnotator(); + final Document bare = Document.of("The dog barks."); + assertThrows(IllegalArgumentException.class, () -> annotator.annotate(bare)); + } + + @Test + void testNullDocumentIsRejected() { + final TermVectorAnnotator annotator = new TermVectorAnnotator(); + assertThrows(IllegalArgumentException.class, () -> annotator.annotate(null)); + } + + @Test + void testNullConstructorArgumentsAreRejected() { + final TermVectorAnnotator.Mode noMode = null; + final OffsetAwareNormalizer noNormalizer = null; + assertThrows(IllegalArgumentException.class, () -> new TermVectorAnnotator(noMode)); + assertThrows(IllegalArgumentException.class, () -> new TermVectorAnnotator(noNormalizer)); + assertThrows(IllegalArgumentException.class, () -> new TermVectorAnnotator(FOLD, null)); + assertThrows(IllegalArgumentException.class, + () -> new TermVectorAnnotator(null, TermVectorAnnotator.Mode.FULL)); + } + + @Test + void testRequiresAndProvides() { + final TermVectorAnnotator annotator = new TermVectorAnnotator(); + assertEquals(Set.of(Layers.TOKENS), annotator.requires()); + assertEquals(Set.of(TermVectorAnnotator.TERM_VECTORS), annotator.provides()); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java new file mode 100644 index 0000000000..2e558f7f27 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.termvector; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnalyzer; +import opennlp.tools.document.Layers; +import opennlp.tools.document.TokenizerAnnotator; +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Wires the {@link TermVectorAnnotator} into a {@link DocumentAnalyzer} behind a + * {@link TokenizerAnnotator}, the way {@code DocumentPipelineExampleTest} demonstrates + * the pipeline: the token layer goes in, the term vector layer comes out, and nothing + * else about the document changes. The tokenizer is a deterministic stand-in defined + * here, so every expected span follows directly from the input text. + */ +public class TermVectorPipelineTest { + + /** + * A deterministic tokenizer that splits on single space characters and keeps all + * other characters, including sentence-final periods, attached to their token. Only + * the span-producing method is implemented because the adapter calls no other method. + */ + private static final Tokenizer SPACE_TOKENIZER = new Tokenizer() { + + @Override + public String[] tokenize(String s) { + throw new UnsupportedOperationException("the adapter only calls tokenizePos"); + } + + @Override + public Span[] tokenizePos(String s) { + final List spans = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= s.length(); i++) { + final boolean boundary = i == s.length() || s.charAt(i) == ' '; + if (boundary && start >= 0) { + spans.add(new Span(start, i)); + start = -1; + } else if (!boundary && start < 0) { + start = i; + } + } + return spans.toArray(new Span[0]); + } + }; + + /** + * Runs tokenizer plus term vector roll-up over a text with repeated tokens and reads + * the aggregated layer back, span by span, in original text coordinates. + */ + @Test + void testTokenizerAndTermVectorPipeline() { + final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() + .add(new TokenizerAnnotator(SPACE_TOKENIZER)) + .add(new TermVectorAnnotator()) + .build(); + + final Document document = analyzer.analyze("The dog barks. The dog naps."); + + // The document carries exactly the two layers the pipeline provides. + assertEquals(Set.of(Layers.TOKENS, TermVectorAnnotator.TERM_VECTORS), document.layers()); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(4, vectors.size()); + assertEquals(new TermVector("The", 2, List.of(new Span(0, 3), new Span(15, 18))), + vectors.get(0).value()); + assertEquals(new TermVector("dog", 2, List.of(new Span(4, 7), new Span(19, 22))), + vectors.get(1).value()); + assertEquals(new TermVector("barks.", 1, List.of(new Span(8, 14))), + vectors.get(2).value()); + assertEquals(new TermVector("naps.", 1, List.of(new Span(23, 28))), + vectors.get(3).value()); + + // Every occurrence span indexes into the original text. + for (final Annotation vector : vectors) { + for (final Span span : vector.value().spans()) { + assertEquals(span.getCoveredText(document.text()).toString(), + document.text().subSequence(span.getStart(), span.getEnd()).toString()); + } + } + } + + /** + * The same pipeline in scoring-only mode: counts survive, offsets are never stored. + */ + @Test + void testScoringOnlyPipeline() { + final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() + .add(new TokenizerAnnotator(SPACE_TOKENIZER)) + .add(new TermVectorAnnotator(TermVectorAnnotator.Mode.SCORING_ONLY)) + .build(); + + final Document document = analyzer.analyze("The dog barks. The dog naps."); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(4, vectors.size()); + assertEquals(TermVector.count("The", 2), vectors.get(0).value()); + assertEquals(TermVector.count("dog", 2), vectors.get(1).value()); + } + + /** + * The analyzer validates the pipeline at build time: the term vector annotator + * requires the token layer, so a pipeline without a tokenizer fails when it is + * assembled. + */ + @Test + void testPipelineWithoutTokenizerIsRejectedAtBuildTime() { + final DocumentAnalyzer.Builder builder = DocumentAnalyzer.builder() + .add(new TermVectorAnnotator()); + assertThrows(IllegalArgumentException.class, builder::build); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java new file mode 100644 index 0000000000..7f01069925 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.termvector; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verifies the {@link TermVector} invariants: the two legal shapes (full with one span + * per occurrence, scoring-only with none), the validation of everything in between, and + * the immutability of the span list. + */ +public class TermVectorTest { + + @Test + void testWithSpansDerivesTheFrequency() { + final TermVector vector = + TermVector.withSpans("dog", List.of(new Span(4, 7), new Span(19, 22))); + assertEquals("dog", vector.term()); + assertEquals(2, vector.frequency()); + assertEquals(List.of(new Span(4, 7), new Span(19, 22)), vector.spans()); + } + + @Test + void testCountCarriesNoSpans() { + final TermVector vector = TermVector.count("dog", 3); + assertEquals("dog", vector.term()); + assertEquals(3, vector.frequency()); + assertTrue(vector.spans().isEmpty()); + } + + @Test + void testSpanListIsDetachedFromTheCallersInput() { + final List spans = new ArrayList<>(List.of(new Span(0, 3))); + final TermVector vector = TermVector.withSpans("the", spans); + spans.add(new Span(4, 7)); + assertEquals(1, vector.spans().size()); + assertThrows(UnsupportedOperationException.class, + () -> vector.spans().add(new Span(8, 11))); + } + + @Test + void testNullTermIsRejected() { + assertThrows(IllegalArgumentException.class, + () -> new TermVector(null, 1, List.of(new Span(0, 1)))); + } + + @Test + void testNullSpanListIsRejected() { + assertThrows(IllegalArgumentException.class, () -> new TermVector("dog", 1, null)); + } + + @Test + void testZeroFrequencyIsRejected() { + assertThrows(IllegalArgumentException.class, () -> TermVector.count("dog", 0)); + } + + @Test + void testEmptySpanListCannotDeriveAFrequency() { + assertThrows(IllegalArgumentException.class, + () -> TermVector.withSpans("dog", List.of())); + } + + @Test + void testPartialSpanListIsRejected() { + // Two occurrences but only one recorded span: neither full nor scoring-only. + assertThrows(IllegalArgumentException.class, + () -> new TermVector("dog", 2, List.of(new Span(0, 3)))); + } +} From c6e788431e1da846d0136bb25249803a159c9771 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 07:00:16 -0400 Subject: [PATCH 2/9] OPENNLP-1897: Address review: fold the constructor chain, split aggregation by mode - Delegate the two convenience constructors of TermVectorAnnotator through this(...) instead of repeating the field assignments, so the no-arg form is defined as FULL mode and the normalizer-only form as normalizer plus FULL. - Drop the requireMode and requireNormalizer helpers and do the null checks inline in the two canonical constructors, matching the argument-validation style used elsewhere in the module and keeping the thrown message next to the parameter it guards. - Split annotate into fullVectors and countVectors, one per Mode, so the scoring-only path no longer carries a null span map as a mode sentinel and the per-term branch inside the emit loop disappears. Each helper is documented and returns the annotations in first-occurrence order, which is the ordering the tests pin. - Make termOf an instance method, since it is now only reached from the two mode helpers and no longer needs to be static to be shared. - Tighten the TermVector class javadoc: the two shapes are told apart by whether spans() is empty, stated once, without the redundant aside about a flag, and the closing sentence now names the invariant instead of repeating the shape list. - Tighten the TermVectorAnnotator class javadoc the same way, and fix the termOf parameter doc that read "or null likewise" to spell out the condition. - Remove the javadoc references to AlignmentTest and DocumentPipelineExampleTest from the test fixtures. Those tests are not part of the contract under test here and the pointers go stale as soon as either file moves. - Add a DigitDeletingNormalizer fixture and a pinning test for a token the normalizer deletes entirely. It groups under the empty term rather than being dropped, which the class javadoc promises but nothing exercised. - Parameterize the frequency rejection test over 0, -1 and Integer.MIN_VALUE instead of only 0, so the guard is pinned across the whole illegal range. - Fix the span assertion in TermVectorPipelineTest. It compared Span.getCoveredText against an equivalent subSequence of the same text, which holds for any span, so it now asserts that the covered text equals the vector's own term. --- .../opennlp/tools/termvector/TermVector.java | 11 +- .../tools/termvector/TermVectorAnnotator.java | 128 ++++++++++-------- .../termvector/TermVectorAnnotatorTest.java | 58 +++++++- .../termvector/TermVectorPipelineTest.java | 12 +- .../tools/termvector/TermVectorTest.java | 9 +- 5 files changed, 139 insertions(+), 79 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java index 7dba46c044..676bf01317 100644 --- a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java +++ b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java @@ -31,12 +31,11 @@ * always point into the document's original text, never into a normalized form, * so an index consumer can highlight every occurrence in what the caller supplied.

* - *

A term vector comes in one of two shapes. A full vector carries one span - * per occurrence, so {@code spans().size() == frequency()}. A scoring-only - * vector carries no spans at all, so consumers that only need term frequencies do not - * pay for offset storage; the empty span list is what distinguishes the shape, never a - * flag. These are the only legal shapes: a non-empty span list must match the frequency - * exactly.

+ *

A term vector comes in one of two shapes, told apart by whether {@link #spans()} is + * empty. A full vector carries one span per occurrence, so + * {@code spans().size() == frequency()}. A scoring-only vector carries no spans + * at all, so consumers that only need term frequencies do not pay for offset storage. + * There is no third shape: a non-empty span list must match the frequency exactly.

* *

Instances are immutable: the span list is copied on construction and the copy is * unmodifiable.

diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java index ff3f2af7b5..e30dd6e7af 100644 --- a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java @@ -37,14 +37,13 @@ * {@link TermVector} per distinct term, carrying the term string, its occurrence count, * and (in {@link Mode#FULL full mode}) the occurrence offsets. * - *

The annotator aggregates; it does not analyze. What makes two tokens the same term - * is decided by the inputs it is given, never by logic of its own. Without a normalizer, - * the term is the token layer's value as-is, that is, the token's covered text in the - * original document. With an {@link OffsetAwareNormalizer}, the whole document text is - * normalized once with its alignment recorded, each token span is mapped forward to the - * normalized form, and the covered normalized text is the term, so tokens that differ - * only by a normalization fold (case, an eszett expansion, collapsed whitespace around - * them) group together. Either way, the occurrence spans emitted in + *

Term identity comes from the annotator's inputs, not from logic of its own. Without + * a normalizer, the term is the token layer's value as-is, that is, the token's covered + * text in the original document. With an {@link OffsetAwareNormalizer}, the whole + * document text is normalized once with its alignment recorded, each token span is + * mapped forward to the normalized form, and the covered normalized text is the term, so + * tokens that differ only by a normalization fold (case, an eszett expansion, collapsed + * whitespace) group together. Either way, the occurrence spans emitted in * {@link Mode#FULL full mode} are the token layer's own spans and therefore always * point into the original text. A token whose normalized form is empty, for example one * the normalizer deleted entirely, groups under the empty string rather than being @@ -90,8 +89,7 @@ public enum Mode { * covered text as-is. */ public TermVectorAnnotator() { - this.normalizer = null; - this.mode = Mode.FULL; + this(Mode.FULL); } /** @@ -101,8 +99,11 @@ public TermVectorAnnotator() { * @throws IllegalArgumentException Thrown if {@code mode} is {@code null}. */ public TermVectorAnnotator(Mode mode) { + if (mode == null) { + throw new IllegalArgumentException("mode must not be null"); + } this.normalizer = null; - this.mode = requireMode(mode); + this.mode = mode; } /** @@ -115,8 +116,7 @@ public TermVectorAnnotator(Mode mode) { * @throws IllegalArgumentException Thrown if {@code normalizer} is {@code null}. */ public TermVectorAnnotator(OffsetAwareNormalizer normalizer) { - this.normalizer = requireNormalizer(normalizer); - this.mode = Mode.FULL; + this(normalizer, Mode.FULL); } /** @@ -130,8 +130,14 @@ public TermVectorAnnotator(OffsetAwareNormalizer normalizer) { * {@code null}. */ public TermVectorAnnotator(OffsetAwareNormalizer normalizer, Mode mode) { - this.normalizer = requireNormalizer(normalizer); - this.mode = requireMode(mode); + if (normalizer == null) { + throw new IllegalArgumentException("normalizer must not be null"); + } + if (mode == null) { + throw new IllegalArgumentException("mode must not be null"); + } + this.normalizer = normalizer; + this.mode = mode; } /** @@ -159,27 +165,57 @@ public Document annotate(Document document) { final AlignedText aligned = normalizer != null ? normalizer.normalizeAligned(document.text()) : null; final String normalized = aligned != null ? aligned.normalizedString() : null; + return document.with(TERM_VECTORS, mode == Mode.FULL + ? fullVectors(tokens, aligned, normalized) + : countVectors(tokens, aligned, normalized)); + } - final Map frequencies = new LinkedHashMap<>(); - // Null in SCORING_ONLY mode, so offset storage is never allocated. - final Map> spansByTerm = mode == Mode.FULL - ? new LinkedHashMap<>() : null; + /** + * Aggregates the tokens into full term vectors, keeping every occurrence span. + * + * @param tokens The token layer. + * @param aligned The normalized document text with its alignment, or {@code null} when + * no normalizer is present. + * @param normalized The normalized document text, or {@code null} when no normalizer is + * present. + * @return One annotation per distinct term, in first-occurrence order. + */ + private List> fullVectors(List> tokens, + AlignedText aligned, String normalized) { + final Map> spansByTerm = new LinkedHashMap<>(); for (final Annotation token : tokens) { - final String term = termOf(token, aligned, normalized); - frequencies.merge(term, 1, Integer::sum); - if (spansByTerm != null) { - spansByTerm.computeIfAbsent(term, key -> new ArrayList<>()).add(token.span()); - } + spansByTerm.computeIfAbsent(termOf(token, aligned, normalized), key -> new ArrayList<>()) + .add(token.span()); + } + final List> vectors = new ArrayList<>(spansByTerm.size()); + for (final Map.Entry> entry : spansByTerm.entrySet()) { + vectors.add(Annotation.of(TermVector.withSpans(entry.getKey(), entry.getValue()))); } + return vectors; + } + /** + * Aggregates the tokens into scoring-only term vectors, so no offset storage is ever + * allocated. + * + * @param tokens The token layer. + * @param aligned The normalized document text with its alignment, or {@code null} when + * no normalizer is present. + * @param normalized The normalized document text, or {@code null} when no normalizer is + * present. + * @return One annotation per distinct term, in first-occurrence order. + */ + private List> countVectors(List> tokens, + AlignedText aligned, String normalized) { + final Map frequencies = new LinkedHashMap<>(); + for (final Annotation token : tokens) { + frequencies.merge(termOf(token, aligned, normalized), 1, Integer::sum); + } final List> vectors = new ArrayList<>(frequencies.size()); for (final Map.Entry entry : frequencies.entrySet()) { - final TermVector vector = spansByTerm != null - ? TermVector.withSpans(entry.getKey(), spansByTerm.get(entry.getKey())) - : TermVector.count(entry.getKey(), entry.getValue()); - vectors.add(Annotation.of(vector)); + vectors.add(Annotation.of(TermVector.count(entry.getKey(), entry.getValue()))); } - return document.with(TERM_VECTORS, vectors); + return vectors; } /** {@inheritDoc} */ @@ -201,43 +237,15 @@ public Set> provides() { * @param token The token annotation. * @param aligned The normalized document text with its alignment, or {@code null} * when no normalizer is present. - * @param normalized The normalized document text, or {@code null} likewise. + * @param normalized The normalized document text, or {@code null} when no normalizer is + * present. * @return The term string. Never {@code null}, possibly empty. */ - private static String termOf(Annotation token, AlignedText aligned, - String normalized) { + private String termOf(Annotation token, AlignedText aligned, String normalized) { if (aligned == null) { return token.value(); } final Span span = aligned.toNormalizedSpan(token.span().getStart(), token.span().getEnd()); return normalized.substring(span.getStart(), span.getEnd()); } - - /** - * Validates a mode argument. - * - * @param mode The mode to validate. - * @return The validated mode. - * @throws IllegalArgumentException Thrown if {@code mode} is {@code null}. - */ - private static Mode requireMode(Mode mode) { - if (mode == null) { - throw new IllegalArgumentException("mode must not be null"); - } - return mode; - } - - /** - * Validates a normalizer argument. - * - * @param normalizer The normalizer to validate. - * @return The validated normalizer. - * @throws IllegalArgumentException Thrown if {@code normalizer} is {@code null}. - */ - private static OffsetAwareNormalizer requireNormalizer(OffsetAwareNormalizer normalizer) { - if (normalizer == null) { - throw new IllegalArgumentException("normalizer must not be null"); - } - return normalizer; - } } diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java index 869431d59c..ee6f852cd3 100644 --- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java @@ -50,9 +50,8 @@ public class TermVectorAnnotatorTest { /** * A deterministic {@link OffsetAwareNormalizer} that collapses every run of * whitespace to one space and applies a full case fold, including the German eszett - * expansion {@code ß -> ss}. Every edit is recorded in an {@link Alignment} built the - * way {@code AlignmentTest} builds them, so the expected spans below follow directly - * from the input text. + * expansion {@code ß -> ss}. Every edit is recorded in an {@link Alignment}, so the + * expected spans below follow directly from the input text. */ private static final class WhitespaceCaseFoldNormalizer implements OffsetAwareNormalizer { @@ -99,8 +98,43 @@ public AlignedText normalizeAligned(CharSequence text) { } } + /** + * A deterministic {@link OffsetAwareNormalizer} that deletes every digit and copies + * every other character unchanged, so a token made up of digits alone normalizes to + * the empty string. + */ + private static final class DigitDeletingNormalizer implements OffsetAwareNormalizer { + + @Serial + private static final long serialVersionUID = 1L; + + @Override + public CharSequence normalize(CharSequence text) { + return normalizeAligned(text).normalized(); + } + + @Override + public AlignedText normalizeAligned(CharSequence text) { + final String source = text.toString(); + final StringBuilder out = new StringBuilder(source.length()); + final Alignment.Builder alignment = new Alignment.Builder(source.length()); + for (int i = 0; i < source.length(); i++) { + final char c = source.charAt(i); + if (Character.isDigit(c)) { + alignment.replace(1, 0); + } else { + out.append(c); + alignment.equal(1); + } + } + return new AlignedText(source, out.toString(), alignment.build(source.length())); + } + } + private static final OffsetAwareNormalizer FOLD = new WhitespaceCaseFoldNormalizer(); + private static final OffsetAwareNormalizer DROP_DIGITS = new DigitDeletingNormalizer(); + /** * Splits a text on single space characters into a token layer, skipping the empty * tokens a collapsed run would produce, so double spaces yield no tokens. @@ -232,6 +266,24 @@ void testScoringOnlyModeWithNormalizerCountsWithoutOffsets() { assertEquals(TermVector.count("gross", 3), vectors.get(0).value()); } + /** + * A token the normalizer deletes entirely groups under the empty string instead of + * being dropped, so the term vector layer still accounts for every token. + */ + @Test + void testTokensNormalizedAwayGroupUnderTheEmptyTerm() { + final Document document = new TermVectorAnnotator(DROP_DIGITS) + .annotate(documentWithTokens("dog 42 dog 7")); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(2, vectors.size()); + assertEquals(new TermVector("dog", 2, List.of(new Span(0, 3), new Span(7, 10))), + vectors.get(0).value()); + assertEquals(new TermVector("", 2, List.of(new Span(4, 6), new Span(11, 12))), + vectors.get(1).value()); + } + @Test void testEmptyTokenLayerYieldsPresentButEmptyLayer() { final Document document = new TermVectorAnnotator() diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java index 2e558f7f27..a00fba96bd 100644 --- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java @@ -36,10 +36,9 @@ /** * Wires the {@link TermVectorAnnotator} into a {@link DocumentAnalyzer} behind a - * {@link TokenizerAnnotator}, the way {@code DocumentPipelineExampleTest} demonstrates - * the pipeline: the token layer goes in, the term vector layer comes out, and nothing - * else about the document changes. The tokenizer is a deterministic stand-in defined - * here, so every expected span follows directly from the input text. + * {@link TokenizerAnnotator}: the token layer goes in, the term vector layer comes out, + * and nothing else about the document changes. The tokenizer is a deterministic stand-in + * defined here, so every expected span follows directly from the input text. */ public class TermVectorPipelineTest { @@ -100,11 +99,10 @@ void testTokenizerAndTermVectorPipeline() { assertEquals(new TermVector("naps.", 1, List.of(new Span(23, 28))), vectors.get(3).value()); - // Every occurrence span indexes into the original text. + // Every occurrence span indexes into the original text and covers its own term. for (final Annotation vector : vectors) { for (final Span span : vector.value().spans()) { - assertEquals(span.getCoveredText(document.text()).toString(), - document.text().subSequence(span.getStart(), span.getEnd()).toString()); + assertEquals(vector.value().term(), span.getCoveredText(document.text()).toString()); } } } diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java index 7f01069925..286b7a2c34 100644 --- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java @@ -21,6 +21,8 @@ import java.util.List; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.util.Span; @@ -73,9 +75,10 @@ void testNullSpanListIsRejected() { assertThrows(IllegalArgumentException.class, () -> new TermVector("dog", 1, null)); } - @Test - void testZeroFrequencyIsRejected() { - assertThrows(IllegalArgumentException.class, () -> TermVector.count("dog", 0)); + @ParameterizedTest + @ValueSource(ints = {0, -1, Integer.MIN_VALUE}) + void testFrequencyBelowOneIsRejected(int frequency) { + assertThrows(IllegalArgumentException.class, () -> TermVector.count("dog", frequency)); } @Test From 5c63d9232ae3d54f779c8a9c79a3acff851ae4b2 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 8 Aug 2026 18:51:03 -0400 Subject: [PATCH 3/9] OPENNLP-1897: Document the term vector layer with a mirror-tested example Adds a term vectors section to the document container chapter, citing TermVectorPipelineTest#testTokenizerAndTermVectorPipeline as the pin for the programlisting and covering the scoring-only mode. --- opennlp-docs/src/docbkx/document.xml | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index 7177fdf0bb..c67a4b6460 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -335,4 +335,37 @@ int firstTokenLength = lengths.get(0).value(); // 3, for "The" at [0..3)]]> adapters use. + +

+ Term vectors + + TermVectorAnnotator rolls the token layer up into a document-scoped + term vector layer for index consumers: one TermVector per distinct + term, carrying the term string, its occurrence count, and the occurrence spans in + original text coordinates. Term identity comes from the annotator's inputs: without + a normalizer the term is the token's covered text as-is; with an + OffsetAwareNormalizer the document text is normalized once and each + token's covered normalized text becomes the term, so tokens differing only by a + normalization fold group together while their spans keep pointing into the original + text. The layer preserves first-occurrence order. + TermVectorPipelineTest#testTokenizerAndTermVectorPipeline asserts the + behavior shown here. + > vectors = document.get(TermVectorAnnotator.TERM_VECTORS); +// vectors.get(1).value() is ("dog", 2, [[4..7), [19..22)])]]> + + + + Scoring-only consumers that never read offsets can skip storing them: + new TermVectorAnnotator(TermVectorAnnotator.Mode.SCORING_ONLY) emits + vectors that carry counts but no spans, and + TermVector.count("dog", 2) builds the matching expectation in tests. + +
From 593b4cc3352a1247b4a4c1c172f648d385d56b10 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:21:41 -0400 Subject: [PATCH 4/9] OPENNLP-1897: Accept a plain per-token normalizer for term identity The documented normalization workflow could not be built with any commonly wanted normalizer: TextNormalizer.Builder.buildAligned() rejects caseFold, nfc, nfkc, and accent folding, so the only OffsetAwareNormalizer chains the annotator accepted were the per-code-point folds. The restriction is unnecessary here because the occurrence spans the annotator emits are always the token's own span in the original text; the normalized text is used only as the term key. Add TermVectorAnnotator(CharSequenceNormalizer) and TermVectorAnnotator(CharSequenceNormalizer, Mode) as the general path: the normalizer is applied to each token's covered text to produce the term, the span stays the token's original span, and any normalizer works (case fold, NFC, accent fold, stemmer-backed). The OffsetAwareNormalizer constructors keep their whole-document aligned behavior unchanged; their javadoc now points at the plain-normalizer constructors as the general path. Tokens that normalize to the empty string still collapse into one empty term on both paths, pinned by matching tests. Red evidence: the new tests cannot compile against the old API (no suitable constructor found for TermVectorAnnotator(CharSequenceNormalizer)), so the tests and the fix land together in this commit. The negative pin that builder().caseFold().buildAligned() throws IllegalStateException already exists in AlignedNormalizerPipelineTest and is not duplicated. --- .../tools/termvector/TermVectorAnnotator.java | 93 +++++++++++++++---- .../termvector/TermVectorAnnotatorTest.java | 80 ++++++++++++++++ 2 files changed, 157 insertions(+), 16 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java index e30dd6e7af..27282db16c 100644 --- a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java @@ -30,6 +30,7 @@ import opennlp.tools.document.Layers; import opennlp.tools.util.Span; import opennlp.tools.util.normalizer.AlignedText; +import opennlp.tools.util.normalizer.CharSequenceNormalizer; import opennlp.tools.util.normalizer.OffsetAwareNormalizer; /** @@ -39,15 +40,18 @@ * *

Term identity comes from the annotator's inputs, not from logic of its own. Without * a normalizer, the term is the token layer's value as-is, that is, the token's covered - * text in the original document. With an {@link OffsetAwareNormalizer}, the whole - * document text is normalized once with its alignment recorded, each token span is - * mapped forward to the normalized form, and the covered normalized text is the term, so - * tokens that differ only by a normalization fold (case, an eszett expansion, collapsed - * whitespace) group together. Either way, the occurrence spans emitted in - * {@link Mode#FULL full mode} are the token layer's own spans and therefore always - * point into the original text. A token whose normalized form is empty, for example one - * the normalizer deleted entirely, groups under the empty string rather than being - * dropped.

+ * text in the original document. With a plain {@link CharSequenceNormalizer}, the general + * path, each token's covered text is normalized on its own to produce the term, so any + * normalizer works: case folding, NFC, accent folding, a stemmer-backed normalizer. With + * an {@link OffsetAwareNormalizer}, the whole document text is normalized once with its + * alignment recorded, each token span is mapped forward to the normalized form, and the + * covered normalized text is the term; this path can see across token boundaries but is + * limited to alignment-reporting normalizers. On every path, tokens that differ only by + * a normalization fold (case, an eszett expansion, collapsed whitespace) group together, + * and the occurrence spans emitted in {@link Mode#FULL full mode} are the token layer's + * own spans and therefore always point into the original text. A token whose normalized + * form is empty, for example one the normalizer deleted entirely, groups under the empty + * string rather than being dropped.

* *

The layer is {@link LayerKey.Scope#DOCUMENT document-scoped}: each {@link TermVector} * is a whole-document statistic, so the annotations carry no span of their own and the @@ -82,6 +86,7 @@ public enum Mode { } private final OffsetAwareNormalizer normalizer; + private final CharSequenceNormalizer tokenNormalizer; private final Mode mode; /** @@ -103,12 +108,57 @@ public TermVectorAnnotator(Mode mode) { throw new IllegalArgumentException("mode must not be null"); } this.normalizer = null; + this.tokenNormalizer = null; this.mode = mode; } /** * Initializes a {@link Mode#FULL full mode} annotator that groups tokens by their - * normalized form. + * per-token normalized form. This is the general path: the normalizer is applied to + * each token's covered text on its own, so any {@link CharSequenceNormalizer} works, + * including the folds that cannot report an alignment (case folding, NFC, accent + * folding, stemmer-backed normalizers). The occurrence spans stay the tokens' own + * spans in the original text. + * + * @param normalizer The normalizer that defines term identity, applied to each token's + * covered text. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code normalizer} is {@code null}. + */ + public TermVectorAnnotator(CharSequenceNormalizer normalizer) { + this(normalizer, Mode.FULL); + } + + /** + * Initializes an annotator that groups tokens by their per-token normalized form. This + * is the general path: the normalizer is applied to each token's covered text on its + * own, so any {@link CharSequenceNormalizer} works, including the folds that cannot + * report an alignment (case folding, NFC, accent folding, stemmer-backed normalizers). + * The occurrence spans stay the tokens' own spans in the original text. + * + * @param normalizer The normalizer that defines term identity, applied to each token's + * covered text. Must not be {@code null}. + * @param mode How much each {@link TermVector} records. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code normalizer} or {@code mode} is + * {@code null}. + */ + public TermVectorAnnotator(CharSequenceNormalizer normalizer, Mode mode) { + if (normalizer == null) { + throw new IllegalArgumentException("normalizer must not be null"); + } + if (mode == null) { + throw new IllegalArgumentException("mode must not be null"); + } + this.normalizer = null; + this.tokenNormalizer = normalizer; + this.mode = mode; + } + + /** + * Initializes a {@link Mode#FULL full mode} annotator that groups tokens by their + * normalized form through a whole-document alignment. Prefer the + * {@link #TermVectorAnnotator(CharSequenceNormalizer) plain-normalizer constructor} + * as the general path; this one only accepts alignment-reporting normalizers but can + * see across token boundaries, for example a whitespace collapse spanning two tokens. * * @param normalizer The normalizer that defines term identity, applied to the whole * document text so token spans can be mapped into the normalized @@ -120,7 +170,12 @@ public TermVectorAnnotator(OffsetAwareNormalizer normalizer) { } /** - * Initializes an annotator that groups tokens by their normalized form. + * Initializes an annotator that groups tokens by their normalized form through a + * whole-document alignment. Prefer the + * {@link #TermVectorAnnotator(CharSequenceNormalizer, Mode) plain-normalizer + * constructor} as the general path; this one only accepts alignment-reporting + * normalizers but can see across token boundaries, for example a whitespace collapse + * spanning two tokens. * * @param normalizer The normalizer that defines term identity, applied to the whole * document text so token spans can be mapped into the normalized @@ -137,6 +192,7 @@ public TermVectorAnnotator(OffsetAwareNormalizer normalizer, Mode mode) { throw new IllegalArgumentException("mode must not be null"); } this.normalizer = normalizer; + this.tokenNormalizer = null; this.mode = mode; } @@ -231,17 +287,22 @@ public Set> provides() { } /** - * Determines the term one token groups under: its covered text as-is, or the covered - * text of its span mapped into the normalized form when a normalizer is present. + * Determines the term one token groups under: its covered text as-is, its covered text + * normalized on its own when a plain per-token normalizer is present, or the covered + * text of its span mapped into the normalized form when an offset-aware normalizer is + * present. * * @param token The token annotation. * @param aligned The normalized document text with its alignment, or {@code null} - * when no normalizer is present. - * @param normalized The normalized document text, or {@code null} when no normalizer is - * present. + * when no offset-aware normalizer is present. + * @param normalized The normalized document text, or {@code null} when no offset-aware + * normalizer is present. * @return The term string. Never {@code null}, possibly empty. */ private String termOf(Annotation token, AlignedText aligned, String normalized) { + if (tokenNormalizer != null) { + return tokenNormalizer.normalize(token.value()).toString(); + } if (aligned == null) { return token.value(); } diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java index ee6f852cd3..03eeb130b9 100644 --- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java @@ -32,6 +32,7 @@ import opennlp.tools.util.Span; import opennlp.tools.util.normalizer.AlignedText; import opennlp.tools.util.normalizer.Alignment; +import opennlp.tools.util.normalizer.CharSequenceNormalizer; import opennlp.tools.util.normalizer.OffsetAwareNormalizer; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -135,6 +136,21 @@ public AlignedText normalizeAligned(CharSequence text) { private static final OffsetAwareNormalizer DROP_DIGITS = new DigitDeletingNormalizer(); + /** + * A plain, alignment-free case folder for the per-token path: it stands in for the + * shipped normalizers (case fold, NFC, accent fold) that cannot report offsets and + * therefore cannot implement {@link OffsetAwareNormalizer}. + */ + private static final CharSequenceNormalizer PLAIN_LOWER = + text -> text.toString().toLowerCase(Locale.ROOT); + + /** + * A plain normalizer that deletes every digit, so a token made up of digits alone + * normalizes to the empty string on the per-token path. + */ + private static final CharSequenceNormalizer PLAIN_DROP_DIGITS = + text -> text.toString().replaceAll("\\d", ""); + /** * Splits a text on single space characters into a token layer, skipping the empty * tokens a collapsed run would produce, so double spaces yield no tokens. @@ -266,6 +282,70 @@ void testScoringOnlyModeWithNormalizerCountsWithoutOffsets() { assertEquals(TermVector.count("gross", 3), vectors.get(0).value()); } + /** + * The general per-token path: a plain {@link CharSequenceNormalizer} defines term + * identity by folding each token's covered text, no alignment involved, while every + * occurrence span stays the token's own span in the original text. This admits the + * folds {@code buildAligned()} rejects (case fold, NFC, accent fold). + */ + @Test + void testPlainNormalizerGroupsFoldedTokensWithOriginalSpans() { + final String text = "Word word WORD"; + final Document document = new TermVectorAnnotator(PLAIN_LOWER) + .annotate(documentWithTokens(text)); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(1, vectors.size()); + assertEquals(new TermVector("word", 3, + List.of(new Span(0, 4), new Span(5, 9), new Span(10, 14))), vectors.get(0).value()); + + // Every occurrence span covers the original surface form, not the folded one. + final List surfaceForms = vectors.get(0).value().spans().stream() + .map(s -> s.getCoveredText(text).toString()).toList(); + assertEquals(List.of("Word", "word", "WORD"), surfaceForms); + } + + @Test + void testPlainNormalizerScoringOnlyModeCountsWithoutOffsets() { + final Document document = new TermVectorAnnotator(PLAIN_LOWER, + TermVectorAnnotator.Mode.SCORING_ONLY).annotate(documentWithTokens("Word word WORD")); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(1, vectors.size()); + assertEquals(TermVector.count("word", 3), vectors.get(0).value()); + } + + /** + * The empty-term bucket behaves identically on the per-token path: tokens a plain + * normalizer folds to the empty string group under one {@code ""} term with their + * original spans, matching {@link #testTokensNormalizedAwayGroupUnderTheEmptyTerm()}. + */ + @Test + void testPlainNormalizerFoldsDeletedTokensIntoTheEmptyTerm() { + final Document document = new TermVectorAnnotator(PLAIN_DROP_DIGITS) + .annotate(documentWithTokens("dog 42 dog 7")); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(2, vectors.size()); + assertEquals(new TermVector("dog", 2, List.of(new Span(0, 3), new Span(7, 10))), + vectors.get(0).value()); + assertEquals(new TermVector("", 2, List.of(new Span(4, 6), new Span(11, 12))), + vectors.get(1).value()); + } + + @Test + void testNullPlainNormalizerIsRejected() { + final CharSequenceNormalizer noNormalizer = null; + assertThrows(IllegalArgumentException.class, () -> new TermVectorAnnotator(noNormalizer)); + assertThrows(IllegalArgumentException.class, + () -> new TermVectorAnnotator(noNormalizer, TermVectorAnnotator.Mode.FULL)); + assertThrows(IllegalArgumentException.class, + () -> new TermVectorAnnotator(PLAIN_LOWER, null)); + } + /** * A token the normalizer deletes entirely groups under the empty string instead of * being dropped, so the term vector layer still accounts for every token. From 333e8948bf203e917dceec77df3a290df25a4874 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:21:47 -0400 Subject: [PATCH 5/9] OPENNLP-1897: Document the normalized-term workflow with a mirror-tested example Extend the term vector section of the manual with the plain-normalizer path: a whitespace tokenizer plus a shipped case folder built by TextNormalizer.builder().caseFold().build(), folding "Word word WORD" into one term with three occurrence spans that stay the tokens' original spans. The example lives in opennlp-runtime because the shipped folds do, and TermVectorNormalizedExampleTest#testCaseFoldedTermsKeepOriginalSpans asserts the behavior shown in the listing. --- .../TermVectorNormalizedExampleTest.java | 69 +++++++++++++++++++ opennlp-docs/src/docbkx/document.xml | 30 ++++++-- 2 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java new file mode 100644 index 0000000000..aa5b7e5db0 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.termvector; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.document.Annotation; +import opennlp.tools.document.Document; +import opennlp.tools.document.DocumentAnalyzer; +import opennlp.tools.document.TokenizerAnnotator; +import opennlp.tools.tokenize.WhitespaceTokenizer; +import opennlp.tools.util.Span; +import opennlp.tools.util.normalizer.CharSequenceNormalizer; +import opennlp.tools.util.normalizer.TextNormalizer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Mirrors the normalized-term workflow shown in the term vector section of the manual: + * a whitespace tokenizer feeding a {@link TermVectorAnnotator} built with a shipped, + * plain {@link CharSequenceNormalizer} case folder. The folder defines term identity per + * token, so case variants group under one term, while every occurrence span stays the + * token's own span in the original text. + */ +public class TermVectorNormalizedExampleTest { + + /** + * The documented example: {@code "Word word WORD"} yields one term, {@code "word"}, + * with three occurrence spans, each the token's exact original span. + */ + @Test + void testCaseFoldedTermsKeepOriginalSpans() { + final CharSequenceNormalizer folder = TextNormalizer.builder().caseFold().build(); + final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() + .add(new TokenizerAnnotator(WhitespaceTokenizer.INSTANCE)) + .add(new TermVectorAnnotator(folder)) + .build(); + + final Document document = analyzer.analyze("Word word WORD"); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(1, vectors.size()); + assertEquals(new TermVector("word", 3, + List.of(new Span(0, 4), new Span(5, 9), new Span(10, 14))), vectors.get(0).value()); + + // The spans point at the original surface forms, not the folded term. + final List surfaceForms = vectors.get(0).value().spans().stream() + .map(s -> s.getCoveredText(document.text()).toString()).toList(); + assertEquals(List.of("Word", "word", "WORD"), surfaceForms); + } +} diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index c67a4b6460..a46d5abe41 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -343,11 +343,13 @@ int firstTokenLength = lengths.get(0).value(); // 3, for "The" at [0..3)]]> term vector layer for index consumers: one TermVector per distinct term, carrying the term string, its occurrence count, and the occurrence spans in original text coordinates. Term identity comes from the annotator's inputs: without - a normalizer the term is the token's covered text as-is; with an + a normalizer the term is the token's covered text as-is; with a plain + CharSequenceNormalizer, the general path, each token's covered text is + normalized on its own to become the term; with an OffsetAwareNormalizer the document text is normalized once and each - token's covered normalized text becomes the term, so tokens differing only by a - normalization fold group together while their spans keep pointing into the original - text. The layer preserves first-occurrence order. + token's covered normalized text becomes the term. On every path tokens differing + only by a normalization fold group together while their spans keep pointing into + the original text. The layer preserves first-occurrence order. TermVectorPipelineTest#testTokenizerAndTermVectorPipeline asserts the behavior shown here. > vectors = document.get(TermVectorAnnotator.TERM_VEC // vectors.get(1).value() is ("dog", 2, [[4..7), [19..22)])]]> + + The normalized-term workflow passes any shipped + CharSequenceNormalizer to the plain-normalizer constructor: the + normalizer is applied to each token's covered text to produce the term, so the + case, NFC, and accent folds that buildAligned() rejects all work + here, and every occurrence span stays the token's own span in the original text. + TermVectorNormalizedExampleTest#testCaseFoldedTermsKeepOriginalSpans + asserts the behavior shown here. + > vectors = document.get(TermVectorAnnotator.TERM_VECTORS); +// vectors.get(0).value() is ("word", 3, [[0..4), [5..9), [10..14)])]]> + + Scoring-only consumers that never read offsets can skip storing them: new TermVectorAnnotator(TermVectorAnnotator.Mode.SCORING_ONLY) emits From 7b6030f4baad46bc996bff5c75d56cf5937d2a78 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 15 Aug 2026 21:08:37 -0400 Subject: [PATCH 6/9] OPENNLP-1897: Review round: reject null span elements, finalize the annotator, share the space-split test fixture, pin supplementary-plane offsets --- .../opennlp/tools/termvector/TermVector.java | 15 +++- .../tools/termvector/TermVectorAnnotator.java | 2 +- .../tools/termvector/SingleSpaceTokens.java | 90 +++++++++++++++++++ .../termvector/TermVectorAnnotatorTest.java | 53 ++++++----- .../termvector/TermVectorPipelineTest.java | 39 +------- .../tools/termvector/TermVectorTest.java | 9 ++ 6 files changed, 147 insertions(+), 61 deletions(-) create mode 100644 opennlp-api/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java index 676bf01317..42c654257d 100644 --- a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java +++ b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVector.java @@ -43,8 +43,9 @@ * @param term The term string. Must not be {@code null}. * @param frequency The number of occurrences in the document. Must be at least one. * @param spans The occurrence spans in original text coordinates, one per occurrence, - * or an empty list for a scoring-only vector. Must not be {@code null} and, - * when non-empty, must contain exactly {@code frequency} spans. + * or an empty list for a scoring-only vector. Must not be {@code null} or + * contain {@code null} and, when non-empty, must hold exactly + * {@code frequency} spans. * * @since 3.0.0 */ @@ -55,8 +56,9 @@ public record TermVector(String term, int frequency, List spans) { * Validates the term vector and detaches the span list from the caller's input. * * @throws IllegalArgumentException Thrown if {@code term} is {@code null}, - * {@code frequency} is below one, {@code spans} is {@code null}, or a - * non-empty {@code spans} list does not hold exactly {@code frequency} spans. + * {@code frequency} is below one, {@code spans} is or contains {@code null}, + * or a non-empty {@code spans} list does not hold exactly {@code frequency} + * spans. */ public TermVector { if (term == null) { @@ -68,6 +70,11 @@ public record TermVector(String term, int frequency, List spans) { if (spans == null) { throw new IllegalArgumentException("spans must not be null"); } + for (final Span span : spans) { + if (span == null) { + throw new IllegalArgumentException("spans must not contain null"); + } + } if (!spans.isEmpty() && spans.size() != frequency) { throw new IllegalArgumentException("a full term vector holds one span per " + "occurrence: frequency is " + frequency + " but spans holds " + spans.size()); diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java index 27282db16c..bec73882fb 100644 --- a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java @@ -63,7 +63,7 @@ * * @since 3.0.0 */ -public class TermVectorAnnotator implements DocumentAnnotator { +public final class TermVectorAnnotator implements DocumentAnnotator { /** * The key of the term vector layer this annotator provides: a document-scoped layer diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java b/opennlp-api/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java new file mode 100644 index 0000000000..4f7beb4643 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.termvector; + +import java.util.ArrayList; +import java.util.List; + +import opennlp.tools.document.Annotation; +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Span; + +/** + * Deterministic single-space tokenization shared by the term vector tests: splits on + * single space characters and keeps all other characters, including sentence-final + * periods, attached to their token. Runs of spaces yield no empty tokens, so every + * expected span follows directly from the input text. + */ +final class SingleSpaceTokens { + + /** + * A {@link Tokenizer} view of {@link #spans(String)}. Only the span-producing method + * is implemented because the annotator adapter calls no other method. + */ + static final Tokenizer TOKENIZER = new Tokenizer() { + + @Override + public String[] tokenize(String s) { + throw new UnsupportedOperationException("the adapter only calls tokenizePos"); + } + + @Override + public Span[] tokenizePos(String s) { + return spans(s).toArray(new Span[0]); + } + }; + + private SingleSpaceTokens() { + } + + /** + * Computes the token spans of a text split on single space characters. + * + * @param text The text to split. + * @return One span per token, in text order. + */ + static List spans(String text) { + final List spans = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= text.length(); i++) { + final boolean boundary = i == text.length() || text.charAt(i) == ' '; + if (boundary && start >= 0) { + spans.add(new Span(start, i)); + start = -1; + } else if (!boundary && start < 0) { + start = i; + } + } + return spans; + } + + /** + * Builds a token layer from {@link #spans(String)}, each token valued with its covered + * text. + * + * @param text The text to split. + * @return One annotation per token, in text order. + */ + static List> tokens(String text) { + final List> tokens = new ArrayList<>(); + for (final Span span : spans(text)) { + tokens.add(new Annotation<>(span, text.substring(span.getStart(), span.getEnd()))); + } + return tokens; + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java index 03eeb130b9..e928d5dfc9 100644 --- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java @@ -18,7 +18,6 @@ package opennlp.tools.termvector; import java.io.Serial; -import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Set; @@ -148,30 +147,18 @@ public AlignedText normalizeAligned(CharSequence text) { * A plain normalizer that deletes every digit, so a token made up of digits alone * normalizes to the empty string on the per-token path. */ - private static final CharSequenceNormalizer PLAIN_DROP_DIGITS = - text -> text.toString().replaceAll("\\d", ""); - - /** - * Splits a text on single space characters into a token layer, skipping the empty - * tokens a collapsed run would produce, so double spaces yield no tokens. - */ - private static List> splitOnSpaces(String text) { - final List> tokens = new ArrayList<>(); - int start = -1; - for (int i = 0; i <= text.length(); i++) { - final boolean boundary = i == text.length() || text.charAt(i) == ' '; - if (boundary && start >= 0) { - tokens.add(new Annotation<>(new Span(start, i), text.substring(start, i))); - start = -1; - } else if (!boundary && start < 0) { - start = i; + private static final CharSequenceNormalizer PLAIN_DROP_DIGITS = text -> { + final StringBuilder out = new StringBuilder(text.length()); + for (int i = 0; i < text.length(); i++) { + if (!Character.isDigit(text.charAt(i))) { + out.append(text.charAt(i)); } } - return tokens; - } + return out.toString(); + }; private static Document documentWithTokens(String text) { - return Document.of(text).with(Layers.TOKENS, splitOnSpaces(text)); + return Document.of(text).with(Layers.TOKENS, SingleSpaceTokens.tokens(text)); } @Test @@ -364,6 +351,30 @@ void testTokensNormalizedAwayGroupUnderTheEmptyTerm() { vectors.get(1).value()); } + /** + * Spans are UTF-16 offsets, so a supplementary-plane character occupies two positions: + * {@code "𝕏 x 𝕏"} tokenizes to spans of width two around the surrogate pairs, and both + * occurrences group under one term whose spans still cover the original text exactly. + */ + @Test + void testSupplementaryPlaneTokensKeepUtf16Offsets() { + final String text = "𝕏 x 𝕏"; + final Document document = new TermVectorAnnotator() + .annotate(documentWithTokens(text)); + + final List> vectors = + document.get(TermVectorAnnotator.TERM_VECTORS); + assertEquals(2, vectors.size()); + assertEquals(new TermVector("𝕏", 2, List.of(new Span(0, 2), new Span(5, 7))), + vectors.get(0).value()); + assertEquals(new TermVector("x", 1, List.of(new Span(3, 4))), vectors.get(1).value()); + for (final Annotation vector : vectors) { + for (final Span span : vector.value().spans()) { + assertEquals(vector.value().term(), span.getCoveredText(text).toString()); + } + } + } + @Test void testEmptyTokenLayerYieldsPresentButEmptyLayer() { final Document document = new TermVectorAnnotator() diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java index a00fba96bd..291b2068c5 100644 --- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java @@ -17,7 +17,6 @@ package opennlp.tools.termvector; -import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -28,7 +27,6 @@ import opennlp.tools.document.DocumentAnalyzer; import opennlp.tools.document.Layers; import opennlp.tools.document.TokenizerAnnotator; -import opennlp.tools.tokenize.Tokenizer; import opennlp.tools.util.Span; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -37,40 +35,11 @@ /** * Wires the {@link TermVectorAnnotator} into a {@link DocumentAnalyzer} behind a * {@link TokenizerAnnotator}: the token layer goes in, the term vector layer comes out, - * and nothing else about the document changes. The tokenizer is a deterministic stand-in - * defined here, so every expected span follows directly from the input text. + * and nothing else about the document changes. The tokenizer is the shared deterministic + * single-space fixture, so every expected span follows directly from the input text. */ public class TermVectorPipelineTest { - /** - * A deterministic tokenizer that splits on single space characters and keeps all - * other characters, including sentence-final periods, attached to their token. Only - * the span-producing method is implemented because the adapter calls no other method. - */ - private static final Tokenizer SPACE_TOKENIZER = new Tokenizer() { - - @Override - public String[] tokenize(String s) { - throw new UnsupportedOperationException("the adapter only calls tokenizePos"); - } - - @Override - public Span[] tokenizePos(String s) { - final List spans = new ArrayList<>(); - int start = -1; - for (int i = 0; i <= s.length(); i++) { - final boolean boundary = i == s.length() || s.charAt(i) == ' '; - if (boundary && start >= 0) { - spans.add(new Span(start, i)); - start = -1; - } else if (!boundary && start < 0) { - start = i; - } - } - return spans.toArray(new Span[0]); - } - }; - /** * Runs tokenizer plus term vector roll-up over a text with repeated tokens and reads * the aggregated layer back, span by span, in original text coordinates. @@ -78,7 +47,7 @@ public Span[] tokenizePos(String s) { @Test void testTokenizerAndTermVectorPipeline() { final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() - .add(new TokenizerAnnotator(SPACE_TOKENIZER)) + .add(new TokenizerAnnotator(SingleSpaceTokens.TOKENIZER)) .add(new TermVectorAnnotator()) .build(); @@ -113,7 +82,7 @@ void testTokenizerAndTermVectorPipeline() { @Test void testScoringOnlyPipeline() { final DocumentAnalyzer analyzer = DocumentAnalyzer.builder() - .add(new TokenizerAnnotator(SPACE_TOKENIZER)) + .add(new TokenizerAnnotator(SingleSpaceTokens.TOKENIZER)) .add(new TermVectorAnnotator(TermVectorAnnotator.Mode.SCORING_ONLY)) .build(); diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java index 286b7a2c34..8448f465d3 100644 --- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorTest.java @@ -75,6 +75,15 @@ void testNullSpanListIsRejected() { assertThrows(IllegalArgumentException.class, () -> new TermVector("dog", 1, null)); } + @Test + void testNullSpanElementIsRejected() { + final List spans = new ArrayList<>(); + spans.add(new Span(0, 3)); + spans.add(null); + assertThrows(IllegalArgumentException.class, () -> new TermVector("dog", 2, spans)); + assertThrows(IllegalArgumentException.class, () -> TermVector.withSpans("dog", spans)); + } + @ParameterizedTest @ValueSource(ints = {0, -1, Integer.MIN_VALUE}) void testFrequencyBelowOneIsRejected(int frequency) { From c3cc8adc1b81e06f939d7263ad8145e704d74263 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 15 Aug 2026 23:14:17 -0400 Subject: [PATCH 7/9] OPENNLP-1897: Omit tokens that normalize to the empty string from the term vector layer An empty string is no term: it cannot be queried and its token stays accounted for in the token layer. Omission also keeps one semantic for term vectors across the library and its search consumers. --- .../tools/termvector/TermVectorAnnotator.java | 16 ++++++--- .../termvector/TermVectorAnnotatorTest.java | 34 ++++++++++++------- opennlp-docs/src/docbkx/document.xml | 3 +- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java index bec73882fb..4ed12d0d13 100644 --- a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java +++ b/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java @@ -50,8 +50,9 @@ * a normalization fold (case, an eszett expansion, collapsed whitespace) group together, * and the occurrence spans emitted in {@link Mode#FULL full mode} are the token layer's * own spans and therefore always point into the original text. A token whose normalized - * form is empty, for example one the normalizer deleted entirely, groups under the empty - * string rather than being dropped.

+ * form is empty, for example one the normalizer deleted entirely, is omitted from the + * layer; an empty string is no term, and the token layer still accounts for the + * token.

* *

The layer is {@link LayerKey.Scope#DOCUMENT document-scoped}: each {@link TermVector} * is a whole-document statistic, so the annotations carry no span of their own and the @@ -240,8 +241,10 @@ private List> fullVectors(List> tokens AlignedText aligned, String normalized) { final Map> spansByTerm = new LinkedHashMap<>(); for (final Annotation token : tokens) { - spansByTerm.computeIfAbsent(termOf(token, aligned, normalized), key -> new ArrayList<>()) - .add(token.span()); + final String term = termOf(token, aligned, normalized); + if (!term.isEmpty()) { + spansByTerm.computeIfAbsent(term, key -> new ArrayList<>()).add(token.span()); + } } final List> vectors = new ArrayList<>(spansByTerm.size()); for (final Map.Entry> entry : spansByTerm.entrySet()) { @@ -265,7 +268,10 @@ private List> countVectors(List> token AlignedText aligned, String normalized) { final Map frequencies = new LinkedHashMap<>(); for (final Annotation token : tokens) { - frequencies.merge(termOf(token, aligned, normalized), 1, Integer::sum); + final String term = termOf(token, aligned, normalized); + if (!term.isEmpty()) { + frequencies.merge(term, 1, Integer::sum); + } } final List> vectors = new ArrayList<>(frequencies.size()); for (final Map.Entry entry : frequencies.entrySet()) { diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java index e928d5dfc9..33559371f9 100644 --- a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java @@ -305,22 +305,20 @@ void testPlainNormalizerScoringOnlyModeCountsWithoutOffsets() { } /** - * The empty-term bucket behaves identically on the per-token path: tokens a plain - * normalizer folds to the empty string group under one {@code ""} term with their - * original spans, matching {@link #testTokensNormalizedAwayGroupUnderTheEmptyTerm()}. + * Empty-term omission behaves identically on the per-token path: tokens a plain + * normalizer folds to the empty string are left out of the layer, matching + * {@link #testTokensNormalizedAwayAreOmitted()}. */ @Test - void testPlainNormalizerFoldsDeletedTokensIntoTheEmptyTerm() { + void testPlainNormalizerOmitsDeletedTokens() { final Document document = new TermVectorAnnotator(PLAIN_DROP_DIGITS) .annotate(documentWithTokens("dog 42 dog 7")); final List> vectors = document.get(TermVectorAnnotator.TERM_VECTORS); - assertEquals(2, vectors.size()); + assertEquals(1, vectors.size()); assertEquals(new TermVector("dog", 2, List.of(new Span(0, 3), new Span(7, 10))), vectors.get(0).value()); - assertEquals(new TermVector("", 2, List.of(new Span(4, 6), new Span(11, 12))), - vectors.get(1).value()); } @Test @@ -334,21 +332,19 @@ void testNullPlainNormalizerIsRejected() { } /** - * A token the normalizer deletes entirely groups under the empty string instead of - * being dropped, so the term vector layer still accounts for every token. + * A token the normalizer deletes entirely is omitted from the layer: an empty string + * is no term, and the token layer still accounts for the token itself. */ @Test - void testTokensNormalizedAwayGroupUnderTheEmptyTerm() { + void testTokensNormalizedAwayAreOmitted() { final Document document = new TermVectorAnnotator(DROP_DIGITS) .annotate(documentWithTokens("dog 42 dog 7")); final List> vectors = document.get(TermVectorAnnotator.TERM_VECTORS); - assertEquals(2, vectors.size()); + assertEquals(1, vectors.size()); assertEquals(new TermVector("dog", 2, List.of(new Span(0, 3), new Span(7, 10))), vectors.get(0).value()); - assertEquals(new TermVector("", 2, List.of(new Span(4, 6), new Span(11, 12))), - vectors.get(1).value()); } /** @@ -375,6 +371,18 @@ void testSupplementaryPlaneTokensKeepUtf16Offsets() { } } + /** + * A document whose every token normalizes to the empty string yields the layer + * present but empty, the same graceful degradation as an empty token layer. + */ + @Test + void testAllTokensNormalizedAwayYieldPresentButEmptyLayer() { + final Document document = new TermVectorAnnotator(DROP_DIGITS) + .annotate(documentWithTokens("42 7")); + assertTrue(document.layers().contains(TermVectorAnnotator.TERM_VECTORS)); + assertTrue(document.get(TermVectorAnnotator.TERM_VECTORS).isEmpty()); + } + @Test void testEmptyTokenLayerYieldsPresentButEmptyLayer() { final Document document = new TermVectorAnnotator() diff --git a/opennlp-docs/src/docbkx/document.xml b/opennlp-docs/src/docbkx/document.xml index a46d5abe41..169a34c2fd 100644 --- a/opennlp-docs/src/docbkx/document.xml +++ b/opennlp-docs/src/docbkx/document.xml @@ -349,7 +349,8 @@ int firstTokenLength = lengths.get(0).value(); // 3, for "The" at [0..3)]]> OffsetAwareNormalizer the document text is normalized once and each token's covered normalized text becomes the term. On every path tokens differing only by a normalization fold group together while their spans keep pointing into - the original text. The layer preserves first-occurrence order. + the original text. A token whose normalized form is empty is omitted from the + layer. The layer preserves first-occurrence order. TermVectorPipelineTest#testTokenizerAndTermVectorPipeline asserts the behavior shown here. Date: Wed, 26 Aug 2026 05:48:50 -0400 Subject: [PATCH 8/9] OPENNLP-1897: Move term vector annotator to runtime --- .../main/java/opennlp/tools/termvector/TermVectorAnnotator.java | 0 .../src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java | 0 .../java/opennlp/tools/termvector/TermVectorAnnotatorTest.java | 0 .../java/opennlp/tools/termvector/TermVectorPipelineTest.java | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java (100%) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java (100%) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java (100%) rename {opennlp-api => opennlp-core/opennlp-runtime}/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java (100%) diff --git a/opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java similarity index 100% rename from opennlp-api/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java rename to opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/termvector/TermVectorAnnotator.java diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java similarity index 100% rename from opennlp-api/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java rename to opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/SingleSpaceTokens.java diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java similarity index 100% rename from opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java rename to opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorAnnotatorTest.java diff --git a/opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java similarity index 100% rename from opennlp-api/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java rename to opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java From 586e573eb19346986dd2fe00e74caf80add032e9 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 26 Aug 2026 07:05:21 -0400 Subject: [PATCH 9/9] OPENNLP-1897: Update annotator imports --- .../tools/termvector/TermVectorNormalizedExampleTest.java | 2 +- .../java/opennlp/tools/termvector/TermVectorPipelineTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java index aa5b7e5db0..2868b021e7 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorNormalizedExampleTest.java @@ -24,7 +24,7 @@ import opennlp.tools.document.Annotation; import opennlp.tools.document.Document; import opennlp.tools.document.DocumentAnalyzer; -import opennlp.tools.document.TokenizerAnnotator; +import opennlp.tools.tokenize.TokenizerAnnotator; import opennlp.tools.tokenize.WhitespaceTokenizer; import opennlp.tools.util.Span; import opennlp.tools.util.normalizer.CharSequenceNormalizer; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java index 291b2068c5..8510cfdac9 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/termvector/TermVectorPipelineTest.java @@ -26,7 +26,7 @@ import opennlp.tools.document.Document; import opennlp.tools.document.DocumentAnalyzer; import opennlp.tools.document.Layers; -import opennlp.tools.document.TokenizerAnnotator; +import opennlp.tools.tokenize.TokenizerAnnotator; import opennlp.tools.util.Span; import static org.junit.jupiter.api.Assertions.assertEquals;