diff --git a/checkstyle-suppressions.xml b/checkstyle-suppressions.xml new file mode 100644 index 0000000000..033f868641 --- /dev/null +++ b/checkstyle-suppressions.xml @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/checkstyle.xml b/checkstyle.xml index 5a573b1c59..554496a098 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -63,8 +63,30 @@ + + + + + + + + + + + + + + + + + diff --git a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java index 44f683d588..c49adfe948 100644 --- a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java +++ b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java @@ -116,6 +116,23 @@ public static boolean isUnicodeWhitespace(int charCode) { return UnicodeWhitespace.isWhitespace(charCode); } + /** + * Determines if the specified {@link Character} is one of the six ASCII whitespace + * characters, the set the regular expression class {@code \s} matches: space, tab, + * line feed, vertical tab, form feed, and carriage return. Unlike + * {@link #isWhitespace(char)}, this ignores the {@link WhitespaceMode} and rejects + * every non-ASCII space. + * + * @param charCode The character to check. + * + * @return {@code true} if {@code charCode} is one of those six characters, + * {@code false} otherwise. + */ + public static boolean isAsciiWhitespace(char charCode) { + return charCode == ' ' || charCode == '\t' || charCode == '\n' || charCode == '\u000B' + || charCode == '\f' || charCode == '\r'; + } + /** * Splits {@code input} on runs of Unicode {@code White_Space}. Leading and trailing * runs are ignored, so whitespace-only input yields an empty array. This is a @@ -151,6 +168,91 @@ public static String[] splitOnUnicodeWhitespace(CharSequence input) { return terms.toArray(new String[0]); } + /** + * Splits {@code input} on runs of ASCII whitespace with the result of + * {@code String.split("\\s+")}: a leading run yields one empty first element, trailing + * empty elements are dropped, whitespace-only input yields an empty array, and empty + * input yields a single empty element. This is a character scan, not a regular + * expression. + * + * @param input The text to split. Must not be {@code null}. + * @return The elements in order. + * @throws IllegalArgumentException If {@code input} is {@code null}. + */ + public static String[] splitOnAsciiWhitespace(String input) { + if (input == null) { + throw new IllegalArgumentException("input must not be null"); + } + if (input.isEmpty()) { + return new String[] {""}; + } + final List elements = new ArrayList<>(); + if (isAsciiWhitespace(input.charAt(0))) { + elements.add(""); + } + int start = 0; + for (int i = 0; i < input.length(); i++) { + if (isAsciiWhitespace(input.charAt(i))) { + if (i > start) { + elements.add(input.substring(start, i)); + } + while (i + 1 < input.length() && isAsciiWhitespace(input.charAt(i + 1))) { + i++; + } + start = i + 1; + } + } + if (input.length() > start) { + elements.add(input.substring(start)); + } + while (!elements.isEmpty() && elements.get(elements.size() - 1).isEmpty()) { + elements.remove(elements.size() - 1); + } + return elements.toArray(new String[0]); + } + + /** + * Tests whether {@code input} contains an ASCII capital letter, {@code A} to {@code Z}. + * Capitals outside ASCII do not count. This is a character scan, not a regular expression. + * + * @param input The text to check. Must not be {@code null}. + * @return {@code true} if at least one character is an ASCII capital letter. + * @throws IllegalArgumentException If {@code input} is {@code null}. + */ + public static boolean containsAsciiUpperCase(CharSequence input) { + if (input == null) { + throw new IllegalArgumentException("input must not be null"); + } + for (int i = 0; i < input.length(); i++) { + final char c = input.charAt(i); + if (c >= 'A' && c <= 'Z') { + return true; + } + } + return false; + } + + /** + * Tests whether {@code input} contains an ASCII digit, {@code 0} to {@code 9}. Digits outside + * ASCII do not count. This is a character scan, not a regular expression. + * + * @param input The text to check. Must not be {@code null}. + * @return {@code true} if at least one character is an ASCII digit. + * @throws IllegalArgumentException If {@code input} is {@code null}. + */ + public static boolean containsAsciiDigit(CharSequence input) { + if (input == null) { + throw new IllegalArgumentException("input must not be null"); + } + for (int i = 0; i < input.length(); i++) { + final char c = input.charAt(i); + if (c >= '0' && c <= '9') { + return true; + } + } + return false; + } + /** * Trims leading and trailing runs of Unicode {@code White_Space}, the same set * {@link #splitOnUnicodeWhitespace(CharSequence)} breaks terms on. diff --git a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java index be1f02e7b8..6ee98995bb 100644 --- a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java +++ b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java @@ -19,7 +19,6 @@ import java.io.File; import java.io.IOException; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,23 +60,71 @@ public String getHelp() { + "Defaults to a WhitespaceTokenizer."; } - private static final Pattern UNTOKENIZED_PAREN_PATTERN_1 = Pattern.compile("([^ ])([({)}])"); - private static final Pattern UNTOKENIZED_PAREN_PATTERN_2 = Pattern.compile("([({)}])([^ ])"); - public static Parse[] parseLine(String line, Parser parser, int numParses) { return parseLine( line, parser, WhitespaceTokenizer.INSTANCE, numParses ); } public static Parse[] parseLine(String line, Parser parser, Tokenizer tokenizer, int numParses) { // fix some parens patterns - line = UNTOKENIZED_PAREN_PATTERN_1.matcher(line).replaceAll("$1 $2"); - line = UNTOKENIZED_PAREN_PATTERN_2.matcher(line).replaceAll("$1 $2"); + line = spaceUntokenizedParens(line); // tokenize String[] tokens = tokenizer.tokenize(line); return parseLine(tokens, parser, numParses); } + /** + * Separates round and curly brackets from adjacent text in two left-to-right passes: the + * first puts a space between a non-space character and a following bracket, the second + * between a bracket and a following non-space character. Each pass resumes after the pair + * it just spaced, so a pair overlapping that match is only seen by the second pass. + * + * @param line The untokenized line. + * @return The spaced line. + */ + static String spaceUntokenizedParens(String line) { + return insertParenSpaces(insertParenSpaces(line, false), true); + } + + /** + * Inserts a space between a bracket and an adjacent non-space character, left to right. + * + * @param line The untokenized line. + * @param parenFirst {@code true} to space a bracket before a character, {@code false} after one. + * @return The spaced line. + */ + private static String insertParenSpaces(String line, boolean parenFirst) { + StringBuilder spaced = new StringBuilder(line.length() + 8); + int i = 0; + while (i < line.length()) { + char c = line.charAt(i); + if (i + 1 < line.length()) { + char next = line.charAt(i + 1); + boolean match = parenFirst + ? isParen(c) && next != ' ' + : c != ' ' && isParen(next); + if (match) { + spaced.append(c).append(' ').append(next); + i += 2; + continue; + } + } + spaced.append(c); + i++; + } + return spaced.toString(); + } + + /** + * Tests for a round or curly bracket. + * + * @param c The character. + * @return {@code true} for {@code (}, {@code )}, {, or }. + */ + private static boolean isParen(char c) { + return c == '(' || c == ')' || c == '{' || c == '}'; + } + /** * Parses the specified pre-tokenized sentence and returns the requested number of parses * or fewer. diff --git a/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/parser/ParserToolTest.java b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/parser/ParserToolTest.java new file mode 100644 index 0000000000..8b03710a93 --- /dev/null +++ b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/parser/ParserToolTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.cmdline.parser; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import opennlp.tools.parser.Parse; +import opennlp.tools.parser.Parser; +import opennlp.tools.tokenize.WhitespaceTokenizer; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ParserToolTest { + + /* + * The first pass puts a space before a bracket that follows a non-space character, the + * second after a bracket that precedes one. Each pass resumes after the pair it spaced, so + * "((" gets its space from the second pass only, and only a space, not a tab, separates. + */ + private static Stream parenLines() { + return Stream.of( + Arguments.of("a(b)c", "a ( b ) c"), + Arguments.of("a (b) c", "a ( b ) c"), + Arguments.of("foo(bar){baz}", "foo ( bar ) {baz }"), + Arguments.of("((a)(b))", "( ( a ) (b ) )"), + Arguments.of("a(b(c)d)e", "a ( b ( c ) d ) e"), + Arguments.of("x((", "x ( ("), + Arguments.of("((x", "( ( x"), + Arguments.of("()", "( )"), + Arguments.of("(", "("), + Arguments.of("", ""), + Arguments.of("no parens here", "no parens here"), + Arguments.of("«quoted»", "«quoted»"), + Arguments.of("tab\there(", "tab\there ("), + Arguments.of("a (b", "a ( b")); + } + + @ParameterizedTest + @MethodSource("parenLines") + void testSpaceUntokenizedParens(String line, String expected) { + assertEquals(expected, ParserTool.spaceUntokenizedParens(line)); + } + + @Test + void testParseLineSeparatesParensBeforeTokenizing() { + Parser echo = new Parser() { + @Override + public Parse[] parse(Parse tokens, int numParses) { + return new Parse[] {tokens}; + } + + @Override + public Parse parse(Parse tokens) { + return tokens; + } + }; + Parse[] parses = ParserTool.parseLine("f(x)+g({y})", echo, WhitespaceTokenizer.INSTANCE, 1); + assertEquals(1, parses.length); + String[] tokens = Stream.of(parses[0].getChildren()).map(Parse::getCoveredText) + .toArray(String[]::new); + // "{y" stays joined: the second pass consumed "{" while spacing "( {" + assertArrayEquals(new String[] {"f", "(", "x", ")", "+g", "(", "{y", "}", ")"}, tokens); + } +} diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADMetadata.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADMetadata.java new file mode 100644 index 0000000000..c377db34d0 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADMetadata.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.ad; + +import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser; + +/** + * Reads the ids from the metadata of an Arvores Deitadas sentence, which differs between + * corpora. The metadata is one line; every method treats metadata with a line terminator as + * invalid. + */ +final class ADMetadata { + + private static final String PARAGRAPH_PREFIX = "p="; + private static final String SOURCE_PREFIX = "source=\""; + + private ADMetadata() { + } + + /** + * Parses the text id and the paragraph id: the text id is the ASCII digit run after any + * leading ASCII letters and hyphens, the paragraph id the digit run after the first + * {@code p=} that at least one digit follows. + * + * @param meta The metadata. + * @return The text id and the paragraph id, or {@code null} if either is missing. + */ + static int[] parseTextAndParagraph(String meta) { + int[] spans = scanTextAndParagraph(meta); + if (spans == null) { + return null; + } + return new int[] {Integer.parseInt(meta.substring(spans[0], spans[1])), + Integer.parseInt(meta.substring(spans[2], spans[3]))}; + } + + /** + * Reads the digits of the text id, see {@link #parseTextAndParagraph(String)}. + * + * @param meta The metadata. + * @return The digits, or {@code null} if the text id or the paragraph id is missing. + */ + static String textId(String meta) { + int[] spans = scanTextAndParagraph(meta); + return spans == null ? null : meta.substring(spans[0], spans[1]); + } + + /** + * Reads the ASCII letters and hyphens before the text id, which name the text in literary + * corpora. + * + * @param meta The metadata. + * @return The prefix, or {@code null} if it is empty or the text id or the paragraph id is + * missing. + */ + static String textPrefix(String meta) { + int[] spans = scanTextAndParagraph(meta); + return spans == null || spans[0] == 0 ? null : meta.substring(0, spans[0]); + } + + /** + * Reads the source: the text between the first {@code source="} and the next double quote. + * + * @param meta The metadata. + * @return The source, or {@code null} if the metadata has none. + */ + static String source(String meta) { + if (hasLineTerminator(meta)) { + return null; + } + int start = meta.indexOf(SOURCE_PREFIX); + if (start == -1) { + return null; + } + start += SOURCE_PREFIX.length(); + int end = meta.indexOf('"', start); + return end == -1 ? null : meta.substring(start, end); + } + + /** + * Scans the text id and the paragraph id, see {@link #parseTextAndParagraph(String)}. + * + * @param meta The metadata. + * @return The start and end of the text id and the start and end of the paragraph id, or + * {@code null} if either is missing. + */ + private static int[] scanTextAndParagraph(String meta) { + int i = 0; + while (i < meta.length() && (isAsciiLetter(meta.charAt(i)) || meta.charAt(i) == '-')) { + i++; + } + int textStart = i; + while (i < meta.length() && isAsciiDigit(meta.charAt(i))) { + i++; + } + if (i == textStart || hasLineTerminator(meta)) { + return null; + } + int textEnd = i; + int from = textEnd; + while (true) { + int prefix = meta.indexOf(PARAGRAPH_PREFIX, from); + if (prefix == -1) { + return null; + } + int paragraphStart = prefix + PARAGRAPH_PREFIX.length(); + int paragraphEnd = paragraphStart; + while (paragraphEnd < meta.length() && isAsciiDigit(meta.charAt(paragraphEnd))) { + paragraphEnd++; + } + if (paragraphEnd > paragraphStart) { + return new int[] {textStart, textEnd, paragraphStart, paragraphEnd}; + } + from = prefix + 1; + } + } + + /** + * Tests whether the metadata contains a line terminator. + * + * @param meta The metadata. + * @return {@code true} if it does. + */ + private static boolean hasLineTerminator(String meta) { + return SentenceParser.indexOfLineTerminator(meta, 0) < meta.length(); + } + + /** + * Tests for an ASCII letter. + * + * @param c The character. + * @return {@code true} for {@code a} to {@code z} or {@code A} to {@code Z}. + */ + private static boolean isAsciiLetter(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + + /** + * Tests for an ASCII digit. + * + * @param c The character. + * @return {@code true} for {@code 0} to {@code 9}. + */ + private static boolean isAsciiDigit(char c) { + return c >= '0' && c <= '9'; + } +} diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java index d2db063515..01c838c7a9 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java @@ -25,8 +25,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import opennlp.tools.commons.Internal; import opennlp.tools.formats.ad.ADSentenceStream.Sentence; @@ -38,6 +36,7 @@ import opennlp.tools.util.ObjectStream; import opennlp.tools.util.PlainTextByLineStream; import opennlp.tools.util.Span; +import opennlp.tools.util.StringUtil; /** * Parser for Floresta Sita(c)tica Arvores Deitadas corpus, output to for the @@ -66,16 +65,6 @@ @Internal public class ADNameSampleStream implements ObjectStream { - /* - * Pattern of a NER tag in Arvores Deitadas - */ - private static final Pattern TAG_PATTERN = Pattern.compile("<(NER:)?(.*?)>"); - private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+"); - private static final Pattern UNDERLINE_PATTERN = Pattern.compile("[_]+"); - private static final Pattern HYPHEN_PATTERN = - Pattern.compile("((\\p{L}+)-$)|(^-(\\p{L}+)(.*))|((\\p{L}+)-(\\p{L}+)(.*))"); - private static final Pattern ALPHANUMERIC_PATTERN = Pattern.compile("^[\\p{L}\\p{Nd}]+$"); - /* * Map to the Arvores Deitadas types to our types. It is read-only. */ @@ -254,7 +243,7 @@ private void processLeaf(Leaf leaf, List sentence, List names) { String c = PortugueseContractionUtility.toContraction( leftContractionPart, right); if (c != null) { - String[] parts = WHITESPACE_PATTERN.split(c); + String[] parts = StringUtil.splitOnAsciiWhitespace(c); sentence.addAll(Arrays.asList(parts)); alreadyAdded = true; } else { @@ -273,7 +262,7 @@ private void processLeaf(Leaf leaf, List sentence, List names) { if (leafTag != null) { if (leafTag.contains("") && !alreadyAdded) { - String[] lexemes = UNDERLINE_PATTERN.split(leaf.getLexeme()); + String[] lexemes = splitOnUnderscores(leaf.getLexeme()); if (lexemes.length > 1) { sentence.addAll(Arrays.asList(lexemes).subList(0, lexemes.length - 1)); } @@ -318,9 +307,9 @@ private void processLeaf(Leaf leaf, List sentence, List names) { private List processLexeme(String lexemeStr) { List out = new ArrayList<>(); - String[] parts = UNDERLINE_PATTERN.split(lexemeStr); + String[] parts = splitOnUnderscores(lexemeStr); for (String tok : parts) { - if (tok.length() > 1 && !ALPHANUMERIC_PATTERN.matcher(tok).matches()) { + if (tok.length() > 1 && !isAlphaNumeric(tok)) { out.addAll(processTok(tok)); } else { out.add(tok); @@ -347,35 +336,19 @@ private List processTok(String tok) { // lets split all hyphens if (this.splitHyphenatedTokens && tok.contains("-") && tok.length() > 1) { - Matcher matcher = HYPHEN_PATTERN.matcher(tok); - - String firstTok = null; - String hyphen = "-"; - String secondTok = null; - String rest = null; - - if (matcher.matches()) { - if (matcher.group(1) != null) { - firstTok = matcher.group(2); - } else if (matcher.group(3) != null) { - secondTok = matcher.group(4); - rest = matcher.group(5); - } else if (matcher.group(6) != null) { - firstTok = matcher.group(7); - secondTok = matcher.group(8); - rest = matcher.group(9); - } + String[] parts = matchHyphenatedToken(tok); - addIfNotEmpty(firstTok, out); - addIfNotEmpty(hyphen, out); - addIfNotEmpty(secondTok, out); - addIfNotEmpty(rest, out); + if (parts != null) { + addIfNotEmpty(parts[0], out); + addIfNotEmpty("-", out); + addIfNotEmpty(parts[1], out); + addIfNotEmpty(parts[2], out); tokAdded = true; } } if (!tokAdded) { if (!original.equals(tok) && tok.length() > 1 - && !ALPHANUMERIC_PATTERN.matcher(tok).matches()) { + && !isAlphaNumeric(tok)) { out.addAll(processTok(tok)); } else { out.add(tok); @@ -391,6 +364,161 @@ private void addIfNotEmpty(String firstTok, List out) { } } + /** + * Splits on runs of underscores with the result of {@code String.split("[_]+")}: a leading + * run yields one empty first element, trailing empty elements are dropped, and + * underscore-only input yields an empty array. + * + * @param s The text. + * @return The elements in order. + */ + static String[] splitOnUnderscores(String s) { + if (s.isEmpty()) { + return new String[] {""}; + } + boolean hasToken = false; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) != '_') { + hasToken = true; + break; + } + } + if (!hasToken) { + return new String[0]; + } + List tokens = new ArrayList<>(); + if (s.charAt(0) == '_') { + tokens.add(""); + } + int start = 0; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) == '_') { + if (i > start) { + tokens.add(s.substring(start, i)); + } + while (i + 1 < s.length() && s.charAt(i + 1) == '_') { + i++; + } + start = i + 1; + } + } + if (s.length() > start) { + tokens.add(s.substring(start)); + } + return tokens.toArray(new String[0]); + } + + /** + * Tests whether a token consists of letters and decimal digits only, by code point. + * + * @param tok The token. + * @return {@code true} if the token is non-empty and every code point is a letter or a + * decimal digit. + */ + static boolean isAlphaNumeric(String tok) { + if (tok.isEmpty()) { + return false; + } + int i = 0; + while (i < tok.length()) { + int cp = tok.codePointAt(i); + if (!Character.isLetter(cp) && !Character.isDigit(cp)) { + return false; + } + i += Character.charCount(cp); + } + return true; + } + + /** + * Splits a hyphenated token into the letters before the hyphen, the letters after it, and + * the rest. Three shapes match: letters followed by a final hyphen, a leading hyphen followed + * by letters and anything, and letters, a hyphen, letters, and anything. + * + * @param tok The token, at least two characters long. + * @return The first token, second token, and rest, each {@code null} when absent, or + * {@code null} if the token has none of the three shapes. + */ + static String[] matchHyphenatedToken(String tok) { + int len = tok.length(); + // (\p{L}+)-$ + if (len > 1 && tok.charAt(len - 1) == '-' && isAllLetters(tok, 0, len - 1)) { + return new String[] {tok.substring(0, len - 1), null, null}; + } + // ^-(\p{L}+)(.*) + if (tok.charAt(0) == '-') { + int lettersEnd = lettersEnd(tok, 1); + if (lettersEnd > 1) { + return new String[] {null, tok.substring(1, lettersEnd), tok.substring(lettersEnd)}; + } + return null; + } + // (\p{L}+)-(\p{L}+)(.*) + int firstEnd = lettersEnd(tok, 0); + if (firstEnd > 0 && firstEnd + 1 < len && tok.charAt(firstEnd) == '-') { + int secondEnd = lettersEnd(tok, firstEnd + 1); + if (secondEnd > firstEnd + 1) { + return new String[] {tok.substring(0, firstEnd), + tok.substring(firstEnd + 1, secondEnd), tok.substring(secondEnd)}; + } + } + return null; + } + + /** + * Finds the end of the run of letters starting at an offset. + * + * @param s The text. + * @param from The start offset. + * @return The offset after the run, or {@code from} if no letter starts there. + */ + private static int lettersEnd(String s, int from) { + int i = from; + while (i < s.length()) { + int cp = s.codePointAt(i); + if (!Character.isLetter(cp)) { + break; + } + i += Character.charCount(cp); + } + return i; + } + + /** + * Tests whether a range holds letters only. + * + * @param s The text. + * @param from The inclusive start. + * @param to The exclusive end. + * @return {@code true} if every code point in the range is a letter. + */ + private static boolean isAllLetters(String s, int from, int to) { + int i = from; + while (i < to) { + int cp = s.codePointAt(i); + if (!Character.isLetter(cp)) { + return false; + } + i += Character.charCount(cp); + } + return true; + } + + /** + * Extracts the content of a NER tag in Arvores Deitadas format, between the optional + * {@code NER:} prefix and the closing angle bracket. + * + * @param t The tag. + * @return The content, or {@code null} if {@code t} is not enclosed in angle brackets. + */ + static String tagContent(String t) { + if (t.length() < 2 || t.charAt(0) != '<' || t.charAt(t.length() - 1) != '>') { + return null; + } + int start = t.startsWith("NER:", 1) ? 5 : 1; + return t.substring(start, t.length() - 1); + } + /** * Parses a NER tag in Arvores Deitadas format. * @@ -401,14 +529,11 @@ private static String getNER(String tags) { if (tags.contains("")) { return null; } - String[] tag = tags.split("\\s+"); + String[] tag = StringUtil.splitOnAsciiWhitespace(tags); for (String t : tag) { - Matcher matcher = TAG_PATTERN.matcher(t); - if (matcher.matches()) { - String ner = matcher.group(2); - if (HAREM.containsKey(ner)) { - return HAREM.get(ner); - } + String ner = tagContent(t); + if (ner != null && HAREM.containsKey(ner)) { + return HAREM.get(ner); } } return null; @@ -424,72 +549,29 @@ public void close() throws IOException { adSentenceStream.close(); } - enum Type { - ama, cie, lit - } - - // works for Amazonia - // private static final Pattern meta1 = Pattern - // .compile("^(?:[a-zA-Z\\-]*(\\d+)).*?p=(\\d+).*"); - // - // // works for selva cie - // private static final Pattern meta2 = Pattern - // .compile("^(?:[a-zA-Z\\-]*(\\d+)).*?p=(\\d+).*"); - private int getTextID(Sentence paragraph) { - + final String meta = paragraph.metadata(); - Type corpusType; - Pattern metaPattern; int textIdMeta2 = -1; String textMeta2 = ""; - if (meta.startsWith("LIT")) { - corpusType = Type.lit; - metaPattern = Pattern.compile("^([a-zA-Z\\-]+)(\\d+).*?p=(\\d+).*"); - } else if (meta.startsWith("CIE")) { - corpusType = Type.cie; - metaPattern = Pattern.compile("^.*?source=\"(.*?)\".*"); - } else { // ama - corpusType = Type.ama; - metaPattern = Pattern.compile("^(?:[a-zA-Z\\-]*(\\d+)).*?p=(\\d+).*"); - } - - if (corpusType.equals(Type.lit)) { - Matcher m2 = metaPattern.matcher(meta); - if (m2.matches()) { - String textId = m2.group(1); - if (!textId.equals(textMeta2)) { - textIdMeta2++; - textMeta2 = textId; - } - return textIdMeta2; - } else { + if (meta.startsWith("LIT") || meta.startsWith("CIE")) { + String textId = meta.startsWith("LIT") ? ADMetadata.textPrefix(meta) : ADMetadata.source(meta); + if (textId == null) { throw new RuntimeException("Invalid metadata: " + meta); } - } else if (corpusType.equals(Type.cie)) { - Matcher m2 = metaPattern.matcher(meta); - if (m2.matches()) { - String textId = m2.group(1); - if (!textId.equals(textMeta2)) { - textIdMeta2++; - textMeta2 = textId; - } - return textIdMeta2; - } else { - throw new RuntimeException("Invalid metadata: " + meta); - } - } else if (corpusType.equals(Type.ama)) { - Matcher m2 = metaPattern.matcher(meta); - if (m2.matches()) { - return Integer.parseInt(m2.group(1)); - // currentPara = Integer.parseInt(m.group(2)); - } else { - throw new RuntimeException("Invalid metadata: " + meta); + if (!textId.equals(textMeta2)) { + textIdMeta2++; + textMeta2 = textId; } + return textIdMeta2; } - - return 0; + // Amazonia + String textId = ADMetadata.textId(meta); + if (textId == null) { + throw new RuntimeException("Invalid metadata: " + meta); + } + return Integer.parseInt(textId); } } diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java index 8b7bd96416..9a66f497f9 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.List; import java.util.StringTokenizer; -import java.util.regex.Pattern; import opennlp.tools.commons.Internal; import opennlp.tools.formats.ad.ADSentenceStream.Sentence; @@ -32,6 +31,7 @@ import opennlp.tools.util.InputStreamFactory; import opennlp.tools.util.ObjectStream; import opennlp.tools.util.PlainTextByLineStream; +import opennlp.tools.util.StringUtil; /** * Note: @@ -40,8 +40,6 @@ @Internal public class ADPOSSampleStream implements ObjectStream { - private static final Pattern WHITESPACES_PATTERN = Pattern.compile("\\s+"); - private final ObjectStream adSentenceStream; private final boolean expandME; private final boolean isIncludeFeatures; @@ -118,7 +116,7 @@ private void processLeaf(Leaf leaf, List sentence, List tags) { if (isIncludeFeatures && leaf.getMorphologicalTag() != null) { tag += " " + leaf.getMorphologicalTag(); } - tag = WHITESPACES_PATTERN.matcher(tag).replaceAll("="); + tag = replaceWhitespaceWithEquals(tag); if (tag == null) tag = lexeme; @@ -152,6 +150,31 @@ private void processLeaf(Leaf leaf, List sentence, List tags) { } + /** + * Replaces every run of ASCII whitespace, leading and trailing runs included, with a single + * equals sign. + * + * @param tag The tag. + * @return The joined tag. + */ + static String replaceWhitespaceWithEquals(String tag) { + StringBuilder replaced = new StringBuilder(tag.length()); + int i = 0; + while (i < tag.length()) { + char c = tag.charAt(i); + if (StringUtil.isAsciiWhitespace(c)) { + replaced.append('='); + while (i + 1 < tag.length() && StringUtil.isAsciiWhitespace(tag.charAt(i + 1))) { + i++; + } + } else { + replaced.append(c); + } + i++; + } + return replaced.toString(); + } + @Override public void reset() throws IOException, UnsupportedOperationException { adSentenceStream.reset(); diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java index c78bf2b9a4..099303e3f0 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java @@ -21,8 +21,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import opennlp.tools.commons.Internal; import opennlp.tools.formats.ad.ADSentenceStream.Sentence; @@ -134,22 +132,15 @@ private boolean hasPunctuation(String text) { return false; } - // there are some different types of metadata depending on the corpus. - // TODO Merge these patterns - private static final Pattern META_1 = Pattern.compile("^(?:[a-zA-Z\\-]*(\\d+)).*?p=(\\d+).*"); - private void updateMeta() { if (this.sent != null) { String meta = this.sent.metadata(); - Matcher m = META_1.matcher(meta); - int currentText; - int currentPara; - if (m.matches()) { - currentText = Integer.parseInt(m.group(1)); - currentPara = Integer.parseInt(m.group(2)); - } else { + int[] textAndPara = ADMetadata.parseTextAndParagraph(meta); + if (textAndPara == null) { throw new RuntimeException("Invalid metadata: " + meta); } + int currentText = textAndPara[0]; + int currentPara = textAndPara[1]; isSamePara = isSameText = false; if (currentText == text) isSameText = true; diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java index 1445f153b5..4cbe29cb16 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java @@ -23,8 +23,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Stack; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +31,7 @@ import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.Node; import opennlp.tools.util.FilterObjectStream; import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.StringUtil; /** * Stream filter which merges text lines into sentences, following the Arvores @@ -63,15 +62,11 @@ public record Sentence (String text, Node root, String metadata) { public static class SentenceParser { private static final Logger logger = LoggerFactory.getLogger(SentenceParser.class); - private static final Pattern NODE_PATTERN = Pattern - .compile("([=-]*)([^:=]+:[^\\(\\s]+)(\\(([^\\)]+)\\))?\\s*(?:(\\((<.+>)\\))*)\\s*$"); - private static final Pattern LEAF_PATTERN = Pattern - .compile("^([=-]*)([^:=]+):([^\\(\\s]+)\\([\"'](.+)[\"']\\s*((?:<.+>)*)\\s*([^\\)]+)?\\)\\s+(.+)"); - private static final Pattern BIZARRE_LEAF_PATTERN = Pattern - .compile("^([=-]*)([^:=]+=[^\\(\\s]+)\\(([\"'].+[\"'])?\\s*([^\\)]+)?\\)\\s+(.+)"); - private static final Pattern PUNCTUATION_PATTERN = Pattern.compile("^(=*)(\\W+)$"); - private static final Pattern PUNCTUATION_DOT_PATTERN = Pattern.compile("\\»\\s+\\."); - private static final Pattern PUNCTUATION_COMMA_PATTERN = Pattern.compile("\\»\\s+\\,"); + + private static final char TAG_SEPARATOR = ':'; + private static final char BIZARRE_TAG_SEPARATOR = '='; + private static final String TAG_GROUP_OPEN = "(<"; + private static final String TAG_GROUP_CLOSE = ">)"; private String text,meta; @@ -208,11 +203,82 @@ public Sentence parse(String sentenceString, int para, boolean isTitle, boolean } private String fixPunctuation(String text) { - text = PUNCTUATION_DOT_PATTERN.matcher(text).replaceAll("»."); - text = PUNCTUATION_COMMA_PATTERN.matcher(text).replaceAll("»,"); + text = replaceGuillemetPunctuation(text, '.', "»."); + text = replaceGuillemetPunctuation(text, ',', "»,"); return text; } + /** + * Removes the ASCII whitespace between a closing guillemet and a following punctuation + * character. + * + * @param text The text. + * @param punct The punctuation character. + * @param replacement The two characters to write in place of guillemet, whitespace, and + * punctuation. + * @return The text with those runs joined. + */ + private String replaceGuillemetPunctuation(String text, char punct, String replacement) { + StringBuilder fixed = new StringBuilder(text.length()); + int i = 0; + while (i < text.length()) { + char c = text.charAt(i); + if (c == '»' && i + 1 < text.length()) { + int j = i + 1; + while (j < text.length() && StringUtil.isAsciiWhitespace(text.charAt(j))) { + j++; + } + if (j > i + 1 && j < text.length() && text.charAt(j) == punct) { + fixed.append(replacement); + i = j + 1; + continue; + } + } + fixed.append(c); + i++; + } + return fixed.toString(); + } + + /** + * Parses a punctuation line: leading equals signs followed by one or more characters that + * are not ASCII letters, digits, or underscores. A line of equals signs only also matches, + * with the last one as lexeme. + * + * @param line The line. + * @return The level, as one more than the count of leading equals signs, and the lexeme, + * or {@code null} if the line is not a punctuation line. + */ + private String[] parsePunctuationLine(String line) { + if (line.isEmpty()) { + return null; + } + for (int i = 0; i < line.length(); i++) { + if (isAsciiWord(line.charAt(i))) { + return null; + } + } + int equals = 0; + while (equals < line.length() && line.charAt(equals) == '=') { + equals++; + } + if (equals == line.length()) { + return new String[] {String.valueOf(equals), "="}; + } + return new String[] {String.valueOf(equals + 1), line.substring(equals)}; + } + + /** + * Tests for an ASCII letter, digit, or underscore. + * + * @param c The character. + * @return {@code true} for a word character. + */ + private boolean isAsciiWord(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') || c == '_'; + } + /** * Parse a tree element from a AD line * @@ -223,45 +289,21 @@ private String fixPunctuation(String text) { public TreeElement getElement(String line) { // Note: all levels are higher than 1, because 0 is reserved for the root. - // try node - Matcher nodeMatcher = NODE_PATTERN.matcher(line); - if (nodeMatcher.matches()) { - int level = nodeMatcher.group(1).length() + 1; - String syntacticTag = nodeMatcher.group(2); - Node node = new Node(); - node.setLevel(level); - node.setSyntacticTag(syntacticTag); + Node node = parseNode(line); + if (node != null) { return node; } - Matcher leafMatcher = LEAF_PATTERN.matcher(line); - if (leafMatcher.matches()) { - int level = leafMatcher.group(1).length() + 1; - String syntacticTag = leafMatcher.group(2); - String funcTag = leafMatcher.group(3); - String lemma = leafMatcher.group(4); - String secondaryTag = leafMatcher.group(5); - String morphologicalTag = leafMatcher.group(6); - String lexeme = leafMatcher.group(7); - Leaf leaf = new Leaf(); - leaf.setLevel(level); - leaf.setSyntacticTag(syntacticTag); - leaf.setFunctionalTag(funcTag); - leaf.setSecondaryTag(secondaryTag); - leaf.setMorphologicalTag(morphologicalTag); - leaf.setLexeme(lexeme); - leaf.setLemma(lemma); - + Leaf leaf = parseLeaf(line); + if (leaf != null) { return leaf; } - Matcher punctuationMatcher = PUNCTUATION_PATTERN.matcher(line); - if (punctuationMatcher.matches()) { - int level = punctuationMatcher.group(1).length() + 1; - String lexeme = punctuationMatcher.group(2); - Leaf leaf = new Leaf(); - leaf.setLevel(level); - leaf.setLexeme(lexeme); + String[] punctuation = parsePunctuationLine(line); + if (punctuation != null) { + leaf = new Leaf(); + leaf.setLevel(Integer.parseInt(punctuation[0])); + leaf.setLexeme(punctuation[1]); return leaf; } @@ -271,47 +313,29 @@ public TreeElement getElement(String line) { } if (line.startsWith("=")) { - Matcher bizarreLeafMatcher = BIZARRE_LEAF_PATTERN.matcher(line); - if (bizarreLeafMatcher.matches()) { - int level = bizarreLeafMatcher.group(1).length() + 1; - String syntacticTag = bizarreLeafMatcher.group(2); - String lemma = bizarreLeafMatcher.group(3); - String morphologicalTag = bizarreLeafMatcher.group(4); - String lexeme = bizarreLeafMatcher.group(5); - Leaf leaf = new Leaf(); - leaf.setLevel(level); - leaf.setSyntacticTag(syntacticTag); - leaf.setMorphologicalTag(morphologicalTag); - leaf.setLexeme(lexeme); - if (lemma != null) { - if (lemma.length() > 2) { - lemma = lemma.substring(1, lemma.length() - 1); - } - leaf.setLemma(lemma); - } - + leaf = parseBizarreLeaf(line); + if (leaf != null) { return leaf; - } else { - int level = line.lastIndexOf("=") + 1; - String lexeme = line.substring(level + 1); + } + int level = line.lastIndexOf("=") + 1; + String lexeme = line.substring(level + 1); - if (lexeme.matches("\\w.*?[\\.<>].*")) { - return null; - } + if (isWordWithMarkup(lexeme)) { + return null; + } - Leaf leaf = new Leaf(); - leaf.setLevel(level + 1); - leaf.setSyntacticTag(""); - leaf.setMorphologicalTag(""); - leaf.setFunctionalTag(""); - leaf.setLexeme(lexeme); + leaf = new Leaf(); + leaf.setLevel(level + 1); + leaf.setSyntacticTag(""); + leaf.setMorphologicalTag(""); + leaf.setFunctionalTag(""); + leaf.setLexeme(lexeme); - return leaf; - } + return leaf; } logger.warn("Couldn't parse leaf: {}", line); - Leaf leaf = new Leaf(); + leaf = new Leaf(); leaf.setLevel(1); leaf.setSyntacticTag(""); leaf.setMorphologicalTag(""); @@ -321,6 +345,412 @@ public TreeElement getElement(String line) { return leaf; } + /** + * Parses a node line: the level prefix, a syntactic tag with a colon, an optional part in + * parentheses, and optional tag groups. + * + * @param line The line. + * @return The node, or {@code null} if the line is not a node line. + */ + private Node parseNode(String line) { + for (int[] tag = scanLevelAndTag(line, TAG_SEPARATOR, line.length()); tag != null; + tag = scanLevelAndTag(line, TAG_SEPARATOR, tag[0] - 1)) { + if (isNodeTail(line, tag[1])) { + Node node = new Node(); + node.setLevel(tag[0] + 1); + node.setSyntacticTag(line.substring(tag[0], tag[1])); + return node; + } + } + return null; + } + + /** + * Parses a leaf line: the level prefix, a syntactic tag, a colon, a functional tag, and in + * parentheses a quoted lemma, secondary tags in angle brackets, and a morphological tag, + * then ASCII whitespace and the lexeme. + * + * @param line The line. + * @return The leaf, or {@code null} if the line is not a leaf line. + */ + private Leaf parseLeaf(String line) { + boolean[] noRestAt = new boolean[line.length() + 1]; + for (int[] tag = scanLevelAndTag(line, TAG_SEPARATOR, line.length()); tag != null; + tag = scanLevelAndTag(line, TAG_SEPARATOR, tag[0] - 1)) { + Leaf leaf = parseLeafAfterTag(line, tag[0], tag[1], noRestAt); + if (leaf != null) { + return leaf; + } + } + return null; + } + + /** + * Parses the part of a leaf line after the tag, see {@link #parseLeaf(String)}. + * + * @param line The line. + * @param start The index where the tag starts. + * @param tagEnd The index after the tag. + * @param noRestAt The indexes after which no rest was found so far, updated here. + * @return The leaf, or {@code null} if the part after the tag does not have that form. + */ + private Leaf parseLeafAfterTag(String line, int start, int tagEnd, boolean[] noRestAt) { + if (tagEnd + 1 >= line.length() || line.charAt(tagEnd) != '(' + || !isQuote(line.charAt(tagEnd + 1))) { + return null; + } + int lemmaStart = tagEnd + 2; + // the longest lemma after which the rest of the line still parses wins + for (int lemmaEnd = indexOfLineTerminator(line, lemmaStart) - 1; lemmaEnd > lemmaStart; + lemmaEnd--) { + if (!isQuote(line.charAt(lemmaEnd))) { + continue; + } + int[] rest = scanLeafRest(line, lemmaEnd + 1, noRestAt); + if (rest != null) { + int separator = line.indexOf(TAG_SEPARATOR, start); + Leaf leaf = new Leaf(); + leaf.setLevel(start + 1); + leaf.setSyntacticTag(line.substring(start, separator)); + leaf.setFunctionalTag(line.substring(separator + 1, tagEnd)); + leaf.setLemma(line.substring(lemmaStart, lemmaEnd)); + leaf.setSecondaryTag(line.substring(rest[0], rest[1])); + leaf.setMorphologicalTag(rest[2] == rest[3] ? null : line.substring(rest[2], rest[3])); + leaf.setLexeme(line.substring(rest[4])); + return leaf; + } + } + return null; + } + + /** + * Parses a leaf line whose tag has an equals sign in place of the colon: the level prefix, + * the tag, and in parentheses an optional quoted lemma and an optional morphological tag, + * then ASCII whitespace and the lexeme. + * + * @param line The line. + * @return The leaf, or {@code null} if the line does not have that form. + */ + private Leaf parseBizarreLeaf(String line) { + for (int[] tag = scanLevelAndTag(line, BIZARRE_TAG_SEPARATOR, line.length()); tag != null; + tag = scanLevelAndTag(line, BIZARRE_TAG_SEPARATOR, tag[0] - 1)) { + Leaf leaf = parseBizarreLeafAfterTag(line, tag[0], tag[1]); + if (leaf != null) { + return leaf; + } + } + return null; + } + + /** + * Parses the part of a leaf line after a tag with an equals sign, see + * {@link #parseBizarreLeaf(String)}. + * + * @param line The line. + * @param start The index where the tag starts. + * @param tagEnd The index after the tag. + * @return The leaf, or {@code null} if the part after the tag does not have that form. + */ + private Leaf parseBizarreLeafAfterTag(String line, int start, int tagEnd) { + if (tagEnd == line.length() || line.charAt(tagEnd) != '(') { + return null; + } + int open = tagEnd + 1; + String lemma = null; + int[] rest = null; + if (open < line.length() && isQuote(line.charAt(open))) { + for (int lemmaEnd = indexOfLineTerminator(line, open + 1) - 1; + lemmaEnd > open + 1 && rest == null; lemmaEnd--) { + if (isQuote(line.charAt(lemmaEnd))) { + rest = scanMorphologyAndLexeme(line, lemmaEnd + 1); + if (rest != null) { + lemma = line.substring(open + 1, lemmaEnd); + } + } + } + } + if (rest == null) { + rest = scanMorphologyAndLexeme(line, open); + if (rest == null) { + return null; + } + } + Leaf leaf = new Leaf(); + leaf.setLevel(start + 1); + leaf.setSyntacticTag(line.substring(start, tagEnd)); + leaf.setMorphologicalTag(rest[0] == rest[1] ? null : line.substring(rest[0], rest[1])); + leaf.setLexeme(line.substring(rest[2])); + leaf.setLemma(lemma); + return leaf; + } + + /** + * Scans the level prefix and the tag at the start of a line. The prefix is the run of equals + * signs and hyphens, the tag one or more characters other than a colon or an equals sign, + * the separator, and one or more characters that are neither an opening parenthesis nor + * ASCII whitespace. A longer prefix is preferred; hyphens at its end may move into the tag, + * so the callers try the next shorter prefix when the rest of the line does not parse. + * + * @param line The line. + * @param separator The character between the two parts of the tag. + * @param maxStart The highest index where the tag may start: the length of the line for the + * first candidate, one less than the previous start for the next one. + * @return The index where the tag starts, which is the length of the prefix, and the index + * after the tag, or {@code null} if there is no further candidate. + */ + private int[] scanLevelAndTag(String line, char separator, int maxStart) { + int run = 0; + while (run < line.length() && (line.charAt(run) == '=' || line.charAt(run) == '-')) { + run++; + } + for (int start = Math.min(run, maxStart); start >= 0; start--) { + if (start == run || line.charAt(start) == '-') { + int end = scanTag(line, start, separator); + if (end != -1) { + return new int[] {start, end}; + } + } + } + return null; + } + + /** + * Scans a tag: one or more characters other than a colon or an equals sign, the separator, + * and one or more characters that are neither an opening parenthesis nor ASCII whitespace. + * + * @param line The line. + * @param from The index where the tag starts. + * @param separator The character between the two parts of the tag. + * @return The index after the tag, or -1 if there is no tag at {@code from}. + */ + private int scanTag(String line, int from, char separator) { + int i = from; + while (i < line.length() && line.charAt(i) != TAG_SEPARATOR + && line.charAt(i) != BIZARRE_TAG_SEPARATOR) { + i++; + } + if (i == from || i == line.length() || line.charAt(i) != separator) { + return -1; + } + int end = i + 1; + while (end < line.length() && line.charAt(end) != '(' + && !StringUtil.isAsciiWhitespace(line.charAt(end))) { + end++; + } + return end == i + 1 ? -1 : end; + } + + /** + * Tests the rest of a node line after the tag: an optional part in parentheses, then the tag + * groups. + * + * @param line The line. + * @param from The index after the tag. + * @return {@code true} if the rest of the line has that form. + */ + private boolean isNodeTail(String line, int from) { + if (from < line.length() && line.charAt(from) == '(') { + int close = line.indexOf(')', from + 1); + if (close > from + 1 && isTagGroupRun(line, close + 1)) { + return true; + } + } + return isTagGroupRun(line, from); + } + + /** + * Tests for tag groups up to the end of the line: after optional ASCII whitespace either + * nothing, or an opening parenthesis and angle bracket, one or more characters other than a + * line terminator, a closing angle bracket and parenthesis, and optional ASCII whitespace. + * + * @param line The line. + * @param from The index where the tag groups start. + * @return {@code true} if the rest of the line has that form. + */ + private boolean isTagGroupRun(String line, int from) { + int start = skipAsciiWhitespace(line, from); + int end = line.length(); + while (end > start && StringUtil.isAsciiWhitespace(line.charAt(end - 1))) { + end--; + } + if (start == end) { + return true; + } + int contentStart = start + TAG_GROUP_OPEN.length(); + int contentEnd = end - TAG_GROUP_CLOSE.length(); + return contentEnd > contentStart && line.startsWith(TAG_GROUP_OPEN, start) + && line.startsWith(TAG_GROUP_CLOSE, contentEnd) + && indexOfLineTerminator(line, contentStart) >= contentEnd; + } + + /** + * Scans the rest of a leaf line after the lemma: optional ASCII whitespace, secondary tags, + * optional ASCII whitespace, an optional morphological tag, the closing parenthesis, ASCII + * whitespace, and the lexeme. + * + * @param line The line. + * @param from The index after the lemma. + * @param noRestAt The indexes after which no rest was found so far, updated here. + * @return The start and end of the secondary tags, the start of the morphological tag, the + * index of the closing parenthesis, and the start of the lexeme, or {@code null} if + * the rest of the line does not have that form. + */ + private int[] scanLeafRest(String line, int from, boolean[] noRestAt) { + int tagsStart = skipAsciiWhitespace(line, from); + int[] rest = scanSecondaryTags(line, tagsStart, noRestAt); + return rest == null ? null : new int[] {tagsStart, rest[0], rest[1], rest[2], rest[3]}; + } + + /** + * Scans secondary tags and the rest of a leaf line after them. Each tag is an opening angle + * bracket, one or more characters other than a line terminator, and a closing angle + * bracket; a longer tag, and then one more tag, is preferred when the rest of the line still + * parses after it. + * + * @param line The line. + * @param from The index where the next secondary tag would start. + * @param noRestAt The indexes after which no rest was found so far, updated here. + * @return The end of the secondary tags, the start of the morphological tag, the index of the + * closing parenthesis, and the start of the lexeme, or {@code null} if the rest of the + * line does not have that form. + */ + private int[] scanSecondaryTags(String line, int from, boolean[] noRestAt) { + if (noRestAt[from]) { + return null; + } + if (from < line.length() && line.charAt(from) == '<') { + for (int close = indexOfLineTerminator(line, from + 1) - 1; close > from + 1; close--) { + if (line.charAt(close) == '>') { + int[] rest = scanSecondaryTags(line, close + 1, noRestAt); + if (rest != null) { + return rest; + } + } + } + } + int[] rest = scanMorphologyAndLexeme(line, from); + if (rest == null) { + noRestAt[from] = true; + return null; + } + return new int[] {from, rest[0], rest[1], rest[2]}; + } + + /** + * Scans the end of a leaf line: optional ASCII whitespace, an optional morphological tag up + * to the first closing parenthesis, that parenthesis, ASCII whitespace, and the lexeme. + * + * @param line The line. + * @param from The index after the secondary tags. + * @return The start of the morphological tag, the index of the closing parenthesis, and the + * start of the lexeme, or {@code null} if the end of the line does not have that form. + */ + private int[] scanMorphologyAndLexeme(String line, int from) { + int morphologyStart = skipAsciiWhitespace(line, from); + int close = line.indexOf(')', morphologyStart); + if (close == -1) { + return null; + } + int lexemeStart = scanLexemeStart(line, close); + return lexemeStart == -1 ? null : new int[] {morphologyStart, close, lexemeStart}; + } + + /** + * Finds the lexeme after the closing parenthesis: ASCII whitespace, then one or more + * characters other than a line terminator up to the end of the line. When only whitespace + * follows the parenthesis, the last character is the lexeme. + * + * @param line The line. + * @param close The index of the closing parenthesis. + * @return The start of the lexeme, or -1 if there is none. + */ + private int scanLexemeStart(String line, int close) { + int lexemeStart = skipAsciiWhitespace(line, close + 1); + if (lexemeStart == close + 1) { + return -1; + } + if (lexemeStart == line.length()) { + lexemeStart--; + return lexemeStart > close + 1 && !isLineTerminator(line.charAt(lexemeStart)) + ? lexemeStart : -1; + } + return indexOfLineTerminator(line, lexemeStart) == line.length() ? lexemeStart : -1; + } + + /** + * Tests whether a lexeme starts with an ASCII letter, digit, or underscore and has a period + * or an angle bracket after it, with no line terminator anywhere. + * + * @param lexeme The lexeme. + * @return {@code true} for such a lexeme. + */ + private boolean isWordWithMarkup(String lexeme) { + if (lexeme.isEmpty() || !isAsciiWord(lexeme.charAt(0)) + || indexOfLineTerminator(lexeme, 0) < lexeme.length()) { + return false; + } + for (int i = 1; i < lexeme.length(); i++) { + char c = lexeme.charAt(i); + if (c == '.' || c == '<' || c == '>') { + return true; + } + } + return false; + } + + /** + * Skips ASCII whitespace. + * + * @param line The line. + * @param from The index to start at. + * @return The index of the first character at or after {@code from} that is not ASCII + * whitespace, or the length of the line. + */ + private int skipAsciiWhitespace(String line, int from) { + int i = from; + while (i < line.length() && StringUtil.isAsciiWhitespace(line.charAt(i))) { + i++; + } + return i; + } + + /** + * Tests for a double or single quote. + * + * @param c The character. + * @return {@code true} for one of the two. + */ + private boolean isQuote(char c) { + return c == '"' || c == '\''; + } + + /** + * Finds the first line terminator at or after an index. + * + * @param text The text. + * @param from The index to start at. + * @return The index of the terminator, or the length of the text if there is none. + */ + static int indexOfLineTerminator(CharSequence text, int from) { + for (int i = from; i < text.length(); i++) { + if (isLineTerminator(text.charAt(i))) { + return i; + } + } + return text.length(); + } + + /** + * Tests for a line terminator: line feed, carriage return, next line, line separator, or + * paragraph separator. + * + * @param c The character. + * @return {@code true} for one of those five characters. + */ + private static boolean isLineTerminator(char c) { + return c == '\n' || c == '\r' || c == '\u0085' || c == '\u2028' || c == '\u2029'; + } + /** Represents a tree element, Node or Leaf */ public abstract static class TreeElement { @@ -455,15 +885,11 @@ public String getLemma() { } - private static final Pattern SENT_START = Pattern.compile("]*>"); - private static final Pattern SENT_END = Pattern.compile(""); - private static final Pattern EXT_END = Pattern.compile(""); - private static final Pattern TITLE_START = Pattern.compile("]*>"); - private static final Pattern TITLE_END = Pattern.compile(""); - private static final Pattern BOX_START = Pattern.compile("]*>"); - private static final Pattern BOX_END = Pattern.compile(""); - private static final Pattern PARA_START = Pattern.compile("]*>"); - private static final Pattern TEXT_START = Pattern.compile("]*>"); + private static final String SENTENCE_TAG = "s"; + private static final String TEXT_TAG = "ext"; + private static final String TITLE_TAG = "t"; + private static final String BOX_TAG = "caixa"; + private static final String PARAGRAPH_TAG = "p"; private final SentenceParser parser; @@ -489,25 +915,25 @@ public Sentence read() throws IOException { if (line != null) { if (sentenceStarted) { - if (SENT_END.matcher(line).matches() || EXT_END.matcher(line).matches()) { + if (isClosingTag(line, SENTENCE_TAG) || isClosingTag(line, TEXT_TAG)) { sentenceStarted = false; } else if (!line.startsWith("A1")) { sentence.append(line).append('\n'); } } else { - if (SENT_START.matcher(line).matches()) { + if (isOpeningTag(line, SENTENCE_TAG)) { sentenceStarted = true; - } else if (PARA_START.matcher(line).matches()) { + } else if (isOpeningTag(line, PARAGRAPH_TAG)) { paraID++; - } else if (TITLE_START.matcher(line).matches()) { + } else if (isOpeningTag(line, TITLE_TAG)) { isTitle = true; - } else if (TITLE_END.matcher(line).matches()) { + } else if (isClosingTag(line, TITLE_TAG)) { isTitle = false; - } else if (TEXT_START.matcher(line).matches()) { + } else if (isOpeningTag(line, TEXT_TAG)) { paraID = 0; - } else if (BOX_START.matcher(line).matches()) { + } else if (isOpeningTag(line, BOX_TAG)) { isBox = true; - } else if (BOX_END.matcher(line).matches()) { + } else if (isClosingTag(line, BOX_TAG)) { isBox = false; } } @@ -529,4 +955,34 @@ public Sentence read() throws IOException { } } } + + /** + * Tests whether a line is an opening markup tag with the given name: the name right after the + * opening angle bracket, then any characters other than a closing angle bracket, then the + * closing angle bracket as the last character. + * + * @param line The line. + * @param name The tag name. + * @return {@code true} if the whole line is such a tag. + */ + static boolean isOpeningTag(String line, String name) { + int last = line.length() - 1; + if (last <= name.length() || line.charAt(0) != '<' || !line.startsWith(name, 1) + || line.charAt(last) != '>') { + return false; + } + return line.indexOf('>', name.length() + 1) == last; + } + + /** + * Tests whether a line is the closing markup tag with the given name and nothing else. + * + * @param line The line. + * @param name The tag name. + * @return {@code true} if the whole line is that closing tag. + */ + static boolean isClosingTag(String line, String name) { + return line.length() == name.length() + 3 && line.startsWith("'; + } } diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java index 17c991e622..1ee1efcb44 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java @@ -27,9 +27,6 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; import opennlp.tools.util.InputStreamFactory; @@ -45,8 +42,6 @@ public class ConlluStream implements ObjectStream { private final ObjectStream sentenceStream; - private static final Pattern regex = Pattern.compile("text_([a-z]{2,3})"); - /** * Initializes a {@link ConlluStream}. * @@ -155,7 +150,7 @@ private List postProcessContractions(List lines) index.put(line.getId(), i); if (line.getId().contains("-")) { List expandedContractions = new ArrayList<>(); - String[] ids = line.getId().split("-"); + String[] ids = splitOnHyphen(line.getId()); int start = Integer.parseInt(ids[0]); int end = Integer.parseInt(ids[1]); for (int j = start; j <= end; j++) { @@ -230,15 +225,7 @@ private ConlluWordLine mergeAnnotation(ConlluWordLine contraction, private Map addTextLang(String firstPart, String secondPart, Map textLang) throws InvalidFormatException { - String lang = ""; - try { - Matcher regexMatcher = regex.matcher(firstPart); - if (regexMatcher.find()) { - lang = regexMatcher.group(1); - } - } catch (PatternSyntaxException e) { - throw new InvalidFormatException(e); - } + String lang = extractTextLang(firstPart); if (!lang.isEmpty()) { textLang.put(Locale.of(lang), secondPart); } @@ -248,6 +235,59 @@ private Map addTextLang(String firstPart, String secondPart, return textLang; } + /** + * Splits a token id on hyphens with the result of {@code String.split("-")}: empty elements + * between consecutive hyphens are kept, trailing empty elements are dropped. + * + * @param id The token id. + * @return The elements in order. + */ + private String[] splitOnHyphen(String id) { + if (id.isEmpty()) { + return new String[] {""}; + } + List parts = new ArrayList<>(); + int start = 0; + for (int i = 0; i < id.length(); i++) { + if (id.charAt(i) == '-') { + parts.add(id.substring(start, i)); + start = i + 1; + } + } + if (id.length() > start) { + parts.add(id.substring(start)); + } + while (!parts.isEmpty() && parts.get(parts.size() - 1).isEmpty()) { + parts.remove(parts.size() - 1); + } + return parts.toArray(new String[0]); + } + + /** + * Extracts the language code from a {@code text_xx} or {@code text_xxx} comment key: the two + * or three ASCII lowercase letters, preferring three, after the first {@code text_} that at + * least two follow. + * + * @param firstPart The comment key. + * @return The language code, or an empty string if there is none. + */ + private String extractTextLang(String firstPart) { + int from = 0; + while ((from = firstPart.indexOf("text_", from)) != -1) { + int i = from + "text_".length(); + int len = 0; + while (len < 3 && i + len < firstPart.length() + && firstPart.charAt(i + len) >= 'a' && firstPart.charAt(i + len) <= 'z') { + len++; + } + if (len >= 2) { + return firstPart.substring(i, i + len); + } + from++; + } + return ""; + } + @Override public void close() throws IOException { sentenceStream.close(); diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStream.java index b2b6199458..3ceef87713 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStream.java @@ -32,6 +32,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; import opennlp.tools.cmdline.TerminateToolException; import opennlp.tools.langdetect.Language; @@ -67,7 +68,10 @@ private class LeipzigSentencesStream implements ObjectStream { // The file name contains the number of lines, but to make this more stable // the file is once scanned for the count even tough this is slower - int totalLineCount = (int) Files.lines(sentencesFile.toPath()).count(); + int totalLineCount; + try (Stream lines = Files.lines(sentencesFile.toPath())) { + totalLineCount = (int) lines.count(); + } int requiredLines = sentencesPerSample * numberOfSamples; if (totalLineCount < requiredLines) @@ -130,6 +134,9 @@ public LanguageSample read() throws IOException { } } + /** The number of leading file name characters that carry the ISO 639-3 language code. */ + private static final int LANG_CODE_LENGTH = 3; + private final int sentencesPerSample; private final Map langSampleCounts; @@ -154,8 +161,8 @@ public LeipzigLanguageSampleStream(File leipzigFolder, final int sentencesPerSam this.sentencesPerSample = sentencesPerSample; sentencesFiles = leipzigFolder.listFiles(pathname -> !pathname.isHidden() && pathname.isFile() - && pathname.getName().length() >= 3 - && pathname.getName().substring(0,3).matches("[a-z]+")); + && pathname.getName().length() >= LANG_CODE_LENGTH + && isAsciiLowerCaseWord(pathname.getName().substring(0, LANG_CODE_LENGTH))); if (null == sentencesFiles) { throw new TerminateToolException(-1 , "Directory " + leipzigFolder + " empty , No files to read!"); @@ -164,7 +171,7 @@ public LeipzigLanguageSampleStream(File leipzigFolder, final int sentencesPerSam Arrays.sort(sentencesFiles); Map langCounts = Arrays.stream(sentencesFiles) - .map(file -> file.getName().substring(0, 3)) + .map(file -> file.getName().substring(0, LANG_CODE_LENGTH)) .collect(Collectors.groupingBy(String::toString, Collectors.summingInt(v -> 1))); langSampleCounts = langCounts.entrySet().stream() @@ -175,6 +182,27 @@ public LeipzigLanguageSampleStream(File leipzigFolder, final int sentencesPerSam reset(); } + /** + * Tests whether {@code text} is a non-empty run of ASCII lower case letters, {@code a} to + * {@code z}. Letters outside that range, digits, and punctuation are rejected. + * + * @param text The text to check. Must not be {@code null}. + * @return {@code true} if {@code text} has at least one character and all of them are + * ASCII lower case letters. + */ + static boolean isAsciiLowerCaseWord(CharSequence text) { + if (text.isEmpty()) { + return false; + } + for (int i = 0; i < text.length(); i++) { + final char c = text.charAt(i); + if (c < 'a' || c > 'z') { + return false; + } + } + return true; + } + @Override public LanguageSample read() throws IOException { LanguageSample sample; @@ -185,7 +213,7 @@ public LanguageSample read() throws IOException { if (sentencesFilesIt.hasNext()) { File sentencesFile = sentencesFilesIt.next(); - String lang = sentencesFile.getName().substring(0, 3); + String lang = sentencesFile.getName().substring(0, LANG_CODE_LENGTH); sampleStream = new LeipzigSentencesStream(lang, sentencesFile, sentencesPerSample, langSampleCounts.get(lang)); diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascIdentifiers.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascIdentifiers.java new file mode 100644 index 0000000000..be77dde778 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascIdentifiers.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.masc; + +/** + * Shared handling of the identifier attributes in MASC annotation files. Node, region, + * and named entity identifiers carry a fixed text prefix followed by a number; the parsers + * remove the prefix before parsing the number. + */ +final class MascIdentifiers { + + /** The prefix of a named entity node identifier, as in {@code ne-n7}. */ + static final String NAMED_ENTITY_ID_PREFIX = "ne-n"; + + /** The prefix of a Penn token node identifier, as in {@code penn-n7}. */ + static final String PENN_TOKEN_ID_PREFIX = "penn-n"; + + /** The prefix of a segmentation region identifier, as in {@code seg-r7}. */ + static final String REGION_ID_PREFIX = "seg-r"; + + private MascIdentifiers() { + } + + /** + * Removes the first occurrence of {@code literal} from {@code input}. Later occurrences + * stay in place, and {@code input} is returned unchanged if it does not contain + * {@code literal}. The search is a plain text comparison. + * + * @param input The text to search. Must not be {@code null}. + * @param literal The text to remove. Must not be {@code null}. + * @return {@code input} without its first occurrence of {@code literal}. + */ + static String removeFirst(String input, String literal) { + final int start = input.indexOf(literal); + if (start < 0) { + return input; + } + return input.substring(0, start) + input.substring(start + literal.length()); + } +} diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascNamedEntityParser.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascNamedEntityParser.java index bd10050756..67385448ae 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascNamedEntityParser.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascNamedEntityParser.java @@ -54,7 +54,7 @@ public void startElement(String uri, String localName, String qName, Attributes try { if (qName.equals("a")) { int entityID = Integer.parseInt( - attributes.getValue("ref").replaceFirst("ne-n", "")); + MascIdentifiers.removeFirst(attributes.getValue("ref"), MascIdentifiers.NAMED_ENTITY_ID_PREFIX)); String label = attributes.getValue("label"); if (entityIDtoEntityType.containsKey(entityID)) { throw new SAXException("Multiple labels for one named entity"); @@ -65,9 +65,9 @@ public void startElement(String uri, String localName, String qName, Attributes if (qName.equals("edge")) { int entityID = Integer.parseInt( - attributes.getValue("from").replaceFirst("ne-n", "")); + MascIdentifiers.removeFirst(attributes.getValue("from"), MascIdentifiers.NAMED_ENTITY_ID_PREFIX)); int tokenID = Integer.parseInt( - attributes.getValue("to").replaceFirst("penn-n", "")); + MascIdentifiers.removeFirst(attributes.getValue("to"), MascIdentifiers.PENN_TOKEN_ID_PREFIX)); if (!entityIDsToTokens.containsKey(entityID)) { List tokens = new ArrayList<>(); diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascPennTagParser.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascPennTagParser.java index 5a423edaa1..57f4ebb598 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascPennTagParser.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascPennTagParser.java @@ -55,8 +55,8 @@ public void startElement(String uri, String localName, String qName, Attributes try { //get the link between region and Penn tag if (qName.equals("node")) { - tokenStack.push(Integer.parseInt(attributes.getValue("xml:id") - .replaceFirst("penn-n", ""))); + tokenStack.push(Integer.parseInt(MascIdentifiers.removeFirst( + attributes.getValue("xml:id"), MascIdentifiers.PENN_TOKEN_ID_PREFIX))); } if (qName.equals("link")) { @@ -65,7 +65,7 @@ public void startElement(String uri, String localName, String qName, Attributes } String[] targets = attributes.getValue("targets") - .replace("seg-r", "").split(" "); + .replace(MascIdentifiers.REGION_ID_PREFIX, "").split(" "); int[] regions = new int[targets.length]; for (int i = 0; i < targets.length; i++) { @@ -76,8 +76,8 @@ public void startElement(String uri, String localName, String qName, Attributes } if (qName.equals("a")) { - tokenStackTag.push(Integer.parseInt(attributes.getValue("ref") - .replaceFirst("penn-n", ""))); + tokenStackTag.push(Integer.parseInt(MascIdentifiers.removeFirst( + attributes.getValue("ref"), MascIdentifiers.PENN_TOKEN_ID_PREFIX))); } if (qName.equals("f")) { diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascWordParser.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascWordParser.java index ea1cb3b757..073536b654 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascWordParser.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/masc/MascWordParser.java @@ -42,7 +42,8 @@ public void startElement(String uri, String localName, String qName, Attributes try { // create a word and put it into the list of words if (qName.equalsIgnoreCase("region")) { - int id = Integer.parseInt(attributes.getValue("xml:id").replaceFirst("seg-r", "")); + int id = Integer.parseInt(MascIdentifiers.removeFirst( + attributes.getValue("xml:id"), MascIdentifiers.REGION_ID_PREFIX)); String[] anchors = attributes.getValue("anchors").split(" "); int left = Integer.parseInt(anchors[0]); diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/util/MarkableFileInputStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/util/MarkableFileInputStream.java index f7cf51f6ac..3febdb8dfc 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/util/MarkableFileInputStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/util/MarkableFileInputStream.java @@ -90,4 +90,14 @@ public int read(byte[] b) throws IOException { public int read(byte[] b, int off, int len) throws IOException { return in.read(b, off, len); } + + /** + * Closes the underlying file. + * + * @throws IOException If the file cannot be closed. + */ + @Override + public void close() throws IOException { + in.close(); + } } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADMetadataTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADMetadataTest.java new file mode 100644 index 0000000000..c47e108a03 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADMetadataTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.ad; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +public class ADMetadataTest { + + @ParameterizedTest + @CsvSource(delimiter = '|', ignoreLeadingAndTrailingWhitespace = false, value = { + "1001 p=1 ref=\"1001.porto-poesia=removeme=-2\" source=\"SELVA 1001.porto\"|1001|1", + "1001 p=12 title box source=\"x\"|1001|12", + "LIT-12 p=3|12|3", + "LIT12 p=3|12|3", + "-12 p=3|12|3", + "12p=9|12|9", + // leading zeros are digits + "0012 p=007|12|7", + // the first p= that a digit follows counts + "12 p= p=8|12|8", + "12 p=a p=8|12|8", + "12 pp=7|12|7", + "1 ap=2|1|2", + // the text id ends at the first character that is no digit + "12x34 p=5 p=6|12|5" + }) + void testParseTextAndParagraph(String meta, int text, int paragraph) { + Assertions.assertArrayEquals(new int[] {text, paragraph}, + ADMetadata.parseTextAndParagraph(meta)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "x p=1", "12", "12 p=", "12 P=1", "12 p=a", " 12 p=1", "p=1", + "1 p==2", "LIT p=1", "LIT-p=1", + // digits from other scripts are no ASCII digits + "١٢ p=1", "12 p=١", + // metadata is one line + "12 p=1\n", "12\u2028 p=1", "12 p=1\u0085", "\r12 p=1"}) + void testParseTextAndParagraphRejects(String meta) { + Assertions.assertNull(ADMetadata.parseTextAndParagraph(meta)); + Assertions.assertNull(ADMetadata.textId(meta)); + Assertions.assertNull(ADMetadata.textPrefix(meta)); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', value = { + "1001 p=1 source=\"x\"|1001", + "LIT-1001 p=1|1001", + "0012 p=1|0012", + "12x34 p=5|12" + }) + void testTextId(String meta, String textId) { + Assertions.assertEquals(textId, ADMetadata.textId(meta)); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', value = { + "LIT-1001 p=1|LIT-", + "LIT1001 p=1|LIT", + "LITx1 p=1|LITx", + "--1 p=1|--", + "a1p=1|a" + }) + void testTextPrefix(String meta, String prefix) { + Assertions.assertEquals(prefix, ADMetadata.textPrefix(meta)); + } + + @ParameterizedTest + @ValueSource(strings = {"1001 p=1", "1001 p=1 LIT", "LIT1", "LIT1 p=", "LITé1 p=1"}) + void testTextPrefixRejects(String meta) { + Assertions.assertNull(ADMetadata.textPrefix(meta)); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', ignoreLeadingAndTrailingWhitespace = false, value = { + "CIE source=\"abc\" x|abc", + "CIE1 p=1 ref=\"r\" source=\"SELVA 1001.porto\"|SELVA 1001.porto", + "CIE source=\"\"|''", + "source=\"a\"|a", + " source=\"a\"|a", + // the first source attribute counts, up to the next double quote + "source=\"a\"source=\"b\"|a", + "source=\"source=\"x\"|source=", + "source=\" a \" |' a '" + }) + void testSource(String meta, String source) { + Assertions.assertEquals(source, ADMetadata.source(meta)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "CIE x", "CIE source=\"a", "CIE source=a\"", "CIE Source=\"a\"", + "source='a'", + // metadata is one line + "source=\"a\nb\"", "source=\"a\"\n", "source=\"a\"\u2028", "\u0085source=\"a\""}) + void testSourceRejects(String meta) { + Assertions.assertNull(ADMetadata.source(meta)); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java index 101b465259..b46f93347b 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java @@ -19,12 +19,21 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.namefind.NameSample; +import opennlp.tools.util.ObjectStream; import opennlp.tools.util.PlainTextByLineStream; import opennlp.tools.util.Span; @@ -117,4 +126,123 @@ void testMissingRightContraction() { Assertions.assertEquals(new Span(5, 6, "person"), samples.get(7).getNames()[2]); } + private static Stream underscoreLexemes() { + return Stream.of( + Arguments.of("Rio_de_Janeiro", new String[] {"Rio", "de", "Janeiro"}), + Arguments.of("a__b", new String[] {"a", "b"}), + Arguments.of("_a", new String[] {"", "a"}), + Arguments.of("a_", new String[] {"a"}), + Arguments.of("__", new String[0]), + Arguments.of("_", new String[0]), + Arguments.of("\uD801\uDC12_\uD83D\uDE00", new String[] {"\uD801\uDC12", "\uD83D\uDE00"}), + Arguments.of("", new String[] {""}), + Arguments.of("casa", new String[] {"casa"})); + } + + @ParameterizedTest + @MethodSource("underscoreLexemes") + void testSplitOnUnderscores(String lexeme, String[] expected) { + Assertions.assertArrayEquals(expected, ADNameSampleStream.splitOnUnderscores(lexeme)); + Assertions.assertArrayEquals(lexeme.split("[_]+"), ADNameSampleStream.splitOnUnderscores(lexeme)); + } + + @ParameterizedTest + @ValueSource(strings = {"casa", "São", "1990", "R2D2", "\uD801\uDC12\u0661"}) + void testIsAlphaNumericAccepts(String token) { + Assertions.assertTrue(ADNameSampleStream.isAlphaNumeric(token)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "guarda-chuva", "R$", "a b", "\u00BD", "\uD83D\uDE00"}) + void testIsAlphaNumericRejects(String token) { + Assertions.assertFalse(ADNameSampleStream.isAlphaNumeric(token)); + } + + private static Stream hyphenatedTokens() { + return Stream.of( + Arguments.of("guarda-", new String[] {"guarda", null, null}), + Arguments.of("a-", new String[] {"a", null, null}), + Arguments.of("-chuva", new String[] {null, "chuva", ""}), + Arguments.of("-chuva2!", new String[] {null, "chuva", "2!"}), + Arguments.of("guarda-chuva", new String[] {"guarda", "chuva", ""}), + Arguments.of("guarda-chuva-sol", new String[] {"guarda", "chuva", "-sol"}), + Arguments.of("São-Paulo", new String[] {"São", "Paulo", ""}), + // supplementary-plane letters are letters, a combining mark ends the letter run + Arguments.of("\uD801\uDC12-\uD801\uDC3A", new String[] {"\uD801\uDC12", "\uD801\uDC3A", ""}), + Arguments.of("e\u0301-a", new String[] {null, null, null})); + } + + @ParameterizedTest + @MethodSource("hyphenatedTokens") + void testMatchHyphenatedToken(String token, String[] expected) { + String[] actual = ADNameSampleStream.matchHyphenatedToken(token); + if (expected[0] == null && expected[1] == null && expected[2] == null) { + Assertions.assertNull(actual); + } else { + Assertions.assertArrayEquals(expected, actual); + } + } + + @ParameterizedTest + @ValueSource(strings = {"-", "--", "-1", "1-", "a1-b", "a-1", "a--b", "ab", "a -"}) + void testMatchHyphenatedTokenRejects(String token) { + Assertions.assertNull(ADNameSampleStream.matchHyphenatedToken(token)); + } + + @ParameterizedTest + @CsvSource({", PROP", ", PROP", "<>, ''", ", ''", ", a, NER:X", ", ner:PROP", "<\uD83D\uDE00>, \uD83D\uDE00"}) + void testTagContent(String tag, String expected) { + Assertions.assertEquals(expected, ADNameSampleStream.tagContent(tag)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "<", ">", "PROP", ""}) + void testTagContentRejects(String tag) { + Assertions.assertNull(ADNameSampleStream.tagContent(tag)); + } + + private static ObjectStream lineStream(List lines) { + Iterator iterator = lines.iterator(); + return new ObjectStream<>() { + @Override + public String read() { + return iterator.hasNext() ? iterator.next() : null; + } + }; + } + + @ParameterizedTest + @CsvSource(delimiter = '|', value = { + "1001|SOURCE: ref=\"x\"", + "LIT-1|SOURCE: ref=\"x\"", + "CIE1|SOURCE: source=\"text\"" + }) + void testTextIdFromCorpusMetadata(String sentenceId, String source) throws IOException { + List lines = List.of("", source, sentenceId + " Olá .", "STA:fcl", + "=H:intj(\"olá\" )\tOlá", ".", ""); + try (ADNameSampleStream stream = new ADNameSampleStream(lineStream(lines), false)) { + NameSample sample = stream.read(); + Assertions.assertNotNull(sample); + Assertions.assertArrayEquals(new String[] {"Olá", "."}, sample.getSentence()); + Assertions.assertNull(stream.read()); + } + } + + @ParameterizedTest + @CsvSource(delimiter = '|', value = { + // no digits after the prefix + "LIT|SOURCE: ref=\"x\"", + // no source attribute + "CIE1|SOURCE: ref=\"x\"", + // no digits + "AX|SOURCE: ref=\"x\"" + }) + void testInvalidMetadataIsRejected(String sentenceId, String source) throws IOException { + List lines = List.of("", source, sentenceId + " Olá .", ""); + try (ADNameSampleStream stream = new ADNameSampleStream(lineStream(lines), false)) { + RuntimeException e = Assertions.assertThrows(RuntimeException.class, stream::read); + Assertions.assertTrue(e.getMessage().startsWith("Invalid metadata: " + sentenceId + " p=")); + } + } } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java index cbbae22012..5fc873a88b 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java @@ -19,10 +19,14 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import opennlp.tools.postag.POSSample; import opennlp.tools.util.PlainTextByLineStream; @@ -108,4 +112,23 @@ void testIncludeFeats() throws IOException { } } + private static Stream tags() { + return Stream.of( + Arguments.of("v-fin", "v-fin"), + Arguments.of("PR 3S IND", "PR=3S=IND"), + Arguments.of("PR \t3S", "PR=3S"), + Arguments.of(" PR 3S ", "=PR=3S="), + Arguments.of(" ", "="), + Arguments.of("", ""), + Arguments.of("PR\u00A03S", "PR\u00A03S"), + Arguments.of("\r\n\u000B\f", "="), + Arguments.of("\uD83D\uDE00 x", "\uD83D\uDE00=x")); + } + + @ParameterizedTest + @MethodSource("tags") + void testReplaceWhitespaceWithEquals(String tag, String expected) { + Assertions.assertEquals(expected, ADPOSSampleStream.replaceWhitespaceWithEquals(tag)); + } + } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java index 58f45635af..2a32e7bb98 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java @@ -19,12 +19,15 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Iterator; +import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import opennlp.tools.sentdetect.SentenceSample; +import opennlp.tools.util.ObjectStream; import opennlp.tools.util.PlainTextByLineStream; import opennlp.tools.util.Span; @@ -59,4 +62,28 @@ void testSentences() { Assertions.assertEquals(new Span(120, 180), samples.get(0).getSentences()[1]); } + @Test + void testInvalidMetadataIsRejected() throws IOException { + // the second sentence id "AX" has no digits, so its metadata cannot be parsed + List lines = List.of( + "", + "SOURCE: src", + "1001 Hello world .", + "", + "", + "SOURCE: src", + "AX Hi there .", + ""); + Iterator iterator = lines.iterator(); + ObjectStream lineStream = new ObjectStream<>() { + @Override + public String read() { + return iterator.hasNext() ? iterator.next() : null; + } + }; + try (ADSentenceSampleStream stream = new ADSentenceSampleStream(lineStream, true)) { + Assertions.assertThrows(RuntimeException.class, stream::read); + } + } + } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java new file mode 100644 index 0000000000..bcee49bf46 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.ad; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import opennlp.tools.formats.ad.ADSentenceStream.Sentence; +import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser; +import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.Leaf; +import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.TreeElement; + +public class ADSentenceStreamTest { + + private static Stream nodeLines() { + return Stream.of( + Arguments.of("STA:fcl", 1, "STA:fcl"), + Arguments.of("=PIV:pp", 2, "PIV:pp"), + Arguments.of("==P<:np", 3, "P<:np"), + Arguments.of("===>N:adjp", 4, ">N:adjp"), + // an optional part in parentheses and tag groups may follow the tag + Arguments.of("=X:y(z)", 2, "X:y"), + Arguments.of("=X:y ()", 2, "X:y"), + Arguments.of("=X:y(z) ()() ", 2, "X:y"), + Arguments.of("==P<:np()", 3, "P<:np"), + // a leaf line followed by a tag group, or without a lexeme, is a node line + Arguments.of("=H:n(\"casa\" M S) ()", 2, "H:n"), + Arguments.of("=H:n(\"casa\" M S)", 2, "H:n"), + Arguments.of("=H:n(\"casa\" M S) ", 2, "H:n"), + // hyphens count as level, but the last one joins the tag when a colon follows it + Arguments.of("=-X:y", 3, "X:y"), + Arguments.of("--:y", 2, "-:y"), + Arguments.of("-:y", 1, "-:y")); + } + + @ParameterizedTest + @MethodSource("nodeLines") + void testNodeLines(String line, int level, String syntacticTag) { + TreeElement element = new SentenceParser().getElement(line); + Assertions.assertNotNull(element); + Assertions.assertFalse(element.isLeaf()); + Assertions.assertEquals(level, element.getLevel()); + Assertions.assertEquals(syntacticTag, element.getSyntacticTag()); + } + + private static Stream leafLines() { + return Stream.of( + Arguments.of("=P:v-fin(\"iniciar\" PR 3S IND VFIN)\tInicia", + 2, "P", "v-fin", "iniciar", " ", "PR 3S IND VFIN", "Inicia"), + Arguments.of("===H:n(\"av.\" M S)\tAv.", + 4, "H", "n", "av.", " ", "M S", "Av."), + Arguments.of("===N<:num(\"6\" M P)\t6", + 4, "N<", "num", "6", " ", "M P", "6"), + // no morphological tag + Arguments.of("==H:prp(\"em\" )\tem", + 3, "H", "prp", "em", " ", null, "em"), + Arguments.of("SUB:conj-s(\"que\" )\tque", + 1, "SUB", "conj-s", "que", "", null, "que"), + // no secondary tags + Arguments.of("=P:v-fin(\"iniciar\" PR 3S IND VFIN)\tInicia", + 2, "P", "v-fin", "iniciar", "", "PR 3S IND VFIN", "Inicia"), + Arguments.of("=H:n('casa') casa", 2, "H", "n", "casa", "", null, "casa"), + // quotes inside lemma and lexeme + Arguments.of("=H:n(\"d'água\" M S)\td'água", + 2, "H", "n", "d'água", "", "M S", "d'água"), + // the lemma extends to the last quote after which the rest of the line still parses + Arguments.of("=H:n(\"a\" \"c\")\tw", 2, "H", "n", "a\" \"c", "", null, "w"), + Arguments.of("=H:n(\"x\" M S)\ta') b", 2, "H", "n", "x\" M S)\ta", "", null, "b"), + // the secondary tags extend to the last closing angle bracket + Arguments.of("=H:n(\"casa\" b M S)\tcasa", + 2, "H", "n", "casa", "b", "M S", "casa"), + Arguments.of("=H:n(\"casa\" M S)\tcasa", + 2, "H", "n", "casa", "", "M S", "casa"), + // the last of trailing whitespace characters is the lexeme + Arguments.of("=H:n(\"a)\" M S) ", 2, "H", "n", "a)", "", "M S", " ")); + } + + @ParameterizedTest + @MethodSource("leafLines") + void testLeafLines(String line, int level, String syntacticTag, String functionalTag, + String lemma, String secondaryTag, String morphologicalTag, String lexeme) { + TreeElement element = new SentenceParser().getElement(line); + Assertions.assertNotNull(element); + Assertions.assertTrue(element.isLeaf()); + Leaf leaf = (Leaf) element; + Assertions.assertEquals(level, leaf.getLevel()); + Assertions.assertEquals(syntacticTag, leaf.getSyntacticTag()); + Assertions.assertEquals(functionalTag, leaf.getFunctionalTag()); + Assertions.assertEquals(lemma, leaf.getLemma()); + Assertions.assertEquals(secondaryTag, leaf.getSecondaryTag()); + Assertions.assertEquals(morphologicalTag, leaf.getMorphologicalTag()); + Assertions.assertEquals(lexeme, leaf.getLexeme()); + } + + private static Stream bizarreLeafLines() { + return Stream.of( + Arguments.of("=x=y(\"q\" a) b", 2, "x=y", "q", "a", "b"), + Arguments.of("=x=y('q')\tb", 2, "x=y", "q", null, "b"), + Arguments.of("=x=y(a b) c", 2, "x=y", null, "a b", "c"), + Arguments.of("=x=y() b", 2, "x=y", null, null, "b"), + // a quoted part without a closing quote is the morphological tag + Arguments.of("=x=y(\"q) b", 2, "x=y", null, "\"q", "b"), + // the level prefix gives up hyphens so that the tag can start + Arguments.of("==-=x(a) b", 3, "-=x", null, "a", "b"), + Arguments.of("=-=x=y(a) b", 4, "x=y", null, "a", "b")); + } + + @ParameterizedTest + @MethodSource("bizarreLeafLines") + void testBizarreLeafLines(String line, int level, String syntacticTag, String lemma, + String morphologicalTag, String lexeme) { + TreeElement element = new SentenceParser().getElement(line); + Assertions.assertNotNull(element); + Assertions.assertTrue(element.isLeaf()); + Leaf leaf = (Leaf) element; + Assertions.assertEquals(level, leaf.getLevel()); + Assertions.assertEquals(syntacticTag, leaf.getSyntacticTag()); + Assertions.assertNull(leaf.getFunctionalTag()); + Assertions.assertEquals(lemma, leaf.getLemma()); + Assertions.assertNull(leaf.getSecondaryTag()); + Assertions.assertEquals(morphologicalTag, leaf.getMorphologicalTag()); + Assertions.assertEquals(lexeme, leaf.getLexeme()); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', ignoreLeadingAndTrailingWhitespace = false, value = { + // no whitespace between the closing parenthesis and the lexeme: not a leaf line + "=H:n(\"casa\" M S)casa|2|:n(\"casa\" M S)casa", + // an empty lemma + "=H:n(\"\" M S) casa|2|:n(\"\" M S) casa", + "=ab|2|b", + "=a.b|2|.b", + "===x|4|''" + }) + void testFallbackLeafLines(String line, int level, String lexeme) { + TreeElement element = new SentenceParser().getElement(line); + Assertions.assertNotNull(element); + Assertions.assertTrue(element.isLeaf()); + Leaf leaf = (Leaf) element; + Assertions.assertEquals(level, leaf.getLevel()); + Assertions.assertEquals("", leaf.getSyntacticTag()); + Assertions.assertEquals("", leaf.getFunctionalTag()); + Assertions.assertEquals("", leaf.getMorphologicalTag()); + Assertions.assertNull(leaf.getLemma()); + Assertions.assertEquals(lexeme, leaf.getLexeme()); + } + + @ParameterizedTest + @ValueSource(strings = {"_", "", "pause", "=ab.", "=xay", "=a_b.c"}) + void testIgnoredLines(String line) { + Assertions.assertNull(new SentenceParser().getElement(line)); + } + + @Test + void testUnparsableLineIsLexeme() { + TreeElement element = new SentenceParser().getElement("random text"); + Assertions.assertTrue(element.isLeaf()); + Leaf leaf = (Leaf) element; + Assertions.assertEquals(1, leaf.getLevel()); + Assertions.assertEquals("", leaf.getSyntacticTag()); + Assertions.assertEquals("random text", leaf.getLexeme()); + } + + @Test + void testPunctuationLeaf() { + SentenceParser parser = new SentenceParser(); + + Leaf leaf = (Leaf) parser.getElement("==,"); + Assertions.assertEquals(3, leaf.getLevel()); + Assertions.assertEquals(",", leaf.getLexeme()); + + leaf = (Leaf) parser.getElement("."); + Assertions.assertEquals(1, leaf.getLevel()); + Assertions.assertEquals(".", leaf.getLexeme()); + + // a line of only equals signs matches, with the last one as lexeme + leaf = (Leaf) parser.getElement("==="); + Assertions.assertEquals(3, leaf.getLevel()); + Assertions.assertEquals("=", leaf.getLexeme()); + + // non-word characters other than equals make up the lexeme + leaf = (Leaf) parser.getElement("=!?"); + Assertions.assertEquals(2, leaf.getLevel()); + Assertions.assertEquals("!?", leaf.getLexeme()); + + // a word character excludes the punctuation parse, the line is treated + // as a bizarre leaf instead + TreeElement element = parser.getElement("=ab"); + Assertions.assertTrue(element.isLeaf()); + leaf = (Leaf) element; + Assertions.assertEquals(2, leaf.getLevel()); + Assertions.assertEquals("b", leaf.getLexeme()); + } + + @Test + void testFixPunctuation() { + SentenceParser parser = new SentenceParser(); + + Sentence sentence = parser.parse( + "\nSOURCE: src\n1001 Olá mundo » .\n\n", 1, false, false); + Assertions.assertEquals("Olá mundo ».", sentence.text()); + + sentence = parser.parse( + "\nSOURCE: src\n1001 Olá » , tudo bem » .\n\n", 1, false, false); + Assertions.assertEquals("Olá », tudo bem ».", sentence.text()); + + // without whitespace between » and the punctuation nothing is replaced + sentence = parser.parse( + "\nSOURCE: src\n1001 Olá mundo ».\n\n", 1, false, false); + Assertions.assertEquals("Olá mundo ».", sentence.text()); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', ignoreLeadingAndTrailingWhitespace = false, value = { + "|s|true", + "|s|true", + "|s|true", + "|ext|true", + "|caixa|true", + "

|p|true", + "|t|true", + ">|s|false", + " |s|false", + " |s|false", + "x|s|false", + "|s|false", + "

|s|false", + "|s|false", + "<>|s|false", + "''|s|false", + "|t|false", + "|ext|false", + "< s>|s|false" + }) + void testIsOpeningTag(String line, String name, boolean expected) { + Assertions.assertEquals(expected, ADSentenceStream.isOpeningTag(line, name)); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', ignoreLeadingAndTrailingWhitespace = false, value = { + "|s|true", + "|ext|true", + "|t|true", + "|caixa|true", + " |s|false", + " |s|false", + "|s|false", + "|s|false", + "|s|false", + ">|s|false", + "|t|false", + "''|s|false", + "|s|false" + }) + void testIsClosingTag(String line, String name, boolean expected) { + Assertions.assertEquals(expected, ADSentenceStream.isClosingTag(line, name)); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluStreamTest.java index 5716a84ab6..afc11909f7 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluStreamTest.java @@ -17,7 +17,9 @@ package opennlp.tools.formats.conllu; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.Locale; @@ -28,6 +30,8 @@ import org.junit.jupiter.api.Test; import opennlp.tools.sentdetect.SentenceSample; +import opennlp.tools.util.InputStreamFactory; +import opennlp.tools.util.InvalidFormatException; import opennlp.tools.util.ObjectStream; public class ConlluStreamTest extends AbstractConlluSampleStreamTest { @@ -106,4 +110,35 @@ void testOptionalComments() throws IOException { Assertions.assertNull(stream.read(), "Stream must be exhausted"); } } + + @Test + void testContractionIdsAreMerged() throws IOException { + try (ObjectStream stream = getStream("es-ud-sample.conllu")) { + ConlluSentence sent1 = stream.read(); + + Assertions.assertEquals(55, sent1.getWordLines().size()); + Assertions.assertEquals("1-3", sent1.getWordLines().get(0).getId()); + Assertions.assertEquals("Digámoslo", sent1.getWordLines().get(0).getForm()); + Assertions.assertEquals("15-16", sent1.getWordLines().get(12).getId()); + for (ConlluWordLine wordLine : sent1.getWordLines()) { + Assertions.assertFalse(wordLine.getId().equals("1") + || wordLine.getId().equals("2") || wordLine.getId().equals("3") + || wordLine.getId().equals("15") || wordLine.getId().equals("16"), + "Expanded contraction parts must be removed"); + } + } + } + + @Test + void testInvalidTextLangCodeIsRejected() throws IOException { + // "text_e" has a single lowercase letter, so no language code can be extracted + InputStreamFactory in = () -> new ByteArrayInputStream( + ("# text_e = Bonjour\n" + + "1\tBonjour\tbonjour\tINTJ\t_\t_\t0\troot\t_\t_\n") + .getBytes(StandardCharsets.UTF_8)); + + try (ObjectStream stream = new ConlluStream(in)) { + Assertions.assertThrows(InvalidFormatException.class, stream::read); + } + } } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamTest.java index a9428bb51a..3e0b18b7e4 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/leipzig/LeipzigLanguageSampleStreamTest.java @@ -19,11 +19,18 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import opennlp.tools.langdetect.LanguageSample; import opennlp.tools.util.InvalidFormatException; /** @@ -89,4 +96,36 @@ void testReadSentenceFilesWithEmptyDir() { } } + @ParameterizedTest + @ValueSource(strings = {"a", "eng", "dan", "abcdefghijklmnopqrstuvwxyz"}) + void testIsAsciiLowerCaseWordAccepts(String text) { + Assertions.assertTrue(LeipzigLanguageSampleStream.isAsciiLowerCaseWord(text)); + Assertions.assertTrue(text.matches("[a-z]+")); + } + + @ParameterizedTest + @ValueSource(strings = {"", "Eng", "eNg", "en1", "123", "e-g", "en ", " en", "\u00e9ng", + "\u0130ng", "\uD835\uDC1Abc", "en\u00A0"}) + void testIsAsciiLowerCaseWordRejects(String text) { + Assertions.assertFalse(LeipzigLanguageSampleStream.isAsciiLowerCaseWord(text)); + Assertions.assertFalse(text.matches("[a-z]+")); + } + + @Test + void testOnlyFilesWithLowerCaseAsciiLanguageCodesAreRead() throws IOException { + String[] names = {"eng-sentences.txt", "Eng-sentences.txt", "en1-sentences.txt", + "e-g-sentences.txt", "\u00e9ng-sentences.txt", "en"}; + for (String name : names) { + Files.writeString(new File(emptyTempDir, name).toPath(), + "1\tThis is a sentence.\n2\tThis is another sentence.\n", StandardCharsets.UTF_8); + } + List languages = new ArrayList<>(); + try (LeipzigLanguageSampleStream stream = new LeipzigLanguageSampleStream(emptyTempDir, 1, 2)) { + LanguageSample sample; + while ((sample = stream.read()) != null) { + languages.add(sample.language().getLang()); + } + } + Assertions.assertEquals(List.of("eng", "eng"), languages); + } } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascIdentifiersTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascIdentifiersTest.java new file mode 100644 index 0000000000..aa32d2ba19 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascIdentifiersTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.masc; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class MascIdentifiersTest { + + private static Stream removals() { + return Stream.of( + Arguments.of("ne-n7", MascIdentifiers.NAMED_ENTITY_ID_PREFIX, "7"), + Arguments.of("penn-n12", MascIdentifiers.PENN_TOKEN_ID_PREFIX, "12"), + Arguments.of("seg-r0", MascIdentifiers.REGION_ID_PREFIX, "0"), + Arguments.of("xne-n7", MascIdentifiers.NAMED_ENTITY_ID_PREFIX, "x7"), + Arguments.of("ne-nne-n7", MascIdentifiers.NAMED_ENTITY_ID_PREFIX, "ne-n7"), + Arguments.of("penn-n7penn-n", MascIdentifiers.PENN_TOKEN_ID_PREFIX, "7penn-n"), + Arguments.of("seg-r", MascIdentifiers.REGION_ID_PREFIX, ""), + Arguments.of("7", MascIdentifiers.NAMED_ENTITY_ID_PREFIX, "7"), + Arguments.of("", MascIdentifiers.NAMED_ENTITY_ID_PREFIX, ""), + Arguments.of("NE-N7", MascIdentifiers.NAMED_ENTITY_ID_PREFIX, "NE-N7"), + Arguments.of("ne\u2011n7", MascIdentifiers.NAMED_ENTITY_ID_PREFIX, "ne\u2011n7"), + Arguments.of("\uD83D\uDE00ne-n1\uD83D\uDE00", MascIdentifiers.NAMED_ENTITY_ID_PREFIX, + "\uD83D\uDE001\uD83D\uDE00"), + Arguments.of("abc", "", "abc")); + } + + @ParameterizedTest + @MethodSource("removals") + void testRemoveFirstRemovesOnlyTheFirstOccurrence(String input, String literal, String expected) { + Assertions.assertEquals(expected, MascIdentifiers.removeFirst(input, literal)); + Assertions.assertEquals(input.replaceFirst(literal, ""), MascIdentifiers.removeFirst(input, literal)); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascNamedEntityParserTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascNamedEntityParserTest.java new file mode 100644 index 0000000000..19ce861a1b --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascNamedEntityParserTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.masc; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import opennlp.tools.util.XmlUtil; + +public class MascNamedEntityParserTest { + + private static MascNamedEntityParser parse(String xml) throws Exception { + MascNamedEntityParser handler = new MascNamedEntityParser(); + XmlUtil.createSaxParser().parse( + new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)), handler); + return handler; + } + + @Test + void testEntityAndTokenIdsLoseTheirPrefix() throws Exception { + MascNamedEntityParser parser = parse("" + + "" + + "" + + "" + + ""); + Assertions.assertEquals("person", parser.getEntityIDtoEntityType().get(3)); + Assertions.assertEquals(List.of(4, 15), parser.getEntityIDsToTokens().get(3)); + } + + @Test + void testOnlyTheFirstPrefixOccurrenceIsRemoved() { + Assertions.assertThrows(SAXException.class, () -> parse( + "")); + Assertions.assertThrows(SAXException.class, () -> parse( + "" + + "")); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascPennTagParserTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascPennTagParserTest.java new file mode 100644 index 0000000000..57b74556dd --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascPennTagParserTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.masc; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import opennlp.tools.util.XmlUtil; + +public class MascPennTagParserTest { + + private static MascPennTagParser parse(String xml) throws Exception { + MascPennTagParser handler = new MascPennTagParser(); + XmlUtil.createSaxParser().parse( + new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)), handler); + return handler; + } + + @Test + void testTokenIdsLoseTheirPrefix() throws Exception { + MascPennTagParser parser = parse("" + + "" + + "" + + "" + + "" + + ""); + Assertions.assertArrayEquals(new int[] {0, 1}, parser.getTokenToQuarks().get(10)); + Assertions.assertEquals("NN", parser.getTags().get(10)); + Assertions.assertEquals("test", parser.getBases().get(10)); + } + + @Test + void testOnlyTheFirstPrefixOccurrenceIsRemoved() { + Assertions.assertThrows(SAXException.class, () -> parse( + "")); + Assertions.assertThrows(SAXException.class, () -> parse( + "")); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascWordParserTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascWordParserTest.java new file mode 100644 index 0000000000..b76c4262e5 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/masc/MascWordParserTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.masc; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.xml.sax.SAXException; + +import opennlp.tools.util.XmlUtil; + +public class MascWordParserTest { + + private static MascWordParser parse(String xml) throws Exception { + MascWordParser handler = new MascWordParser(); + XmlUtil.createSaxParser().parse( + new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)), handler); + return handler; + } + + @Test + void testRegionIdsLoseTheirPrefix() throws Exception { + List words = parse("" + + "" + + "" + + "").getAnchors(); + Assertions.assertEquals(2, words.size()); + Assertions.assertEquals(0, words.get(0).getId()); + Assertions.assertEquals(0, words.get(0).getStart()); + Assertions.assertEquals(4, words.get(0).getEnd()); + Assertions.assertEquals(11, words.get(1).getId()); + Assertions.assertEquals(5, words.get(1).getStart()); + Assertions.assertEquals(7, words.get(1).getEnd()); + } + + @Test + void testOnlyTheFirstPrefixOccurrenceIsRemoved() { + Assertions.assertThrows(SAXException.class, () -> parse( + "")); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/util/MarkableFileInputStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/util/MarkableFileInputStreamTest.java new file mode 100644 index 0000000000..74dc6ae328 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/util/MarkableFileInputStreamTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class MarkableFileInputStreamTest { + + @TempDir + Path tempDir; + + private Path file() throws IOException { + Path file = tempDir.resolve("lines.txt"); + Files.writeString(file, "first\nsecond\n", StandardCharsets.UTF_8); + return file; + } + + @Test + void testMarkAndReset() throws IOException { + try (InputStream in = new MarkableFileInputStreamFactory(file().toFile()).createInputStream()) { + Assertions.assertTrue(in.markSupported()); + Assertions.assertEquals('f', in.read()); + in.mark(0); + Assertions.assertEquals('i', in.read()); + in.reset(); + Assertions.assertEquals('i', in.read()); + } + } + + @Test + void testResetWithoutMarkIsRejected() throws IOException { + try (InputStream in = new MarkableFileInputStreamFactory(file().toFile()).createInputStream()) { + Assertions.assertThrows(IOException.class, in::reset); + } + } + + @Test + void testCloseClosesTheFile() throws IOException { + InputStream in = new MarkableFileInputStreamFactory(file().toFile()).createInputStream(); + Assertions.assertEquals('f', in.read()); + in.close(); + Assertions.assertThrows(IOException.class, in::read, "reading a closed stream must fail"); + } + + @Test + void testCloseThroughReaderClosesTheFile() throws IOException { + InputStream in = new MarkableFileInputStreamFactory(file().toFile()).createInputStream(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { + Assertions.assertEquals("first", reader.readLine()); + } + Assertions.assertThrows(IOException.class, in::read, "the reader must close the file"); + } +} diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java index a598199270..fe48133551 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java @@ -28,8 +28,6 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Stream; import ai.onnxruntime.OrtEnvironment; @@ -74,9 +72,6 @@ protected record ChunkRange(int start, int end) { protected record TextChunk(String text, int start, int end) { } - private static final Pattern JSON_ENTRY_PATTERN = - Pattern.compile("\"((?:[^\"\\\\]|\\\\.)*)\"\\s*:\\s*(\\d+)"); - /** * Initializes the shared, immutable inference state: the ONNX environment and session, * the loaded vocabulary and the configured tokenizer. These fields are {@code final} @@ -499,16 +494,37 @@ protected static List chunkRanges(final int tokenCount, final int do return List.copyOf(ranges); } - private static Map loadJsonVocab(final String json) { + /** + * Collects every string literal that is followed by a colon and a run of ASCII digits, + * wherever it occurs in the text, mapping the unescaped string to the integer. A literal + * followed by anything else is not an entry; the scan then resumes with the character + * after its opening quote, so a quote inside it may open the next candidate. A later + * entry for the same token overwrites an earlier one. + * + * @param json The JSON text of the vocabulary. + * @return A map of vocabulary tokens to IDs. + * @throws IllegalArgumentException Thrown if a token contains an invalid escape. + * @throws NumberFormatException Thrown if an ID does not fit into an {@code int}. + */ + static Map loadJsonVocab(final String json) { final Map vocab = new HashMap<>(); - final Matcher matcher = JSON_ENTRY_PATTERN.matcher(json); - while (matcher.find()) { - final String token = matcher.group(1) - .transform(AbstractDL::unescapeJsonString); - final int id = Integer.parseInt(matcher.group(2)); - vocab.put(token, id); + int open = json.indexOf('"'); + while (open >= 0) { + int next = open + 1; + final int close = JsonScan.closingQuote(json, open); + if (close >= 0) { + final int idStart = JsonScan.afterColon(json, close + 1); + final int idEnd = idStart < 0 ? -1 : JsonScan.endOfDigits(json, idStart); + if (idEnd > idStart) { + final String token = unescapeJsonString(json.substring(open + 1, close)); + final int id = Integer.parseInt(json.substring(idStart, idEnd)); + vocab.put(token, id); + next = idEnd; + } + } + open = json.indexOf('"', next); } return vocab; diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/JsonScan.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/JsonScan.java new file mode 100644 index 0000000000..06f073c336 --- /dev/null +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/JsonScan.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.dl; + +import opennlp.tools.commons.Internal; +import opennlp.tools.util.StringUtil; + +/** + * Cursor helpers for picking string literals, colons, and digit runs out of the small, + * HuggingFace-shaped JSON files the deep-learning components read (vocabularies and + * model configurations). The helpers work on offsets into the text and never build a + * document tree; they are not a general-purpose JSON parser. Whitespace means the six + * ASCII whitespace characters, and digits the ten ASCII digits. + */ +@Internal +public final class JsonScan { + + private JsonScan() { + } + + /** + * Finds the closing quote of a string literal, honoring backslash escapes: a backslash + * and the character after it never close the literal. + * + * @param text The text to scan. + * @param openQuote The offset of the opening quote. + * @return The offset of the first unescaped quote after {@code openQuote}, or {@code -1} + * if there is none or a backslash is followed by a line terminator or the end of the + * text. + */ + static int closingQuote(String text, int openQuote) { + int i = openQuote + 1; + while (i < text.length()) { + final char c = text.charAt(i); + if (c == '"') { + return i; + } + if (c == '\\') { + if (i + 1 >= text.length() || isLineTerminator(text.charAt(i + 1))) { + return -1; + } + i += 2; + } else { + i++; + } + } + return -1; + } + + /** + * Finds the closing quote of a string literal that must not span lines. Escapes are not + * honored, so a backslash-quote pair closes the literal. + * + * @param text The text to scan. + * @param openQuote The offset of the opening quote. + * @return The offset of the first quote after {@code openQuote}, or {@code -1} if there is + * none or a line terminator comes before it. + */ + public static int closingQuoteOnLine(String text, int openQuote) { + for (int i = openQuote + 1; i < text.length(); i++) { + final char c = text.charAt(i); + if (c == '"') { + return i; + } + if (isLineTerminator(c)) { + return -1; + } + } + return -1; + } + + /** + * Skips a colon that may be surrounded by whitespace. + * + * @param text The text to scan. + * @param from The offset to start at. + * @return The offset of the first non-whitespace character after the colon, which may be + * the length of the text, or {@code -1} if the first non-whitespace character at or + * after {@code from} is not a colon. + */ + public static int afterColon(String text, int from) { + final int colon = skipWhitespace(text, from); + if (colon >= text.length() || text.charAt(colon) != ':') { + return -1; + } + return skipWhitespace(text, colon + 1); + } + + /** + * Skips whitespace. + * + * @param text The text to scan. + * @param from The offset to start at. + * @return The offset of the first non-whitespace character at or after {@code from}, or the + * length of the text if only whitespace remains. + */ + static int skipWhitespace(String text, int from) { + int i = from; + while (i < text.length() && StringUtil.isAsciiWhitespace(text.charAt(i))) { + i++; + } + return i; + } + + /** + * Reads a run of digits. + * + * @param text The text to scan. + * @param from The offset to start at. + * @return The offset after the last digit of the run starting at {@code from}, or + * {@code from} itself if no digit is there. + */ + static int endOfDigits(String text, int from) { + int i = from; + while (i < text.length() && text.charAt(i) >= '0' && text.charAt(i) <= '9') { + i++; + } + return i; + } + + /** + * Tells whether a character ends a line: line feed, carriage return, next line, line + * separator, or paragraph separator. + * + * @param c The character to check. + * @return {@code true} if {@code c} is one of those five characters, {@code false} otherwise. + */ + private static boolean isLineTerminator(char c) { + return c == '\n' || c == '\r' || c == '\u0085' || c == '\u2028' || c == '\u2029'; + } +} diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerConfig.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerConfig.java index 00805f16e4..0da532dac4 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerConfig.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerConfig.java @@ -21,38 +21,87 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.regex.Matcher; -import java.util.regex.Pattern; + +import opennlp.dl.JsonScan; public record DocumentCategorizerConfig(Map id2label) { - private static final Pattern ID_TO_LABEL_PATTERN = - Pattern.compile("\"id2label\"\\s*:\\s*\\{(.*?)\\}", Pattern.DOTALL); - private static final Pattern ENTRY_PATTERN = - Pattern.compile("\"([^\"]+)\"\\s*:\\s*\"(.*?)\""); + private static final String ID_TO_LABEL_KEY = "\"id2label\""; @Override public Map id2label() { return Collections.unmodifiableMap(id2label); } + /** + * Reads the {@code id2label} object of a model configuration. The object is the text between + * the opening brace after the first {@code "id2label"} key that has one and the first closing + * brace after it, so a nested object is cut short there. Every string literal in it that is + * followed by a colon and another string literal on the same line is an entry; the key is + * taken up to the next quote, the value up to the next quote, neither honoring escapes. + * + * @param json The JSON text of the configuration. + * @return The configuration, holding an empty map if no {@code id2label} object is present. + */ public static DocumentCategorizerConfig fromJson(String json) { Objects.requireNonNull(json, "json must not be null"); final Map id2label = new HashMap<>(); - final Matcher matcher = ID_TO_LABEL_PATTERN.matcher(json); + final String id2labelContent = id2labelContent(json); + if (id2labelContent != null) { + putStringEntries(id2labelContent, id2label); + } - if (matcher.find()) { - final String id2labelContent = matcher.group(1); - final Matcher entryMatcher = ENTRY_PATTERN.matcher(id2labelContent); + return new DocumentCategorizerConfig(id2label); + } - while (entryMatcher.find()) { - final String key = entryMatcher.group(1); - final String value = entryMatcher.group(2); - id2label.put(key, value); + /** + * Finds the text between the braces of the {@code id2label} object. + * + * @param json The JSON text of the configuration. + * @return The text between the opening brace and the first closing brace after it, or + * {@code null} if no {@code "id2label"} key is followed by a colon, an opening brace, + * and a later closing brace. + */ + private static String id2labelContent(String json) { + int at = json.indexOf(ID_TO_LABEL_KEY); + while (at >= 0) { + final int brace = JsonScan.afterColon(json, at + ID_TO_LABEL_KEY.length()); + if (brace >= 0 && brace < json.length() && json.charAt(brace) == '{') { + final int end = json.indexOf('}', brace + 1); + if (end >= 0) { + return json.substring(brace + 1, end); + } } + at = json.indexOf(ID_TO_LABEL_KEY, at + 1); } + return null; + } - return new DocumentCategorizerConfig(id2label); + /** + * Adds every string-to-string entry of the text to the map. A quote that does not open an + * entry is skipped, and the scan resumes with the character after it. + * + * @param content The text between the braces of an object. + * @param entries The map to add the entries to, a later key overwriting an earlier one. + */ + private static void putStringEntries(String content, Map entries) { + int open = content.indexOf('"'); + while (open >= 0) { + int next = open + 1; + final int keyEnd = content.indexOf('"', open + 1); + if (keyEnd > open + 1) { + final int valueOpen = JsonScan.afterColon(content, keyEnd + 1); + if (valueOpen >= 0 && valueOpen < content.length() && content.charAt(valueOpen) == '"') { + final int valueEnd = JsonScan.closingQuoteOnLine(content, valueOpen); + if (valueEnd >= 0) { + entries.put(content.substring(open + 1, keyEnd), + content.substring(valueOpen + 1, valueEnd)); + next = valueEnd + 1; + } + } + } + open = content.indexOf('"', next); + } } } diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/JsonScanTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/JsonScanTest.java new file mode 100644 index 0000000000..41c4308f03 --- /dev/null +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/JsonScanTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.dl; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the {@link JsonScan} class. + */ +public class JsonScanTest { + + // ------------------------------------------------------------------------- + // closingQuote + // ------------------------------------------------------------------------- + + static Stream closingQuotes() { + return Stream.of( + Arguments.of("\"a\"", 0, 2), + Arguments.of("\"\"", 0, 1), + Arguments.of("x\"abc\": 1", 1, 5), + Arguments.of("\"a\\\"b\"", 0, 5), + Arguments.of("\"a\\\\\"", 0, 4), + Arguments.of("\"\\u0120\"", 0, 7), + Arguments.of("\"a\nb\"", 0, 4), + Arguments.of("\"\uD83D\uDE00\"", 0, 3), + Arguments.of("\"\\\uD83D\uDE00\"", 0, 4), + Arguments.of("\"a", 0, -1), + Arguments.of("\"", 0, -1), + Arguments.of("\"a\\", 0, -1), + Arguments.of("\"a\\\"", 0, -1), + Arguments.of("\"a\\\nb\"", 0, -1), + Arguments.of("\"a\\\rb\"", 0, -1), + Arguments.of("\"a\\\u0085b\"", 0, -1), + Arguments.of("\"a\\\u2028b\"", 0, -1), + Arguments.of("\"a\\\u2029b\"", 0, -1), + Arguments.of("\"a\\\tb\"", 0, 5)); + } + + @ParameterizedTest + @MethodSource("closingQuotes") + void testClosingQuote(String text, int openQuote, int expected) { + Assertions.assertEquals(expected, JsonScan.closingQuote(text, openQuote)); + } + + // ------------------------------------------------------------------------- + // closingQuoteOnLine + // ------------------------------------------------------------------------- + + static Stream closingQuotesOnLine() { + return Stream.of( + Arguments.of("\"a\"", 0, 2), + Arguments.of("\"\"", 0, 1), + Arguments.of("x\"a b\"c\"", 1, 5), + Arguments.of("\"a\\\"b\"", 0, 3), + Arguments.of("\"a\tb\"", 0, 4), + Arguments.of("\"a\u00A0\u3000b\"", 0, 5), + Arguments.of("\"\uD83D\uDE00\"", 0, 3), + Arguments.of("\"a", 0, -1), + Arguments.of("\"", 0, -1), + Arguments.of("\"a\nb\"", 0, -1), + Arguments.of("\"a\rb\"", 0, -1), + Arguments.of("\"a\u0085b\"", 0, -1), + Arguments.of("\"a\u2028b\"", 0, -1), + Arguments.of("\"a\u2029b\"", 0, -1)); + } + + @ParameterizedTest + @MethodSource("closingQuotesOnLine") + void testClosingQuoteOnLine(String text, int openQuote, int expected) { + Assertions.assertEquals(expected, JsonScan.closingQuoteOnLine(text, openQuote)); + } + + // ------------------------------------------------------------------------- + // afterColon + // ------------------------------------------------------------------------- + + static Stream colons() { + return Stream.of( + Arguments.of(":", 0, 1), + Arguments.of(":1", 0, 1), + Arguments.of(" : 1", 0, 3), + Arguments.of("\t\n\r\u000B\f:\t\n\r\u000B\f1", 0, 11), + Arguments.of("x: 1", 1, 3), + Arguments.of(": ", 0, 2), + Arguments.of("", 0, -1), + Arguments.of(" ", 0, -1), + Arguments.of("1", 0, -1), + Arguments.of("x:", 0, -1), + Arguments.of("::", 1, 2), + Arguments.of("\u00A0:", 0, -1), + Arguments.of("\u2003:", 0, -1), + Arguments.of(":\u00A01", 0, 1)); + } + + @ParameterizedTest + @MethodSource("colons") + void testAfterColon(String text, int from, int expected) { + Assertions.assertEquals(expected, JsonScan.afterColon(text, from)); + } + + // ------------------------------------------------------------------------- + // skipWhitespace + // ------------------------------------------------------------------------- + + static Stream whitespaceRuns() { + return Stream.of( + Arguments.of("", 0, 0), + Arguments.of("a", 0, 0), + Arguments.of(" a", 0, 1), + Arguments.of(" \t\n\u000B\f\ra", 0, 6), + Arguments.of(" ", 0, 3), + Arguments.of("a b", 1, 3), + Arguments.of("a b", 3, 3), + Arguments.of("\u00A0a", 0, 0), + Arguments.of("\u0085a", 0, 0), + Arguments.of("\u2003a", 0, 0), + Arguments.of("\u3000a", 0, 0), + Arguments.of("\u001Ca", 0, 0)); + } + + @ParameterizedTest + @MethodSource("whitespaceRuns") + void testSkipWhitespace(String text, int from, int expected) { + Assertions.assertEquals(expected, JsonScan.skipWhitespace(text, from)); + } + + // ------------------------------------------------------------------------- + // endOfDigits + // ------------------------------------------------------------------------- + + static Stream digitRuns() { + return Stream.of( + Arguments.of("", 0, 0), + Arguments.of("0", 0, 1), + Arguments.of("0123456789", 0, 10), + Arguments.of("12abc", 0, 2), + Arguments.of("a12", 0, 0), + Arguments.of("a12", 1, 3), + Arguments.of("-1", 0, 0), + Arguments.of("1.5", 0, 1), + Arguments.of("\u0661\u0662", 0, 0), + Arguments.of("\uFF11", 0, 0), + Arguments.of("\uD835\uDFCE", 0, 0), + Arguments.of("1\u0661", 0, 1)); + } + + @ParameterizedTest + @MethodSource("digitRuns") + void testEndOfDigits(String text, int from, int expected) { + Assertions.assertEquals(expected, JsonScan.endOfDigits(text, from)); + } +} diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/LoadVocabTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/LoadVocabTest.java index 8b3961e787..0c52be905a 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/LoadVocabTest.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/LoadVocabTest.java @@ -24,8 +24,13 @@ import java.nio.file.StandardCopyOption; import java.util.Map; import java.util.Objects; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -121,4 +126,52 @@ void testJsonAndPlainTextVocabProduceSameResult() throws IOException { assertEquals(plainVocab, jsonVocab); } + + static Stream jsonVocabs() { + return Stream.of( + Arguments.of("", Map.of()), + Arguments.of("{}", Map.of()), + Arguments.of("{\"a\": 1, \"b\": 2}", Map.of("a", 1, "b", 2)), + Arguments.of("{\"a\"\n:\r\n 3}", Map.of("a", 3)), + Arguments.of("{\"a\":\t4\u000B}", Map.of("a", 4)), + Arguments.of("{\"a\":\u00A05}", Map.of()), + Arguments.of("{\"a\": 1, \"b\": \"x\", \"c\": 2}", Map.of("a", 1, "c", 2)), + Arguments.of("{\"a\": 1.5, \"b\": 2}", Map.of("a", 1, "b", 2)), + Arguments.of("{\"a\": -1, \"b\": 2}", Map.of("b", 2)), + Arguments.of("{\"a\": 12abc}", Map.of("a", 12)), + Arguments.of("{\"a\": \u0661, \"b\": 2}", Map.of("b", 2)), + Arguments.of("{\"a\\\"b\": 1}", Map.of("a\"b", 1)), + Arguments.of("{\"a\\\"b\": x, \"c\": 1}", Map.of("c", 1)), + Arguments.of("{\"a\\\\\": 1}", Map.of("a\\", 1)), + Arguments.of("{\"\\u0120x\": 7, \"\\u00e9\": 8}", Map.of("\u0120x", 7, "\u00E9", 8)), + Arguments.of("{\"\uD83D\uDE00\": 1}", Map.of("\uD83D\uDE00", 1)), + Arguments.of("{\"a\nb\": 1}", Map.of("a\nb", 1)), + Arguments.of("{\"a\\\nb\": 1}", Map.of()), + Arguments.of("{\"a\\\u2028b\": 1, \"c\": 2}", Map.of("c", 2)), + Arguments.of("{\"\": 1}", Map.of("", 1)), + Arguments.of("{\"a\": 1, \"a\": 2}", Map.of("a", 2)), + Arguments.of("{\"a\": 1, \"\\u0061\": 2}", Map.of("a", 2)), + Arguments.of("{\"x\": {\"a\": 1}, \"b\": 2}", Map.of("a", 1, "b", 2)), + Arguments.of("\"a\":1\"b\":2", Map.of("a", 1, "b", 2)), + Arguments.of("\"a\": ", Map.of()), + Arguments.of("\"a\"", Map.of()), + Arguments.of("\"", Map.of())); + } + + @ParameterizedTest + @MethodSource("jsonVocabs") + void testLoadJsonVocab(String json, Map expected) { + assertEquals(expected, AbstractDL.loadJsonVocab(json)); + } + + @Test + void testLoadJsonVocabRejectsOverflowingId() { + assertThrows(NumberFormatException.class, () -> AbstractDL.loadJsonVocab("{\"a\": 99999999999}")); + } + + @ParameterizedTest + @ValueSource(strings = {"{\"a\\q\": 1}", "{\"\\\uD83D\uDE00\": 2}", "{\"\\u12\": 3}"}) + void testLoadJsonVocabRejectsInvalidEscape(String json) { + assertThrows(IllegalArgumentException.class, () -> AbstractDL.loadJsonVocab(json)); + } } diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/doccat/DocumentCategorizerConfigTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/doccat/DocumentCategorizerConfigTest.java index d381b22262..1a4e6d0abb 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/doccat/DocumentCategorizerConfigTest.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/doccat/DocumentCategorizerConfigTest.java @@ -17,15 +17,63 @@ package opennlp.dl.doccat; import java.util.Map; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; public class DocumentCategorizerConfigTest { + static Stream id2labels() { + return Stream.of( + Arguments.of("{\"id2label\": {\"0\": \"x\"}}", Map.of("0", "x")), + Arguments.of("{\"id2label\":{}}", Map.of()), + Arguments.of("{\"id2label\" : {\n\"0\" : \"neg\" ,\n\"1\"\t:\r\n\"pos\"\n}}", + Map.of("0", "neg", "1", "pos")), + Arguments.of("{\"id2label\":\u00A0{\"0\": \"x\"}}", Map.of()), + Arguments.of("{\"vocab_size\": 5}", Map.of()), + Arguments.of("{\"id2label\": {\"0\": \"x\"", Map.of()), + Arguments.of("{\"id2label\": \"nope\", \"id2label\": {\"0\": \"x\"}}", Map.of("0", "x")), + Arguments.of("{\"id2label\"id2label\": {\"0\": \"x\"}}", Map.of("0", "x")), + Arguments.of("{\"other\": {\"id2label\": {\"0\": \"x\"}}}", Map.of("0", "x")), + Arguments.of("{\"id2label\": [\"0\", \"x\"]}", Map.of()), + Arguments.of("{\"id2label\": {\"0\": \"x\", \"1\": {\"n\": \"y\"}, \"2\": \"z\"}}", + Map.of("0", "x", "n", "y")), + Arguments.of("{\"id2label\": {\"0\": \"x\", \"1\": \"y}\", \"2\": \"z\"}}", Map.of("0", "x")), + Arguments.of("{\"id2label\": {\"0\": \"say \\\"hi\\\"\", \"1\": \"ok\"}}", + Map.of("0", "say \\", "1", "ok")), + Arguments.of("{\"id2label\": {\"a\\\"b\": \"c\"}}", Map.of("b", "c")), + Arguments.of("{\"id2label\": {\"0\": \"li\nne\", \"1\": \"ok\"}}", Map.of("1", "ok")), + Arguments.of("{\"id2label\": {\"0\": \"li\u2028ne\", \"1\": \"ok\"}}", Map.of("1", "ok")), + Arguments.of("{\"id2label\": {\"0\": \"tab\there\"}}", Map.of("0", "tab\there")), + Arguments.of("{\"id2label\": {\"k\ney\": \"v\"}}", Map.of("k\ney", "v")), + Arguments.of("{\"id2label\": {\"\": \"x\", \"1\": \"y\"}}", Map.of("1", "y")), + Arguments.of("{\"id2label\": {\"0\": 5, \"1\": \"y\"}}", Map.of("1", "y")), + Arguments.of("{\"id2label\": {\"0\":\"\"}}", Map.of("0", "")), + Arguments.of("{\"id2label\": {\"0\": \"x\" \"1\": \"y\"}}", Map.of("0", "x", "1", "y")), + Arguments.of("{\"id2label\": {\"0\": \"x\", \"0\": \"y\"}}", Map.of("0", "y")), + Arguments.of("{\"id2label\": {\"\uD83D\uDE00\": \"\uD801\uDC12\", \"\u00E9\": \"\u3000x\"}}", + Map.of("\uD83D\uDE00", "\uD801\uDC12", "\u00E9", "\u3000x"))); + } + + @ParameterizedTest + @MethodSource("id2labels") + public void testId2LabelsFromJson(String json, Map expected) { + assertEquals(expected, DocumentCategorizerConfig.fromJson(json).id2label()); + } + + @Test + public void testId2LabelsFromJsonNullThrows() { + assertThrows(NullPointerException.class, () -> DocumentCategorizerConfig.fromJson(null)); + } + @Test public void testId2LabelsFromJsonPrettyValid() { final String json = """ diff --git a/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/RealValueFileEventStream.java b/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/RealValueFileEventStream.java index 141d0e0389..f5b116ac34 100644 --- a/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/RealValueFileEventStream.java +++ b/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/RealValueFileEventStream.java @@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory; import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.StringUtil; /** * Class for using a file of real-valued {@link Event events} as an @@ -137,7 +138,7 @@ public Event read() throws IOException { if ((line = reader.readLine()) != null) { int si = line.indexOf(' '); String outcome = line.substring(0, si); - String[] contexts = line.substring(si + 1).split("\\s+"); + String[] contexts = StringUtil.splitOnAsciiWhitespace(line.substring(si + 1)); float[] values = parseContexts(contexts); return new Event(outcome, contexts, values); } diff --git a/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/SimpleEventStreamBuilder.java b/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/SimpleEventStreamBuilder.java index 93ccbdc7ff..2376f0ceda 100644 --- a/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/SimpleEventStreamBuilder.java +++ b/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/SimpleEventStreamBuilder.java @@ -22,6 +22,7 @@ import java.util.List; import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.StringUtil; public class SimpleEventStreamBuilder { @@ -40,7 +41,7 @@ public SimpleEventStreamBuilder add(String event) { } // look for context (and values) - String[] cvPairs = ss[1].split("\\s+"); + String[] cvPairs = StringUtil.splitOnAsciiWhitespace(ss[1]); if (cvPairs[0].contains(";")) { // has values? String[] context = new String[cvPairs.length]; float[] values = new float[cvPairs.length]; diff --git a/opennlp-core/opennlp-ml/opennlp-ml-commons/src/test/java/opennlp/tools/ml/model/RealValueFileEventStreamTest.java b/opennlp-core/opennlp-ml/opennlp-ml-commons/src/test/java/opennlp/tools/ml/model/RealValueFileEventStreamTest.java index b4071b3de1..96a05c9a9b 100644 --- a/opennlp-core/opennlp-ml/opennlp-ml-commons/src/test/java/opennlp/tools/ml/model/RealValueFileEventStreamTest.java +++ b/opennlp-core/opennlp-ml/opennlp-ml-commons/src/test/java/opennlp/tools/ml/model/RealValueFileEventStreamTest.java @@ -92,4 +92,20 @@ void testReset() throws IOException { } catch (UnsupportedOperationException expected) { } } + + @Test + void testReadSplitsContextsOnAsciiWhitespaceRuns() throws IOException { + String input = "other wc=ic=1.0\t\tw&c=he,ic=2.0 n1wc=lc=3.0 \t \n" + + "other wc=lc=1.0 w&c=belongs,lc=2.0\u00A0p1wc=ic=3.0\n"; + try (ObjectStream eventStream = createEventStream(input)) { + Event e = eventStream.read(); + Assertions.assertArrayEquals( + new String[] {"wc=ic", "w&c=he,ic", "n1wc=lc"}, e.getContext()); + Assertions.assertArrayEquals(new float[] {1.0f, 2.0f, 3.0f}, e.getValues()); + e = eventStream.read(); + Assertions.assertArrayEquals( + new String[] {"wc=lc", "w&c=belongs,lc=2.0\u00A0p1wc=ic"}, e.getContext()); + Assertions.assertNull(eventStream.read()); + } + } } diff --git a/opennlp-core/opennlp-ml/opennlp-ml-commons/src/test/java/opennlp/tools/ml/model/SimpleEventStreamBuilderTest.java b/opennlp-core/opennlp-ml/opennlp-ml-commons/src/test/java/opennlp/tools/ml/model/SimpleEventStreamBuilderTest.java new file mode 100644 index 0000000000..f964ce90f6 --- /dev/null +++ b/opennlp-core/opennlp-ml/opennlp-ml-commons/src/test/java/opennlp/tools/ml/model/SimpleEventStreamBuilderTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.ml.model; + +import java.io.IOException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.ObjectStream; + +public class SimpleEventStreamBuilderTest { + + @Test + void testAddSplitsContextsOnAsciiWhitespaceRuns() throws IOException { + try (ObjectStream events = new SimpleEventStreamBuilder() + .add("other/w=he\t\tn1w=belongs n2w=to \t po=other") + .build()) { + Event e = events.read(); + Assertions.assertEquals("other", e.getOutcome()); + Assertions.assertArrayEquals( + new String[] {"w=he", "n1w=belongs", "n2w=to", "po=other"}, e.getContext()); + Assertions.assertNull(e.getValues()); + Assertions.assertNull(events.read()); + } + } + + @Test + void testAddDropsTrailingWhitespaceAndKeepsLeadingEmptyContext() throws IOException { + try (ObjectStream events = new SimpleEventStreamBuilder() + .add("other/w=he n1w=belongs ") + .add("other/ w=he") + .build()) { + Assertions.assertArrayEquals(new String[] {"w=he", "n1w=belongs"}, events.read().getContext()); + Assertions.assertArrayEquals(new String[] {"", "w=he"}, events.read().getContext()); + } + } + + @Test + void testAddWithValuesSplitsOnAsciiWhitespaceRuns() throws IOException { + try (ObjectStream events = new SimpleEventStreamBuilder() + .add("other/w=he;0.5\tn1w=belongs;0.4 n2w=to;0.3") + .build()) { + Event e = events.read(); + Assertions.assertArrayEquals(new String[] {"w=he", "n1w=belongs", "n2w=to"}, e.getContext()); + Assertions.assertArrayEquals(new float[] {0.5f, 0.4f, 0.3f}, e.getValues()); + } + } + + @Test + void testAddDoesNotSplitOnNonAsciiSpace() throws IOException { + try (ObjectStream events = new SimpleEventStreamBuilder() + .add("other/w=he n1w=belongs n2w=to") + .build()) { + Assertions.assertArrayEquals( + new String[] {"w=he n1w=belongs", "n2w=to"}, events.read().getContext()); + } + } + + @Test + void testAddRejectsMissingSlash() { + Assertions.assertThrows(RuntimeException.class, + () -> new SimpleEventStreamBuilder().add("other w=he")); + } +} diff --git a/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java b/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java index 408e97fd72..9deeb6c786 100644 --- a/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java +++ b/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java @@ -17,6 +17,8 @@ package opennlp.tools.ml.maxent; +import java.util.ArrayList; +import java.util.List; /** * A {@link ContextGenerator} implementation for maxent decisions, assuming that the input @@ -28,24 +30,53 @@ */ public class BasicContextGenerator implements ContextGenerator { - private String separator = " "; + private static final String DEFAULT_SEPARATOR = " "; - public BasicContextGenerator() {} + private final String separator; + + public BasicContextGenerator() { + this(DEFAULT_SEPARATOR); + } /** - * Initializes a {@link BasicContextGenerator} with a different separator char. - * This overwrites the default whitespace separator. + * Initializes a {@link BasicContextGenerator} with a different separator. + * This overwrites the default single space. * - * @param sep The {@link String separator character} to use. + * @param sep The separator, taken as written and not as a regular expression. + * Must not be {@code null} or empty. + * @throws IllegalArgumentException If {@code sep} is {@code null} or empty. */ public BasicContextGenerator(String sep) { + if (sep == null || sep.isEmpty()) { + throw new IllegalArgumentException("sep must not be null or empty"); + } separator = sep; } + /** + * {@inheritDoc} + * Splits at each occurrence of the separator with the result {@code String.split} gives for + * a literal: a leading occurrence gives an empty first element, trailing empty elements + * are removed. + */ @Override public String[] getContext(String o) { - return o.split(separator); + final List contexts = new ArrayList<>(); + int start = 0; + int next; + while ((next = o.indexOf(separator, start)) != -1) { + contexts.add(o.substring(start, next)); + start = next + separator.length(); + } + contexts.add(o.substring(start)); + int end = contexts.size(); + while (end > 0 && contexts.get(end - 1).isEmpty()) { + end--; + } + if (end == 0 && o.isEmpty()) { + return new String[] {""}; + } + return contexts.subList(0, end).toArray(new String[0]); } } - diff --git a/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java b/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java index 1f477c3809..83b793fa54 100644 --- a/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java +++ b/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java @@ -22,6 +22,7 @@ import opennlp.tools.ml.model.Event; import opennlp.tools.ml.model.RealValueFileEventStream; import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.StringUtil; /** * Class for real-valued {@link Event events} as an @@ -60,7 +61,7 @@ private Event createEvent(String obs) { return null; else { String outcome = obs.substring(0, si); - String[] contexts = obs.substring(si + 1).split("\\s+"); + String[] contexts = StringUtil.splitOnAsciiWhitespace(obs.substring(si + 1)); float[] values = RealValueFileEventStream.parseContexts(contexts); return new Event(outcome,contexts,values); } diff --git a/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/test/java/opennlp/tools/ml/maxent/BasicContextGeneratorTest.java b/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/test/java/opennlp/tools/ml/maxent/BasicContextGeneratorTest.java new file mode 100644 index 0000000000..955c6af36c --- /dev/null +++ b/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/test/java/opennlp/tools/ml/maxent/BasicContextGeneratorTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.ml.maxent; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +public class BasicContextGeneratorTest { + + private static Stream contexts() { + return Stream.of( + Arguments.of(" ", "cp_1 cp_2 cp_3", new String[] {"cp_1", "cp_2", "cp_3"}), + Arguments.of(" ", "single", new String[] {"single"}), + Arguments.of(" ", "", new String[] {""}), + // the separator is taken as written, not as a regular expression + Arguments.of("|", "a|b|c", new String[] {"a", "b", "c"}), + Arguments.of(".", "a.b", new String[] {"a", "b"}), + Arguments.of("+", "a+b", new String[] {"a", "b"}), + Arguments.of("(", "a(b", new String[] {"a", "b"}), + Arguments.of("\\s", "a\\sb", new String[] {"a", "b"}), + Arguments.of("::", "a::b::c", new String[] {"a", "b", "c"}), + Arguments.of("::", "a:b", new String[] {"a:b"}), + // String.split shape: leading empty element kept, trailing empty elements dropped + Arguments.of(",", ",a,,b,,", new String[] {"", "a", "", "b"}), + Arguments.of(",", ",,", new String[0]), + Arguments.of(" ", "a b", new String[] {"a", "b"}), + Arguments.of("😀", "a😀b", new String[] {"a", "b"})); + } + + @ParameterizedTest + @MethodSource("contexts") + void testSplitsOnTheLiteralSeparator(String separator, String input, String[] expected) { + Assertions.assertArrayEquals(expected, new BasicContextGenerator(separator).getContext(input)); + } + + @Test + void testDefaultSeparatorIsSpace() { + Assertions.assertArrayEquals(new String[] {"a", "b"}, new BasicContextGenerator().getContext("a b")); + // a tab is not a separator by default + Assertions.assertArrayEquals(new String[] {"a\tb"}, new BasicContextGenerator().getContext("a\tb")); + } + + @ParameterizedTest + @ValueSource(strings = {""}) + void testEmptySeparatorIsRejected(String separator) { + Assertions.assertThrows(IllegalArgumentException.class, () -> new BasicContextGenerator(separator)); + } + + @Test + void testNullSeparatorIsRejected() { + Assertions.assertThrows(IllegalArgumentException.class, () -> new BasicContextGenerator(null)); + } +} diff --git a/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/test/java/opennlp/tools/ml/maxent/RealBasicEventStreamTest.java b/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/test/java/opennlp/tools/ml/maxent/RealBasicEventStreamTest.java index d7224d7d1d..0687fa0815 100644 --- a/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/test/java/opennlp/tools/ml/maxent/RealBasicEventStreamTest.java +++ b/opennlp-core/opennlp-ml/opennlp-ml-maxent/src/test/java/opennlp/tools/ml/maxent/RealBasicEventStreamTest.java @@ -104,4 +104,19 @@ void testReset() { } } + @Test + void testReadSplitsContextsOnAsciiWhitespaceRuns() throws IOException { + String input = "other wc=ic=1.0\t\tw&c=he,ic=2.0 n1wc=lc=3.0 \t \n" + + "other wc=lc=1.0 w&c=belongs,lc=2.0\u00A0p1wc=ic=3.0\n"; + try (ObjectStream eventStream = createEventStream(input)) { + Event e = eventStream.read(); + Assertions.assertArrayEquals( + new String[] {"wc=ic", "w&c=he,ic", "n1wc=lc"}, e.getContext()); + Assertions.assertArrayEquals(new float[] {1.0f, 2.0f, 3.0f}, e.getValues()); + e = eventStream.read(); + Assertions.assertArrayEquals( + new String[] {"wc=lc", "w&c=belongs,lc=2.0\u00A0p1wc=ic"}, e.getContext()); + Assertions.assertNull(eventStream.read()); + } + } } diff --git a/opennlp-core/opennlp-model-resolver/pom.xml b/opennlp-core/opennlp-model-resolver/pom.xml index 7c328e64b0..b869bed40d 100644 --- a/opennlp-core/opennlp-model-resolver/pom.xml +++ b/opennlp-core/opennlp-model-resolver/pom.xml @@ -62,6 +62,11 @@ junit-jupiter-api test + + org.junit.jupiter + junit-jupiter-params + test + org.junit.jupiter junit-jupiter-engine diff --git a/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/AbstractClassPathModelFinder.java b/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/AbstractClassPathModelFinder.java index 2050892c88..79a74c1c47 100644 --- a/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/AbstractClassPathModelFinder.java +++ b/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/AbstractClassPathModelFinder.java @@ -31,7 +31,6 @@ import java.util.Set; import java.util.jar.JarEntry; import java.util.jar.JarFile; -import java.util.regex.Pattern; /** * A base implementation of a {@link ClassPathModelFinder} for the detection of @@ -137,20 +136,16 @@ protected String getJarModelPrefix() { } /** - * Escapes a {@code wildcard} expressions for usage as a Java regular expression. + * Tests whether the file part of {@code url} matches {@code wildcard} from start to end, + * where {@code *} stands for any run of characters and {@code ?} for exactly one. * - * @param wildcard A valid expression. It must not be {@code null}. - * @return The escaped regex. + * @param url The {@link URL} whose {@link URL#getFile() file part} is tested. + * Must not be {@code null}. + * @param wildcard The wildcard expression. Must not be {@code null}. + * @return {@code true} if the file part matches, {@code false} otherwise. */ - protected String asRegex(String wildcard) { - return wildcard - .replace(".", "\\.") - .replace("*", ".*") - .replace("?", "."); - } - - protected boolean matchesPattern(URL url, Pattern pattern) { - return pattern.matcher(url.getFile()).matches(); + protected boolean matchesWildcard(URL url, String wildcard) { + return GlobMatcher.matches(wildcard, url.getFile()); } /** diff --git a/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/GlobMatcher.java b/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/GlobMatcher.java new file mode 100644 index 0000000000..ec784e1abe --- /dev/null +++ b/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/GlobMatcher.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.models; + +/** + * Matches file names against the wildcard globs accepted by {@link ClassPathModelFinder} + * implementations. A {@code *} matches any run of characters, including none, a {@code ?} + * matches exactly one character, and every other character stands for itself. Neither + * wildcard crosses a line terminator (line feed, carriage return, next line, line + * separator, or paragraph separator). The glob must cover the whole input. + */ +final class GlobMatcher { + + private static final int ANY_RUN = '*'; + private static final int ANY_ONE = '?'; + + private GlobMatcher() { + } + + /** + * Tests whether the whole {@code input} is covered by {@code glob}. Both are compared by + * code point, so a supplementary character counts as one character for {@code ?}. + * + * @param glob The wildcard expression. Must not be {@code null}. + * @param input The text to test. Must not be {@code null}. + * @return {@code true} if {@code input} matches {@code glob} from start to end, + * {@code false} otherwise. + */ + static boolean matches(String glob, String input) { + final int[] g = glob.codePoints().toArray(); + final int[] in = input.codePoints().toArray(); + int gi = 0; + int ii = 0; + int runStartG = -1; + int runStartI = -1; + while (ii < in.length) { + if (gi < g.length && g[gi] == ANY_RUN) { + runStartG = gi++; + runStartI = ii; + } else if (gi < g.length && matchesOne(g[gi], in[ii])) { + gi++; + ii++; + } else if (runStartG >= 0 && !isLineTerminator(in[runStartI])) { + // let the most recent '*' absorb one more character and retry after it + gi = runStartG + 1; + ii = ++runStartI; + } else { + return false; + } + } + while (gi < g.length && g[gi] == ANY_RUN) { + gi++; + } + return gi == g.length; + } + + private static boolean matchesOne(int globCodePoint, int inputCodePoint) { + if (globCodePoint == ANY_ONE) { + return !isLineTerminator(inputCodePoint); + } + return globCodePoint == inputCodePoint; + } + + private static boolean isLineTerminator(int codePoint) { + return codePoint == '\n' || codePoint == '\r' || codePoint == '\u0085' + || codePoint == '\u2028' || codePoint == '\u2029'; + } +} diff --git a/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/dir/DirectoryModelFinder.java b/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/dir/DirectoryModelFinder.java index 9a7b3d1d62..22951fdf5b 100644 --- a/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/dir/DirectoryModelFinder.java +++ b/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/dir/DirectoryModelFinder.java @@ -25,7 +25,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.regex.Pattern; import java.util.stream.Stream; import org.slf4j.LoggerFactory; @@ -57,9 +56,7 @@ public class DirectoryModelFinder extends AbstractClassPathModelFinder implement private final Path directory; private final boolean recursive; - private final Pattern jarPattern; - private Pattern filePattern; - private String prevFilePattern; + private final String jarWildcard; /** * Instantiates a new {@link DirectoryModelFinder} with the specified parameters. @@ -78,7 +75,7 @@ public DirectoryModelFinder(String jarModelPrefix, Path directory, boolean recur } this.directory = directory; this.recursive = recursive; - this.jarPattern = Pattern.compile(asRegex("*" + getJarModelPrefix())); + this.jarWildcard = "*" + getJarModelPrefix(); } /** @@ -101,17 +98,13 @@ protected List getMatchingURIs(String wildcardPattern, Object context) { final boolean isWindows = isWindows(); final List cp = getDirectoryContent(); final List cpu = new ArrayList<>(); - final String filePatternString = asRegex("*" + wildcardPattern); - if (!filePatternString.equals(prevFilePattern)) { - this.filePattern = Pattern.compile(filePatternString); - this.prevFilePattern = filePatternString; - } + final String fileWildcard = "*" + wildcardPattern; for (URL url : cp) { - if (matchesPattern(url, jarPattern)) { + if (matchesWildcard(url, jarWildcard)) { try { for (URI u : getURIsFromJar(url, isWindows)) { - if (matchesPattern(u.toURL(), filePattern)) { + if (matchesWildcard(u.toURL(), fileWildcard)) { cpu.add(u); } } diff --git a/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/simple/SimpleClassPathModelFinder.java b/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/simple/SimpleClassPathModelFinder.java index 10b83f9381..b427cf9ea2 100644 --- a/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/simple/SimpleClassPathModelFinder.java +++ b/opennlp-core/opennlp-model-resolver/src/main/java/opennlp/tools/models/simple/SimpleClassPathModelFinder.java @@ -28,7 +28,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,9 +62,8 @@ public class SimpleClassPathModelFinder extends AbstractClassPathModelFinder implements ClassPathModelFinder { private static final Logger logger = LoggerFactory.getLogger(SimpleClassPathModelFinder.class); - private static final Pattern CLASSPATH_SEPARATOR_PATTERN_WINDOWS = Pattern.compile(";"); - private static final Pattern CLASSPATH_SEPARATOR_PATTERN_UNIX = Pattern.compile(":"); - // ; for Windows, : for Linux/OSX + private static final char CLASSPATH_SEPARATOR_WINDOWS = ';'; + private static final char CLASSPATH_SEPARATOR_UNIX = ':'; /** * By default, it scans for {@link #OPENNLP_MODEL_JAR_PREFIX}. @@ -105,14 +103,14 @@ protected List getMatchingURIs(String wildcardPattern, Object context) { final boolean isWindows = isWindows(); final List cp = getClassPathElements(); final List cpu = new ArrayList<>(); - final Pattern jarPattern = Pattern.compile(asRegex("*" + getJarModelPrefix())); - final Pattern filePattern = Pattern.compile(asRegex("*" + wildcardPattern)); + final String jarWildcard = "*" + getJarModelPrefix(); + final String fileWildcard = "*" + wildcardPattern; for (URL url : cp) { - if (matchesPattern(url, jarPattern)) { + if (matchesWildcard(url, jarWildcard)) { try { for (URI u : getURIsFromJar(url, isWindows)) { - if (matchesPattern(u.toURL(), filePattern)) { + if (matchesWildcard(u.toURL(), fileWildcard)) { cpu.add(u); } } @@ -154,11 +152,8 @@ private List getClassPathElements() { private List getClassPathUrlsFromSystemProperty() { final String cp = System.getProperty("java.class.path", ""); - final String[] matches = isWindows() - ? CLASSPATH_SEPARATOR_PATTERN_WINDOWS.split(cp) - : CLASSPATH_SEPARATOR_PATTERN_UNIX.split(cp); final List jarUrls = new ArrayList<>(); - for (String classPath: matches) { + for (String classPath : splitClassPath(cp, isWindows())) { try { jarUrls.add(Path.of(classPath).toUri().toURL()); } catch (MalformedURLException ignored) { @@ -169,6 +164,39 @@ private List getClassPathUrlsFromSystemProperty() { return jarUrls; } + /** + * Splits {@code classPath} on the platform's path separator, {@code ;} on Windows and + * {@code :} elsewhere, with the result of {@code String.split} for that separator: a + * leading separator yields one empty first element, empty elements between consecutive + * separators are kept, trailing empty elements are dropped, separator-only input yields + * an empty array, and empty input yields a single empty element. + * + * @param classPath The class path value to split. Must not be {@code null}. + * @param isWindows {@code true} to split on {@code ;}, {@code false} to split on {@code :}. + * @return The class path elements in order. + */ + static String[] splitClassPath(String classPath, boolean isWindows) { + final char separator = isWindows ? CLASSPATH_SEPARATOR_WINDOWS : CLASSPATH_SEPARATOR_UNIX; + final List elements = new ArrayList<>(); + int start = 0; + for (int i = 0; i < classPath.length(); i++) { + if (classPath.charAt(i) == separator) { + elements.add(classPath.substring(start, i)); + start = i + 1; + } + } + elements.add(classPath.substring(start)); + int size = elements.size(); + while (size > 0 && elements.get(size - 1).isEmpty()) { + size--; + } + // String.split keeps the whole input when no separator occurs, even if it is empty + if (size == 0 && elements.size() == 1) { + size = 1; + } + return elements.subList(0, size).toArray(new String[0]); + } + /* * Java 9+ Bridge to obtain URLs from classpath. * This requires "--add-opens java.base/jdk.internal.loader=ALL-UNNAMED" as JVM argument diff --git a/opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/GlobMatcherTest.java b/opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/GlobMatcherTest.java new file mode 100644 index 0000000000..63c3d3703f --- /dev/null +++ b/opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/GlobMatcherTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.models; + +import java.net.URI; +import java.net.URL; +import java.util.List; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class GlobMatcherTest { + + private static final String MODEL_URL = + "jar:file:/repo/opennlp-models-pos-en-1.2.0.jar!/opennlp/models/en-pos.bin"; + + private static final String SMILEY = "\uD83D\uDE00"; + + private static Stream accepted() { + return Stream.of( + Arguments.of("*", ""), + Arguments.of("*", "anything at all"), + Arguments.of("**", "ab"), + Arguments.of("", ""), + Arguments.of("a", "a"), + Arguments.of("a*", "a"), + Arguments.of("a*", "abc"), + Arguments.of("*a", "a"), + Arguments.of("*a", "bca"), + Arguments.of("*a*b*", "xaybz"), + Arguments.of("a?c", "abc"), + Arguments.of("a?c", "a.c"), + Arguments.of("?", "a"), + Arguments.of("?", SMILEY), + Arguments.of("*.bin", "en-pos.bin"), + Arguments.of("*.bin", ".bin"), + Arguments.of("*model.properties", "/x/opennlp-models-pos-en-1.2.0.jar!/model.properties"), + Arguments.of("*opennlp-models-*", "/repo/opennlp-models-pos-en-1.2.0.jar"), + Arguments.of("*opennlp-models-*.jar", "/repo/opennlp-models-pos-en-1.2.0.jar"), + Arguments.of("*-en-*.jar", "/repo/opennlp-models-pos-en-1.2.0.jar"), + Arguments.of("(a)", "(a)"), + Arguments.of("[ab]", "[ab]"), + Arguments.of("a$", "a$"), + Arguments.of("a+", "a+"), + Arguments.of("a\\b", "a\\b"), + Arguments.of("*" + SMILEY + "*", "a" + SMILEY + "b"), + Arguments.of(SMILEY + "?", SMILEY + SMILEY), + Arguments.of("*\n*", "a\nb"), + Arguments.of("a\nb", "a\nb"), + Arguments.of("*\n*\n*", "a\nb\nc"), + Arguments.of("*\r\n*", "a\r\nb")); + } + + @ParameterizedTest + @MethodSource("accepted") + void testMatchesAccepts(String glob, String input) { + Assertions.assertTrue(GlobMatcher.matches(glob, input), + "glob '" + glob + "' should accept '" + input + "'"); + } + + private static Stream rejected() { + return Stream.of( + Arguments.of("a", ""), + Arguments.of("", "a"), + Arguments.of("a", "b"), + Arguments.of("a", "ab"), + Arguments.of("ab", "a"), + Arguments.of("a*", "ba"), + Arguments.of("*a", "ab"), + Arguments.of("a?c", "ac"), + Arguments.of("a?c", "abbc"), + Arguments.of("?", ""), + Arguments.of("?", "ab"), + Arguments.of("*.bin", "en-pos.bini"), + Arguments.of("*.bin", "en-posxbin"), + Arguments.of("*.bin", "en-pos.BIN"), + Arguments.of("*model.properties", "/x/model.properties.bak"), + Arguments.of("*opennlp-models-*", "/repo/opennlp-model-pos-en-1.2.0.jar"), + Arguments.of("(a)", "a"), + Arguments.of("[ab]", "a"), + Arguments.of("a+", "aa"), + Arguments.of("a\\b", "a"), + Arguments.of(SMILEY, "\uD83D"), + Arguments.of("??", SMILEY), + // neither wildcard crosses a line terminator + Arguments.of("*", "a\nb"), + Arguments.of("*", "\n"), + Arguments.of("*.bin", "a\n.bin"), + Arguments.of("*a*b", "a\nab"), + Arguments.of("?", "\n"), + Arguments.of("a?b", "a\nb"), + Arguments.of("*", "a\rb"), + Arguments.of("*", "a\u0085b"), + Arguments.of("*", "a\u2028b"), + Arguments.of("*", "a\u2029b")); + } + + @ParameterizedTest + @MethodSource("rejected") + void testMatchesRejects(String glob, String input) { + Assertions.assertFalse(GlobMatcher.matches(glob, input), + "glob '" + glob + "' should reject '" + input + "'"); + } + + @Test + void testMatchesWildcardUsesFilePart() throws Exception { + final AbstractClassPathModelFinder finder = new AbstractClassPathModelFinder() { + @Override + protected Object getContext() { + return null; + } + + @Override + protected List getMatchingURIs(String wildcardPattern, Object context) { + return List.of(); + } + }; + final URL url = new URI(MODEL_URL).toURL(); + Assertions.assertTrue(finder.matchesWildcard(url, "*.bin")); + Assertions.assertTrue(finder.matchesWildcard(url, "*opennlp-models-*")); + Assertions.assertTrue(finder.matchesWildcard(url, "*en-pos.bin")); + Assertions.assertFalse(finder.matchesWildcard(url, "en-pos.bin")); + Assertions.assertFalse(finder.matchesWildcard(url, "*.properties")); + Assertions.assertFalse(finder.matchesWildcard(url, "jar:*")); + } +} diff --git a/opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/dir/DirectoryModelFinderTest.java b/opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/dir/DirectoryModelFinderTest.java new file mode 100644 index 0000000000..d3e006989f --- /dev/null +++ b/opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/dir/DirectoryModelFinderTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.models.dir; + +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.models.AbstractClassPathFinderTest; +import opennlp.tools.models.ClassPathModelEntry; +import opennlp.tools.models.ClassPathModelFinder; + +/** + * Runs the shared finder tests against a directory holding copies of the model jars + * from the test classpath, placed one level below the scanned root. + */ +public class DirectoryModelFinderTest extends AbstractClassPathFinderTest { + + private static final String MODEL_JAR_PREFIX = "opennlp-models-"; + private static final String JAR_SUFFIX = ".jar"; + + @TempDir + private static Path root; + private static Path modelDir; + + @BeforeAll + static void copyModelJars() throws IOException, URISyntaxException { + modelDir = Files.createDirectory(root.resolve("models")); + final List jars = modelJarsOnClassPath(); + Assertions.assertFalse(jars.isEmpty(), "no model jars found on the test classpath"); + for (Path jar : jars) { + Files.copy(jar, modelDir.resolve(jar.getFileName())); + } + } + + private static List modelJarsOnClassPath() throws URISyntaxException { + final List jars = new ArrayList<>(); + final ClassLoader cl = Thread.currentThread().getContextClassLoader(); + if (cl instanceof URLClassLoader ucl) { + for (URL url : ucl.getURLs()) { + if ("file".equals(url.getProtocol())) { + addIfModelJar(Path.of(url.toURI()), jars); + } + } + } + if (jars.isEmpty()) { + for (String element : System.getProperty("java.class.path", "").split(File.pathSeparator)) { + addIfModelJar(Path.of(element), jars); + } + } + return jars; + } + + private static void addIfModelJar(Path candidate, List jars) { + final String name = String.valueOf(candidate.getFileName()); + if (name.startsWith(MODEL_JAR_PREFIX) && name.endsWith(JAR_SUFFIX) && Files.isRegularFile(candidate)) { + jars.add(candidate); + } + } + + @Override + protected ClassPathModelFinder getModelFinder() { + return new DirectoryModelFinder(null, root, true); + } + + @Override + protected ClassPathModelFinder getModelFinder(String pattern) { + return new DirectoryModelFinder(pattern, root, true); + } + + @Test + void testNonRecursiveSkipsSubdirectories() { + final Set models = new DirectoryModelFinder(null, root, false).findModels(false); + Assertions.assertNotNull(models); + Assertions.assertTrue(models.isEmpty()); + } + + @Test + void testNonRecursiveFindsDirectChildren() { + final Set models = new DirectoryModelFinder(null, modelDir, false).findModels(false); + Assertions.assertEquals(4, models.size()); + } + + @Test + void testJarPrefixNarrowsTheResult() { + final Set pos = + new DirectoryModelFinder("opennlp-models-pos-*", root, true).findModels(false); + Assertions.assertEquals(1, pos.size()); + Assertions.assertTrue(pos.iterator().next().model().toString().contains("opennlp-models-pos-")); + + final Set none = + new DirectoryModelFinder("opennlp-models-unknown-*", root, true).findModels(false); + Assertions.assertTrue(none.isEmpty()); + } + + @Test + void testNullDirectoryIsRejected() { + Assertions.assertThrows(IllegalArgumentException.class, () -> new DirectoryModelFinder(null, null, true)); + } +} diff --git a/opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/simple/SimpleClassPathModelFinderTest.java b/opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/simple/SimpleClassPathModelFinderTest.java index faab41d78e..f488b0e999 100644 --- a/opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/simple/SimpleClassPathModelFinderTest.java +++ b/opennlp-core/opennlp-model-resolver/src/test/java/opennlp/tools/models/simple/SimpleClassPathModelFinderTest.java @@ -16,6 +16,13 @@ */ package opennlp.tools.models.simple; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + import opennlp.tools.models.AbstractClassPathFinderTest; import opennlp.tools.models.ClassPathModelFinder; @@ -31,4 +38,54 @@ protected ClassPathModelFinder getModelFinder() { protected ClassPathModelFinder getModelFinder(String pattern) { return new SimpleClassPathModelFinder(pattern); } + + private static Stream unixClassPaths() { + return Stream.of( + Arguments.of("", new String[] {""}), + Arguments.of(":", new String[0]), + Arguments.of("::", new String[0]), + Arguments.of("a.jar", new String[] {"a.jar"}), + Arguments.of("a.jar:b.jar", new String[] {"a.jar", "b.jar"}), + Arguments.of(":a.jar", new String[] {"", "a.jar"}), + Arguments.of("a.jar:", new String[] {"a.jar"}), + Arguments.of("a.jar::", new String[] {"a.jar"}), + Arguments.of("a.jar::b.jar", new String[] {"a.jar", "", "b.jar"}), + Arguments.of("::a.jar:", new String[] {"", "", "a.jar"}), + Arguments.of("/usr/lib/a.jar:/opt/b.jar", new String[] {"/usr/lib/a.jar", "/opt/b.jar"}), + Arguments.of("C:\\lib\\a.jar;C:\\lib\\b.jar", new String[] {"C", "\\lib\\a.jar;C", "\\lib\\b.jar"}), + Arguments.of("\uD801\uDC12.jar:b.jar", new String[] {"\uD801\uDC12.jar", "b.jar"})); + } + + @ParameterizedTest + @MethodSource("unixClassPaths") + void testSplitClassPathUnix(String classPath, String[] expected) { + Assertions.assertArrayEquals(expected, SimpleClassPathModelFinder.splitClassPath(classPath, false)); + Assertions.assertArrayEquals(classPath.split(":"), + SimpleClassPathModelFinder.splitClassPath(classPath, false)); + } + + private static Stream windowsClassPaths() { + return Stream.of( + Arguments.of("", new String[] {""}), + Arguments.of(";", new String[0]), + Arguments.of(";;", new String[0]), + Arguments.of("a.jar", new String[] {"a.jar"}), + Arguments.of("a.jar;b.jar", new String[] {"a.jar", "b.jar"}), + Arguments.of(";a.jar", new String[] {"", "a.jar"}), + Arguments.of("a.jar;", new String[] {"a.jar"}), + Arguments.of("a.jar;;", new String[] {"a.jar"}), + Arguments.of("a.jar;;b.jar", new String[] {"a.jar", "", "b.jar"}), + Arguments.of(";;a.jar;", new String[] {"", "", "a.jar"}), + Arguments.of("C:\\lib\\a.jar;C:\\lib\\b.jar", new String[] {"C:\\lib\\a.jar", "C:\\lib\\b.jar"}), + Arguments.of("/usr/lib/a.jar:/opt/b.jar", new String[] {"/usr/lib/a.jar:/opt/b.jar"}), + Arguments.of("\uD801\uDC12.jar;b.jar", new String[] {"\uD801\uDC12.jar", "b.jar"})); + } + + @ParameterizedTest + @MethodSource("windowsClassPaths") + void testSplitClassPathWindows(String classPath, String[] expected) { + Assertions.assertArrayEquals(expected, SimpleClassPathModelFinder.splitClassPath(classPath, true)); + Assertions.assertArrayEquals(classPath.split(";"), + SimpleClassPathModelFinder.splitClassPath(classPath, true)); + } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java index 04007f9109..e8639af23e 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java @@ -19,7 +19,8 @@ import java.util.ArrayList; import java.util.List; -import java.util.regex.Pattern; + +import opennlp.tools.util.StringUtil; /** * Simple feature generator for learning statistical lemmatizers. @@ -34,9 +35,6 @@ public class DefaultLemmatizerContextGenerator implements LemmatizerContextGener private static final int PREFIX_LENGTH = 5; private static final int SUFFIX_LENGTH = 7; - private static final Pattern PATTERN_HAS_CAP = Pattern.compile("[A-Z]"); - private static final Pattern PATTERN_HAS_NUM = Pattern.compile("[0-9]"); - public DefaultLemmatizerContextGenerator() { } @@ -105,11 +103,11 @@ public String[] getContext(int index, String[] toks, String[] tags, String[] pre features.add("h"); } - if (PATTERN_HAS_CAP.matcher(lex).find()) { + if (StringUtil.containsAsciiUpperCase(lex)) { features.add("c"); } - if (PATTERN_HAS_NUM.matcher(lex).find()) { + if (StringUtil.containsAsciiDigit(lex)) { features.add("d"); } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/BioCodec.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/BioCodec.java index f7d6c5e852..563c9bd55f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/BioCodec.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/BioCodec.java @@ -20,8 +20,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import opennlp.tools.util.SequenceCodec; import opennlp.tools.util.Span; @@ -47,17 +45,37 @@ public class BioCodec implements SequenceCodec { public static final String CONTINUE = "cont"; public static final String OTHER = "other"; - private static final Pattern TYPED_OUTCOME_PATTERN = Pattern.compile("(.+)-\\w+"); - static String extractNameType(String outcome) { - Matcher matcher = TYPED_OUTCOME_PATTERN.matcher(outcome); - if (matcher.matches()) { - return matcher.group(1); + int separator = outcome.lastIndexOf('-'); + if (separator > 0 && isWordChars(outcome, separator + 1)) { + return outcome.substring(0, separator); } return null; } + /** + * Tests whether the rest of an outcome is a non-empty run of ASCII letters, digits, or + * underscores. + * + * @param outcome The outcome label. + * @param from The offset the run starts at. + * @return {@code true} if at least one character follows and all are word characters. + */ + private static boolean isWordChars(String outcome, int from) { + if (from >= outcome.length()) { + return false; + } + for (int i = from; i < outcome.length(); i++) { + char c = outcome.charAt(i); + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') || c == '_')) { + return false; + } + } + return true; + } + @Override public Span[] decode(List c) { int start = -1; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderME.java index f1dec349c1..b8fa9ef937 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderME.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderME.java @@ -24,8 +24,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import opennlp.tools.commons.ThreadSafe; import opennlp.tools.ml.BeamSearch; @@ -66,7 +64,6 @@ public class NameFinderME implements TokenNameFinder, Probabilistic { private static final String[][] EMPTY = new String[0][0]; public static final int DEFAULT_BEAM_SIZE = 3; - private static final Pattern typedOutcomePattern = Pattern.compile("(.+)-\\w+"); public static final String START = "start"; public static final String CONTINUE = "cont"; @@ -327,12 +324,7 @@ else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) { * @return The name type, or {@code null} if not set. */ static String extractNameType(String outcome) { - Matcher matcher = typedOutcomePattern.matcher(outcome); - if (matcher.matches()) { - return matcher.group(1); - } - - return null; + return BioCodec.extractNameType(outcome); } /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java index 93c256c60f..1060b990ad 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java @@ -20,10 +20,10 @@ import java.util.ArrayList; import java.util.List; -import java.util.regex.Pattern; import opennlp.tools.dictionary.Dictionary; import opennlp.tools.util.StringList; +import opennlp.tools.util.StringUtil; /** * A default {@link POSContextGenerator context generator} for a {@link POSTagger}. @@ -38,9 +38,6 @@ public class DefaultPOSContextGenerator implements POSContextGenerator { private static final int PREFIX_LENGTH = 4; private static final int SUFFIX_LENGTH = 4; - private static final Pattern hasCap = Pattern.compile("[A-Z]"); - private static final Pattern hasNum = Pattern.compile("[0-9]"); - private final Dictionary dict; /** @@ -165,11 +162,11 @@ public String[] getContext(int index, Object[] tokens, String[] tags) { e.add("h"); } - if (hasCap.matcher(lex).find()) { + if (StringUtil.containsAsciiUpperCase(lex)) { e.add("c"); } - if (hasNum.matcher(lex).find()) { + if (StringUtil.containsAsciiDigit(lex)) { e.add("d"); } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/AlphaNumericCheck.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/AlphaNumericCheck.java new file mode 100644 index 0000000000..d9bbf5cbab --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/AlphaNumericCheck.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize; + +import java.util.BitSet; +import java.util.regex.Pattern; + +/** + * Decides whether a token is alphanumeric under a tokenizer model's alphanumeric + * {@link Pattern}. A pattern of the shape {@code ^[...]+$} whose class holds only literal + * characters and simple ranges, which covers every built-in language default, is evaluated + * as a character set lookup. Any other pattern is evaluated by the regular expression engine, + * so the result is the same as {@code pattern.matcher(token).matches()} in both cases. + */ +final class AlphaNumericCheck { + + private static final String CLASS_PREFIX = "^["; + private static final String CLASS_SUFFIX = "]+$"; + + private final BitSet characters; + private final Pattern pattern; + + private AlphaNumericCheck(BitSet characters, Pattern pattern) { + this.characters = characters; + this.pattern = pattern; + } + + /** + * Creates the check for a pattern. + * + * @param pattern The alphanumeric pattern. Must not be {@code null}. + * @return A check that accepts exactly the tokens the pattern matches as a whole. + * @throws IllegalArgumentException If {@code pattern} is {@code null}. + */ + static AlphaNumericCheck of(Pattern pattern) { + if (pattern == null) { + throw new IllegalArgumentException("pattern must not be null"); + } + if (pattern.flags() != 0) { + return new AlphaNumericCheck(null, pattern); + } + final BitSet characters = parseCharacterClass(pattern.pattern()); + return new AlphaNumericCheck(characters, characters == null ? pattern : null); + } + + /** + * Tests a token. + * + * @param token The token. + * @return {@code true} if the whole token matches the pattern. + */ + boolean test(CharSequence token) { + if (characters == null) { + return pattern.matcher(token).matches(); + } + if (token.isEmpty()) { + return false; + } + for (int i = 0; i < token.length(); i++) { + if (!characters.get(token.charAt(i))) { + return false; + } + } + return true; + } + + /** + * Tells whether the check runs as a character set lookup. + * + * @return {@code true} for a set lookup, {@code false} when the pattern is evaluated as + * a regular expression. + */ + boolean isCharacterSet() { + return characters != null; + } + + /** + * Reads a pattern of the shape {@code ^[...]+$} into the set of characters its class accepts. + * Inside the class only literal characters and ranges written as {@code x-y} are understood; a + * hyphen in first or last position is literal. Escapes, negation, nested classes, + * intersections, and anything outside the Basic Multilingual Plane make the pattern + * ineligible. + * + * @param regex The pattern text. + * @return The accepted characters, or {@code null} if the pattern is not of that shape. + */ + private static BitSet parseCharacterClass(String regex) { + if (!regex.startsWith(CLASS_PREFIX) || !regex.endsWith(CLASS_SUFFIX) + || regex.length() <= CLASS_PREFIX.length() + CLASS_SUFFIX.length()) { + return null; + } + final String body = regex.substring(CLASS_PREFIX.length(), regex.length() - CLASS_SUFFIX.length()); + final BitSet characters = new BitSet(); + int i = 0; + while (i < body.length()) { + final char c = body.charAt(i); + if (!isLiteral(c)) { + return null; + } + if (i + 2 < body.length() && body.charAt(i + 1) == '-') { + final char to = body.charAt(i + 2); + if (!isLiteral(to) || to < c) { + return null; + } + characters.set(c, to + 1); + i += 3; + } else { + characters.set(c); + i++; + } + } + return characters; + } + + /** + * Tests whether a character stands for itself inside a character class. + * + * @param c The character. + * @return {@code false} for class syntax and for surrogates, {@code true} otherwise. + */ + private static boolean isLiteral(char c) { + return c != '[' && c != ']' && c != '\\' && c != '^' && c != '&' && !Character.isSurrogate(c); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokSpanEventStream.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokSpanEventStream.java index 36f4a98de4..a84120cdb3 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokSpanEventStream.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokSpanEventStream.java @@ -44,7 +44,7 @@ public class TokSpanEventStream extends AbstractEventStream { private final boolean skipAlphaNumerics; - private final Pattern alphaNumeric; + private final AlphaNumericCheck alphaNumeric; /** * Initializes a new event stream based on the data stream using a {@link TokenContextGenerator}. @@ -59,7 +59,7 @@ public class TokSpanEventStream extends AbstractEventStream { public TokSpanEventStream(ObjectStream tokenSamples, boolean skipAlphaNumerics, Pattern alphaNumeric, TokenContextGenerator cg) { super(tokenSamples); - this.alphaNumeric = alphaNumeric; + this.alphaNumeric = AlphaNumericCheck.of(alphaNumeric); this.skipAlphaNumerics = skipAlphaNumerics; this.cg = cg; } @@ -119,7 +119,7 @@ protected Iterator createEvents(TokenSample tokenSample) { //adjust cSpan to text offsets cSpan = new Span(cSpan.getStart() + start, cSpan.getEnd() + start); //should we skip this token - if (ctok.length() > 1 && (!skipAlphaNumerics || !alphaNumeric.matcher(ctok).matches())) { + if (ctok.length() > 1 && (!skipAlphaNumerics || !alphaNumeric.test(ctok))) { //find offsets of annotated tokens inside of candidate tokens boolean foundTrainingTokens = false; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerME.java index de9ee4ab67..f42df609e6 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerME.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerME.java @@ -22,7 +22,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.regex.Pattern; import opennlp.tools.commons.ThreadSafe; import opennlp.tools.dictionary.Dictionary; @@ -93,7 +92,7 @@ public class TokenizerME extends AbstractTokenizer implements Probabilistic { */ public static final String NO_SPLIT = "F"; - private final Pattern alphanumeric; + private final AlphaNumericCheck alphanumeric; /* * The maximum entropy model to use to evaluate contexts. @@ -156,7 +155,7 @@ public TokenizerME(TokenizerModel model, Dictionary abbDict) { this.abbDict = abbDict; TokenizerFactory factory = model.getFactory(); this.cg = factory.getContextGenerator(); - this.alphanumeric = factory.getAlphaNumericPattern(); + this.alphanumeric = AlphaNumericCheck.of(factory.getAlphaNumericPattern()); this.useAlphaNumericOptimization = factory.isUseAlphaNumericOptimization(); } @@ -203,7 +202,7 @@ public Span[] tokenizePos(String d) { if (tok.length() < 2) { localTokens.add(s); localProbs.add(1d); - } else if (useAlphaNumericOptimization() && alphanumeric.matcher(tok).matches()) { + } else if (useAlphaNumericOptimization() && alphanumeric.test(tok)) { localTokens.add(s); localProbs.add(1d); } else { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java index 951fca5b5d..ab0d7add09 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java @@ -24,13 +24,13 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import opennlp.tools.tokenize.TokenSample; import opennlp.tools.util.Span; +import opennlp.tools.util.StringUtil; /** * Class which produces an Iterator<TokenSample> from a file of space delimited token. @@ -42,7 +42,6 @@ public class TokenSampleStream implements Iterator { private static final Logger logger = LoggerFactory.getLogger(TokenSampleStream.class); private final BufferedReader in; private String line; - private final Pattern alphaNumeric = Pattern.compile("[A-Za-z0-9]"); private boolean evenq = true; public TokenSampleStream(InputStream is) throws IOException { @@ -55,7 +54,7 @@ public boolean hasNext() { } public TokenSample next() { - String[] tokens = line.split("\\s+"); + String[] tokens = StringUtil.splitOnAsciiWhitespace(line); if (tokens.length == 0) { evenq = true; } @@ -73,7 +72,7 @@ public TokenSample next() { default -> token; }; if (sb.length() != 0) { - if (!alphaNumeric.matcher(token).find() || token.startsWith("'") || token.equalsIgnoreCase("n't")) { + if (!containsAsciiAlphaNum(token) || token.startsWith("'") || token.equalsIgnoreCase("n't")) { if ((token.equals("``") || token.equals("--") || token.equals("$") || token.equals("(") || token.equals("&") || token.equals("#") || (token.equals("\"") && (evenq && ti != tokens.length - 1))) @@ -113,6 +112,23 @@ public void remove() { throw new UnsupportedOperationException(); } + + /** + * Tests whether a token contains an ASCII letter or digit. + * + * @param token The token. + * @return {@code true} if one is present. + */ + private boolean containsAsciiAlphaNum(String token) { + for (int i = 0; i < token.length(); i++) { + char c = token.charAt(i); + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { + return true; + } + } + return false; + } + private static void usage() { logger.info("TokenSampleStream [-spans] < in"); logger.info("Where in is a space delimited list of tokens."); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java index 0f6f6800d0..195e932447 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java @@ -40,8 +40,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -256,13 +254,23 @@ private static String downloadChecksumFile(String sha512, Path model) throws IOE /** * Extracts the hash from the content of a checksum file, which holds the hash followed by the * name of the file it applies to. + * + * @param checksumFileContent The file content. + * @return The hash, or {@code null} if the content is {@code null} or blank. */ - private static String parseChecksum(String checksumFileContent) { + static String parseChecksum(String checksumFileContent) { if (checksumFileContent == null) { return null; } final String trimmed = checksumFileContent.trim(); - return trimmed.isEmpty() ? null : trimmed.split("\\s")[0]; + if (trimmed.isEmpty()) { + return null; + } + int end = 0; + while (end < trimmed.length() && !StringUtil.isAsciiWhitespace(trimmed.charAt(end))) { + end++; + } + return trimmed.substring(0, end); } private static void verifyChecksum(Path model, String expectedChecksum) throws IOException { @@ -324,7 +332,6 @@ private static Path getDownloadHome() { @Internal static class DownloadParser { - private static final Pattern LINK_PATTERN = Pattern.compile("(.*?)", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); private final URL indexUrl; DownloadParser(URL indexUrl) { @@ -334,14 +341,60 @@ static class DownloadParser { Map> getAvailableModels() throws MalformedURLException, URISyntaxException { - final Matcher matcher = LINK_PATTERN.matcher(fetchPageIndex()); + return toMap(extractLinks(fetchPageIndex())); + } + /** + * Collects the href values of the anchor elements in an index page. The tag name and + * attribute are matched ignoring ASCII case, a value ends at the first {@code ">}, a link + * ends at the first {@code }, and both may span lines. An anchor without a closing tag + * is skipped. + * + * @param page The page content. + * @return The href values in order. + */ + static List extractLinks(String page) { final List links = new ArrayList<>(); - while (matcher.find()) { - links.add(matcher.group(1)); + int from = 0; + while ((from = indexOfIgnoreCase(page, "", valueStart); + if (valueEnd != -1) { + final int close = indexOfIgnoreCase(page, "", valueEnd + 2); + if (close != -1) { + links.add(page.substring(valueStart, valueEnd)); + from = close + "".length(); + continue; + } + } + from++; } + return links; + } - return toMap(links); + /** + * Finds a lowercase ASCII literal, ignoring the case of ASCII letters in the text. + * + * @param text The text. + * @param literal The lowercase literal. + * @param from The start offset. + * @return The first match offset, or {@code -1}. + */ + private static int indexOfIgnoreCase(String text, String literal, int from) { + outer: + for (int i = from; i + literal.length() <= text.length(); i++) { + for (int j = 0; j < literal.length(); j++) { + char c = text.charAt(i + j); + if (c >= 'A' && c <= 'Z') { + c += 'a' - 'A'; + } + if (c != literal.charAt(j)) { + continue outer; + } + } + return i; + } + return -1; } private Map> toMap(List links) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/BrownCluster.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/BrownCluster.java index 221816b49a..d40044ab99 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/BrownCluster.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/BrownCluster.java @@ -26,9 +26,10 @@ import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.regex.Pattern; import opennlp.tools.util.model.ArtifactSerializer; import opennlp.tools.util.model.SerializableArtifact; @@ -47,8 +48,6 @@ */ public class BrownCluster implements SerializableArtifact { - private static final Pattern tabPattern = Pattern.compile("\t"); - public static class BrownClusterSerializer implements ArtifactSerializer { @Override @@ -82,7 +81,7 @@ public BrownCluster(InputStream in) throws IOException { String line; while ((line = breader.readLine()) != null) { - String[] lineArray = tabPattern.split(line); + String[] lineArray = splitTabs(line); if (lineArray.length == 3) { int freq = Integer.parseInt(lineArray[2]); if (freq > 5 ) { @@ -96,6 +95,31 @@ else if (lineArray.length == 2) { } } + /** + * Splits on tabs with the result of {@code String.split("\\t")}: trailing empty fields are + * dropped and an empty line yields a single empty field. + * + * @param line The line. + * @return The fields in order. + */ + private String[] splitTabs(String line) { + if (line.isEmpty()) { + return new String[] {""}; + } + List fields = new ArrayList<>(); + int start = 0; + int separator; + while ((separator = line.indexOf('\t', start)) != -1) { + fields.add(line.substring(start, separator)); + start = separator + 1; + } + fields.add(line.substring(start)); + while (!fields.isEmpty() && fields.get(fields.size() - 1).isEmpty()) { + fields.remove(fields.size() - 1); + } + return fields.toArray(new String[0]); + } + /** * Check if a token is in the Brown:paths, token map. * diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java index 2237302191..b585f207d6 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java @@ -18,15 +18,11 @@ package opennlp.tools.util.featuregen; -import java.util.regex.Pattern; - /** * This class provide common utilities for feature generation. */ public class FeatureGeneratorUtil { - private static final Pattern capPeriod = Pattern.compile("^[A-ZÄÖÜ]\\.$"); - /** * Generates a class name for the specified token. * The classes are as follows where the first matching class is used: @@ -98,7 +94,7 @@ else if (pattern.isAllCapitalLetter()) { feat = "ac"; } } - else if (capPeriod.matcher(token).find()) { + else if (isCapPeriod(token)) { feat = "cp"; } else if (pattern.isInitialCapitalLetter()) { @@ -110,4 +106,24 @@ else if (pattern.isInitialCapitalLetter()) { return (feat); } + + /** + * Tests for a single capital followed by a period. + * + * @param token The token. + * @return {@code true} for exactly that shape. + */ + private static boolean isCapPeriod(String token) { + return token.length() == 2 && isCapPeriodStart(token.charAt(0)) && token.charAt(1) == '.'; + } + + /** + * Tests for an ASCII capital or a German umlaut capital. + * + * @param c The character. + * @return {@code true} for one of those capitals. + */ + private static boolean isCapPeriodStart(char c) { + return (c >= 'A' && c <= 'Z') || c == 'Ä' || c == 'Ö' || c == 'Ü'; + } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java index 8596244de7..6f71abaa5a 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java @@ -19,7 +19,6 @@ package opennlp.tools.util.featuregen; import java.util.List; -import java.util.regex.Pattern; import opennlp.tools.tokenize.SimpleTokenizer; import opennlp.tools.tokenize.Tokenizer; @@ -38,7 +37,6 @@ public class TokenPatternFeatureGenerator implements AdaptiveFeatureGenerator { private static final String SUB_TOKEN_PART2_PREFIX = "pt2=" ; private static final String SUB_TOKEN_PART3_PREFIX = "pt3=" ; - private final Pattern noLetters = Pattern.compile("[^a-zA-Z]"); private final Tokenizer tokenizer; /** @@ -87,11 +85,27 @@ public void createFeatures(List feats, String[] toks, int index, String[ pattern.append(FeatureGeneratorUtil.tokenFeature(tokenized[i])); - if (!noLetters.matcher(tokenized[i]).find()) { + if (!containsNonLetter(tokenized[i])) { feats.add(SUB_TOKEN_PREFIX + StringUtil.toLowerCase(tokenized[i])); } } feats.add("pta=" + pattern); } + + /** + * Tests whether a token contains a character outside the ASCII letters. + * + * @param token The token. + * @return {@code true} if one is present. + */ + private boolean containsNonLetter(String token) { + for (int i = 0; i < token.length(); i++) { + char c = token.charAt(i); + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'))) { + return true; + } + } + return false; + } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java index d2b75335ae..95dc047d85 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java @@ -16,8 +16,6 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** * A {@link EmojiCharSequenceNormalizer} implementation that normalizes text * in terms of emojis. Every encounter will be replaced by a whitespace. @@ -36,15 +34,44 @@ public static EmojiCharSequenceNormalizer getInstance() { return INSTANCE; } - private static final Pattern EMOJI_REGEX = - Pattern.compile("[\\uD83C-\\uDBFF\\uDC00-\\uDFFF]+"); + /** + * The lowest code point that is replaced: the first high surrogate of the emoji planes. + * Lone surrogates and every BMP character from here up count as well. + */ + private static final int LOWER_CODE_POINT = 0xD83C; + + /** The highest code point that is replaced. */ + private static final int UPPER_CODE_POINT = 0x10FC00; - /** {@inheritDoc} */ + /** + * {@inheritDoc} + * Every maximal run of code points in {@code [U+D83C, U+10FC00]} becomes one space. + */ @Override public CharSequence normalize (CharSequence text) { if (text == null) { throw new IllegalArgumentException("The text must not be null."); } - return EMOJI_REGEX.matcher(text).replaceAll(" "); + StringBuilder normalized = new StringBuilder(text.length()); + int i = 0; + while (i < text.length()) { + int cp = Character.codePointAt(text, i); + if (cp >= LOWER_CODE_POINT && cp <= UPPER_CODE_POINT) { + i += Character.charCount(cp); + while (i < text.length()) { + int next = Character.codePointAt(text, i); + if (next < LOWER_CODE_POINT || next > UPPER_CODE_POINT) { + break; + } + i += Character.charCount(next); + } + normalized.append(' '); + } + else { + normalized.appendCodePoint(cp); + i += Character.charCount(cp); + } + } + return normalized.toString(); } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGeneratorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGeneratorTest.java new file mode 100644 index 0000000000..ea0743a91d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGeneratorTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.lemmatizer; + +import java.util.Arrays; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +public class DefaultLemmatizerContextGeneratorTest { + + private static final DefaultLemmatizerContextGenerator GENERATOR = + new DefaultLemmatizerContextGenerator(); + + @ParameterizedTest + @CsvSource({ + "Token9, true, true", + "token, false, false", + "TOKEN, true, false", + "1990, false, true", + // accented capitals and non-ASCII digits are not matched + "Étudiant, false, false", + "x٥, false, false", + "well-known, false, false"}) + void testCapitalAndDigitFeatures(String token, boolean capital, boolean digit) { + List features = Arrays.asList(GENERATOR.getContext(0, + new String[] {token}, new String[] {"tag"}, null)); + Assertions.assertEquals(capital, features.contains("c"), "capital feature of " + token); + Assertions.assertEquals(digit, features.contains("d"), "digit feature of " + token); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/BioCodecTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/BioCodecTest.java index 55334fd190..641e785881 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/BioCodecTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/BioCodecTest.java @@ -23,6 +23,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.util.Span; @@ -262,4 +265,17 @@ void testCompatibilityRepeated() { new String[] {A_START, A_START, A_CONTINUE, A_CONTINUE, B_START, B_START, OTHER, OTHER})); } + @ParameterizedTest + @CsvSource({"atype-start, atype", "a-b-start, a-b", "type_1-cont, type_1", "Type9-X_1, Type9"}) + void testExtractNameType(String outcome, String type) { + Assertions.assertEquals(type, BioCodec.extractNameType(outcome)); + } + + @ParameterizedTest + @ValueSource(strings = {"start", "other", "-start", "atype-", "atype-st.art", "atype-st art", + "atype-stärt"}) + void testExtractNameTypeWithoutType(String outcome) { + Assertions.assertNull(BioCodec.extractNameType(outcome)); + } + } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderMETest.java index 3c8c8e6bc1..7266f3008f 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderMETest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderMETest.java @@ -22,6 +22,9 @@ import java.util.Collections; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.util.MockInputStreamFactory; import opennlp.tools.util.ObjectStream; @@ -32,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -325,4 +329,16 @@ void testNameFinderWithMultipleTypes() throws Exception { assertEquals("organization", names2[1].getType()); } + @ParameterizedTest + @CsvSource({"atype-start, atype", "a-b-start, a-b", "type_1-cont, type_1"}) + void testExtractNameType(String outcome, String type) { + assertEquals(type, NameFinderME.extractNameType(outcome)); + } + + @ParameterizedTest + @ValueSource(strings = {"start", "other", "-start", "atype-", "atype-st.art"}) + void testExtractNameTypeWithoutType(String outcome) { + assertNull(NameFinderME.extractNameType(outcome)); + } + } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/DefaultPOSContextGeneratorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/DefaultPOSContextGeneratorTest.java index 9e84538b3e..3f728182b2 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/DefaultPOSContextGeneratorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/DefaultPOSContextGeneratorTest.java @@ -102,6 +102,23 @@ void noDictionaryMatch() { + Arrays.toString(actual)); } + @Test + void capitalAndDigitFeatures() { + DefaultPOSContextGenerator generator = new DefaultPOSContextGenerator(null); + + // accept sides: ASCII capital letter and ASCII digit + final String[] withCapAndNum = generator.getContext(0, + new Object[] {"Token9"}, new String[] {"tag"}); + Assertions.assertTrue(Arrays.asList(withCapAndNum).contains("c")); + Assertions.assertTrue(Arrays.asList(withCapAndNum).contains("d")); + + // reject sides: accented capitals and non-ASCII digits are not matched + final String[] accented = generator.getContext(0, + new Object[] {"Étudiant٥"}, new String[] {"tag"}); + Assertions.assertFalse(Arrays.asList(accented).contains("c")); + Assertions.assertFalse(Arrays.asList(accented).contains("d")); + } + @Test void dictionaryMatch() { int indexWithDictionaryMatch = 2; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/AlphaNumericCheckTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/AlphaNumericCheckTest.java new file mode 100644 index 0000000000..d1b2f5e68e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/AlphaNumericCheckTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize; + +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import opennlp.tools.tokenize.lang.Factory; + +public class AlphaNumericCheckTest { + + private static final List LANGUAGES = + List.of("en", "es", "it", "pt", "ca", "pl", "de", "fr", "nl", "xx"); + + private static final List TOKENS = List.of( + "", "a", "Z", "0", "abc123", "Straße", "Café", "señor", "łódź", "ijs", "Ÿ", "-", "a-b", + "a b", "a\nb", "\n", "abc\n", "ñ", "Ç", "ß", "é", " ", "١٢٣", "Ａ", "𐐒", "😀a", + "aé", "AB.", "x_y", "[", "]", "\\", "^", "&", "$"); + + private static Stream builtInPatternsAndTokens() { + return LANGUAGES.stream().flatMap(language -> { + Pattern pattern = new Factory().getAlphanumeric(language); + return TOKENS.stream().map(token -> Arguments.of(language, pattern, token)); + }); + } + + @ParameterizedTest(name = "{0}: \"{2}\"") + @MethodSource("builtInPatternsAndTokens") + void testBuiltInPatternsAgreeWithRegex(String language, Pattern pattern, String token) { + AlphaNumericCheck check = AlphaNumericCheck.of(pattern); + Assertions.assertTrue(check.isCharacterSet(), language + " default runs as a set lookup"); + Assertions.assertEquals(pattern.matcher(token).matches(), check.test(token)); + } + + private static Stream customPatternsAndTokens() { + List patterns = List.of( + "^[a-z-]+$", "^[-a-z]+$", "^[a-c1-3]+$", "^[a]+$", "^[a-zA-Z0-9_]+$", + "^[\\p{L}]+$", "^[^a-z]+$", "^[a-z&&[^b]]+$", "^[\\d]+$", "^[a-z]+$|^[0-9]+$", + "^[a-z]*$", "[a-z]+", "^(?i)[a-z]+$", "^[a-z]+\\d$", "^[ab\\]]+$", "^[a-]+$", + "^[😀]+$"); + return patterns.stream().flatMap(regex -> { + Pattern pattern = Pattern.compile(regex); + return TOKENS.stream().map(token -> Arguments.of(regex, pattern, token)); + }); + } + + @ParameterizedTest(name = "{0}: \"{2}\"") + @MethodSource("customPatternsAndTokens") + void testCustomPatternsAgreeWithRegex(String regex, Pattern pattern, String token) { + Assertions.assertEquals(pattern.matcher(token).matches(), AlphaNumericCheck.of(pattern).test(token)); + } + + @ParameterizedTest + @ValueSource(strings = {"^[A-Za-z0-9]+$", "^[a-z-]+$", "^[-a-z]+$", "^[a-]+$", "^[a-c1-3]+$", + "^[a]+$", "^[a-zA-Z0-9_]+$", "^[0-9a-záãâàéêíóõôúüçA-ZÁÃÂÀÉÊÍÓÕÔÚÜÇ]+$"}) + void testEligiblePatternsRunAsSetLookup(String regex) { + Assertions.assertTrue(AlphaNumericCheck.of(Pattern.compile(regex)).isCharacterSet()); + } + + @ParameterizedTest + @ValueSource(strings = {"^[\\p{L}]+$", "^[^a-z]+$", "^[a-z&&[^b]]+$", "^[\\d]+$", + "^[a-z]+$|^[0-9]+$", "^[a-z]*$", "[a-z]+", "^(?i)[a-z]+$", "^[a-z]+\\d$", "^[ab\\]]+$", + "^[😀]+$"}) + void testOtherPatternsFallBackToRegex(String regex) { + Assertions.assertFalse(AlphaNumericCheck.of(Pattern.compile(regex)).isCharacterSet()); + } + + @Test + void testFlagsForceRegex() { + Pattern pattern = Pattern.compile("^[a-z]+$", Pattern.CASE_INSENSITIVE); + AlphaNumericCheck check = AlphaNumericCheck.of(pattern); + Assertions.assertFalse(check.isCharacterSet()); + Assertions.assertTrue(check.test("ABC")); + } + + @Test + void testNullPatternIsRejected() { + Assertions.assertThrows(IllegalArgumentException.class, () -> AlphaNumericCheck.of(null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lang/en/TokenSampleStreamTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lang/en/TokenSampleStreamTest.java new file mode 100644 index 0000000000..f63d1d1d3d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lang/en/TokenSampleStreamTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lang.en; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.tokenize.TokenSample; +import opennlp.tools.util.Span; + +public class TokenSampleStreamTest { + + private static TokenSampleStream stream(String text) throws IOException { + return new TokenSampleStream(new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8))); + } + + @Test + void testReadsTokensAndSpans() throws IOException { + TokenSampleStream stream = stream("The dog 's -LRB- big -RRB- .\n"); + Assertions.assertTrue(stream.hasNext()); + TokenSample sample = stream.next(); + Assertions.assertEquals("The dog's ( big).", sample.getText()); + Assertions.assertArrayEquals(new Span[] {new Span(0, 3), new Span(4, 7), new Span(7, 9), + new Span(10, 11), new Span(12, 15), new Span(15, 16), new Span(16, 17)}, + sample.getTokenSpans()); + Assertions.assertFalse(stream.hasNext()); + } + + @Test + void testCollapsesWhitespaceRuns() throws IOException { + TokenSample sample = stream("a \t b\n").next(); + Assertions.assertEquals("a b", sample.getText()); + Assertions.assertEquals(2, sample.getTokenSpans().length); + } + + @Test + void testLeadingWhitespaceYieldsEmptyFirstToken() throws IOException { + // a leading run gives one empty token, as String.split("\\s+") does + TokenSample sample = stream(" a\n").next(); + Assertions.assertEquals("a", sample.getText()); + Assertions.assertArrayEquals(new Span[] {new Span(0, 0), new Span(0, 1)}, + sample.getTokenSpans()); + } + + @Test + void testWhitespaceOnlyLineHasNoTokens() throws IOException { + TokenSample sample = stream(" \n").next(); + Assertions.assertEquals("", sample.getText()); + Assertions.assertEquals(0, sample.getTokenSpans().length); + } + + @Test + void testNonAsciiWhitespaceIsNotASeparator() throws IOException { + // no-break space is not in the ASCII whitespace set + TokenSample sample = stream("a b c\n").next(); + Assertions.assertEquals(2, sample.getTokenSpans().length); + Assertions.assertEquals("a b", sample.getText().substring(0, 3)); + } + + @Test + void testQuoteAndPunctuationAttachment() throws IOException { + // a token without an ASCII letter or digit attaches to the previous token + TokenSample sample = stream("Hello , world !\n").next(); + Assertions.assertEquals("Hello, world!", sample.getText()); + Assertions.assertArrayEquals(new Span[] {new Span(0, 5), new Span(5, 6), new Span(7, 12), + new Span(12, 13)}, sample.getTokenSpans()); + + // a token holding a digit is a word and gets a space in front + sample = stream("Room 101 .\n").next(); + Assertions.assertEquals("Room 101.", sample.getText()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/BrownClusterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/BrownClusterTest.java new file mode 100644 index 0000000000..4fdb46295f --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/BrownClusterTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util.featuregen; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class BrownClusterTest { + + private static BrownCluster cluster(String lexicon) throws IOException { + return new BrownCluster(new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8))); + } + + @Test + void testThreeColumnLinesNeedFrequencyAboveFive() throws IOException { + BrownCluster cluster = cluster("0101\tthe\t10\n0110\trare\t5\n"); + Assertions.assertEquals("0101", cluster.lookupToken("the")); + Assertions.assertNull(cluster.lookupToken("rare")); + } + + @Test + void testTwoColumnLinesMapFirstFieldToSecond() throws IOException { + BrownCluster cluster = cluster("dog\t0111\n"); + Assertions.assertEquals("0111", cluster.lookupToken("dog")); + } + + @Test + void testTrailingTabsAreDropped() throws IOException { + // trailing empty fields do not count, as with String.split("\t") + BrownCluster cluster = cluster("cat\t0100\t\t\n0011\tbird\t7\t\n"); + Assertions.assertEquals("0100", cluster.lookupToken("cat")); + Assertions.assertEquals("0011", cluster.lookupToken("bird")); + } + + @Test + void testLinesWithOtherFieldCountsAreIgnored() throws IOException { + BrownCluster cluster = cluster("\n\t\nsingle\na\tb\tc\td\n0101\tfish\t10\n"); + Assertions.assertNull(cluster.lookupToken("single")); + Assertions.assertNull(cluster.lookupToken("a")); + Assertions.assertNull(cluster.lookupToken("")); + Assertions.assertEquals("0101", cluster.lookupToken("fish")); + } + + @Test + void testSpacesAreNotSeparators() throws IOException { + BrownCluster cluster = cluster("0101 the 10\n"); + Assertions.assertNull(cluster.lookupToken("the")); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java index cd35f092ad..6aa668c807 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java @@ -19,6 +19,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; public class FeatureGeneratorUtilTest { @@ -70,6 +72,14 @@ void testGerman() { Assertions.assertEquals("sc", FeatureGeneratorUtil.tokenFeature("Ü")); } + @ParameterizedTest + @CsvSource({"A., cp", "Z., cp", "Ä., cp", "Ö., cp", "Ü., cp", + // lower case initial, other capital, other second character, longer tokens + "a., other", "É., ic", "'A,', ic", "Ab., ic", "AB., ic"}) + void testCapPeriod(String token, String feature) { + Assertions.assertEquals(feature, FeatureGeneratorUtil.tokenFeature(token)); + } + @Test void testJapanese() { // Hiragana diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/TokenPatternFeatureGeneratorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/TokenPatternFeatureGeneratorTest.java index 28e8e8c9d1..47c96696a2 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/TokenPatternFeatureGeneratorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/TokenPatternFeatureGeneratorTest.java @@ -71,4 +71,19 @@ void testSentence() { Assertions.assertEquals("st=sentence", features.get(12)); Assertions.assertEquals("pta=iclclclclc", features.get(13)); } + + @Test + void testSkipsNonLetterSubTokens() { + + String[] testSentence = new String[] {"well-known"}; + final int testTokenIndex = 0; + + AdaptiveFeatureGenerator generator = new TokenPatternFeatureGenerator(); + + generator.createFeatures(features, testSentence, testTokenIndex, null); + // the hyphen sub-token must not produce an "st=" feature + Assertions.assertFalse(features.contains("st=-")); + Assertions.assertTrue(features.contains("st=well")); + Assertions.assertTrue(features.contains("st=known")); + } } diff --git a/opennlp-extensions/opennlp-morfologik/pom.xml b/opennlp-extensions/opennlp-morfologik/pom.xml index ddc4415ffd..e506dc60ac 100644 --- a/opennlp-extensions/opennlp-morfologik/pom.xml +++ b/opennlp-extensions/opennlp-morfologik/pom.xml @@ -81,6 +81,12 @@ test + + org.junit.jupiter + junit-jupiter-params + test + + org.slf4j slf4j-simple diff --git a/opennlp-extensions/opennlp-morfologik/src/main/java/opennlp/morfologik/builder/MorfologikDictionaryBuilder.java b/opennlp-extensions/opennlp-morfologik/src/main/java/opennlp/morfologik/builder/MorfologikDictionaryBuilder.java index 8fdf4bdb57..3fdf63b883 100644 --- a/opennlp-extensions/opennlp-morfologik/src/main/java/opennlp/morfologik/builder/MorfologikDictionaryBuilder.java +++ b/opennlp-extensions/opennlp-morfologik/src/main/java/opennlp/morfologik/builder/MorfologikDictionaryBuilder.java @@ -36,6 +36,12 @@ */ public class MorfologikDictionaryBuilder { + /** The file name suffix of a dictionary metadata file, a dot and the Morfologik extension. */ + private static final String METADATA_FILE_SUFFIX = "." + DictionaryMetadata.METADATA_FILE_EXTENSION; + + /** The file name suffix of a compiled dictionary automaton. */ + private static final String DICTIONARY_FILE_SUFFIX = ".dict"; + /** * Helper to compile a morphological dictionary automaton. * @@ -61,8 +67,23 @@ public Path build(Path input, boolean overwrite, boolean validate, Path metadataPath = DictionaryMetadata.getExpectedMetadataLocation(input); return metadataPath.resolveSibling( - metadataPath.getFileName().toString().replaceAll( - "\\." + DictionaryMetadata.METADATA_FILE_EXTENSION + "$", ".dict")); + toDictionaryFileName(metadataPath.getFileName().toString())); + } + + /** + * Derives the compiled dictionary file name from a metadata file name by exchanging the + * trailing {@code .info} for {@code .dict}. A name without that trailing suffix is + * returned unchanged. + * + * @param metadataFileName The metadata file name. Must not be {@code null}. + * @return The dictionary file name. + */ + static String toDictionaryFileName(String metadataFileName) { + if (metadataFileName.endsWith(METADATA_FILE_SUFFIX)) { + return metadataFileName.substring(0, metadataFileName.length() - METADATA_FILE_SUFFIX.length()) + + DICTIONARY_FILE_SUFFIX; + } + return metadataFileName; } /** diff --git a/opennlp-extensions/opennlp-morfologik/src/test/java/opennlp/morfologik/builder/MorfologikDictionaryBuilderTest.java b/opennlp-extensions/opennlp-morfologik/src/test/java/opennlp/morfologik/builder/MorfologikDictionaryBuilderTest.java index 9228e36723..8934c992e8 100644 --- a/opennlp-extensions/opennlp-morfologik/src/test/java/opennlp/morfologik/builder/MorfologikDictionaryBuilderTest.java +++ b/opennlp-extensions/opennlp-morfologik/src/test/java/opennlp/morfologik/builder/MorfologikDictionaryBuilderTest.java @@ -23,8 +23,11 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import morfologik.stemming.DictionaryMetadata; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import opennlp.morfologik.AbstractMorfologikTest; import opennlp.morfologik.lemmatizer.MorfologikLemmatizer; @@ -72,4 +75,33 @@ public void testBuildDictionary() throws Exception { output.toFile().deleteOnExit(); } + @Test + public void testBuildNamesTheDictionaryAfterTheMetadataFile() throws Exception { + final Path rawLemmaDictionary = + new File(getResource("/dictionaryWithLemma.txt").getFile()).toPath(); + Path output = new MorfologikDictionaryBuilder().build(rawLemmaDictionary); + Assertions.assertEquals("dictionaryWithLemma.dict", output.getFileName().toString()); + Assertions.assertEquals(rawLemmaDictionary.getParent(), output.getParent()); + output.toFile().deleteOnExit(); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', value = { + "dictionaryWithLemma.info|dictionaryWithLemma.dict", + "a.info.info|a.info.dict", + ".info|.dict", + "info.info|info.dict", + "dictionary.txt|dictionary.txt", + "dictionary.info.bak|dictionary.info.bak", + "dictionaryXinfo|dictionaryXinfo", + "dictionary.INFO|dictionary.INFO", + "info|info", + "''|''"}) + public void testToDictionaryFileNameExchangesTheTrailingSuffixOnly(String input, String expected) { + Assertions.assertEquals(expected, MorfologikDictionaryBuilder.toDictionaryFileName(input)); + Assertions.assertEquals(input.replaceAll( + "\\." + DictionaryMetadata.METADATA_FILE_EXTENSION + "$", ".dict"), + MorfologikDictionaryBuilder.toDictionaryFileName(input)); + } + } diff --git a/opennlp-extensions/opennlp-spellcheck/pom.xml b/opennlp-extensions/opennlp-spellcheck/pom.xml index 07280c17c5..b06d31861c 100644 --- a/opennlp-extensions/opennlp-spellcheck/pom.xml +++ b/opennlp-extensions/opennlp-spellcheck/pom.xml @@ -72,6 +72,11 @@ junit-jupiter-engine test + + org.junit.jupiter + junit-jupiter-params + test + org.slf4j diff --git a/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/dictionary/FrequencyDictionaryLoader.java b/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/dictionary/FrequencyDictionaryLoader.java index 0f77645b30..d03ffd9b64 100644 --- a/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/dictionary/FrequencyDictionaryLoader.java +++ b/opennlp-extensions/opennlp-spellcheck/src/main/java/opennlp/spellcheck/dictionary/FrequencyDictionaryLoader.java @@ -20,9 +20,10 @@ import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.regex.Pattern; import opennlp.spellcheck.symspell.SymSpell; import opennlp.tools.util.InputStreamFactory; @@ -62,9 +63,6 @@ public final class FrequencyDictionaryLoader { /** The default character set used when none is supplied. */ public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8; - /** Splits a line into columns on a TAB or a run of spaces. */ - private static final Pattern COLUMN_SEPARATOR = Pattern.compile("[\\t ]+"); - /** UTF-8 byte-order mark (U+FEFF); stripped if it leads a line. */ private static final char BOM = (char) 0xFEFF; @@ -158,7 +156,7 @@ private long readUnigrams(InputStreamFactory factory, UnigramSink sink) throws I if (isSkippable(content)) { continue; } - final String[] columns = COLUMN_SEPARATOR.split(content.strip()); + final String[] columns = splitColumns(content.strip()); if (columns.length < 2 || columns[0].isEmpty()) { throw new MalformedDictionaryLineException(lineNo, line, "expected 'wordcount'"); } @@ -182,7 +180,7 @@ private long readBigrams(InputStreamFactory factory, BigramSink sink) throws IOE if (isSkippable(content)) { continue; } - final String[] columns = COLUMN_SEPARATOR.split(content.strip()); + final String[] columns = splitColumns(content.strip()); if (columns.length < 3 || columns[0].isEmpty() || columns[1].isEmpty()) { throw new MalformedDictionaryLineException(lineNo, line, "expected 'w1w2count'"); } @@ -194,6 +192,54 @@ private long readBigrams(InputStreamFactory factory, BigramSink sink) throws IOE return read; } + /** + * Splits {@code line} into columns on runs of TAB and space characters. A leading run + * yields one empty first column, trailing empty columns are dropped, a line of only + * separators yields an empty array, and an empty line yields a single empty column. Other + * whitespace, such as a no-break space or a vertical tab, stays inside a column. + * + * @param line The line to split. Must not be {@code null}. + * @return The columns in order. + */ + static String[] splitColumns(String line) { + if (line.isEmpty()) { + return new String[] {""}; + } + final List columns = new ArrayList<>(); + if (isColumnSeparator(line.charAt(0))) { + columns.add(""); + } + int start = 0; + for (int i = 0; i < line.length(); i++) { + if (isColumnSeparator(line.charAt(i))) { + if (i > start) { + columns.add(line.substring(start, i)); + } + while (i + 1 < line.length() && isColumnSeparator(line.charAt(i + 1))) { + i++; + } + start = i + 1; + } + } + if (line.length() > start) { + columns.add(line.substring(start)); + } + while (!columns.isEmpty() && columns.get(columns.size() - 1).isEmpty()) { + columns.remove(columns.size() - 1); + } + return columns.toArray(new String[0]); + } + + /** + * Tests whether {@code c} separates dictionary columns, which only a TAB or a space does. + * + * @param c The character to check. + * @return {@code true} if {@code c} is a TAB or a space. + */ + private static boolean isColumnSeparator(char c) { + return c == '\t' || c == ' '; + } + private static String stripBom(String line) { if (!line.isEmpty() && line.charAt(0) == BOM) { return line.substring(1); diff --git a/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/dictionary/FrequencyDictionaryLoaderTest.java b/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/dictionary/FrequencyDictionaryLoaderTest.java new file mode 100644 index 0000000000..a85ad2e87d --- /dev/null +++ b/opennlp-extensions/opennlp-spellcheck/src/test/java/opennlp/spellcheck/dictionary/FrequencyDictionaryLoaderTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.spellcheck.dictionary; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import opennlp.tools.util.InputStreamFactory; + +public class FrequencyDictionaryLoaderTest { + + private static final Pattern FORMER_COLUMN_SEPARATOR = Pattern.compile("[\\t ]+"); + + private static Stream columnSplits() { + return Stream.of( + Arguments.of("", new String[] {""}), + Arguments.of(" ", new String[0]), + Arguments.of("\t", new String[0]), + Arguments.of(" \t \t", new String[0]), + Arguments.of("a", new String[] {"a"}), + Arguments.of("a b", new String[] {"a", "b"}), + Arguments.of("a\tb", new String[] {"a", "b"}), + Arguments.of("a \t \t b", new String[] {"a", "b"}), + Arguments.of(" a", new String[] {"", "a"}), + Arguments.of("\t\ta b", new String[] {"", "a", "b"}), + Arguments.of("a b \t", new String[] {"a", "b"}), + Arguments.of("a b", new String[] {"a b"}), + Arguments.of("a\fb\rc\nd", new String[] {"a\fb\rc\nd"}), + Arguments.of("a\u00A0b 5", new String[] {"a\u00A0b", "5"}), + Arguments.of("a b\u3000c", new String[] {"a", "b\u3000c"}), + Arguments.of("\uD83D\uDE00 5\t\uD83D\uDE00", + new String[] {"\uD83D\uDE00", "5", "\uD83D\uDE00"})); + } + + @ParameterizedTest + @MethodSource("columnSplits") + void testSplitColumnsOnTabAndSpaceRuns(String line, String[] expected) { + Assertions.assertArrayEquals(expected, FrequencyDictionaryLoader.splitColumns(line)); + Assertions.assertArrayEquals(FORMER_COLUMN_SEPARATOR.split(line), + FrequencyDictionaryLoader.splitColumns(line)); + } + + @Test + void testUnigramColumnsSplitOnTabAndSpaceRunsOnly() throws IOException { + final String text = "the \t 100\nworld\t\t50\n hello 7 \na b 5\nc\u00A0d\t9\n"; + final Map into = new LinkedHashMap<>(); + final long read = new FrequencyDictionaryLoader().parseUnigrams(stringResource(text), into); + Assertions.assertEquals(5, read); + Assertions.assertEquals(100L, into.get("the")); + Assertions.assertEquals(50L, into.get("world")); + Assertions.assertEquals(7L, into.get("hello")); + Assertions.assertEquals(5L, into.get("a b")); + Assertions.assertEquals(9L, into.get("c\u00A0d")); + } + + @Test + void testBigramColumnsSplitOnTabAndSpaceRuns() throws IOException { + final String text = "the world\t3\nhello \t there 4\n"; + final Map into = new LinkedHashMap<>(); + final long read = new FrequencyDictionaryLoader().parseBigrams(stringResource(text), into); + Assertions.assertEquals(2, read); + Assertions.assertEquals(3L, into.get("the world")); + Assertions.assertEquals(4L, into.get("hello there")); + } + + @Test + void testUnigramLineWithoutTabOrSpaceIsMalformed() { + final String text = "the\u00A0100\n"; + final Map into = new LinkedHashMap<>(); + final FrequencyDictionaryLoader loader = new FrequencyDictionaryLoader(); + final MalformedDictionaryLineException ex = Assertions.assertThrows( + MalformedDictionaryLineException.class, () -> loader.parseUnigrams(stringResource(text), into)); + Assertions.assertEquals(1, ex.getLineNumber()); + } + + private static InputStreamFactory stringResource(String text) { + return () -> new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java index ac04d7d4ef..0365d00303 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java @@ -21,6 +21,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -73,6 +74,37 @@ void testNullUrl() { ); } + private static Stream indexPages() { + return Stream.of( + Arguments.of("xy", List.of("a.bin", "b.bin")), + // tag and attribute names ignore case + Arguments.of("z", List.of("c.bin")), + Arguments.of("xy", List.of("a.bin", "b.bin")), + // an anchor without closing tag is skipped, an earlier one is still found + Arguments.of("x", List.of()), + Arguments.of("xy", List.of("a.bin")), + // the first "" closes the link, so nested anchor markup is swallowed + Arguments.of("x y", List.of("d.bin")), + // the href value ends at the first "\">", so it may hold other markup + Arguments.of("g.bin\">f", List.of("fg.bin")), + // values and link text may span lines + Arguments.of("x y\nz", List.of("a.bin", "h\ni.bin")), + Arguments.of("no links here", List.of()), + Arguments.of("", List.of()), + // an empty href value is kept, a single-quoted one is not an anchor + Arguments.of("", List.of("")), + Arguments.of("x", List.of()), + // the closing tag is matched case-insensitively and only as "" + Arguments.of("xyz", List.of("a.bin")), + Arguments.of("x", List.of("\uD83D\uDE00.bin"))); + } + + @ParameterizedTest + @MethodSource("indexPages") + void testExtractLinks(String page, List expected) { + assertEquals(expected, DownloadUtil.DownloadParser.extractLinks(page)); + } + @Test void testInvalidUrl() { try { diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java index f92ada9c99..299487596c 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java @@ -36,6 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -65,6 +66,31 @@ public void testDownloadModelByURL(String language, URL url) throws IOException assertTrue(model.isLoadedFromSerialized()); } + @ParameterizedTest + @MethodSource("checksumFiles") + void testParseChecksum(String content, String expected) { + assertEquals(expected, DownloadUtil.parseChecksum(content)); + } + + private static Stream checksumFiles() { + return Stream.of( + Arguments.of("abc123 model.bin", "abc123"), + Arguments.of("abc123\tmodel.bin\n", "abc123"), + Arguments.of(" abc123 model.bin", "abc123"), + Arguments.of("abc123", "abc123"), + Arguments.of("abc123 *model.bin\r\n", "abc123"), + Arguments.of("abc123 model.bin\ndef456 other.bin\n", "abc123"), + Arguments.of("abc123\u000Bmodel.bin", "abc123"), + Arguments.of("abc123\u00A0model.bin", "abc123\u00A0model.bin")); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {" ", "\t\n"}) + void testParseChecksumOfBlankContent(String content) { + assertNull(DownloadUtil.parseChecksum(content)); + } + @Test @EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org") public void testExistsModel() throws IOException { diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java index 1951c114a0..353ecfbd8c 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; @@ -185,6 +186,95 @@ void testIsUnicodeWhitespaceDelegates() { Assertions.assertFalse(StringUtil.isUnicodeWhitespace(0x200B)); } + // ------------------------------------------------------------------------- + // isAsciiWhitespace + // ------------------------------------------------------------------------- + + @ParameterizedTest + @ValueSource(chars = {' ', '\t', '\n', '\u000B', '\f', '\r'}) + void testIsAsciiWhitespaceAccepts(char c) { + Assertions.assertTrue(StringUtil.isAsciiWhitespace(c)); + } + + @ParameterizedTest + @ValueSource(chars = {'a', '0', '_', '\u0000', '\u001C', '\u0085', '\u00A0', '\u2003', '\u3000'}) + void testIsAsciiWhitespaceRejects(char c) { + Assertions.assertFalse(StringUtil.isAsciiWhitespace(c)); + } + + // ------------------------------------------------------------------------- + // splitOnAsciiWhitespace + // ------------------------------------------------------------------------- + + @Test + void testSplitOnAsciiWhitespaceNullThrows() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> StringUtil.splitOnAsciiWhitespace(null)); + } + + private static Stream asciiSplits() { + return Stream.of( + Arguments.of("", new String[] {""}), + Arguments.of(" ", new String[0]), + Arguments.of("\t\n\r", new String[0]), + Arguments.of("a", new String[] {"a"}), + Arguments.of("a b", new String[] {"a", "b"}), + Arguments.of("hello world", new String[] {"hello", "world"}), + Arguments.of("a\t\u000B\fb", new String[] {"a", "b"}), + Arguments.of(" a", new String[] {"", "a"}), + Arguments.of("a ", new String[] {"a"}), + Arguments.of(" a\tb ", new String[] {"", "a", "b"}), + Arguments.of("a\u00A0b", new String[] {"a\u00A0b"}), + Arguments.of("a\u2003b c", new String[] {"a\u2003b", "c"}), + Arguments.of("\uD801\uDC12 \uD83D\uDE00", new String[] {"\uD801\uDC12", "\uD83D\uDE00"}), + Arguments.of(" \r\n ", new String[0])); + } + + @ParameterizedTest + @MethodSource("asciiSplits") + void testSplitOnAsciiWhitespaceMatchesStringSplit(String input, String[] expected) { + Assertions.assertArrayEquals(expected, StringUtil.splitOnAsciiWhitespace(input)); + Assertions.assertArrayEquals(input.split("\\s+"), StringUtil.splitOnAsciiWhitespace(input)); + } + + // ------------------------------------------------------------------------- + // containsAsciiUpperCase, containsAsciiDigit + // ------------------------------------------------------------------------- + + @Test + void testContainsAsciiNullThrows() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> StringUtil.containsAsciiUpperCase(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> StringUtil.containsAsciiDigit(null)); + } + + @ParameterizedTest + @ValueSource(strings = {"A", "Z", "aBc", "token-X", "1A", "Q\uD801\uDC12"}) + void testContainsAsciiUpperCaseAccepts(String input) { + Assertions.assertTrue(StringUtil.containsAsciiUpperCase(input)); + Assertions.assertTrue(StringUtil.containsAsciiUpperCase(new StringBuilder(input))); + } + + @ParameterizedTest + @ValueSource(strings = {"", "abc", "123", "Étudiant", "ÄÖÜ", "\uFF21", "\uD801\uDC12", "_-."}) + void testContainsAsciiUpperCaseRejects(String input) { + Assertions.assertFalse(StringUtil.containsAsciiUpperCase(input)); + } + + @ParameterizedTest + @ValueSource(strings = {"0", "9", "a1b", "x-2", "9A", "\uD801\uDC120"}) + void testContainsAsciiDigitAccepts(String input) { + Assertions.assertTrue(StringUtil.containsAsciiDigit(input)); + Assertions.assertTrue(StringUtil.containsAsciiDigit(new StringBuilder(input))); + } + + @ParameterizedTest + @ValueSource(strings = {"", "abc", "ABC", "\u0661", "\uFF11", "\u00BD", "\uD835\uDFCE", "_-."}) + void testContainsAsciiDigitRejects(String input) { + Assertions.assertFalse(StringUtil.containsAsciiDigit(input)); + } + // ------------------------------------------------------------------------- // splitOnUnicodeWhitespace // ------------------------------------------------------------------------- diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java index f63d2d1f9e..b965cf1536 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java @@ -40,4 +40,29 @@ void normalizeEmoji() { "Any funny text goes here ", normalizer.normalize(s)); } + @Test + void normalizeUnpairedSurrogates() { + // a lone high surrogate and a lone low surrogate are matched individually + String s = "a" + '\uD83C' + "b" + '\uDC00' + "c"; + Assertions.assertEquals("a b c", normalizer.normalize(s)); + + // adjacent surrogates, paired or not, collapse into a single space + StringBuilder sb = new StringBuilder(); + sb.append("x").append('\uD83C').append('\uDC00').append("y"); + Assertions.assertEquals("x y", normalizer.normalize(sb)); + } + + @Test + void normalizeMatchesCodePointsNotOnlyEmoji() { + // the matched code point range is [U+D83C, U+10FC00], so BMP characters + // from U+D83C up are replaced as well + Assertions.assertEquals("a b", normalizer.normalize("a" + '\uE000' + "b")); + + // supplementary code points beyond U+10FC00 are kept verbatim + StringBuilder sb = new StringBuilder(); + sb.append("a").appendCodePoint(0x10FFFF).append("b"); + Assertions.assertEquals("a" + new String(Character.toChars(0x10FFFF)) + "b", + normalizer.normalize(sb)); + } + } diff --git a/pom.xml b/pom.xml index 32b36c2097..306e7adb21 100644 --- a/pom.xml +++ b/pom.xml @@ -303,6 +303,7 @@ validate ${maven.multiModuleProjectDirectory}/checkstyle.xml + ${maven.multiModuleProjectDirectory}/checkstyle-suppressions.xml true true ${project.basedir}/src/test/java