diff --git a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java index 44f683d588..c49adfe948 100644 --- a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java +++ b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java @@ -116,6 +116,23 @@ public static boolean isUnicodeWhitespace(int charCode) { return UnicodeWhitespace.isWhitespace(charCode); } + /** + * Determines if the specified {@link Character} is one of the six ASCII whitespace + * characters, the set the regular expression class {@code \s} matches: space, tab, + * line feed, vertical tab, form feed, and carriage return. Unlike + * {@link #isWhitespace(char)}, this ignores the {@link WhitespaceMode} and rejects + * every non-ASCII space. + * + * @param charCode The character to check. + * + * @return {@code true} if {@code charCode} is one of those six characters, + * {@code false} otherwise. + */ + public static boolean isAsciiWhitespace(char charCode) { + return charCode == ' ' || charCode == '\t' || charCode == '\n' || charCode == '\u000B' + || charCode == '\f' || charCode == '\r'; + } + /** * Splits {@code input} on runs of Unicode {@code White_Space}. Leading and trailing * runs are ignored, so whitespace-only input yields an empty array. This is a @@ -151,6 +168,91 @@ public static String[] splitOnUnicodeWhitespace(CharSequence input) { return terms.toArray(new String[0]); } + /** + * Splits {@code input} on runs of ASCII whitespace with the result of + * {@code String.split("\\s+")}: a leading run yields one empty first element, trailing + * empty elements are dropped, whitespace-only input yields an empty array, and empty + * input yields a single empty element. This is a character scan, not a regular + * expression. + * + * @param input The text to split. Must not be {@code null}. + * @return The elements in order. + * @throws IllegalArgumentException If {@code input} is {@code null}. + */ + public static String[] splitOnAsciiWhitespace(String input) { + if (input == null) { + throw new IllegalArgumentException("input must not be null"); + } + if (input.isEmpty()) { + return new String[] {""}; + } + final List elements = new ArrayList<>(); + if (isAsciiWhitespace(input.charAt(0))) { + elements.add(""); + } + int start = 0; + for (int i = 0; i < input.length(); i++) { + if (isAsciiWhitespace(input.charAt(i))) { + if (i > start) { + elements.add(input.substring(start, i)); + } + while (i + 1 < input.length() && isAsciiWhitespace(input.charAt(i + 1))) { + i++; + } + start = i + 1; + } + } + if (input.length() > start) { + elements.add(input.substring(start)); + } + while (!elements.isEmpty() && elements.get(elements.size() - 1).isEmpty()) { + elements.remove(elements.size() - 1); + } + return elements.toArray(new String[0]); + } + + /** + * Tests whether {@code input} contains an ASCII capital letter, {@code A} to {@code Z}. + * Capitals outside ASCII do not count. This is a character scan, not a regular expression. + * + * @param input The text to check. Must not be {@code null}. + * @return {@code true} if at least one character is an ASCII capital letter. + * @throws IllegalArgumentException If {@code input} is {@code null}. + */ + public static boolean containsAsciiUpperCase(CharSequence input) { + if (input == null) { + throw new IllegalArgumentException("input must not be null"); + } + for (int i = 0; i < input.length(); i++) { + final char c = input.charAt(i); + if (c >= 'A' && c <= 'Z') { + return true; + } + } + return false; + } + + /** + * Tests whether {@code input} contains an ASCII digit, {@code 0} to {@code 9}. Digits outside + * ASCII do not count. This is a character scan, not a regular expression. + * + * @param input The text to check. Must not be {@code null}. + * @return {@code true} if at least one character is an ASCII digit. + * @throws IllegalArgumentException If {@code input} is {@code null}. + */ + public static boolean containsAsciiDigit(CharSequence input) { + if (input == null) { + throw new IllegalArgumentException("input must not be null"); + } + for (int i = 0; i < input.length(); i++) { + final char c = input.charAt(i); + if (c >= '0' && c <= '9') { + return true; + } + } + return false; + } + /** * Trims leading and trailing runs of Unicode {@code White_Space}, the same set * {@link #splitOnUnicodeWhitespace(CharSequence)} breaks terms on. diff --git a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java index be1f02e7b8..6ee98995bb 100644 --- a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java +++ b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java @@ -19,7 +19,6 @@ import java.io.File; import java.io.IOException; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,23 +60,71 @@ public String getHelp() { + "Defaults to a WhitespaceTokenizer."; } - private static final Pattern UNTOKENIZED_PAREN_PATTERN_1 = Pattern.compile("([^ ])([({)}])"); - private static final Pattern UNTOKENIZED_PAREN_PATTERN_2 = Pattern.compile("([({)}])([^ ])"); - public static Parse[] parseLine(String line, Parser parser, int numParses) { return parseLine( line, parser, WhitespaceTokenizer.INSTANCE, numParses ); } public static Parse[] parseLine(String line, Parser parser, Tokenizer tokenizer, int numParses) { // fix some parens patterns - line = UNTOKENIZED_PAREN_PATTERN_1.matcher(line).replaceAll("$1 $2"); - line = UNTOKENIZED_PAREN_PATTERN_2.matcher(line).replaceAll("$1 $2"); + line = spaceUntokenizedParens(line); // tokenize String[] tokens = tokenizer.tokenize(line); return parseLine(tokens, parser, numParses); } + /** + * Separates round and curly brackets from adjacent text in two left-to-right passes: the + * first puts a space between a non-space character and a following bracket, the second + * between a bracket and a following non-space character. Each pass resumes after the pair + * it just spaced, so a pair overlapping that match is only seen by the second pass. + * + * @param line The untokenized line. + * @return The spaced line. + */ + static String spaceUntokenizedParens(String line) { + return insertParenSpaces(insertParenSpaces(line, false), true); + } + + /** + * Inserts a space between a bracket and an adjacent non-space character, left to right. + * + * @param line The untokenized line. + * @param parenFirst {@code true} to space a bracket before a character, {@code false} after one. + * @return The spaced line. + */ + private static String insertParenSpaces(String line, boolean parenFirst) { + StringBuilder spaced = new StringBuilder(line.length() + 8); + int i = 0; + while (i < line.length()) { + char c = line.charAt(i); + if (i + 1 < line.length()) { + char next = line.charAt(i + 1); + boolean match = parenFirst + ? isParen(c) && next != ' ' + : c != ' ' && isParen(next); + if (match) { + spaced.append(c).append(' ').append(next); + i += 2; + continue; + } + } + spaced.append(c); + i++; + } + return spaced.toString(); + } + + /** + * Tests for a round or curly bracket. + * + * @param c The character. + * @return {@code true} for {@code (}, {@code )}, {, or }. + */ + private static boolean isParen(char c) { + return c == '(' || c == ')' || c == '{' || c == '}'; + } + /** * Parses the specified pre-tokenized sentence and returns the requested number of parses * or fewer. diff --git a/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/parser/ParserToolTest.java b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/parser/ParserToolTest.java new file mode 100644 index 0000000000..8b03710a93 --- /dev/null +++ b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/parser/ParserToolTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.cmdline.parser; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import opennlp.tools.parser.Parse; +import opennlp.tools.parser.Parser; +import opennlp.tools.tokenize.WhitespaceTokenizer; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ParserToolTest { + + /* + * The first pass puts a space before a bracket that follows a non-space character, the + * second after a bracket that precedes one. Each pass resumes after the pair it spaced, so + * "((" gets its space from the second pass only, and only a space, not a tab, separates. + */ + private static Stream parenLines() { + return Stream.of( + Arguments.of("a(b)c", "a ( b ) c"), + Arguments.of("a (b) c", "a ( b ) c"), + Arguments.of("foo(bar){baz}", "foo ( bar ) {baz }"), + Arguments.of("((a)(b))", "( ( a ) (b ) )"), + Arguments.of("a(b(c)d)e", "a ( b ( c ) d ) e"), + Arguments.of("x((", "x ( ("), + Arguments.of("((x", "( ( x"), + Arguments.of("()", "( )"), + Arguments.of("(", "("), + Arguments.of("", ""), + Arguments.of("no parens here", "no parens here"), + Arguments.of("«quoted»", "«quoted»"), + Arguments.of("tab\there(", "tab\there ("), + Arguments.of("a (b", "a ( b")); + } + + @ParameterizedTest + @MethodSource("parenLines") + void testSpaceUntokenizedParens(String line, String expected) { + assertEquals(expected, ParserTool.spaceUntokenizedParens(line)); + } + + @Test + void testParseLineSeparatesParensBeforeTokenizing() { + Parser echo = new Parser() { + @Override + public Parse[] parse(Parse tokens, int numParses) { + return new Parse[] {tokens}; + } + + @Override + public Parse parse(Parse tokens) { + return tokens; + } + }; + Parse[] parses = ParserTool.parseLine("f(x)+g({y})", echo, WhitespaceTokenizer.INSTANCE, 1); + assertEquals(1, parses.length); + String[] tokens = Stream.of(parses[0].getChildren()).map(Parse::getCoveredText) + .toArray(String[]::new); + // "{y" stays joined: the second pass consumed "{" while spacing "( {" + assertArrayEquals(new String[] {"f", "(", "x", ")", "+g", "(", "{y", "}", ")"}, tokens); + } +} diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADMetadata.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADMetadata.java new file mode 100644 index 0000000000..c377db34d0 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADMetadata.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.ad; + +import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser; + +/** + * Reads the ids from the metadata of an Arvores Deitadas sentence, which differs between + * corpora. The metadata is one line; every method treats metadata with a line terminator as + * invalid. + */ +final class ADMetadata { + + private static final String PARAGRAPH_PREFIX = "p="; + private static final String SOURCE_PREFIX = "source=\""; + + private ADMetadata() { + } + + /** + * Parses the text id and the paragraph id: the text id is the ASCII digit run after any + * leading ASCII letters and hyphens, the paragraph id the digit run after the first + * {@code p=} that at least one digit follows. + * + * @param meta The metadata. + * @return The text id and the paragraph id, or {@code null} if either is missing. + */ + static int[] parseTextAndParagraph(String meta) { + int[] spans = scanTextAndParagraph(meta); + if (spans == null) { + return null; + } + return new int[] {Integer.parseInt(meta.substring(spans[0], spans[1])), + Integer.parseInt(meta.substring(spans[2], spans[3]))}; + } + + /** + * Reads the digits of the text id, see {@link #parseTextAndParagraph(String)}. + * + * @param meta The metadata. + * @return The digits, or {@code null} if the text id or the paragraph id is missing. + */ + static String textId(String meta) { + int[] spans = scanTextAndParagraph(meta); + return spans == null ? null : meta.substring(spans[0], spans[1]); + } + + /** + * Reads the ASCII letters and hyphens before the text id, which name the text in literary + * corpora. + * + * @param meta The metadata. + * @return The prefix, or {@code null} if it is empty or the text id or the paragraph id is + * missing. + */ + static String textPrefix(String meta) { + int[] spans = scanTextAndParagraph(meta); + return spans == null || spans[0] == 0 ? null : meta.substring(0, spans[0]); + } + + /** + * Reads the source: the text between the first {@code source="} and the next double quote. + * + * @param meta The metadata. + * @return The source, or {@code null} if the metadata has none. + */ + static String source(String meta) { + if (hasLineTerminator(meta)) { + return null; + } + int start = meta.indexOf(SOURCE_PREFIX); + if (start == -1) { + return null; + } + start += SOURCE_PREFIX.length(); + int end = meta.indexOf('"', start); + return end == -1 ? null : meta.substring(start, end); + } + + /** + * Scans the text id and the paragraph id, see {@link #parseTextAndParagraph(String)}. + * + * @param meta The metadata. + * @return The start and end of the text id and the start and end of the paragraph id, or + * {@code null} if either is missing. + */ + private static int[] scanTextAndParagraph(String meta) { + int i = 0; + while (i < meta.length() && (isAsciiLetter(meta.charAt(i)) || meta.charAt(i) == '-')) { + i++; + } + int textStart = i; + while (i < meta.length() && isAsciiDigit(meta.charAt(i))) { + i++; + } + if (i == textStart || hasLineTerminator(meta)) { + return null; + } + int textEnd = i; + int from = textEnd; + while (true) { + int prefix = meta.indexOf(PARAGRAPH_PREFIX, from); + if (prefix == -1) { + return null; + } + int paragraphStart = prefix + PARAGRAPH_PREFIX.length(); + int paragraphEnd = paragraphStart; + while (paragraphEnd < meta.length() && isAsciiDigit(meta.charAt(paragraphEnd))) { + paragraphEnd++; + } + if (paragraphEnd > paragraphStart) { + return new int[] {textStart, textEnd, paragraphStart, paragraphEnd}; + } + from = prefix + 1; + } + } + + /** + * Tests whether the metadata contains a line terminator. + * + * @param meta The metadata. + * @return {@code true} if it does. + */ + private static boolean hasLineTerminator(String meta) { + return SentenceParser.indexOfLineTerminator(meta, 0) < meta.length(); + } + + /** + * Tests for an ASCII letter. + * + * @param c The character. + * @return {@code true} for {@code a} to {@code z} or {@code A} to {@code Z}. + */ + private static boolean isAsciiLetter(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + + /** + * Tests for an ASCII digit. + * + * @param c The character. + * @return {@code true} for {@code 0} to {@code 9}. + */ + private static boolean isAsciiDigit(char c) { + return c >= '0' && c <= '9'; + } +} diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java index d2db063515..01c838c7a9 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java @@ -25,8 +25,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import opennlp.tools.commons.Internal; import opennlp.tools.formats.ad.ADSentenceStream.Sentence; @@ -38,6 +36,7 @@ import opennlp.tools.util.ObjectStream; import opennlp.tools.util.PlainTextByLineStream; import opennlp.tools.util.Span; +import opennlp.tools.util.StringUtil; /** * Parser for Floresta Sita(c)tica Arvores Deitadas corpus, output to for the @@ -66,16 +65,6 @@ @Internal public class ADNameSampleStream implements ObjectStream { - /* - * Pattern of a NER tag in Arvores Deitadas - */ - private static final Pattern TAG_PATTERN = Pattern.compile("<(NER:)?(.*?)>"); - private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+"); - private static final Pattern UNDERLINE_PATTERN = Pattern.compile("[_]+"); - private static final Pattern HYPHEN_PATTERN = - Pattern.compile("((\\p{L}+)-$)|(^-(\\p{L}+)(.*))|((\\p{L}+)-(\\p{L}+)(.*))"); - private static final Pattern ALPHANUMERIC_PATTERN = Pattern.compile("^[\\p{L}\\p{Nd}]+$"); - /* * Map to the Arvores Deitadas types to our types. It is read-only. */ @@ -254,7 +243,7 @@ private void processLeaf(Leaf leaf, List sentence, List names) { String c = PortugueseContractionUtility.toContraction( leftContractionPart, right); if (c != null) { - String[] parts = WHITESPACE_PATTERN.split(c); + String[] parts = StringUtil.splitOnAsciiWhitespace(c); sentence.addAll(Arrays.asList(parts)); alreadyAdded = true; } else { @@ -273,7 +262,7 @@ private void processLeaf(Leaf leaf, List sentence, List names) { if (leafTag != null) { if (leafTag.contains("") && !alreadyAdded) { - String[] lexemes = UNDERLINE_PATTERN.split(leaf.getLexeme()); + String[] lexemes = splitOnUnderscores(leaf.getLexeme()); if (lexemes.length > 1) { sentence.addAll(Arrays.asList(lexemes).subList(0, lexemes.length - 1)); } @@ -318,9 +307,9 @@ private void processLeaf(Leaf leaf, List sentence, List names) { private List processLexeme(String lexemeStr) { List out = new ArrayList<>(); - String[] parts = UNDERLINE_PATTERN.split(lexemeStr); + String[] parts = splitOnUnderscores(lexemeStr); for (String tok : parts) { - if (tok.length() > 1 && !ALPHANUMERIC_PATTERN.matcher(tok).matches()) { + if (tok.length() > 1 && !isAlphaNumeric(tok)) { out.addAll(processTok(tok)); } else { out.add(tok); @@ -347,35 +336,19 @@ private List processTok(String tok) { // lets split all hyphens if (this.splitHyphenatedTokens && tok.contains("-") && tok.length() > 1) { - Matcher matcher = HYPHEN_PATTERN.matcher(tok); - - String firstTok = null; - String hyphen = "-"; - String secondTok = null; - String rest = null; - - if (matcher.matches()) { - if (matcher.group(1) != null) { - firstTok = matcher.group(2); - } else if (matcher.group(3) != null) { - secondTok = matcher.group(4); - rest = matcher.group(5); - } else if (matcher.group(6) != null) { - firstTok = matcher.group(7); - secondTok = matcher.group(8); - rest = matcher.group(9); - } + String[] parts = matchHyphenatedToken(tok); - addIfNotEmpty(firstTok, out); - addIfNotEmpty(hyphen, out); - addIfNotEmpty(secondTok, out); - addIfNotEmpty(rest, out); + if (parts != null) { + addIfNotEmpty(parts[0], out); + addIfNotEmpty("-", out); + addIfNotEmpty(parts[1], out); + addIfNotEmpty(parts[2], out); tokAdded = true; } } if (!tokAdded) { if (!original.equals(tok) && tok.length() > 1 - && !ALPHANUMERIC_PATTERN.matcher(tok).matches()) { + && !isAlphaNumeric(tok)) { out.addAll(processTok(tok)); } else { out.add(tok); @@ -391,6 +364,161 @@ private void addIfNotEmpty(String firstTok, List out) { } } + /** + * Splits on runs of underscores with the result of {@code String.split("[_]+")}: a leading + * run yields one empty first element, trailing empty elements are dropped, and + * underscore-only input yields an empty array. + * + * @param s The text. + * @return The elements in order. + */ + static String[] splitOnUnderscores(String s) { + if (s.isEmpty()) { + return new String[] {""}; + } + boolean hasToken = false; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) != '_') { + hasToken = true; + break; + } + } + if (!hasToken) { + return new String[0]; + } + List tokens = new ArrayList<>(); + if (s.charAt(0) == '_') { + tokens.add(""); + } + int start = 0; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) == '_') { + if (i > start) { + tokens.add(s.substring(start, i)); + } + while (i + 1 < s.length() && s.charAt(i + 1) == '_') { + i++; + } + start = i + 1; + } + } + if (s.length() > start) { + tokens.add(s.substring(start)); + } + return tokens.toArray(new String[0]); + } + + /** + * Tests whether a token consists of letters and decimal digits only, by code point. + * + * @param tok The token. + * @return {@code true} if the token is non-empty and every code point is a letter or a + * decimal digit. + */ + static boolean isAlphaNumeric(String tok) { + if (tok.isEmpty()) { + return false; + } + int i = 0; + while (i < tok.length()) { + int cp = tok.codePointAt(i); + if (!Character.isLetter(cp) && !Character.isDigit(cp)) { + return false; + } + i += Character.charCount(cp); + } + return true; + } + + /** + * Splits a hyphenated token into the letters before the hyphen, the letters after it, and + * the rest. Three shapes match: letters followed by a final hyphen, a leading hyphen followed + * by letters and anything, and letters, a hyphen, letters, and anything. + * + * @param tok The token, at least two characters long. + * @return The first token, second token, and rest, each {@code null} when absent, or + * {@code null} if the token has none of the three shapes. + */ + static String[] matchHyphenatedToken(String tok) { + int len = tok.length(); + // (\p{L}+)-$ + if (len > 1 && tok.charAt(len - 1) == '-' && isAllLetters(tok, 0, len - 1)) { + return new String[] {tok.substring(0, len - 1), null, null}; + } + // ^-(\p{L}+)(.*) + if (tok.charAt(0) == '-') { + int lettersEnd = lettersEnd(tok, 1); + if (lettersEnd > 1) { + return new String[] {null, tok.substring(1, lettersEnd), tok.substring(lettersEnd)}; + } + return null; + } + // (\p{L}+)-(\p{L}+)(.*) + int firstEnd = lettersEnd(tok, 0); + if (firstEnd > 0 && firstEnd + 1 < len && tok.charAt(firstEnd) == '-') { + int secondEnd = lettersEnd(tok, firstEnd + 1); + if (secondEnd > firstEnd + 1) { + return new String[] {tok.substring(0, firstEnd), + tok.substring(firstEnd + 1, secondEnd), tok.substring(secondEnd)}; + } + } + return null; + } + + /** + * Finds the end of the run of letters starting at an offset. + * + * @param s The text. + * @param from The start offset. + * @return The offset after the run, or {@code from} if no letter starts there. + */ + private static int lettersEnd(String s, int from) { + int i = from; + while (i < s.length()) { + int cp = s.codePointAt(i); + if (!Character.isLetter(cp)) { + break; + } + i += Character.charCount(cp); + } + return i; + } + + /** + * Tests whether a range holds letters only. + * + * @param s The text. + * @param from The inclusive start. + * @param to The exclusive end. + * @return {@code true} if every code point in the range is a letter. + */ + private static boolean isAllLetters(String s, int from, int to) { + int i = from; + while (i < to) { + int cp = s.codePointAt(i); + if (!Character.isLetter(cp)) { + return false; + } + i += Character.charCount(cp); + } + return true; + } + + /** + * Extracts the content of a NER tag in Arvores Deitadas format, between the optional + * {@code NER:} prefix and the closing angle bracket. + * + * @param t The tag. + * @return The content, or {@code null} if {@code t} is not enclosed in angle brackets. + */ + static String tagContent(String t) { + if (t.length() < 2 || t.charAt(0) != '<' || t.charAt(t.length() - 1) != '>') { + return null; + } + int start = t.startsWith("NER:", 1) ? 5 : 1; + return t.substring(start, t.length() - 1); + } + /** * Parses a NER tag in Arvores Deitadas format. * @@ -401,14 +529,11 @@ private static String getNER(String tags) { if (tags.contains("")) { return null; } - String[] tag = tags.split("\\s+"); + String[] tag = StringUtil.splitOnAsciiWhitespace(tags); for (String t : tag) { - Matcher matcher = TAG_PATTERN.matcher(t); - if (matcher.matches()) { - String ner = matcher.group(2); - if (HAREM.containsKey(ner)) { - return HAREM.get(ner); - } + String ner = tagContent(t); + if (ner != null && HAREM.containsKey(ner)) { + return HAREM.get(ner); } } return null; @@ -424,72 +549,29 @@ public void close() throws IOException { adSentenceStream.close(); } - enum Type { - ama, cie, lit - } - - // works for Amazonia - // private static final Pattern meta1 = Pattern - // .compile("^(?:[a-zA-Z\\-]*(\\d+)).*?p=(\\d+).*"); - // - // // works for selva cie - // private static final Pattern meta2 = Pattern - // .compile("^(?:[a-zA-Z\\-]*(\\d+)).*?p=(\\d+).*"); - private int getTextID(Sentence paragraph) { - + final String meta = paragraph.metadata(); - Type corpusType; - Pattern metaPattern; int textIdMeta2 = -1; String textMeta2 = ""; - if (meta.startsWith("LIT")) { - corpusType = Type.lit; - metaPattern = Pattern.compile("^([a-zA-Z\\-]+)(\\d+).*?p=(\\d+).*"); - } else if (meta.startsWith("CIE")) { - corpusType = Type.cie; - metaPattern = Pattern.compile("^.*?source=\"(.*?)\".*"); - } else { // ama - corpusType = Type.ama; - metaPattern = Pattern.compile("^(?:[a-zA-Z\\-]*(\\d+)).*?p=(\\d+).*"); - } - - if (corpusType.equals(Type.lit)) { - Matcher m2 = metaPattern.matcher(meta); - if (m2.matches()) { - String textId = m2.group(1); - if (!textId.equals(textMeta2)) { - textIdMeta2++; - textMeta2 = textId; - } - return textIdMeta2; - } else { + if (meta.startsWith("LIT") || meta.startsWith("CIE")) { + String textId = meta.startsWith("LIT") ? ADMetadata.textPrefix(meta) : ADMetadata.source(meta); + if (textId == null) { throw new RuntimeException("Invalid metadata: " + meta); } - } else if (corpusType.equals(Type.cie)) { - Matcher m2 = metaPattern.matcher(meta); - if (m2.matches()) { - String textId = m2.group(1); - if (!textId.equals(textMeta2)) { - textIdMeta2++; - textMeta2 = textId; - } - return textIdMeta2; - } else { - throw new RuntimeException("Invalid metadata: " + meta); - } - } else if (corpusType.equals(Type.ama)) { - Matcher m2 = metaPattern.matcher(meta); - if (m2.matches()) { - return Integer.parseInt(m2.group(1)); - // currentPara = Integer.parseInt(m.group(2)); - } else { - throw new RuntimeException("Invalid metadata: " + meta); + if (!textId.equals(textMeta2)) { + textIdMeta2++; + textMeta2 = textId; } + return textIdMeta2; } - - return 0; + // Amazonia + String textId = ADMetadata.textId(meta); + if (textId == null) { + throw new RuntimeException("Invalid metadata: " + meta); + } + return Integer.parseInt(textId); } } diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java index 8b7bd96416..9a66f497f9 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.List; import java.util.StringTokenizer; -import java.util.regex.Pattern; import opennlp.tools.commons.Internal; import opennlp.tools.formats.ad.ADSentenceStream.Sentence; @@ -32,6 +31,7 @@ import opennlp.tools.util.InputStreamFactory; import opennlp.tools.util.ObjectStream; import opennlp.tools.util.PlainTextByLineStream; +import opennlp.tools.util.StringUtil; /** * Note: @@ -40,8 +40,6 @@ @Internal public class ADPOSSampleStream implements ObjectStream { - private static final Pattern WHITESPACES_PATTERN = Pattern.compile("\\s+"); - private final ObjectStream adSentenceStream; private final boolean expandME; private final boolean isIncludeFeatures; @@ -118,7 +116,7 @@ private void processLeaf(Leaf leaf, List sentence, List tags) { if (isIncludeFeatures && leaf.getMorphologicalTag() != null) { tag += " " + leaf.getMorphologicalTag(); } - tag = WHITESPACES_PATTERN.matcher(tag).replaceAll("="); + tag = replaceWhitespaceWithEquals(tag); if (tag == null) tag = lexeme; @@ -152,6 +150,31 @@ private void processLeaf(Leaf leaf, List sentence, List tags) { } + /** + * Replaces every run of ASCII whitespace, leading and trailing runs included, with a single + * equals sign. + * + * @param tag The tag. + * @return The joined tag. + */ + static String replaceWhitespaceWithEquals(String tag) { + StringBuilder replaced = new StringBuilder(tag.length()); + int i = 0; + while (i < tag.length()) { + char c = tag.charAt(i); + if (StringUtil.isAsciiWhitespace(c)) { + replaced.append('='); + while (i + 1 < tag.length() && StringUtil.isAsciiWhitespace(tag.charAt(i + 1))) { + i++; + } + } else { + replaced.append(c); + } + i++; + } + return replaced.toString(); + } + @Override public void reset() throws IOException, UnsupportedOperationException { adSentenceStream.reset(); diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java index c78bf2b9a4..099303e3f0 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java @@ -21,8 +21,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import opennlp.tools.commons.Internal; import opennlp.tools.formats.ad.ADSentenceStream.Sentence; @@ -134,22 +132,15 @@ private boolean hasPunctuation(String text) { return false; } - // there are some different types of metadata depending on the corpus. - // TODO Merge these patterns - private static final Pattern META_1 = Pattern.compile("^(?:[a-zA-Z\\-]*(\\d+)).*?p=(\\d+).*"); - private void updateMeta() { if (this.sent != null) { String meta = this.sent.metadata(); - Matcher m = META_1.matcher(meta); - int currentText; - int currentPara; - if (m.matches()) { - currentText = Integer.parseInt(m.group(1)); - currentPara = Integer.parseInt(m.group(2)); - } else { + int[] textAndPara = ADMetadata.parseTextAndParagraph(meta); + if (textAndPara == null) { throw new RuntimeException("Invalid metadata: " + meta); } + int currentText = textAndPara[0]; + int currentPara = textAndPara[1]; isSamePara = isSameText = false; if (currentText == text) isSameText = true; diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java index 1445f153b5..4cbe29cb16 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java @@ -23,8 +23,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Stack; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +31,7 @@ import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.Node; import opennlp.tools.util.FilterObjectStream; import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.StringUtil; /** * Stream filter which merges text lines into sentences, following the Arvores @@ -63,15 +62,11 @@ public record Sentence (String text, Node root, String metadata) { public static class SentenceParser { private static final Logger logger = LoggerFactory.getLogger(SentenceParser.class); - private static final Pattern NODE_PATTERN = Pattern - .compile("([=-]*)([^:=]+:[^\\(\\s]+)(\\(([^\\)]+)\\))?\\s*(?:(\\((<.+>)\\))*)\\s*$"); - private static final Pattern LEAF_PATTERN = Pattern - .compile("^([=-]*)([^:=]+):([^\\(\\s]+)\\([\"'](.+)[\"']\\s*((?:<.+>)*)\\s*([^\\)]+)?\\)\\s+(.+)"); - private static final Pattern BIZARRE_LEAF_PATTERN = Pattern - .compile("^([=-]*)([^:=]+=[^\\(\\s]+)\\(([\"'].+[\"'])?\\s*([^\\)]+)?\\)\\s+(.+)"); - private static final Pattern PUNCTUATION_PATTERN = Pattern.compile("^(=*)(\\W+)$"); - private static final Pattern PUNCTUATION_DOT_PATTERN = Pattern.compile("\\»\\s+\\."); - private static final Pattern PUNCTUATION_COMMA_PATTERN = Pattern.compile("\\»\\s+\\,"); + + private static final char TAG_SEPARATOR = ':'; + private static final char BIZARRE_TAG_SEPARATOR = '='; + private static final String TAG_GROUP_OPEN = "(<"; + private static final String TAG_GROUP_CLOSE = ">)"; private String text,meta; @@ -208,11 +203,82 @@ public Sentence parse(String sentenceString, int para, boolean isTitle, boolean } private String fixPunctuation(String text) { - text = PUNCTUATION_DOT_PATTERN.matcher(text).replaceAll("»."); - text = PUNCTUATION_COMMA_PATTERN.matcher(text).replaceAll("»,"); + text = replaceGuillemetPunctuation(text, '.', "»."); + text = replaceGuillemetPunctuation(text, ',', "»,"); return text; } + /** + * Removes the ASCII whitespace between a closing guillemet and a following punctuation + * character. + * + * @param text The text. + * @param punct The punctuation character. + * @param replacement The two characters to write in place of guillemet, whitespace, and + * punctuation. + * @return The text with those runs joined. + */ + private String replaceGuillemetPunctuation(String text, char punct, String replacement) { + StringBuilder fixed = new StringBuilder(text.length()); + int i = 0; + while (i < text.length()) { + char c = text.charAt(i); + if (c == '»' && i + 1 < text.length()) { + int j = i + 1; + while (j < text.length() && StringUtil.isAsciiWhitespace(text.charAt(j))) { + j++; + } + if (j > i + 1 && j < text.length() && text.charAt(j) == punct) { + fixed.append(replacement); + i = j + 1; + continue; + } + } + fixed.append(c); + i++; + } + return fixed.toString(); + } + + /** + * Parses a punctuation line: leading equals signs followed by one or more characters that + * are not ASCII letters, digits, or underscores. A line of equals signs only also matches, + * with the last one as lexeme. + * + * @param line The line. + * @return The level, as one more than the count of leading equals signs, and the lexeme, + * or {@code null} if the line is not a punctuation line. + */ + private String[] parsePunctuationLine(String line) { + if (line.isEmpty()) { + return null; + } + for (int i = 0; i < line.length(); i++) { + if (isAsciiWord(line.charAt(i))) { + return null; + } + } + int equals = 0; + while (equals < line.length() && line.charAt(equals) == '=') { + equals++; + } + if (equals == line.length()) { + return new String[] {String.valueOf(equals), "="}; + } + return new String[] {String.valueOf(equals + 1), line.substring(equals)}; + } + + /** + * Tests for an ASCII letter, digit, or underscore. + * + * @param c The character. + * @return {@code true} for a word character. + */ + private boolean isAsciiWord(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') || c == '_'; + } + /** * Parse a tree element from a AD line * @@ -223,45 +289,21 @@ private String fixPunctuation(String text) { public TreeElement getElement(String line) { // Note: all levels are higher than 1, because 0 is reserved for the root. - // try node - Matcher nodeMatcher = NODE_PATTERN.matcher(line); - if (nodeMatcher.matches()) { - int level = nodeMatcher.group(1).length() + 1; - String syntacticTag = nodeMatcher.group(2); - Node node = new Node(); - node.setLevel(level); - node.setSyntacticTag(syntacticTag); + Node node = parseNode(line); + if (node != null) { return node; } - Matcher leafMatcher = LEAF_PATTERN.matcher(line); - if (leafMatcher.matches()) { - int level = leafMatcher.group(1).length() + 1; - String syntacticTag = leafMatcher.group(2); - String funcTag = leafMatcher.group(3); - String lemma = leafMatcher.group(4); - String secondaryTag = leafMatcher.group(5); - String morphologicalTag = leafMatcher.group(6); - String lexeme = leafMatcher.group(7); - Leaf leaf = new Leaf(); - leaf.setLevel(level); - leaf.setSyntacticTag(syntacticTag); - leaf.setFunctionalTag(funcTag); - leaf.setSecondaryTag(secondaryTag); - leaf.setMorphologicalTag(morphologicalTag); - leaf.setLexeme(lexeme); - leaf.setLemma(lemma); - + Leaf leaf = parseLeaf(line); + if (leaf != null) { return leaf; } - Matcher punctuationMatcher = PUNCTUATION_PATTERN.matcher(line); - if (punctuationMatcher.matches()) { - int level = punctuationMatcher.group(1).length() + 1; - String lexeme = punctuationMatcher.group(2); - Leaf leaf = new Leaf(); - leaf.setLevel(level); - leaf.setLexeme(lexeme); + String[] punctuation = parsePunctuationLine(line); + if (punctuation != null) { + leaf = new Leaf(); + leaf.setLevel(Integer.parseInt(punctuation[0])); + leaf.setLexeme(punctuation[1]); return leaf; } @@ -271,47 +313,29 @@ public TreeElement getElement(String line) { } if (line.startsWith("=")) { - Matcher bizarreLeafMatcher = BIZARRE_LEAF_PATTERN.matcher(line); - if (bizarreLeafMatcher.matches()) { - int level = bizarreLeafMatcher.group(1).length() + 1; - String syntacticTag = bizarreLeafMatcher.group(2); - String lemma = bizarreLeafMatcher.group(3); - String morphologicalTag = bizarreLeafMatcher.group(4); - String lexeme = bizarreLeafMatcher.group(5); - Leaf leaf = new Leaf(); - leaf.setLevel(level); - leaf.setSyntacticTag(syntacticTag); - leaf.setMorphologicalTag(morphologicalTag); - leaf.setLexeme(lexeme); - if (lemma != null) { - if (lemma.length() > 2) { - lemma = lemma.substring(1, lemma.length() - 1); - } - leaf.setLemma(lemma); - } - + leaf = parseBizarreLeaf(line); + if (leaf != null) { return leaf; - } else { - int level = line.lastIndexOf("=") + 1; - String lexeme = line.substring(level + 1); + } + int level = line.lastIndexOf("=") + 1; + String lexeme = line.substring(level + 1); - if (lexeme.matches("\\w.*?[\\.<>].*")) { - return null; - } + if (isWordWithMarkup(lexeme)) { + return null; + } - Leaf leaf = new Leaf(); - leaf.setLevel(level + 1); - leaf.setSyntacticTag(""); - leaf.setMorphologicalTag(""); - leaf.setFunctionalTag(""); - leaf.setLexeme(lexeme); + leaf = new Leaf(); + leaf.setLevel(level + 1); + leaf.setSyntacticTag(""); + leaf.setMorphologicalTag(""); + leaf.setFunctionalTag(""); + leaf.setLexeme(lexeme); - return leaf; - } + return leaf; } logger.warn("Couldn't parse leaf: {}", line); - Leaf leaf = new Leaf(); + leaf = new Leaf(); leaf.setLevel(1); leaf.setSyntacticTag(""); leaf.setMorphologicalTag(""); @@ -321,6 +345,412 @@ public TreeElement getElement(String line) { return leaf; } + /** + * Parses a node line: the level prefix, a syntactic tag with a colon, an optional part in + * parentheses, and optional tag groups. + * + * @param line The line. + * @return The node, or {@code null} if the line is not a node line. + */ + private Node parseNode(String line) { + for (int[] tag = scanLevelAndTag(line, TAG_SEPARATOR, line.length()); tag != null; + tag = scanLevelAndTag(line, TAG_SEPARATOR, tag[0] - 1)) { + if (isNodeTail(line, tag[1])) { + Node node = new Node(); + node.setLevel(tag[0] + 1); + node.setSyntacticTag(line.substring(tag[0], tag[1])); + return node; + } + } + return null; + } + + /** + * Parses a leaf line: the level prefix, a syntactic tag, a colon, a functional tag, and in + * parentheses a quoted lemma, secondary tags in angle brackets, and a morphological tag, + * then ASCII whitespace and the lexeme. + * + * @param line The line. + * @return The leaf, or {@code null} if the line is not a leaf line. + */ + private Leaf parseLeaf(String line) { + boolean[] noRestAt = new boolean[line.length() + 1]; + for (int[] tag = scanLevelAndTag(line, TAG_SEPARATOR, line.length()); tag != null; + tag = scanLevelAndTag(line, TAG_SEPARATOR, tag[0] - 1)) { + Leaf leaf = parseLeafAfterTag(line, tag[0], tag[1], noRestAt); + if (leaf != null) { + return leaf; + } + } + return null; + } + + /** + * Parses the part of a leaf line after the tag, see {@link #parseLeaf(String)}. + * + * @param line The line. + * @param start The index where the tag starts. + * @param tagEnd The index after the tag. + * @param noRestAt The indexes after which no rest was found so far, updated here. + * @return The leaf, or {@code null} if the part after the tag does not have that form. + */ + private Leaf parseLeafAfterTag(String line, int start, int tagEnd, boolean[] noRestAt) { + if (tagEnd + 1 >= line.length() || line.charAt(tagEnd) != '(' + || !isQuote(line.charAt(tagEnd + 1))) { + return null; + } + int lemmaStart = tagEnd + 2; + // the longest lemma after which the rest of the line still parses wins + for (int lemmaEnd = indexOfLineTerminator(line, lemmaStart) - 1; lemmaEnd > lemmaStart; + lemmaEnd--) { + if (!isQuote(line.charAt(lemmaEnd))) { + continue; + } + int[] rest = scanLeafRest(line, lemmaEnd + 1, noRestAt); + if (rest != null) { + int separator = line.indexOf(TAG_SEPARATOR, start); + Leaf leaf = new Leaf(); + leaf.setLevel(start + 1); + leaf.setSyntacticTag(line.substring(start, separator)); + leaf.setFunctionalTag(line.substring(separator + 1, tagEnd)); + leaf.setLemma(line.substring(lemmaStart, lemmaEnd)); + leaf.setSecondaryTag(line.substring(rest[0], rest[1])); + leaf.setMorphologicalTag(rest[2] == rest[3] ? null : line.substring(rest[2], rest[3])); + leaf.setLexeme(line.substring(rest[4])); + return leaf; + } + } + return null; + } + + /** + * Parses a leaf line whose tag has an equals sign in place of the colon: the level prefix, + * the tag, and in parentheses an optional quoted lemma and an optional morphological tag, + * then ASCII whitespace and the lexeme. + * + * @param line The line. + * @return The leaf, or {@code null} if the line does not have that form. + */ + private Leaf parseBizarreLeaf(String line) { + for (int[] tag = scanLevelAndTag(line, BIZARRE_TAG_SEPARATOR, line.length()); tag != null; + tag = scanLevelAndTag(line, BIZARRE_TAG_SEPARATOR, tag[0] - 1)) { + Leaf leaf = parseBizarreLeafAfterTag(line, tag[0], tag[1]); + if (leaf != null) { + return leaf; + } + } + return null; + } + + /** + * Parses the part of a leaf line after a tag with an equals sign, see + * {@link #parseBizarreLeaf(String)}. + * + * @param line The line. + * @param start The index where the tag starts. + * @param tagEnd The index after the tag. + * @return The leaf, or {@code null} if the part after the tag does not have that form. + */ + private Leaf parseBizarreLeafAfterTag(String line, int start, int tagEnd) { + if (tagEnd == line.length() || line.charAt(tagEnd) != '(') { + return null; + } + int open = tagEnd + 1; + String lemma = null; + int[] rest = null; + if (open < line.length() && isQuote(line.charAt(open))) { + for (int lemmaEnd = indexOfLineTerminator(line, open + 1) - 1; + lemmaEnd > open + 1 && rest == null; lemmaEnd--) { + if (isQuote(line.charAt(lemmaEnd))) { + rest = scanMorphologyAndLexeme(line, lemmaEnd + 1); + if (rest != null) { + lemma = line.substring(open + 1, lemmaEnd); + } + } + } + } + if (rest == null) { + rest = scanMorphologyAndLexeme(line, open); + if (rest == null) { + return null; + } + } + Leaf leaf = new Leaf(); + leaf.setLevel(start + 1); + leaf.setSyntacticTag(line.substring(start, tagEnd)); + leaf.setMorphologicalTag(rest[0] == rest[1] ? null : line.substring(rest[0], rest[1])); + leaf.setLexeme(line.substring(rest[2])); + leaf.setLemma(lemma); + return leaf; + } + + /** + * Scans the level prefix and the tag at the start of a line. The prefix is the run of equals + * signs and hyphens, the tag one or more characters other than a colon or an equals sign, + * the separator, and one or more characters that are neither an opening parenthesis nor + * ASCII whitespace. A longer prefix is preferred; hyphens at its end may move into the tag, + * so the callers try the next shorter prefix when the rest of the line does not parse. + * + * @param line The line. + * @param separator The character between the two parts of the tag. + * @param maxStart The highest index where the tag may start: the length of the line for the + * first candidate, one less than the previous start for the next one. + * @return The index where the tag starts, which is the length of the prefix, and the index + * after the tag, or {@code null} if there is no further candidate. + */ + private int[] scanLevelAndTag(String line, char separator, int maxStart) { + int run = 0; + while (run < line.length() && (line.charAt(run) == '=' || line.charAt(run) == '-')) { + run++; + } + for (int start = Math.min(run, maxStart); start >= 0; start--) { + if (start == run || line.charAt(start) == '-') { + int end = scanTag(line, start, separator); + if (end != -1) { + return new int[] {start, end}; + } + } + } + return null; + } + + /** + * Scans a tag: one or more characters other than a colon or an equals sign, the separator, + * and one or more characters that are neither an opening parenthesis nor ASCII whitespace. + * + * @param line The line. + * @param from The index where the tag starts. + * @param separator The character between the two parts of the tag. + * @return The index after the tag, or -1 if there is no tag at {@code from}. + */ + private int scanTag(String line, int from, char separator) { + int i = from; + while (i < line.length() && line.charAt(i) != TAG_SEPARATOR + && line.charAt(i) != BIZARRE_TAG_SEPARATOR) { + i++; + } + if (i == from || i == line.length() || line.charAt(i) != separator) { + return -1; + } + int end = i + 1; + while (end < line.length() && line.charAt(end) != '(' + && !StringUtil.isAsciiWhitespace(line.charAt(end))) { + end++; + } + return end == i + 1 ? -1 : end; + } + + /** + * Tests the rest of a node line after the tag: an optional part in parentheses, then the tag + * groups. + * + * @param line The line. + * @param from The index after the tag. + * @return {@code true} if the rest of the line has that form. + */ + private boolean isNodeTail(String line, int from) { + if (from < line.length() && line.charAt(from) == '(') { + int close = line.indexOf(')', from + 1); + if (close > from + 1 && isTagGroupRun(line, close + 1)) { + return true; + } + } + return isTagGroupRun(line, from); + } + + /** + * Tests for tag groups up to the end of the line: after optional ASCII whitespace either + * nothing, or an opening parenthesis and angle bracket, one or more characters other than a + * line terminator, a closing angle bracket and parenthesis, and optional ASCII whitespace. + * + * @param line The line. + * @param from The index where the tag groups start. + * @return {@code true} if the rest of the line has that form. + */ + private boolean isTagGroupRun(String line, int from) { + int start = skipAsciiWhitespace(line, from); + int end = line.length(); + while (end > start && StringUtil.isAsciiWhitespace(line.charAt(end - 1))) { + end--; + } + if (start == end) { + return true; + } + int contentStart = start + TAG_GROUP_OPEN.length(); + int contentEnd = end - TAG_GROUP_CLOSE.length(); + return contentEnd > contentStart && line.startsWith(TAG_GROUP_OPEN, start) + && line.startsWith(TAG_GROUP_CLOSE, contentEnd) + && indexOfLineTerminator(line, contentStart) >= contentEnd; + } + + /** + * Scans the rest of a leaf line after the lemma: optional ASCII whitespace, secondary tags, + * optional ASCII whitespace, an optional morphological tag, the closing parenthesis, ASCII + * whitespace, and the lexeme. + * + * @param line The line. + * @param from The index after the lemma. + * @param noRestAt The indexes after which no rest was found so far, updated here. + * @return The start and end of the secondary tags, the start of the morphological tag, the + * index of the closing parenthesis, and the start of the lexeme, or {@code null} if + * the rest of the line does not have that form. + */ + private int[] scanLeafRest(String line, int from, boolean[] noRestAt) { + int tagsStart = skipAsciiWhitespace(line, from); + int[] rest = scanSecondaryTags(line, tagsStart, noRestAt); + return rest == null ? null : new int[] {tagsStart, rest[0], rest[1], rest[2], rest[3]}; + } + + /** + * Scans secondary tags and the rest of a leaf line after them. Each tag is an opening angle + * bracket, one or more characters other than a line terminator, and a closing angle + * bracket; a longer tag, and then one more tag, is preferred when the rest of the line still + * parses after it. + * + * @param line The line. + * @param from The index where the next secondary tag would start. + * @param noRestAt The indexes after which no rest was found so far, updated here. + * @return The end of the secondary tags, the start of the morphological tag, the index of the + * closing parenthesis, and the start of the lexeme, or {@code null} if the rest of the + * line does not have that form. + */ + private int[] scanSecondaryTags(String line, int from, boolean[] noRestAt) { + if (noRestAt[from]) { + return null; + } + if (from < line.length() && line.charAt(from) == '<') { + for (int close = indexOfLineTerminator(line, from + 1) - 1; close > from + 1; close--) { + if (line.charAt(close) == '>') { + int[] rest = scanSecondaryTags(line, close + 1, noRestAt); + if (rest != null) { + return rest; + } + } + } + } + int[] rest = scanMorphologyAndLexeme(line, from); + if (rest == null) { + noRestAt[from] = true; + return null; + } + return new int[] {from, rest[0], rest[1], rest[2]}; + } + + /** + * Scans the end of a leaf line: optional ASCII whitespace, an optional morphological tag up + * to the first closing parenthesis, that parenthesis, ASCII whitespace, and the lexeme. + * + * @param line The line. + * @param from The index after the secondary tags. + * @return The start of the morphological tag, the index of the closing parenthesis, and the + * start of the lexeme, or {@code null} if the end of the line does not have that form. + */ + private int[] scanMorphologyAndLexeme(String line, int from) { + int morphologyStart = skipAsciiWhitespace(line, from); + int close = line.indexOf(')', morphologyStart); + if (close == -1) { + return null; + } + int lexemeStart = scanLexemeStart(line, close); + return lexemeStart == -1 ? null : new int[] {morphologyStart, close, lexemeStart}; + } + + /** + * Finds the lexeme after the closing parenthesis: ASCII whitespace, then one or more + * characters other than a line terminator up to the end of the line. When only whitespace + * follows the parenthesis, the last character is the lexeme. + * + * @param line The line. + * @param close The index of the closing parenthesis. + * @return The start of the lexeme, or -1 if there is none. + */ + private int scanLexemeStart(String line, int close) { + int lexemeStart = skipAsciiWhitespace(line, close + 1); + if (lexemeStart == close + 1) { + return -1; + } + if (lexemeStart == line.length()) { + lexemeStart--; + return lexemeStart > close + 1 && !isLineTerminator(line.charAt(lexemeStart)) + ? lexemeStart : -1; + } + return indexOfLineTerminator(line, lexemeStart) == line.length() ? lexemeStart : -1; + } + + /** + * Tests whether a lexeme starts with an ASCII letter, digit, or underscore and has a period + * or an angle bracket after it, with no line terminator anywhere. + * + * @param lexeme The lexeme. + * @return {@code true} for such a lexeme. + */ + private boolean isWordWithMarkup(String lexeme) { + if (lexeme.isEmpty() || !isAsciiWord(lexeme.charAt(0)) + || indexOfLineTerminator(lexeme, 0) < lexeme.length()) { + return false; + } + for (int i = 1; i < lexeme.length(); i++) { + char c = lexeme.charAt(i); + if (c == '.' || c == '<' || c == '>') { + return true; + } + } + return false; + } + + /** + * Skips ASCII whitespace. + * + * @param line The line. + * @param from The index to start at. + * @return The index of the first character at or after {@code from} that is not ASCII + * whitespace, or the length of the line. + */ + private int skipAsciiWhitespace(String line, int from) { + int i = from; + while (i < line.length() && StringUtil.isAsciiWhitespace(line.charAt(i))) { + i++; + } + return i; + } + + /** + * Tests for a double or single quote. + * + * @param c The character. + * @return {@code true} for one of the two. + */ + private boolean isQuote(char c) { + return c == '"' || c == '\''; + } + + /** + * Finds the first line terminator at or after an index. + * + * @param text The text. + * @param from The index to start at. + * @return The index of the terminator, or the length of the text if there is none. + */ + static int indexOfLineTerminator(CharSequence text, int from) { + for (int i = from; i < text.length(); i++) { + if (isLineTerminator(text.charAt(i))) { + return i; + } + } + return text.length(); + } + + /** + * Tests for a line terminator: line feed, carriage return, next line, line separator, or + * paragraph separator. + * + * @param c The character. + * @return {@code true} for one of those five characters. + */ + private static boolean isLineTerminator(char c) { + return c == '\n' || c == '\r' || c == '\u0085' || c == '\u2028' || c == '\u2029'; + } + /** Represents a tree element, Node or Leaf */ public abstract static class TreeElement { @@ -455,15 +885,11 @@ public String getLemma() { } - private static final Pattern SENT_START = Pattern.compile("]*>"); - private static final Pattern SENT_END = Pattern.compile(""); - private static final Pattern EXT_END = Pattern.compile(""); - private static final Pattern TITLE_START = Pattern.compile("]*>"); - private static final Pattern TITLE_END = Pattern.compile(""); - private static final Pattern BOX_START = Pattern.compile("]*>"); - private static final Pattern BOX_END = Pattern.compile(""); - private static final Pattern PARA_START = Pattern.compile("]*>"); - private static final Pattern TEXT_START = Pattern.compile("]*>"); + private static final String SENTENCE_TAG = "s"; + private static final String TEXT_TAG = "ext"; + private static final String TITLE_TAG = "t"; + private static final String BOX_TAG = "caixa"; + private static final String PARAGRAPH_TAG = "p"; private final SentenceParser parser; @@ -489,25 +915,25 @@ public Sentence read() throws IOException { if (line != null) { if (sentenceStarted) { - if (SENT_END.matcher(line).matches() || EXT_END.matcher(line).matches()) { + if (isClosingTag(line, SENTENCE_TAG) || isClosingTag(line, TEXT_TAG)) { sentenceStarted = false; } else if (!line.startsWith("A1")) { sentence.append(line).append('\n'); } } else { - if (SENT_START.matcher(line).matches()) { + if (isOpeningTag(line, SENTENCE_TAG)) { sentenceStarted = true; - } else if (PARA_START.matcher(line).matches()) { + } else if (isOpeningTag(line, PARAGRAPH_TAG)) { paraID++; - } else if (TITLE_START.matcher(line).matches()) { + } else if (isOpeningTag(line, TITLE_TAG)) { isTitle = true; - } else if (TITLE_END.matcher(line).matches()) { + } else if (isClosingTag(line, TITLE_TAG)) { isTitle = false; - } else if (TEXT_START.matcher(line).matches()) { + } else if (isOpeningTag(line, TEXT_TAG)) { paraID = 0; - } else if (BOX_START.matcher(line).matches()) { + } else if (isOpeningTag(line, BOX_TAG)) { isBox = true; - } else if (BOX_END.matcher(line).matches()) { + } else if (isClosingTag(line, BOX_TAG)) { isBox = false; } } @@ -529,4 +955,34 @@ public Sentence read() throws IOException { } } } + + /** + * Tests whether a line is an opening markup tag with the given name: the name right after the + * opening angle bracket, then any characters other than a closing angle bracket, then the + * closing angle bracket as the last character. + * + * @param line The line. + * @param name The tag name. + * @return {@code true} if the whole line is such a tag. + */ + static boolean isOpeningTag(String line, String name) { + int last = line.length() - 1; + if (last <= name.length() || line.charAt(0) != '<' || !line.startsWith(name, 1) + || line.charAt(last) != '>') { + return false; + } + return line.indexOf('>', name.length() + 1) == last; + } + + /** + * Tests whether a line is the closing markup tag with the given name and nothing else. + * + * @param line The line. + * @param name The tag name. + * @return {@code true} if the whole line is that closing tag. + */ + static boolean isClosingTag(String line, String name) { + return line.length() == name.length() + 3 && line.startsWith("'; + } } diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java index 17c991e622..1ee1efcb44 100644 --- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java @@ -27,9 +27,6 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; import opennlp.tools.util.InputStreamFactory; @@ -45,8 +42,6 @@ public class ConlluStream implements ObjectStream { private final ObjectStream sentenceStream; - private static final Pattern regex = Pattern.compile("text_([a-z]{2,3})"); - /** * Initializes a {@link ConlluStream}. * @@ -155,7 +150,7 @@ private List postProcessContractions(List lines) index.put(line.getId(), i); if (line.getId().contains("-")) { List expandedContractions = new ArrayList<>(); - String[] ids = line.getId().split("-"); + String[] ids = splitOnHyphen(line.getId()); int start = Integer.parseInt(ids[0]); int end = Integer.parseInt(ids[1]); for (int j = start; j <= end; j++) { @@ -230,15 +225,7 @@ private ConlluWordLine mergeAnnotation(ConlluWordLine contraction, private Map addTextLang(String firstPart, String secondPart, Map textLang) throws InvalidFormatException { - String lang = ""; - try { - Matcher regexMatcher = regex.matcher(firstPart); - if (regexMatcher.find()) { - lang = regexMatcher.group(1); - } - } catch (PatternSyntaxException e) { - throw new InvalidFormatException(e); - } + String lang = extractTextLang(firstPart); if (!lang.isEmpty()) { textLang.put(Locale.of(lang), secondPart); } @@ -248,6 +235,59 @@ private Map addTextLang(String firstPart, String secondPart, return textLang; } + /** + * Splits a token id on hyphens with the result of {@code String.split("-")}: empty elements + * between consecutive hyphens are kept, trailing empty elements are dropped. + * + * @param id The token id. + * @return The elements in order. + */ + private String[] splitOnHyphen(String id) { + if (id.isEmpty()) { + return new String[] {""}; + } + List parts = new ArrayList<>(); + int start = 0; + for (int i = 0; i < id.length(); i++) { + if (id.charAt(i) == '-') { + parts.add(id.substring(start, i)); + start = i + 1; + } + } + if (id.length() > start) { + parts.add(id.substring(start)); + } + while (!parts.isEmpty() && parts.get(parts.size() - 1).isEmpty()) { + parts.remove(parts.size() - 1); + } + return parts.toArray(new String[0]); + } + + /** + * Extracts the language code from a {@code text_xx} or {@code text_xxx} comment key: the two + * or three ASCII lowercase letters, preferring three, after the first {@code text_} that at + * least two follow. + * + * @param firstPart The comment key. + * @return The language code, or an empty string if there is none. + */ + private String extractTextLang(String firstPart) { + int from = 0; + while ((from = firstPart.indexOf("text_", from)) != -1) { + int i = from + "text_".length(); + int len = 0; + while (len < 3 && i + len < firstPart.length() + && firstPart.charAt(i + len) >= 'a' && firstPart.charAt(i + len) <= 'z') { + len++; + } + if (len >= 2) { + return firstPart.substring(i, i + len); + } + from++; + } + return ""; + } + @Override public void close() throws IOException { sentenceStream.close(); diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADMetadataTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADMetadataTest.java new file mode 100644 index 0000000000..c47e108a03 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADMetadataTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.ad; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +public class ADMetadataTest { + + @ParameterizedTest + @CsvSource(delimiter = '|', ignoreLeadingAndTrailingWhitespace = false, value = { + "1001 p=1 ref=\"1001.porto-poesia=removeme=-2\" source=\"SELVA 1001.porto\"|1001|1", + "1001 p=12 title box source=\"x\"|1001|12", + "LIT-12 p=3|12|3", + "LIT12 p=3|12|3", + "-12 p=3|12|3", + "12p=9|12|9", + // leading zeros are digits + "0012 p=007|12|7", + // the first p= that a digit follows counts + "12 p= p=8|12|8", + "12 p=a p=8|12|8", + "12 pp=7|12|7", + "1 ap=2|1|2", + // the text id ends at the first character that is no digit + "12x34 p=5 p=6|12|5" + }) + void testParseTextAndParagraph(String meta, int text, int paragraph) { + Assertions.assertArrayEquals(new int[] {text, paragraph}, + ADMetadata.parseTextAndParagraph(meta)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "x p=1", "12", "12 p=", "12 P=1", "12 p=a", " 12 p=1", "p=1", + "1 p==2", "LIT p=1", "LIT-p=1", + // digits from other scripts are no ASCII digits + "١٢ p=1", "12 p=١", + // metadata is one line + "12 p=1\n", "12\u2028 p=1", "12 p=1\u0085", "\r12 p=1"}) + void testParseTextAndParagraphRejects(String meta) { + Assertions.assertNull(ADMetadata.parseTextAndParagraph(meta)); + Assertions.assertNull(ADMetadata.textId(meta)); + Assertions.assertNull(ADMetadata.textPrefix(meta)); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', value = { + "1001 p=1 source=\"x\"|1001", + "LIT-1001 p=1|1001", + "0012 p=1|0012", + "12x34 p=5|12" + }) + void testTextId(String meta, String textId) { + Assertions.assertEquals(textId, ADMetadata.textId(meta)); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', value = { + "LIT-1001 p=1|LIT-", + "LIT1001 p=1|LIT", + "LITx1 p=1|LITx", + "--1 p=1|--", + "a1p=1|a" + }) + void testTextPrefix(String meta, String prefix) { + Assertions.assertEquals(prefix, ADMetadata.textPrefix(meta)); + } + + @ParameterizedTest + @ValueSource(strings = {"1001 p=1", "1001 p=1 LIT", "LIT1", "LIT1 p=", "LITé1 p=1"}) + void testTextPrefixRejects(String meta) { + Assertions.assertNull(ADMetadata.textPrefix(meta)); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', ignoreLeadingAndTrailingWhitespace = false, value = { + "CIE source=\"abc\" x|abc", + "CIE1 p=1 ref=\"r\" source=\"SELVA 1001.porto\"|SELVA 1001.porto", + "CIE source=\"\"|''", + "source=\"a\"|a", + " source=\"a\"|a", + // the first source attribute counts, up to the next double quote + "source=\"a\"source=\"b\"|a", + "source=\"source=\"x\"|source=", + "source=\" a \" |' a '" + }) + void testSource(String meta, String source) { + Assertions.assertEquals(source, ADMetadata.source(meta)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "CIE x", "CIE source=\"a", "CIE source=a\"", "CIE Source=\"a\"", + "source='a'", + // metadata is one line + "source=\"a\nb\"", "source=\"a\"\n", "source=\"a\"\u2028", "\u0085source=\"a\""}) + void testSourceRejects(String meta) { + Assertions.assertNull(ADMetadata.source(meta)); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java index 101b465259..b46f93347b 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java @@ -19,12 +19,21 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.namefind.NameSample; +import opennlp.tools.util.ObjectStream; import opennlp.tools.util.PlainTextByLineStream; import opennlp.tools.util.Span; @@ -117,4 +126,123 @@ void testMissingRightContraction() { Assertions.assertEquals(new Span(5, 6, "person"), samples.get(7).getNames()[2]); } + private static Stream underscoreLexemes() { + return Stream.of( + Arguments.of("Rio_de_Janeiro", new String[] {"Rio", "de", "Janeiro"}), + Arguments.of("a__b", new String[] {"a", "b"}), + Arguments.of("_a", new String[] {"", "a"}), + Arguments.of("a_", new String[] {"a"}), + Arguments.of("__", new String[0]), + Arguments.of("_", new String[0]), + Arguments.of("\uD801\uDC12_\uD83D\uDE00", new String[] {"\uD801\uDC12", "\uD83D\uDE00"}), + Arguments.of("", new String[] {""}), + Arguments.of("casa", new String[] {"casa"})); + } + + @ParameterizedTest + @MethodSource("underscoreLexemes") + void testSplitOnUnderscores(String lexeme, String[] expected) { + Assertions.assertArrayEquals(expected, ADNameSampleStream.splitOnUnderscores(lexeme)); + Assertions.assertArrayEquals(lexeme.split("[_]+"), ADNameSampleStream.splitOnUnderscores(lexeme)); + } + + @ParameterizedTest + @ValueSource(strings = {"casa", "São", "1990", "R2D2", "\uD801\uDC12\u0661"}) + void testIsAlphaNumericAccepts(String token) { + Assertions.assertTrue(ADNameSampleStream.isAlphaNumeric(token)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "guarda-chuva", "R$", "a b", "\u00BD", "\uD83D\uDE00"}) + void testIsAlphaNumericRejects(String token) { + Assertions.assertFalse(ADNameSampleStream.isAlphaNumeric(token)); + } + + private static Stream hyphenatedTokens() { + return Stream.of( + Arguments.of("guarda-", new String[] {"guarda", null, null}), + Arguments.of("a-", new String[] {"a", null, null}), + Arguments.of("-chuva", new String[] {null, "chuva", ""}), + Arguments.of("-chuva2!", new String[] {null, "chuva", "2!"}), + Arguments.of("guarda-chuva", new String[] {"guarda", "chuva", ""}), + Arguments.of("guarda-chuva-sol", new String[] {"guarda", "chuva", "-sol"}), + Arguments.of("São-Paulo", new String[] {"São", "Paulo", ""}), + // supplementary-plane letters are letters, a combining mark ends the letter run + Arguments.of("\uD801\uDC12-\uD801\uDC3A", new String[] {"\uD801\uDC12", "\uD801\uDC3A", ""}), + Arguments.of("e\u0301-a", new String[] {null, null, null})); + } + + @ParameterizedTest + @MethodSource("hyphenatedTokens") + void testMatchHyphenatedToken(String token, String[] expected) { + String[] actual = ADNameSampleStream.matchHyphenatedToken(token); + if (expected[0] == null && expected[1] == null && expected[2] == null) { + Assertions.assertNull(actual); + } else { + Assertions.assertArrayEquals(expected, actual); + } + } + + @ParameterizedTest + @ValueSource(strings = {"-", "--", "-1", "1-", "a1-b", "a-1", "a--b", "ab", "a -"}) + void testMatchHyphenatedTokenRejects(String token) { + Assertions.assertNull(ADNameSampleStream.matchHyphenatedToken(token)); + } + + @ParameterizedTest + @CsvSource({", PROP", ", PROP", "<>, ''", ", ''", ", a, NER:X", ", ner:PROP", "<\uD83D\uDE00>, \uD83D\uDE00"}) + void testTagContent(String tag, String expected) { + Assertions.assertEquals(expected, ADNameSampleStream.tagContent(tag)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "<", ">", "PROP", ""}) + void testTagContentRejects(String tag) { + Assertions.assertNull(ADNameSampleStream.tagContent(tag)); + } + + private static ObjectStream lineStream(List lines) { + Iterator iterator = lines.iterator(); + return new ObjectStream<>() { + @Override + public String read() { + return iterator.hasNext() ? iterator.next() : null; + } + }; + } + + @ParameterizedTest + @CsvSource(delimiter = '|', value = { + "1001|SOURCE: ref=\"x\"", + "LIT-1|SOURCE: ref=\"x\"", + "CIE1|SOURCE: source=\"text\"" + }) + void testTextIdFromCorpusMetadata(String sentenceId, String source) throws IOException { + List lines = List.of("", source, sentenceId + " Olá .", "STA:fcl", + "=H:intj(\"olá\" )\tOlá", ".", ""); + try (ADNameSampleStream stream = new ADNameSampleStream(lineStream(lines), false)) { + NameSample sample = stream.read(); + Assertions.assertNotNull(sample); + Assertions.assertArrayEquals(new String[] {"Olá", "."}, sample.getSentence()); + Assertions.assertNull(stream.read()); + } + } + + @ParameterizedTest + @CsvSource(delimiter = '|', value = { + // no digits after the prefix + "LIT|SOURCE: ref=\"x\"", + // no source attribute + "CIE1|SOURCE: ref=\"x\"", + // no digits + "AX|SOURCE: ref=\"x\"" + }) + void testInvalidMetadataIsRejected(String sentenceId, String source) throws IOException { + List lines = List.of("", source, sentenceId + " Olá .", ""); + try (ADNameSampleStream stream = new ADNameSampleStream(lineStream(lines), false)) { + RuntimeException e = Assertions.assertThrows(RuntimeException.class, stream::read); + Assertions.assertTrue(e.getMessage().startsWith("Invalid metadata: " + sentenceId + " p=")); + } + } } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java index cbbae22012..5fc873a88b 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java @@ -19,10 +19,14 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import opennlp.tools.postag.POSSample; import opennlp.tools.util.PlainTextByLineStream; @@ -108,4 +112,23 @@ void testIncludeFeats() throws IOException { } } + private static Stream tags() { + return Stream.of( + Arguments.of("v-fin", "v-fin"), + Arguments.of("PR 3S IND", "PR=3S=IND"), + Arguments.of("PR \t3S", "PR=3S"), + Arguments.of(" PR 3S ", "=PR=3S="), + Arguments.of(" ", "="), + Arguments.of("", ""), + Arguments.of("PR\u00A03S", "PR\u00A03S"), + Arguments.of("\r\n\u000B\f", "="), + Arguments.of("\uD83D\uDE00 x", "\uD83D\uDE00=x")); + } + + @ParameterizedTest + @MethodSource("tags") + void testReplaceWhitespaceWithEquals(String tag, String expected) { + Assertions.assertEquals(expected, ADPOSSampleStream.replaceWhitespaceWithEquals(tag)); + } + } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java index 58f45635af..2a32e7bb98 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java @@ -19,12 +19,15 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Iterator; +import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import opennlp.tools.sentdetect.SentenceSample; +import opennlp.tools.util.ObjectStream; import opennlp.tools.util.PlainTextByLineStream; import opennlp.tools.util.Span; @@ -59,4 +62,28 @@ void testSentences() { Assertions.assertEquals(new Span(120, 180), samples.get(0).getSentences()[1]); } + @Test + void testInvalidMetadataIsRejected() throws IOException { + // the second sentence id "AX" has no digits, so its metadata cannot be parsed + List lines = List.of( + "", + "SOURCE: src", + "1001 Hello world .", + "", + "", + "SOURCE: src", + "AX Hi there .", + ""); + Iterator iterator = lines.iterator(); + ObjectStream lineStream = new ObjectStream<>() { + @Override + public String read() { + return iterator.hasNext() ? iterator.next() : null; + } + }; + try (ADSentenceSampleStream stream = new ADSentenceSampleStream(lineStream, true)) { + Assertions.assertThrows(RuntimeException.class, stream::read); + } + } + } diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java new file mode 100644 index 0000000000..bcee49bf46 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.ad; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import opennlp.tools.formats.ad.ADSentenceStream.Sentence; +import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser; +import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.Leaf; +import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.TreeElement; + +public class ADSentenceStreamTest { + + private static Stream nodeLines() { + return Stream.of( + Arguments.of("STA:fcl", 1, "STA:fcl"), + Arguments.of("=PIV:pp", 2, "PIV:pp"), + Arguments.of("==P<:np", 3, "P<:np"), + Arguments.of("===>N:adjp", 4, ">N:adjp"), + // an optional part in parentheses and tag groups may follow the tag + Arguments.of("=X:y(z)", 2, "X:y"), + Arguments.of("=X:y ()", 2, "X:y"), + Arguments.of("=X:y(z) ()() ", 2, "X:y"), + Arguments.of("==P<:np()", 3, "P<:np"), + // a leaf line followed by a tag group, or without a lexeme, is a node line + Arguments.of("=H:n(\"casa\" M S) ()", 2, "H:n"), + Arguments.of("=H:n(\"casa\" M S)", 2, "H:n"), + Arguments.of("=H:n(\"casa\" M S) ", 2, "H:n"), + // hyphens count as level, but the last one joins the tag when a colon follows it + Arguments.of("=-X:y", 3, "X:y"), + Arguments.of("--:y", 2, "-:y"), + Arguments.of("-:y", 1, "-:y")); + } + + @ParameterizedTest + @MethodSource("nodeLines") + void testNodeLines(String line, int level, String syntacticTag) { + TreeElement element = new SentenceParser().getElement(line); + Assertions.assertNotNull(element); + Assertions.assertFalse(element.isLeaf()); + Assertions.assertEquals(level, element.getLevel()); + Assertions.assertEquals(syntacticTag, element.getSyntacticTag()); + } + + private static Stream leafLines() { + return Stream.of( + Arguments.of("=P:v-fin(\"iniciar\" PR 3S IND VFIN)\tInicia", + 2, "P", "v-fin", "iniciar", " ", "PR 3S IND VFIN", "Inicia"), + Arguments.of("===H:n(\"av.\" M S)\tAv.", + 4, "H", "n", "av.", " ", "M S", "Av."), + Arguments.of("===N<:num(\"6\" M P)\t6", + 4, "N<", "num", "6", " ", "M P", "6"), + // no morphological tag + Arguments.of("==H:prp(\"em\" )\tem", + 3, "H", "prp", "em", " ", null, "em"), + Arguments.of("SUB:conj-s(\"que\" )\tque", + 1, "SUB", "conj-s", "que", "", null, "que"), + // no secondary tags + Arguments.of("=P:v-fin(\"iniciar\" PR 3S IND VFIN)\tInicia", + 2, "P", "v-fin", "iniciar", "", "PR 3S IND VFIN", "Inicia"), + Arguments.of("=H:n('casa') casa", 2, "H", "n", "casa", "", null, "casa"), + // quotes inside lemma and lexeme + Arguments.of("=H:n(\"d'água\" M S)\td'água", + 2, "H", "n", "d'água", "", "M S", "d'água"), + // the lemma extends to the last quote after which the rest of the line still parses + Arguments.of("=H:n(\"a\" \"c\")\tw", 2, "H", "n", "a\" \"c", "", null, "w"), + Arguments.of("=H:n(\"x\" M S)\ta') b", 2, "H", "n", "x\" M S)\ta", "", null, "b"), + // the secondary tags extend to the last closing angle bracket + Arguments.of("=H:n(\"casa\" b M S)\tcasa", + 2, "H", "n", "casa", "b", "M S", "casa"), + Arguments.of("=H:n(\"casa\" M S)\tcasa", + 2, "H", "n", "casa", "", "M S", "casa"), + // the last of trailing whitespace characters is the lexeme + Arguments.of("=H:n(\"a)\" M S) ", 2, "H", "n", "a)", "", "M S", " ")); + } + + @ParameterizedTest + @MethodSource("leafLines") + void testLeafLines(String line, int level, String syntacticTag, String functionalTag, + String lemma, String secondaryTag, String morphologicalTag, String lexeme) { + TreeElement element = new SentenceParser().getElement(line); + Assertions.assertNotNull(element); + Assertions.assertTrue(element.isLeaf()); + Leaf leaf = (Leaf) element; + Assertions.assertEquals(level, leaf.getLevel()); + Assertions.assertEquals(syntacticTag, leaf.getSyntacticTag()); + Assertions.assertEquals(functionalTag, leaf.getFunctionalTag()); + Assertions.assertEquals(lemma, leaf.getLemma()); + Assertions.assertEquals(secondaryTag, leaf.getSecondaryTag()); + Assertions.assertEquals(morphologicalTag, leaf.getMorphologicalTag()); + Assertions.assertEquals(lexeme, leaf.getLexeme()); + } + + private static Stream bizarreLeafLines() { + return Stream.of( + Arguments.of("=x=y(\"q\" a) b", 2, "x=y", "q", "a", "b"), + Arguments.of("=x=y('q')\tb", 2, "x=y", "q", null, "b"), + Arguments.of("=x=y(a b) c", 2, "x=y", null, "a b", "c"), + Arguments.of("=x=y() b", 2, "x=y", null, null, "b"), + // a quoted part without a closing quote is the morphological tag + Arguments.of("=x=y(\"q) b", 2, "x=y", null, "\"q", "b"), + // the level prefix gives up hyphens so that the tag can start + Arguments.of("==-=x(a) b", 3, "-=x", null, "a", "b"), + Arguments.of("=-=x=y(a) b", 4, "x=y", null, "a", "b")); + } + + @ParameterizedTest + @MethodSource("bizarreLeafLines") + void testBizarreLeafLines(String line, int level, String syntacticTag, String lemma, + String morphologicalTag, String lexeme) { + TreeElement element = new SentenceParser().getElement(line); + Assertions.assertNotNull(element); + Assertions.assertTrue(element.isLeaf()); + Leaf leaf = (Leaf) element; + Assertions.assertEquals(level, leaf.getLevel()); + Assertions.assertEquals(syntacticTag, leaf.getSyntacticTag()); + Assertions.assertNull(leaf.getFunctionalTag()); + Assertions.assertEquals(lemma, leaf.getLemma()); + Assertions.assertNull(leaf.getSecondaryTag()); + Assertions.assertEquals(morphologicalTag, leaf.getMorphologicalTag()); + Assertions.assertEquals(lexeme, leaf.getLexeme()); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', ignoreLeadingAndTrailingWhitespace = false, value = { + // no whitespace between the closing parenthesis and the lexeme: not a leaf line + "=H:n(\"casa\" M S)casa|2|:n(\"casa\" M S)casa", + // an empty lemma + "=H:n(\"\" M S) casa|2|:n(\"\" M S) casa", + "=ab|2|b", + "=a.b|2|.b", + "===x|4|''" + }) + void testFallbackLeafLines(String line, int level, String lexeme) { + TreeElement element = new SentenceParser().getElement(line); + Assertions.assertNotNull(element); + Assertions.assertTrue(element.isLeaf()); + Leaf leaf = (Leaf) element; + Assertions.assertEquals(level, leaf.getLevel()); + Assertions.assertEquals("", leaf.getSyntacticTag()); + Assertions.assertEquals("", leaf.getFunctionalTag()); + Assertions.assertEquals("", leaf.getMorphologicalTag()); + Assertions.assertNull(leaf.getLemma()); + Assertions.assertEquals(lexeme, leaf.getLexeme()); + } + + @ParameterizedTest + @ValueSource(strings = {"_", "", "pause", "=ab.", "=xay", "=a_b.c"}) + void testIgnoredLines(String line) { + Assertions.assertNull(new SentenceParser().getElement(line)); + } + + @Test + void testUnparsableLineIsLexeme() { + TreeElement element = new SentenceParser().getElement("random text"); + Assertions.assertTrue(element.isLeaf()); + Leaf leaf = (Leaf) element; + Assertions.assertEquals(1, leaf.getLevel()); + Assertions.assertEquals("", leaf.getSyntacticTag()); + Assertions.assertEquals("random text", leaf.getLexeme()); + } + + @Test + void testPunctuationLeaf() { + SentenceParser parser = new SentenceParser(); + + Leaf leaf = (Leaf) parser.getElement("==,"); + Assertions.assertEquals(3, leaf.getLevel()); + Assertions.assertEquals(",", leaf.getLexeme()); + + leaf = (Leaf) parser.getElement("."); + Assertions.assertEquals(1, leaf.getLevel()); + Assertions.assertEquals(".", leaf.getLexeme()); + + // a line of only equals signs matches, with the last one as lexeme + leaf = (Leaf) parser.getElement("==="); + Assertions.assertEquals(3, leaf.getLevel()); + Assertions.assertEquals("=", leaf.getLexeme()); + + // non-word characters other than equals make up the lexeme + leaf = (Leaf) parser.getElement("=!?"); + Assertions.assertEquals(2, leaf.getLevel()); + Assertions.assertEquals("!?", leaf.getLexeme()); + + // a word character excludes the punctuation parse, the line is treated + // as a bizarre leaf instead + TreeElement element = parser.getElement("=ab"); + Assertions.assertTrue(element.isLeaf()); + leaf = (Leaf) element; + Assertions.assertEquals(2, leaf.getLevel()); + Assertions.assertEquals("b", leaf.getLexeme()); + } + + @Test + void testFixPunctuation() { + SentenceParser parser = new SentenceParser(); + + Sentence sentence = parser.parse( + "\nSOURCE: src\n1001 Olá mundo » .\n\n", 1, false, false); + Assertions.assertEquals("Olá mundo ».", sentence.text()); + + sentence = parser.parse( + "\nSOURCE: src\n1001 Olá » , tudo bem » .\n\n", 1, false, false); + Assertions.assertEquals("Olá », tudo bem ».", sentence.text()); + + // without whitespace between » and the punctuation nothing is replaced + sentence = parser.parse( + "\nSOURCE: src\n1001 Olá mundo ».\n\n", 1, false, false); + Assertions.assertEquals("Olá mundo ».", sentence.text()); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', ignoreLeadingAndTrailingWhitespace = false, value = { + "|s|true", + "|s|true", + "|s|true", + "|ext|true", + "|caixa|true", + "

|p|true", + "|t|true", + ">|s|false", + " |s|false", + " |s|false", + "x|s|false", + "|s|false", + "

|s|false", + "|s|false", + "<>|s|false", + "''|s|false", + "|t|false", + "|ext|false", + "< s>|s|false" + }) + void testIsOpeningTag(String line, String name, boolean expected) { + Assertions.assertEquals(expected, ADSentenceStream.isOpeningTag(line, name)); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', ignoreLeadingAndTrailingWhitespace = false, value = { + "|s|true", + "|ext|true", + "|t|true", + "|caixa|true", + " |s|false", + " |s|false", + "|s|false", + "|s|false", + "|s|false", + ">|s|false", + "|t|false", + "''|s|false", + "|s|false" + }) + void testIsClosingTag(String line, String name, boolean expected) { + Assertions.assertEquals(expected, ADSentenceStream.isClosingTag(line, name)); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluStreamTest.java index 5716a84ab6..afc11909f7 100644 --- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluStreamTest.java +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluStreamTest.java @@ -17,7 +17,9 @@ package opennlp.tools.formats.conllu; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.Locale; @@ -28,6 +30,8 @@ import org.junit.jupiter.api.Test; import opennlp.tools.sentdetect.SentenceSample; +import opennlp.tools.util.InputStreamFactory; +import opennlp.tools.util.InvalidFormatException; import opennlp.tools.util.ObjectStream; public class ConlluStreamTest extends AbstractConlluSampleStreamTest { @@ -106,4 +110,35 @@ void testOptionalComments() throws IOException { Assertions.assertNull(stream.read(), "Stream must be exhausted"); } } + + @Test + void testContractionIdsAreMerged() throws IOException { + try (ObjectStream stream = getStream("es-ud-sample.conllu")) { + ConlluSentence sent1 = stream.read(); + + Assertions.assertEquals(55, sent1.getWordLines().size()); + Assertions.assertEquals("1-3", sent1.getWordLines().get(0).getId()); + Assertions.assertEquals("Digámoslo", sent1.getWordLines().get(0).getForm()); + Assertions.assertEquals("15-16", sent1.getWordLines().get(12).getId()); + for (ConlluWordLine wordLine : sent1.getWordLines()) { + Assertions.assertFalse(wordLine.getId().equals("1") + || wordLine.getId().equals("2") || wordLine.getId().equals("3") + || wordLine.getId().equals("15") || wordLine.getId().equals("16"), + "Expanded contraction parts must be removed"); + } + } + } + + @Test + void testInvalidTextLangCodeIsRejected() throws IOException { + // "text_e" has a single lowercase letter, so no language code can be extracted + InputStreamFactory in = () -> new ByteArrayInputStream( + ("# text_e = Bonjour\n" + + "1\tBonjour\tbonjour\tINTJ\t_\t_\t0\troot\t_\t_\n") + .getBytes(StandardCharsets.UTF_8)); + + try (ObjectStream stream = new ConlluStream(in)) { + Assertions.assertThrows(InvalidFormatException.class, stream::read); + } + } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java index 04007f9109..e8639af23e 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java @@ -19,7 +19,8 @@ import java.util.ArrayList; import java.util.List; -import java.util.regex.Pattern; + +import opennlp.tools.util.StringUtil; /** * Simple feature generator for learning statistical lemmatizers. @@ -34,9 +35,6 @@ public class DefaultLemmatizerContextGenerator implements LemmatizerContextGener private static final int PREFIX_LENGTH = 5; private static final int SUFFIX_LENGTH = 7; - private static final Pattern PATTERN_HAS_CAP = Pattern.compile("[A-Z]"); - private static final Pattern PATTERN_HAS_NUM = Pattern.compile("[0-9]"); - public DefaultLemmatizerContextGenerator() { } @@ -105,11 +103,11 @@ public String[] getContext(int index, String[] toks, String[] tags, String[] pre features.add("h"); } - if (PATTERN_HAS_CAP.matcher(lex).find()) { + if (StringUtil.containsAsciiUpperCase(lex)) { features.add("c"); } - if (PATTERN_HAS_NUM.matcher(lex).find()) { + if (StringUtil.containsAsciiDigit(lex)) { features.add("d"); } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/BioCodec.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/BioCodec.java index f7d6c5e852..563c9bd55f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/BioCodec.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/BioCodec.java @@ -20,8 +20,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import opennlp.tools.util.SequenceCodec; import opennlp.tools.util.Span; @@ -47,17 +45,37 @@ public class BioCodec implements SequenceCodec { public static final String CONTINUE = "cont"; public static final String OTHER = "other"; - private static final Pattern TYPED_OUTCOME_PATTERN = Pattern.compile("(.+)-\\w+"); - static String extractNameType(String outcome) { - Matcher matcher = TYPED_OUTCOME_PATTERN.matcher(outcome); - if (matcher.matches()) { - return matcher.group(1); + int separator = outcome.lastIndexOf('-'); + if (separator > 0 && isWordChars(outcome, separator + 1)) { + return outcome.substring(0, separator); } return null; } + /** + * Tests whether the rest of an outcome is a non-empty run of ASCII letters, digits, or + * underscores. + * + * @param outcome The outcome label. + * @param from The offset the run starts at. + * @return {@code true} if at least one character follows and all are word characters. + */ + private static boolean isWordChars(String outcome, int from) { + if (from >= outcome.length()) { + return false; + } + for (int i = from; i < outcome.length(); i++) { + char c = outcome.charAt(i); + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') || c == '_')) { + return false; + } + } + return true; + } + @Override public Span[] decode(List c) { int start = -1; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderME.java index f1dec349c1..b8fa9ef937 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderME.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderME.java @@ -24,8 +24,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import opennlp.tools.commons.ThreadSafe; import opennlp.tools.ml.BeamSearch; @@ -66,7 +64,6 @@ public class NameFinderME implements TokenNameFinder, Probabilistic { private static final String[][] EMPTY = new String[0][0]; public static final int DEFAULT_BEAM_SIZE = 3; - private static final Pattern typedOutcomePattern = Pattern.compile("(.+)-\\w+"); public static final String START = "start"; public static final String CONTINUE = "cont"; @@ -327,12 +324,7 @@ else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) { * @return The name type, or {@code null} if not set. */ static String extractNameType(String outcome) { - Matcher matcher = typedOutcomePattern.matcher(outcome); - if (matcher.matches()) { - return matcher.group(1); - } - - return null; + return BioCodec.extractNameType(outcome); } /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java index 93c256c60f..1060b990ad 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java @@ -20,10 +20,10 @@ import java.util.ArrayList; import java.util.List; -import java.util.regex.Pattern; import opennlp.tools.dictionary.Dictionary; import opennlp.tools.util.StringList; +import opennlp.tools.util.StringUtil; /** * A default {@link POSContextGenerator context generator} for a {@link POSTagger}. @@ -38,9 +38,6 @@ public class DefaultPOSContextGenerator implements POSContextGenerator { private static final int PREFIX_LENGTH = 4; private static final int SUFFIX_LENGTH = 4; - private static final Pattern hasCap = Pattern.compile("[A-Z]"); - private static final Pattern hasNum = Pattern.compile("[0-9]"); - private final Dictionary dict; /** @@ -165,11 +162,11 @@ public String[] getContext(int index, Object[] tokens, String[] tags) { e.add("h"); } - if (hasCap.matcher(lex).find()) { + if (StringUtil.containsAsciiUpperCase(lex)) { e.add("c"); } - if (hasNum.matcher(lex).find()) { + if (StringUtil.containsAsciiDigit(lex)) { e.add("d"); } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java index 951fca5b5d..ab0d7add09 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java @@ -24,13 +24,13 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import opennlp.tools.tokenize.TokenSample; import opennlp.tools.util.Span; +import opennlp.tools.util.StringUtil; /** * Class which produces an Iterator<TokenSample> from a file of space delimited token. @@ -42,7 +42,6 @@ public class TokenSampleStream implements Iterator { private static final Logger logger = LoggerFactory.getLogger(TokenSampleStream.class); private final BufferedReader in; private String line; - private final Pattern alphaNumeric = Pattern.compile("[A-Za-z0-9]"); private boolean evenq = true; public TokenSampleStream(InputStream is) throws IOException { @@ -55,7 +54,7 @@ public boolean hasNext() { } public TokenSample next() { - String[] tokens = line.split("\\s+"); + String[] tokens = StringUtil.splitOnAsciiWhitespace(line); if (tokens.length == 0) { evenq = true; } @@ -73,7 +72,7 @@ public TokenSample next() { default -> token; }; if (sb.length() != 0) { - if (!alphaNumeric.matcher(token).find() || token.startsWith("'") || token.equalsIgnoreCase("n't")) { + if (!containsAsciiAlphaNum(token) || token.startsWith("'") || token.equalsIgnoreCase("n't")) { if ((token.equals("``") || token.equals("--") || token.equals("$") || token.equals("(") || token.equals("&") || token.equals("#") || (token.equals("\"") && (evenq && ti != tokens.length - 1))) @@ -113,6 +112,23 @@ public void remove() { throw new UnsupportedOperationException(); } + + /** + * Tests whether a token contains an ASCII letter or digit. + * + * @param token The token. + * @return {@code true} if one is present. + */ + private boolean containsAsciiAlphaNum(String token) { + for (int i = 0; i < token.length(); i++) { + char c = token.charAt(i); + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { + return true; + } + } + return false; + } + private static void usage() { logger.info("TokenSampleStream [-spans] < in"); logger.info("Where in is a space delimited list of tokens."); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java index 0f6f6800d0..195e932447 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java @@ -40,8 +40,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -256,13 +254,23 @@ private static String downloadChecksumFile(String sha512, Path model) throws IOE /** * Extracts the hash from the content of a checksum file, which holds the hash followed by the * name of the file it applies to. + * + * @param checksumFileContent The file content. + * @return The hash, or {@code null} if the content is {@code null} or blank. */ - private static String parseChecksum(String checksumFileContent) { + static String parseChecksum(String checksumFileContent) { if (checksumFileContent == null) { return null; } final String trimmed = checksumFileContent.trim(); - return trimmed.isEmpty() ? null : trimmed.split("\\s")[0]; + if (trimmed.isEmpty()) { + return null; + } + int end = 0; + while (end < trimmed.length() && !StringUtil.isAsciiWhitespace(trimmed.charAt(end))) { + end++; + } + return trimmed.substring(0, end); } private static void verifyChecksum(Path model, String expectedChecksum) throws IOException { @@ -324,7 +332,6 @@ private static Path getDownloadHome() { @Internal static class DownloadParser { - private static final Pattern LINK_PATTERN = Pattern.compile("(.*?)", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); private final URL indexUrl; DownloadParser(URL indexUrl) { @@ -334,14 +341,60 @@ static class DownloadParser { Map> getAvailableModels() throws MalformedURLException, URISyntaxException { - final Matcher matcher = LINK_PATTERN.matcher(fetchPageIndex()); + return toMap(extractLinks(fetchPageIndex())); + } + /** + * Collects the href values of the anchor elements in an index page. The tag name and + * attribute are matched ignoring ASCII case, a value ends at the first {@code ">}, a link + * ends at the first {@code }, and both may span lines. An anchor without a closing tag + * is skipped. + * + * @param page The page content. + * @return The href values in order. + */ + static List extractLinks(String page) { final List links = new ArrayList<>(); - while (matcher.find()) { - links.add(matcher.group(1)); + int from = 0; + while ((from = indexOfIgnoreCase(page, "", valueStart); + if (valueEnd != -1) { + final int close = indexOfIgnoreCase(page, "", valueEnd + 2); + if (close != -1) { + links.add(page.substring(valueStart, valueEnd)); + from = close + "".length(); + continue; + } + } + from++; } + return links; + } - return toMap(links); + /** + * Finds a lowercase ASCII literal, ignoring the case of ASCII letters in the text. + * + * @param text The text. + * @param literal The lowercase literal. + * @param from The start offset. + * @return The first match offset, or {@code -1}. + */ + private static int indexOfIgnoreCase(String text, String literal, int from) { + outer: + for (int i = from; i + literal.length() <= text.length(); i++) { + for (int j = 0; j < literal.length(); j++) { + char c = text.charAt(i + j); + if (c >= 'A' && c <= 'Z') { + c += 'a' - 'A'; + } + if (c != literal.charAt(j)) { + continue outer; + } + } + return i; + } + return -1; } private Map> toMap(List links) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/BrownCluster.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/BrownCluster.java index 221816b49a..d40044ab99 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/BrownCluster.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/BrownCluster.java @@ -26,9 +26,10 @@ import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.regex.Pattern; import opennlp.tools.util.model.ArtifactSerializer; import opennlp.tools.util.model.SerializableArtifact; @@ -47,8 +48,6 @@ */ public class BrownCluster implements SerializableArtifact { - private static final Pattern tabPattern = Pattern.compile("\t"); - public static class BrownClusterSerializer implements ArtifactSerializer { @Override @@ -82,7 +81,7 @@ public BrownCluster(InputStream in) throws IOException { String line; while ((line = breader.readLine()) != null) { - String[] lineArray = tabPattern.split(line); + String[] lineArray = splitTabs(line); if (lineArray.length == 3) { int freq = Integer.parseInt(lineArray[2]); if (freq > 5 ) { @@ -96,6 +95,31 @@ else if (lineArray.length == 2) { } } + /** + * Splits on tabs with the result of {@code String.split("\\t")}: trailing empty fields are + * dropped and an empty line yields a single empty field. + * + * @param line The line. + * @return The fields in order. + */ + private String[] splitTabs(String line) { + if (line.isEmpty()) { + return new String[] {""}; + } + List fields = new ArrayList<>(); + int start = 0; + int separator; + while ((separator = line.indexOf('\t', start)) != -1) { + fields.add(line.substring(start, separator)); + start = separator + 1; + } + fields.add(line.substring(start)); + while (!fields.isEmpty() && fields.get(fields.size() - 1).isEmpty()) { + fields.remove(fields.size() - 1); + } + return fields.toArray(new String[0]); + } + /** * Check if a token is in the Brown:paths, token map. * diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java index 2237302191..b585f207d6 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java @@ -18,15 +18,11 @@ package opennlp.tools.util.featuregen; -import java.util.regex.Pattern; - /** * This class provide common utilities for feature generation. */ public class FeatureGeneratorUtil { - private static final Pattern capPeriod = Pattern.compile("^[A-ZÄÖÜ]\\.$"); - /** * Generates a class name for the specified token. * The classes are as follows where the first matching class is used: @@ -98,7 +94,7 @@ else if (pattern.isAllCapitalLetter()) { feat = "ac"; } } - else if (capPeriod.matcher(token).find()) { + else if (isCapPeriod(token)) { feat = "cp"; } else if (pattern.isInitialCapitalLetter()) { @@ -110,4 +106,24 @@ else if (pattern.isInitialCapitalLetter()) { return (feat); } + + /** + * Tests for a single capital followed by a period. + * + * @param token The token. + * @return {@code true} for exactly that shape. + */ + private static boolean isCapPeriod(String token) { + return token.length() == 2 && isCapPeriodStart(token.charAt(0)) && token.charAt(1) == '.'; + } + + /** + * Tests for an ASCII capital or a German umlaut capital. + * + * @param c The character. + * @return {@code true} for one of those capitals. + */ + private static boolean isCapPeriodStart(char c) { + return (c >= 'A' && c <= 'Z') || c == 'Ä' || c == 'Ö' || c == 'Ü'; + } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java index 8596244de7..6f71abaa5a 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java @@ -19,7 +19,6 @@ package opennlp.tools.util.featuregen; import java.util.List; -import java.util.regex.Pattern; import opennlp.tools.tokenize.SimpleTokenizer; import opennlp.tools.tokenize.Tokenizer; @@ -38,7 +37,6 @@ public class TokenPatternFeatureGenerator implements AdaptiveFeatureGenerator { private static final String SUB_TOKEN_PART2_PREFIX = "pt2=" ; private static final String SUB_TOKEN_PART3_PREFIX = "pt3=" ; - private final Pattern noLetters = Pattern.compile("[^a-zA-Z]"); private final Tokenizer tokenizer; /** @@ -87,11 +85,27 @@ public void createFeatures(List feats, String[] toks, int index, String[ pattern.append(FeatureGeneratorUtil.tokenFeature(tokenized[i])); - if (!noLetters.matcher(tokenized[i]).find()) { + if (!containsNonLetter(tokenized[i])) { feats.add(SUB_TOKEN_PREFIX + StringUtil.toLowerCase(tokenized[i])); } } feats.add("pta=" + pattern); } + + /** + * Tests whether a token contains a character outside the ASCII letters. + * + * @param token The token. + * @return {@code true} if one is present. + */ + private boolean containsNonLetter(String token) { + for (int i = 0; i < token.length(); i++) { + char c = token.charAt(i); + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'))) { + return true; + } + } + return false; + } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java index d2b75335ae..95dc047d85 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java @@ -16,8 +16,6 @@ */ package opennlp.tools.util.normalizer; -import java.util.regex.Pattern; - /** * A {@link EmojiCharSequenceNormalizer} implementation that normalizes text * in terms of emojis. Every encounter will be replaced by a whitespace. @@ -36,15 +34,44 @@ public static EmojiCharSequenceNormalizer getInstance() { return INSTANCE; } - private static final Pattern EMOJI_REGEX = - Pattern.compile("[\\uD83C-\\uDBFF\\uDC00-\\uDFFF]+"); + /** + * The lowest code point that is replaced: the first high surrogate of the emoji planes. + * Lone surrogates and every BMP character from here up count as well. + */ + private static final int LOWER_CODE_POINT = 0xD83C; + + /** The highest code point that is replaced. */ + private static final int UPPER_CODE_POINT = 0x10FC00; - /** {@inheritDoc} */ + /** + * {@inheritDoc} + * Every maximal run of code points in {@code [U+D83C, U+10FC00]} becomes one space. + */ @Override public CharSequence normalize (CharSequence text) { if (text == null) { throw new IllegalArgumentException("The text must not be null."); } - return EMOJI_REGEX.matcher(text).replaceAll(" "); + StringBuilder normalized = new StringBuilder(text.length()); + int i = 0; + while (i < text.length()) { + int cp = Character.codePointAt(text, i); + if (cp >= LOWER_CODE_POINT && cp <= UPPER_CODE_POINT) { + i += Character.charCount(cp); + while (i < text.length()) { + int next = Character.codePointAt(text, i); + if (next < LOWER_CODE_POINT || next > UPPER_CODE_POINT) { + break; + } + i += Character.charCount(next); + } + normalized.append(' '); + } + else { + normalized.appendCodePoint(cp); + i += Character.charCount(cp); + } + } + return normalized.toString(); } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGeneratorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGeneratorTest.java new file mode 100644 index 0000000000..ea0743a91d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGeneratorTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.lemmatizer; + +import java.util.Arrays; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +public class DefaultLemmatizerContextGeneratorTest { + + private static final DefaultLemmatizerContextGenerator GENERATOR = + new DefaultLemmatizerContextGenerator(); + + @ParameterizedTest + @CsvSource({ + "Token9, true, true", + "token, false, false", + "TOKEN, true, false", + "1990, false, true", + // accented capitals and non-ASCII digits are not matched + "Étudiant, false, false", + "x٥, false, false", + "well-known, false, false"}) + void testCapitalAndDigitFeatures(String token, boolean capital, boolean digit) { + List features = Arrays.asList(GENERATOR.getContext(0, + new String[] {token}, new String[] {"tag"}, null)); + Assertions.assertEquals(capital, features.contains("c"), "capital feature of " + token); + Assertions.assertEquals(digit, features.contains("d"), "digit feature of " + token); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/BioCodecTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/BioCodecTest.java index 55334fd190..641e785881 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/BioCodecTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/BioCodecTest.java @@ -23,6 +23,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.util.Span; @@ -262,4 +265,17 @@ void testCompatibilityRepeated() { new String[] {A_START, A_START, A_CONTINUE, A_CONTINUE, B_START, B_START, OTHER, OTHER})); } + @ParameterizedTest + @CsvSource({"atype-start, atype", "a-b-start, a-b", "type_1-cont, type_1", "Type9-X_1, Type9"}) + void testExtractNameType(String outcome, String type) { + Assertions.assertEquals(type, BioCodec.extractNameType(outcome)); + } + + @ParameterizedTest + @ValueSource(strings = {"start", "other", "-start", "atype-", "atype-st.art", "atype-st art", + "atype-stärt"}) + void testExtractNameTypeWithoutType(String outcome) { + Assertions.assertNull(BioCodec.extractNameType(outcome)); + } + } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderMETest.java index 3c8c8e6bc1..7266f3008f 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderMETest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderMETest.java @@ -22,6 +22,9 @@ import java.util.Collections; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.util.MockInputStreamFactory; import opennlp.tools.util.ObjectStream; @@ -32,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -325,4 +329,16 @@ void testNameFinderWithMultipleTypes() throws Exception { assertEquals("organization", names2[1].getType()); } + @ParameterizedTest + @CsvSource({"atype-start, atype", "a-b-start, a-b", "type_1-cont, type_1"}) + void testExtractNameType(String outcome, String type) { + assertEquals(type, NameFinderME.extractNameType(outcome)); + } + + @ParameterizedTest + @ValueSource(strings = {"start", "other", "-start", "atype-", "atype-st.art"}) + void testExtractNameTypeWithoutType(String outcome) { + assertNull(NameFinderME.extractNameType(outcome)); + } + } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/DefaultPOSContextGeneratorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/DefaultPOSContextGeneratorTest.java index 9e84538b3e..3f728182b2 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/DefaultPOSContextGeneratorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/DefaultPOSContextGeneratorTest.java @@ -102,6 +102,23 @@ void noDictionaryMatch() { + Arrays.toString(actual)); } + @Test + void capitalAndDigitFeatures() { + DefaultPOSContextGenerator generator = new DefaultPOSContextGenerator(null); + + // accept sides: ASCII capital letter and ASCII digit + final String[] withCapAndNum = generator.getContext(0, + new Object[] {"Token9"}, new String[] {"tag"}); + Assertions.assertTrue(Arrays.asList(withCapAndNum).contains("c")); + Assertions.assertTrue(Arrays.asList(withCapAndNum).contains("d")); + + // reject sides: accented capitals and non-ASCII digits are not matched + final String[] accented = generator.getContext(0, + new Object[] {"Étudiant٥"}, new String[] {"tag"}); + Assertions.assertFalse(Arrays.asList(accented).contains("c")); + Assertions.assertFalse(Arrays.asList(accented).contains("d")); + } + @Test void dictionaryMatch() { int indexWithDictionaryMatch = 2; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lang/en/TokenSampleStreamTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lang/en/TokenSampleStreamTest.java new file mode 100644 index 0000000000..f63d1d1d3d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lang/en/TokenSampleStreamTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lang.en; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.tokenize.TokenSample; +import opennlp.tools.util.Span; + +public class TokenSampleStreamTest { + + private static TokenSampleStream stream(String text) throws IOException { + return new TokenSampleStream(new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8))); + } + + @Test + void testReadsTokensAndSpans() throws IOException { + TokenSampleStream stream = stream("The dog 's -LRB- big -RRB- .\n"); + Assertions.assertTrue(stream.hasNext()); + TokenSample sample = stream.next(); + Assertions.assertEquals("The dog's ( big).", sample.getText()); + Assertions.assertArrayEquals(new Span[] {new Span(0, 3), new Span(4, 7), new Span(7, 9), + new Span(10, 11), new Span(12, 15), new Span(15, 16), new Span(16, 17)}, + sample.getTokenSpans()); + Assertions.assertFalse(stream.hasNext()); + } + + @Test + void testCollapsesWhitespaceRuns() throws IOException { + TokenSample sample = stream("a \t b\n").next(); + Assertions.assertEquals("a b", sample.getText()); + Assertions.assertEquals(2, sample.getTokenSpans().length); + } + + @Test + void testLeadingWhitespaceYieldsEmptyFirstToken() throws IOException { + // a leading run gives one empty token, as String.split("\\s+") does + TokenSample sample = stream(" a\n").next(); + Assertions.assertEquals("a", sample.getText()); + Assertions.assertArrayEquals(new Span[] {new Span(0, 0), new Span(0, 1)}, + sample.getTokenSpans()); + } + + @Test + void testWhitespaceOnlyLineHasNoTokens() throws IOException { + TokenSample sample = stream(" \n").next(); + Assertions.assertEquals("", sample.getText()); + Assertions.assertEquals(0, sample.getTokenSpans().length); + } + + @Test + void testNonAsciiWhitespaceIsNotASeparator() throws IOException { + // no-break space is not in the ASCII whitespace set + TokenSample sample = stream("a b c\n").next(); + Assertions.assertEquals(2, sample.getTokenSpans().length); + Assertions.assertEquals("a b", sample.getText().substring(0, 3)); + } + + @Test + void testQuoteAndPunctuationAttachment() throws IOException { + // a token without an ASCII letter or digit attaches to the previous token + TokenSample sample = stream("Hello , world !\n").next(); + Assertions.assertEquals("Hello, world!", sample.getText()); + Assertions.assertArrayEquals(new Span[] {new Span(0, 5), new Span(5, 6), new Span(7, 12), + new Span(12, 13)}, sample.getTokenSpans()); + + // a token holding a digit is a word and gets a space in front + sample = stream("Room 101 .\n").next(); + Assertions.assertEquals("Room 101.", sample.getText()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/BrownClusterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/BrownClusterTest.java new file mode 100644 index 0000000000..4fdb46295f --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/BrownClusterTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util.featuregen; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class BrownClusterTest { + + private static BrownCluster cluster(String lexicon) throws IOException { + return new BrownCluster(new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8))); + } + + @Test + void testThreeColumnLinesNeedFrequencyAboveFive() throws IOException { + BrownCluster cluster = cluster("0101\tthe\t10\n0110\trare\t5\n"); + Assertions.assertEquals("0101", cluster.lookupToken("the")); + Assertions.assertNull(cluster.lookupToken("rare")); + } + + @Test + void testTwoColumnLinesMapFirstFieldToSecond() throws IOException { + BrownCluster cluster = cluster("dog\t0111\n"); + Assertions.assertEquals("0111", cluster.lookupToken("dog")); + } + + @Test + void testTrailingTabsAreDropped() throws IOException { + // trailing empty fields do not count, as with String.split("\t") + BrownCluster cluster = cluster("cat\t0100\t\t\n0011\tbird\t7\t\n"); + Assertions.assertEquals("0100", cluster.lookupToken("cat")); + Assertions.assertEquals("0011", cluster.lookupToken("bird")); + } + + @Test + void testLinesWithOtherFieldCountsAreIgnored() throws IOException { + BrownCluster cluster = cluster("\n\t\nsingle\na\tb\tc\td\n0101\tfish\t10\n"); + Assertions.assertNull(cluster.lookupToken("single")); + Assertions.assertNull(cluster.lookupToken("a")); + Assertions.assertNull(cluster.lookupToken("")); + Assertions.assertEquals("0101", cluster.lookupToken("fish")); + } + + @Test + void testSpacesAreNotSeparators() throws IOException { + BrownCluster cluster = cluster("0101 the 10\n"); + Assertions.assertNull(cluster.lookupToken("the")); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java index cd35f092ad..6aa668c807 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java @@ -19,6 +19,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; public class FeatureGeneratorUtilTest { @@ -70,6 +72,14 @@ void testGerman() { Assertions.assertEquals("sc", FeatureGeneratorUtil.tokenFeature("Ü")); } + @ParameterizedTest + @CsvSource({"A., cp", "Z., cp", "Ä., cp", "Ö., cp", "Ü., cp", + // lower case initial, other capital, other second character, longer tokens + "a., other", "É., ic", "'A,', ic", "Ab., ic", "AB., ic"}) + void testCapPeriod(String token, String feature) { + Assertions.assertEquals(feature, FeatureGeneratorUtil.tokenFeature(token)); + } + @Test void testJapanese() { // Hiragana diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/TokenPatternFeatureGeneratorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/TokenPatternFeatureGeneratorTest.java index 28e8e8c9d1..47c96696a2 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/TokenPatternFeatureGeneratorTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/TokenPatternFeatureGeneratorTest.java @@ -71,4 +71,19 @@ void testSentence() { Assertions.assertEquals("st=sentence", features.get(12)); Assertions.assertEquals("pta=iclclclclc", features.get(13)); } + + @Test + void testSkipsNonLetterSubTokens() { + + String[] testSentence = new String[] {"well-known"}; + final int testTokenIndex = 0; + + AdaptiveFeatureGenerator generator = new TokenPatternFeatureGenerator(); + + generator.createFeatures(features, testSentence, testTokenIndex, null); + // the hyphen sub-token must not produce an "st=" feature + Assertions.assertFalse(features.contains("st=-")); + Assertions.assertTrue(features.contains("st=well")); + Assertions.assertTrue(features.contains("st=known")); + } } diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java index ac04d7d4ef..0365d00303 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java @@ -21,6 +21,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -73,6 +74,37 @@ void testNullUrl() { ); } + private static Stream indexPages() { + return Stream.of( + Arguments.of("xy", List.of("a.bin", "b.bin")), + // tag and attribute names ignore case + Arguments.of("z", List.of("c.bin")), + Arguments.of("xy", List.of("a.bin", "b.bin")), + // an anchor without closing tag is skipped, an earlier one is still found + Arguments.of("x", List.of()), + Arguments.of("xy", List.of("a.bin")), + // the first "" closes the link, so nested anchor markup is swallowed + Arguments.of("x y", List.of("d.bin")), + // the href value ends at the first "\">", so it may hold other markup + Arguments.of("g.bin\">f", List.of("fg.bin")), + // values and link text may span lines + Arguments.of("x y\nz", List.of("a.bin", "h\ni.bin")), + Arguments.of("no links here", List.of()), + Arguments.of("", List.of()), + // an empty href value is kept, a single-quoted one is not an anchor + Arguments.of("", List.of("")), + Arguments.of("x", List.of()), + // the closing tag is matched case-insensitively and only as "" + Arguments.of("xyz", List.of("a.bin")), + Arguments.of("x", List.of("\uD83D\uDE00.bin"))); + } + + @ParameterizedTest + @MethodSource("indexPages") + void testExtractLinks(String page, List expected) { + assertEquals(expected, DownloadUtil.DownloadParser.extractLinks(page)); + } + @Test void testInvalidUrl() { try { diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java index f92ada9c99..299487596c 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java @@ -36,6 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -65,6 +66,31 @@ public void testDownloadModelByURL(String language, URL url) throws IOException assertTrue(model.isLoadedFromSerialized()); } + @ParameterizedTest + @MethodSource("checksumFiles") + void testParseChecksum(String content, String expected) { + assertEquals(expected, DownloadUtil.parseChecksum(content)); + } + + private static Stream checksumFiles() { + return Stream.of( + Arguments.of("abc123 model.bin", "abc123"), + Arguments.of("abc123\tmodel.bin\n", "abc123"), + Arguments.of(" abc123 model.bin", "abc123"), + Arguments.of("abc123", "abc123"), + Arguments.of("abc123 *model.bin\r\n", "abc123"), + Arguments.of("abc123 model.bin\ndef456 other.bin\n", "abc123"), + Arguments.of("abc123\u000Bmodel.bin", "abc123"), + Arguments.of("abc123\u00A0model.bin", "abc123\u00A0model.bin")); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource(strings = {" ", "\t\n"}) + void testParseChecksumOfBlankContent(String content) { + assertNull(DownloadUtil.parseChecksum(content)); + } + @Test @EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org") public void testExistsModel() throws IOException { diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java index 1951c114a0..353ecfbd8c 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; @@ -185,6 +186,95 @@ void testIsUnicodeWhitespaceDelegates() { Assertions.assertFalse(StringUtil.isUnicodeWhitespace(0x200B)); } + // ------------------------------------------------------------------------- + // isAsciiWhitespace + // ------------------------------------------------------------------------- + + @ParameterizedTest + @ValueSource(chars = {' ', '\t', '\n', '\u000B', '\f', '\r'}) + void testIsAsciiWhitespaceAccepts(char c) { + Assertions.assertTrue(StringUtil.isAsciiWhitespace(c)); + } + + @ParameterizedTest + @ValueSource(chars = {'a', '0', '_', '\u0000', '\u001C', '\u0085', '\u00A0', '\u2003', '\u3000'}) + void testIsAsciiWhitespaceRejects(char c) { + Assertions.assertFalse(StringUtil.isAsciiWhitespace(c)); + } + + // ------------------------------------------------------------------------- + // splitOnAsciiWhitespace + // ------------------------------------------------------------------------- + + @Test + void testSplitOnAsciiWhitespaceNullThrows() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> StringUtil.splitOnAsciiWhitespace(null)); + } + + private static Stream asciiSplits() { + return Stream.of( + Arguments.of("", new String[] {""}), + Arguments.of(" ", new String[0]), + Arguments.of("\t\n\r", new String[0]), + Arguments.of("a", new String[] {"a"}), + Arguments.of("a b", new String[] {"a", "b"}), + Arguments.of("hello world", new String[] {"hello", "world"}), + Arguments.of("a\t\u000B\fb", new String[] {"a", "b"}), + Arguments.of(" a", new String[] {"", "a"}), + Arguments.of("a ", new String[] {"a"}), + Arguments.of(" a\tb ", new String[] {"", "a", "b"}), + Arguments.of("a\u00A0b", new String[] {"a\u00A0b"}), + Arguments.of("a\u2003b c", new String[] {"a\u2003b", "c"}), + Arguments.of("\uD801\uDC12 \uD83D\uDE00", new String[] {"\uD801\uDC12", "\uD83D\uDE00"}), + Arguments.of(" \r\n ", new String[0])); + } + + @ParameterizedTest + @MethodSource("asciiSplits") + void testSplitOnAsciiWhitespaceMatchesStringSplit(String input, String[] expected) { + Assertions.assertArrayEquals(expected, StringUtil.splitOnAsciiWhitespace(input)); + Assertions.assertArrayEquals(input.split("\\s+"), StringUtil.splitOnAsciiWhitespace(input)); + } + + // ------------------------------------------------------------------------- + // containsAsciiUpperCase, containsAsciiDigit + // ------------------------------------------------------------------------- + + @Test + void testContainsAsciiNullThrows() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> StringUtil.containsAsciiUpperCase(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> StringUtil.containsAsciiDigit(null)); + } + + @ParameterizedTest + @ValueSource(strings = {"A", "Z", "aBc", "token-X", "1A", "Q\uD801\uDC12"}) + void testContainsAsciiUpperCaseAccepts(String input) { + Assertions.assertTrue(StringUtil.containsAsciiUpperCase(input)); + Assertions.assertTrue(StringUtil.containsAsciiUpperCase(new StringBuilder(input))); + } + + @ParameterizedTest + @ValueSource(strings = {"", "abc", "123", "Étudiant", "ÄÖÜ", "\uFF21", "\uD801\uDC12", "_-."}) + void testContainsAsciiUpperCaseRejects(String input) { + Assertions.assertFalse(StringUtil.containsAsciiUpperCase(input)); + } + + @ParameterizedTest + @ValueSource(strings = {"0", "9", "a1b", "x-2", "9A", "\uD801\uDC120"}) + void testContainsAsciiDigitAccepts(String input) { + Assertions.assertTrue(StringUtil.containsAsciiDigit(input)); + Assertions.assertTrue(StringUtil.containsAsciiDigit(new StringBuilder(input))); + } + + @ParameterizedTest + @ValueSource(strings = {"", "abc", "ABC", "\u0661", "\uFF11", "\u00BD", "\uD835\uDFCE", "_-."}) + void testContainsAsciiDigitRejects(String input) { + Assertions.assertFalse(StringUtil.containsAsciiDigit(input)); + } + // ------------------------------------------------------------------------- // splitOnUnicodeWhitespace // ------------------------------------------------------------------------- diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java index f63d2d1f9e..b965cf1536 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java @@ -40,4 +40,29 @@ void normalizeEmoji() { "Any funny text goes here ", normalizer.normalize(s)); } + @Test + void normalizeUnpairedSurrogates() { + // a lone high surrogate and a lone low surrogate are matched individually + String s = "a" + '\uD83C' + "b" + '\uDC00' + "c"; + Assertions.assertEquals("a b c", normalizer.normalize(s)); + + // adjacent surrogates, paired or not, collapse into a single space + StringBuilder sb = new StringBuilder(); + sb.append("x").append('\uD83C').append('\uDC00').append("y"); + Assertions.assertEquals("x y", normalizer.normalize(sb)); + } + + @Test + void normalizeMatchesCodePointsNotOnlyEmoji() { + // the matched code point range is [U+D83C, U+10FC00], so BMP characters + // from U+D83C up are replaced as well + Assertions.assertEquals("a b", normalizer.normalize("a" + '\uE000' + "b")); + + // supplementary code points beyond U+10FC00 are kept verbatim + StringBuilder sb = new StringBuilder(); + sb.append("a").appendCodePoint(0x10FFFF).append("b"); + Assertions.assertEquals("a" + new String(Character.toChars(0x10FFFF)) + "b", + normalizer.normalize(sb)); + } + }