OPENNLP-1930: Parse Arvores Deitadas markup with cursor scans

opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java

@@ -116,6 +116,23 @@ public static boolean isUnicodeWhitespace(int charCode) {
     return UnicodeWhitespace.isWhitespace(charCode);
   }
 
+  /**
+   * Determines if the specified {@link Character} is one of the six ASCII whitespace
+   * characters, the set the regular expression class {@code \s} matches: space, tab,
+   * line feed, vertical tab, form feed, and carriage return. Unlike
+   * {@link #isWhitespace(char)}, this ignores the {@link WhitespaceMode} and rejects
+   * every non-ASCII space.
+   *
+   * @param charCode The character to check.
+   *
+   * @return {@code true} if {@code charCode} is one of those six characters,
+   *     {@code false} otherwise.
+   */
+  public static boolean isAsciiWhitespace(char charCode) {
+    return charCode == ' ' || charCode == '\t' || charCode == '\n' || charCode == '\u000B'
+        || charCode == '\f' || charCode == '\r';
+  }
+
   /**
    * Splits {@code input} on runs of Unicode {@code White_Space}. Leading and trailing
    * runs are ignored, so whitespace-only input yields an empty array. This is a
@@ -151,6 +168,91 @@ public static String[] splitOnUnicodeWhitespace(CharSequence input) {
     return terms.toArray(new String[0]);
   }
 
+  /**
+   * Splits {@code input} on runs of ASCII whitespace with the result of
+   * {@code String.split("\\s+")}: a leading run yields one empty first element, trailing
+   * empty elements are dropped, whitespace-only input yields an empty array, and empty
+   * input yields a single empty element. This is a character scan, not a regular
+   * expression.
+   *
+   * @param input The text to split. Must not be {@code null}.
+   * @return The elements in order.
+   * @throws IllegalArgumentException If {@code input} is {@code null}.
+   */
+  public static String[] splitOnAsciiWhitespace(String input) {
+    if (input == null) {
+      throw new IllegalArgumentException("input must not be null");
+    }
+    if (input.isEmpty()) {
+      return new String[] {""};
+    }
+    final List<String> elements = new ArrayList<>();
+    if (isAsciiWhitespace(input.charAt(0))) {
+      elements.add("");
+    }
+    int start = 0;
+    for (int i = 0; i < input.length(); i++) {
+      if (isAsciiWhitespace(input.charAt(i))) {
+        if (i > start) {
+          elements.add(input.substring(start, i));
+        }
+        while (i + 1 < input.length() && isAsciiWhitespace(input.charAt(i + 1))) {
+          i++;
+        }
+        start = i + 1;
+      }
+    }
+    if (input.length() > start) {
+      elements.add(input.substring(start));
+    }
+    while (!elements.isEmpty() && elements.get(elements.size() - 1).isEmpty()) {
+      elements.remove(elements.size() - 1);
+    }
+    return elements.toArray(new String[0]);
+  }
+
+  /**
+   * Tests whether {@code input} contains an ASCII capital letter, {@code A} to {@code Z}.
+   * Capitals outside ASCII do not count. This is a character scan, not a regular expression.
+   *
+   * @param input The text to check. Must not be {@code null}.
+   * @return {@code true} if at least one character is an ASCII capital letter.
+   * @throws IllegalArgumentException If {@code input} is {@code null}.
+   */
+  public static boolean containsAsciiUpperCase(CharSequence input) {
+    if (input == null) {
+      throw new IllegalArgumentException("input must not be null");
+    }
+    for (int i = 0; i < input.length(); i++) {
+      final char c = input.charAt(i);
+      if (c >= 'A' && c <= 'Z') {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Tests whether {@code input} contains an ASCII digit, {@code 0} to {@code 9}. Digits outside
+   * ASCII do not count. This is a character scan, not a regular expression.
+   *
+   * @param input The text to check. Must not be {@code null}.
+   * @return {@code true} if at least one character is an ASCII digit.
+   * @throws IllegalArgumentException If {@code input} is {@code null}.
+   */
+  public static boolean containsAsciiDigit(CharSequence input) {
+    if (input == null) {
+      throw new IllegalArgumentException("input must not be null");
+    }
+    for (int i = 0; i < input.length(); i++) {
+      final char c = input.charAt(i);
+      if (c >= '0' && c <= '9') {
+        return true;
+      }
+    }
+    return false;
+  }
+
   /**
    * Trims leading and trailing runs of Unicode {@code White_Space}, the same set
    * {@link #splitOnUnicodeWhitespace(CharSequence)} breaks terms on.

opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java

@@ -19,7 +19,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.util.regex.Pattern;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -61,23 +60,71 @@ public String getHelp() {
             + "Defaults to a WhitespaceTokenizer.";
   }
 
-  private static final Pattern UNTOKENIZED_PAREN_PATTERN_1 = Pattern.compile("([^ ])([({)}])");
-  private static final Pattern UNTOKENIZED_PAREN_PATTERN_2 = Pattern.compile("([({)}])([^ ])");
-
   public static Parse[] parseLine(String line, Parser parser, int numParses) {
     return parseLine( line, parser, WhitespaceTokenizer.INSTANCE, numParses );
   }
 
   public static Parse[] parseLine(String line, Parser parser, Tokenizer tokenizer, int numParses) {
     // fix some parens patterns
-    line = UNTOKENIZED_PAREN_PATTERN_1.matcher(line).replaceAll("$1 $2");
-    line = UNTOKENIZED_PAREN_PATTERN_2.matcher(line).replaceAll("$1 $2");
+    line = spaceUntokenizedParens(line);
 
     // tokenize
     String[] tokens = tokenizer.tokenize(line);
     return parseLine(tokens, parser, numParses);
   }
 
+  /**
+   * Separates round and curly brackets from adjacent text in two left-to-right passes: the
+   * first puts a space between a non-space character and a following bracket, the second
+   * between a bracket and a following non-space character. Each pass resumes after the pair
+   * it just spaced, so a pair overlapping that match is only seen by the second pass.
+   *
+   * @param line The untokenized line.
+   * @return The spaced line.
+   */
+  static String spaceUntokenizedParens(String line) {
+    return insertParenSpaces(insertParenSpaces(line, false), true);
+  }
+
+  /**
+   * Inserts a space between a bracket and an adjacent non-space character, left to right.
+   *
+   * @param line The untokenized line.
+   * @param parenFirst {@code true} to space a bracket before a character, {@code false} after one.
+   * @return The spaced line.
+   */
+  private static String insertParenSpaces(String line, boolean parenFirst) {
+    StringBuilder spaced = new StringBuilder(line.length() + 8);
+    int i = 0;
+    while (i < line.length()) {
+      char c = line.charAt(i);
+      if (i + 1 < line.length()) {
+        char next = line.charAt(i + 1);
+        boolean match = parenFirst
+            ? isParen(c) && next != ' '
+            : c != ' ' && isParen(next);
+        if (match) {
+          spaced.append(c).append(' ').append(next);
+          i += 2;
+          continue;
+        }
+      }
+      spaced.append(c);
+      i++;
+    }
+    return spaced.toString();
+  }
+
+  /**
+   * Tests for a round or curly bracket.
+   *
+   * @param c The character.
+   * @return {@code true} for {@code (}, {@code )}, <code>{</code>, or <code>}</code>.
+   */
+  private static boolean isParen(char c) {
+    return c == '(' || c == ')' || c == '{' || c == '}';
+  }
+
   /**
    * Parses the specified pre-tokenized sentence and returns the requested number of parses
    * or fewer.

opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/parser/ParserToolTest.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.parser;
+
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import opennlp.tools.parser.Parse;
+import opennlp.tools.parser.Parser;
+import opennlp.tools.tokenize.WhitespaceTokenizer;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class ParserToolTest {
+
+  /*
+   * The first pass puts a space before a bracket that follows a non-space character, the
+   * second after a bracket that precedes one. Each pass resumes after the pair it spaced, so
+   * "((" gets its space from the second pass only, and only a space, not a tab, separates.
+   */
+  private static Stream<Arguments> parenLines() {
+    return Stream.of(
+        Arguments.of("a(b)c", "a ( b ) c"),
+        Arguments.of("a (b) c", "a ( b ) c"),
+        Arguments.of("foo(bar){baz}", "foo ( bar ) {baz }"),
+        Arguments.of("((a)(b))", "( ( a ) (b ) )"),
+        Arguments.of("a(b(c)d)e", "a ( b ( c ) d ) e"),
+        Arguments.of("x((", "x ( ("),
+        Arguments.of("((x", "( ( x"),
+        Arguments.of("()", "( )"),
+        Arguments.of("(", "("),
+        Arguments.of("", ""),
+        Arguments.of("no parens here", "no parens here"),
+        Arguments.of("«quoted»", "«quoted»"),
+        Arguments.of("tab\there(", "tab\there ("),
+        Arguments.of("a  (b", "a  ( b"));
+  }
+
+  @ParameterizedTest
+  @MethodSource("parenLines")
+  void testSpaceUntokenizedParens(String line, String expected) {
+    assertEquals(expected, ParserTool.spaceUntokenizedParens(line));
+  }
+
+  @Test
+  void testParseLineSeparatesParensBeforeTokenizing() {
+    Parser echo = new Parser() {
+      @Override
+      public Parse[] parse(Parse tokens, int numParses) {
+        return new Parse[] {tokens};
+      }
+
+      @Override
+      public Parse parse(Parse tokens) {
+        return tokens;
+      }
+    };
+    Parse[] parses = ParserTool.parseLine("f(x)+g({y})", echo, WhitespaceTokenizer.INSTANCE, 1);
+    assertEquals(1, parses.length);
+    String[] tokens = Stream.of(parses[0].getChildren()).map(Parse::getCoveredText)
+        .toArray(String[]::new);
+    // "{y" stays joined: the second pass consumed "{" while spacing "( {"
+    assertArrayEquals(new String[] {"f", "(", "x", ")", "+g", "(", "{y", "}", ")"}, tokens);
+  }
+}