diff --git a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
index 44f683d588..c49adfe948 100644
--- a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
+++ b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java
@@ -116,6 +116,23 @@ public static boolean isUnicodeWhitespace(int charCode) {
     return UnicodeWhitespace.isWhitespace(charCode);
   }
 
+  /**
+   * Determines if the specified {@link Character} is one of the six ASCII whitespace
+   * characters, the set the regular expression class {@code \s} matches: space, tab,
+   * line feed, vertical tab, form feed, and carriage return. Unlike
+   * {@link #isWhitespace(char)}, this ignores the {@link WhitespaceMode} and rejects
+   * every non-ASCII space.
+   *
+   * @param charCode The character to check.
+   *
+   * @return {@code true} if {@code charCode} is one of those six characters,
+   *     {@code false} otherwise.
+   */
+  public static boolean isAsciiWhitespace(char charCode) {
+    return charCode == ' ' || charCode == '\t' || charCode == '\n' || charCode == '\u000B'
+        || charCode == '\f' || charCode == '\r';
+  }
+
   /**
    * Splits {@code input} on runs of Unicode {@code White_Space}. Leading and trailing
    * runs are ignored, so whitespace-only input yields an empty array. This is a
@@ -151,6 +168,91 @@ public static String[] splitOnUnicodeWhitespace(CharSequence input) {
     return terms.toArray(new String[0]);
   }
 
+  /**
+   * Splits {@code input} on runs of ASCII whitespace with the result of
+   * {@code String.split("\\s+")}: a leading run yields one empty first element, trailing
+   * empty elements are dropped, whitespace-only input yields an empty array, and empty
+   * input yields a single empty element. This is a character scan, not a regular
+   * expression.
+   *
+   * @param input The text to split. Must not be {@code null}.
+   * @return The elements in order.
+   * @throws IllegalArgumentException If {@code input} is {@code null}.
+   */
+  public static String[] splitOnAsciiWhitespace(String input) {
+    if (input == null) {
+      throw new IllegalArgumentException("input must not be null");
+    }
+    if (input.isEmpty()) {
+      return new String[] {""};
+    }
+    final List<String> elements = new ArrayList<>();
+    if (isAsciiWhitespace(input.charAt(0))) {
+      elements.add("");
+    }
+    int start = 0;
+    for (int i = 0; i < input.length(); i++) {
+      if (isAsciiWhitespace(input.charAt(i))) {
+        if (i > start) {
+          elements.add(input.substring(start, i));
+        }
+        while (i + 1 < input.length() && isAsciiWhitespace(input.charAt(i + 1))) {
+          i++;
+        }
+        start = i + 1;
+      }
+    }
+    if (input.length() > start) {
+      elements.add(input.substring(start));
+    }
+    while (!elements.isEmpty() && elements.get(elements.size() - 1).isEmpty()) {
+      elements.remove(elements.size() - 1);
+    }
+    return elements.toArray(new String[0]);
+  }
+
+  /**
+   * Tests whether {@code input} contains an ASCII capital letter, {@code A} to {@code Z}.
+   * Capitals outside ASCII do not count. This is a character scan, not a regular expression.
+   *
+   * @param input The text to check. Must not be {@code null}.
+   * @return {@code true} if at least one character is an ASCII capital letter.
+   * @throws IllegalArgumentException If {@code input} is {@code null}.
+   */
+  public static boolean containsAsciiUpperCase(CharSequence input) {
+    if (input == null) {
+      throw new IllegalArgumentException("input must not be null");
+    }
+    for (int i = 0; i < input.length(); i++) {
+      final char c = input.charAt(i);
+      if (c >= 'A' && c <= 'Z') {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Tests whether {@code input} contains an ASCII digit, {@code 0} to {@code 9}. Digits outside
+   * ASCII do not count. This is a character scan, not a regular expression.
+   *
+   * @param input The text to check. Must not be {@code null}.
+   * @return {@code true} if at least one character is an ASCII digit.
+   * @throws IllegalArgumentException If {@code input} is {@code null}.
+   */
+  public static boolean containsAsciiDigit(CharSequence input) {
+    if (input == null) {
+      throw new IllegalArgumentException("input must not be null");
+    }
+    for (int i = 0; i < input.length(); i++) {
+      final char c = input.charAt(i);
+      if (c >= '0' && c <= '9') {
+        return true;
+      }
+    }
+    return false;
+  }
+
   /**
    * Trims leading and trailing runs of Unicode {@code White_Space}, the same set
    * {@link #splitOnUnicodeWhitespace(CharSequence)} breaks terms on.
diff --git a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java
index be1f02e7b8..6ee98995bb 100644
--- a/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java
+++ b/opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/parser/ParserTool.java
@@ -19,7 +19,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.util.regex.Pattern;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -61,23 +60,71 @@ public String getHelp() {
             + "Defaults to a WhitespaceTokenizer.";
   }
 
-  private static final Pattern UNTOKENIZED_PAREN_PATTERN_1 = Pattern.compile("([^ ])([({)}])");
-  private static final Pattern UNTOKENIZED_PAREN_PATTERN_2 = Pattern.compile("([({)}])([^ ])");
-
   public static Parse[] parseLine(String line, Parser parser, int numParses) {
     return parseLine( line, parser, WhitespaceTokenizer.INSTANCE, numParses );
   }
 
   public static Parse[] parseLine(String line, Parser parser, Tokenizer tokenizer, int numParses) {
     // fix some parens patterns
-    line = UNTOKENIZED_PAREN_PATTERN_1.matcher(line).replaceAll("$1 $2");
-    line = UNTOKENIZED_PAREN_PATTERN_2.matcher(line).replaceAll("$1 $2");
+    line = spaceUntokenizedParens(line);
 
     // tokenize
     String[] tokens = tokenizer.tokenize(line);
     return parseLine(tokens, parser, numParses);
   }
 
+  /**
+   * Separates round and curly brackets from adjacent text in two left-to-right passes: the
+   * first puts a space between a non-space character and a following bracket, the second
+   * between a bracket and a following non-space character. Each pass resumes after the pair
+   * it just spaced, so a pair overlapping that match is only seen by the second pass.
+   *
+   * @param line The untokenized line.
+   * @return The spaced line.
+   */
+  static String spaceUntokenizedParens(String line) {
+    return insertParenSpaces(insertParenSpaces(line, false), true);
+  }
+
+  /**
+   * Inserts a space between a bracket and an adjacent non-space character, left to right.
+   *
+   * @param line The untokenized line.
+   * @param parenFirst {@code true} to space a bracket before a character, {@code false} after one.
+   * @return The spaced line.
+   */
+  private static String insertParenSpaces(String line, boolean parenFirst) {
+    StringBuilder spaced = new StringBuilder(line.length() + 8);
+    int i = 0;
+    while (i < line.length()) {
+      char c = line.charAt(i);
+      if (i + 1 < line.length()) {
+        char next = line.charAt(i + 1);
+        boolean match = parenFirst
+            ? isParen(c) && next != ' '
+            : c != ' ' && isParen(next);
+        if (match) {
+          spaced.append(c).append(' ').append(next);
+          i += 2;
+          continue;
+        }
+      }
+      spaced.append(c);
+      i++;
+    }
+    return spaced.toString();
+  }
+
+  /**
+   * Tests for a round or curly bracket.
+   *
+   * @param c The character.
+   * @return {@code true} for {@code (}, {@code )}, <code>{</code>, or <code>}</code>.
+   */
+  private static boolean isParen(char c) {
+    return c == '(' || c == ')' || c == '{' || c == '}';
+  }
+
   /**
    * Parses the specified pre-tokenized sentence and returns the requested number of parses
    * or fewer.
diff --git a/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/parser/ParserToolTest.java b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/parser/ParserToolTest.java
new file mode 100644
index 0000000000..8b03710a93
--- /dev/null
+++ b/opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/parser/ParserToolTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.parser;
+
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import opennlp.tools.parser.Parse;
+import opennlp.tools.parser.Parser;
+import opennlp.tools.tokenize.WhitespaceTokenizer;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class ParserToolTest {
+
+  /*
+   * The first pass puts a space before a bracket that follows a non-space character, the
+   * second after a bracket that precedes one. Each pass resumes after the pair it spaced, so
+   * "((" gets its space from the second pass only, and only a space, not a tab, separates.
+   */
+  private static Stream<Arguments> parenLines() {
+    return Stream.of(
+        Arguments.of("a(b)c", "a ( b ) c"),
+        Arguments.of("a (b) c", "a ( b ) c"),
+        Arguments.of("foo(bar){baz}", "foo ( bar ) {baz }"),
+        Arguments.of("((a)(b))", "( ( a ) (b ) )"),
+        Arguments.of("a(b(c)d)e", "a ( b ( c ) d ) e"),
+        Arguments.of("x((", "x ( ("),
+        Arguments.of("((x", "( ( x"),
+        Arguments.of("()", "( )"),
+        Arguments.of("(", "("),
+        Arguments.of("", ""),
+        Arguments.of("no parens here", "no parens here"),
+        Arguments.of("«quoted»", "«quoted»"),
+        Arguments.of("tab\there(", "tab\there ("),
+        Arguments.of("a  (b", "a  ( b"));
+  }
+
+  @ParameterizedTest
+  @MethodSource("parenLines")
+  void testSpaceUntokenizedParens(String line, String expected) {
+    assertEquals(expected, ParserTool.spaceUntokenizedParens(line));
+  }
+
+  @Test
+  void testParseLineSeparatesParensBeforeTokenizing() {
+    Parser echo = new Parser() {
+      @Override
+      public Parse[] parse(Parse tokens, int numParses) {
+        return new Parse[] {tokens};
+      }
+
+      @Override
+      public Parse parse(Parse tokens) {
+        return tokens;
+      }
+    };
+    Parse[] parses = ParserTool.parseLine("f(x)+g({y})", echo, WhitespaceTokenizer.INSTANCE, 1);
+    assertEquals(1, parses.length);
+    String[] tokens = Stream.of(parses[0].getChildren()).map(Parse::getCoveredText)
+        .toArray(String[]::new);
+    // "{y" stays joined: the second pass consumed "{" while spacing "( {"
+    assertArrayEquals(new String[] {"f", "(", "x", ")", "+g", "(", "{y", "}", ")"}, tokens);
+  }
+}
diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
index d2db063515..5c19511dcf 100644
--- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
+++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
@@ -38,6 +38,7 @@
 import opennlp.tools.util.ObjectStream;
 import opennlp.tools.util.PlainTextByLineStream;
 import opennlp.tools.util.Span;
+import opennlp.tools.util.StringUtil;
 
 /**
  * Parser for Floresta Sita(c)tica Arvores Deitadas corpus, output to for the
@@ -66,16 +67,6 @@
 @Internal
 public class ADNameSampleStream implements ObjectStream<NameSample> {
 
-  /*
-   * Pattern of a NER tag in Arvores Deitadas
-   */
-  private static final Pattern TAG_PATTERN = Pattern.compile("<(NER:)?(.*?)>");
-  private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
-  private static final Pattern UNDERLINE_PATTERN = Pattern.compile("[_]+");
-  private static final Pattern HYPHEN_PATTERN =
-      Pattern.compile("((\\p{L}+)-$)|(^-(\\p{L}+)(.*))|((\\p{L}+)-(\\p{L}+)(.*))");
-  private static final Pattern ALPHANUMERIC_PATTERN = Pattern.compile("^[\\p{L}\\p{Nd}]+$");
-
   /*
    * Map to the Arvores Deitadas types to our types. It is read-only.
    */
@@ -254,7 +245,7 @@ private void processLeaf(Leaf leaf, List<String> sentence, List<Span> names) {
       String c = PortugueseContractionUtility.toContraction(
           leftContractionPart, right);
       if (c != null) {
-        String[] parts = WHITESPACE_PATTERN.split(c);
+        String[] parts = StringUtil.splitOnAsciiWhitespace(c);
         sentence.addAll(Arrays.asList(parts));
         alreadyAdded = true;
       } else {
@@ -273,7 +264,7 @@ private void processLeaf(Leaf leaf, List<String> sentence, List<Span> names) {
 
     if (leafTag != null) {
       if (leafTag.contains("<sam->") && !alreadyAdded) {
-        String[] lexemes = UNDERLINE_PATTERN.split(leaf.getLexeme());
+        String[] lexemes = splitOnUnderscores(leaf.getLexeme());
         if (lexemes.length > 1) {
           sentence.addAll(Arrays.asList(lexemes).subList(0, lexemes.length - 1));
         }
@@ -318,9 +309,9 @@ private void processLeaf(Leaf leaf, List<String> sentence, List<Span> names) {
 
   private List<String> processLexeme(String lexemeStr) {
     List<String> out = new ArrayList<>();
-    String[] parts = UNDERLINE_PATTERN.split(lexemeStr);
+    String[] parts = splitOnUnderscores(lexemeStr);
     for (String tok : parts) {
-      if (tok.length() > 1 && !ALPHANUMERIC_PATTERN.matcher(tok).matches()) {
+      if (tok.length() > 1 && !isAlphaNumeric(tok)) {
         out.addAll(processTok(tok));
       } else {
         out.add(tok);
@@ -347,35 +338,19 @@ private List<String> processTok(String tok) {
 
     // lets split all hyphens
     if (this.splitHyphenatedTokens && tok.contains("-") && tok.length() > 1) {
-      Matcher matcher = HYPHEN_PATTERN.matcher(tok);
-
-      String firstTok = null;
-      String hyphen = "-";
-      String secondTok = null;
-      String rest = null;
-
-      if (matcher.matches()) {
-        if (matcher.group(1) != null) {
-          firstTok = matcher.group(2);
-        } else if (matcher.group(3) != null) {
-          secondTok = matcher.group(4);
-          rest = matcher.group(5);
-        } else if (matcher.group(6) != null) {
-          firstTok = matcher.group(7);
-          secondTok = matcher.group(8);
-          rest = matcher.group(9);
-        }
+      String[] parts = matchHyphenatedToken(tok);
 
-        addIfNotEmpty(firstTok, out);
-        addIfNotEmpty(hyphen, out);
-        addIfNotEmpty(secondTok, out);
-        addIfNotEmpty(rest, out);
+      if (parts != null) {
+        addIfNotEmpty(parts[0], out);
+        addIfNotEmpty("-", out);
+        addIfNotEmpty(parts[1], out);
+        addIfNotEmpty(parts[2], out);
         tokAdded = true;
       }
     }
     if (!tokAdded) {
       if (!original.equals(tok) && tok.length() > 1
-          && !ALPHANUMERIC_PATTERN.matcher(tok).matches()) {
+          && !isAlphaNumeric(tok)) {
         out.addAll(processTok(tok));
       } else {
         out.add(tok);
@@ -391,6 +366,161 @@ private void addIfNotEmpty(String firstTok, List<String> out) {
     }
   }
 
+  /**
+   * Splits on runs of underscores with the result of {@code String.split("[_]+")}: a leading
+   * run yields one empty first element, trailing empty elements are dropped, and
+   * underscore-only input yields an empty array.
+   *
+   * @param s The text.
+   * @return The elements in order.
+   */
+  static String[] splitOnUnderscores(String s) {
+    if (s.isEmpty()) {
+      return new String[] {""};
+    }
+    boolean hasToken = false;
+    for (int i = 0; i < s.length(); i++) {
+      if (s.charAt(i) != '_') {
+        hasToken = true;
+        break;
+      }
+    }
+    if (!hasToken) {
+      return new String[0];
+    }
+    List<String> tokens = new ArrayList<>();
+    if (s.charAt(0) == '_') {
+      tokens.add("");
+    }
+    int start = 0;
+    for (int i = 0; i < s.length(); i++) {
+      if (s.charAt(i) == '_') {
+        if (i > start) {
+          tokens.add(s.substring(start, i));
+        }
+        while (i + 1 < s.length() && s.charAt(i + 1) == '_') {
+          i++;
+        }
+        start = i + 1;
+      }
+    }
+    if (s.length() > start) {
+      tokens.add(s.substring(start));
+    }
+    return tokens.toArray(new String[0]);
+  }
+
+  /**
+   * Tests whether a token consists of letters and decimal digits only, by code point.
+   *
+   * @param tok The token.
+   * @return {@code true} if the token is non-empty and every code point is a letter or a
+   *         decimal digit.
+   */
+  static boolean isAlphaNumeric(String tok) {
+    if (tok.isEmpty()) {
+      return false;
+    }
+    int i = 0;
+    while (i < tok.length()) {
+      int cp = tok.codePointAt(i);
+      if (!Character.isLetter(cp) && !Character.isDigit(cp)) {
+        return false;
+      }
+      i += Character.charCount(cp);
+    }
+    return true;
+  }
+
+  /**
+   * Splits a hyphenated token into the letters before the hyphen, the letters after it, and
+   * the rest. Three shapes match: letters followed by a final hyphen, a leading hyphen followed
+   * by letters and anything, and letters, a hyphen, letters, and anything.
+   *
+   * @param tok The token, at least two characters long.
+   * @return The first token, second token, and rest, each {@code null} when absent, or
+   *         {@code null} if the token has none of the three shapes.
+   */
+  static String[] matchHyphenatedToken(String tok) {
+    int len = tok.length();
+    // (\p{L}+)-$
+    if (len > 1 && tok.charAt(len - 1) == '-' && isAllLetters(tok, 0, len - 1)) {
+      return new String[] {tok.substring(0, len - 1), null, null};
+    }
+    // ^-(\p{L}+)(.*)
+    if (tok.charAt(0) == '-') {
+      int lettersEnd = lettersEnd(tok, 1);
+      if (lettersEnd > 1) {
+        return new String[] {null, tok.substring(1, lettersEnd), tok.substring(lettersEnd)};
+      }
+      return null;
+    }
+    // (\p{L}+)-(\p{L}+)(.*)
+    int firstEnd = lettersEnd(tok, 0);
+    if (firstEnd > 0 && firstEnd + 1 < len && tok.charAt(firstEnd) == '-') {
+      int secondEnd = lettersEnd(tok, firstEnd + 1);
+      if (secondEnd > firstEnd + 1) {
+        return new String[] {tok.substring(0, firstEnd),
+            tok.substring(firstEnd + 1, secondEnd), tok.substring(secondEnd)};
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Finds the end of the run of letters starting at an offset.
+   *
+   * @param s The text.
+   * @param from The start offset.
+   * @return The offset after the run, or {@code from} if no letter starts there.
+   */
+  private static int lettersEnd(String s, int from) {
+    int i = from;
+    while (i < s.length()) {
+      int cp = s.codePointAt(i);
+      if (!Character.isLetter(cp)) {
+        break;
+      }
+      i += Character.charCount(cp);
+    }
+    return i;
+  }
+
+  /**
+   * Tests whether a range holds letters only.
+   *
+   * @param s The text.
+   * @param from The inclusive start.
+   * @param to The exclusive end.
+   * @return {@code true} if every code point in the range is a letter.
+   */
+  private static boolean isAllLetters(String s, int from, int to) {
+    int i = from;
+    while (i < to) {
+      int cp = s.codePointAt(i);
+      if (!Character.isLetter(cp)) {
+        return false;
+      }
+      i += Character.charCount(cp);
+    }
+    return true;
+  }
+
+  /**
+   * Extracts the content of a NER tag in Arvores Deitadas format, between the optional
+   * {@code NER:} prefix and the closing angle bracket.
+   *
+   * @param t The tag.
+   * @return The content, or {@code null} if {@code t} is not enclosed in angle brackets.
+   */
+  static String tagContent(String t) {
+    if (t.length() < 2 || t.charAt(0) != '<' || t.charAt(t.length() - 1) != '>') {
+      return null;
+    }
+    int start = t.startsWith("NER:", 1) ? 5 : 1;
+    return t.substring(start, t.length() - 1);
+  }
+
   /**
    * Parses a NER tag in Arvores Deitadas format.
    *
@@ -401,14 +531,11 @@ private static String getNER(String tags) {
     if (tags.contains("<NER2>")) {
       return null;
     }
-    String[] tag = tags.split("\\s+");
+    String[] tag = StringUtil.splitOnAsciiWhitespace(tags);
     for (String t : tag) {
-      Matcher matcher = TAG_PATTERN.matcher(t);
-      if (matcher.matches()) {
-        String ner = matcher.group(2);
-        if (HAREM.containsKey(ner)) {
-          return HAREM.get(ner);
-        }
+      String ner = tagContent(t);
+      if (ner != null && HAREM.containsKey(ner)) {
+        return HAREM.get(ner);
       }
     }
     return null;
diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
index 8b7bd96416..9a66f497f9 100644
--- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
+++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
@@ -21,7 +21,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.StringTokenizer;
-import java.util.regex.Pattern;
 
 import opennlp.tools.commons.Internal;
 import opennlp.tools.formats.ad.ADSentenceStream.Sentence;
@@ -32,6 +31,7 @@
 import opennlp.tools.util.InputStreamFactory;
 import opennlp.tools.util.ObjectStream;
 import opennlp.tools.util.PlainTextByLineStream;
+import opennlp.tools.util.StringUtil;
 
 /**
  * <b>Note:</b>
@@ -40,8 +40,6 @@
 @Internal
 public class ADPOSSampleStream implements ObjectStream<POSSample> {
 
-  private static final Pattern WHITESPACES_PATTERN = Pattern.compile("\\s+");
-
   private final ObjectStream<ADSentenceStream.Sentence> adSentenceStream;
   private final boolean expandME;
   private final boolean isIncludeFeatures;
@@ -118,7 +116,7 @@ private void processLeaf(Leaf leaf, List<String> sentence, List<String> tags) {
       if (isIncludeFeatures && leaf.getMorphologicalTag() != null) {
         tag += " " + leaf.getMorphologicalTag();
       }
-      tag = WHITESPACES_PATTERN.matcher(tag).replaceAll("=");
+      tag = replaceWhitespaceWithEquals(tag);
 
       if (tag == null)
         tag = lexeme;
@@ -152,6 +150,31 @@ private void processLeaf(Leaf leaf, List<String> sentence, List<String> tags) {
 
   }
 
+  /**
+   * Replaces every run of ASCII whitespace, leading and trailing runs included, with a single
+   * equals sign.
+   *
+   * @param tag The tag.
+   * @return The joined tag.
+   */
+  static String replaceWhitespaceWithEquals(String tag) {
+    StringBuilder replaced = new StringBuilder(tag.length());
+    int i = 0;
+    while (i < tag.length()) {
+      char c = tag.charAt(i);
+      if (StringUtil.isAsciiWhitespace(c)) {
+        replaced.append('=');
+        while (i + 1 < tag.length() && StringUtil.isAsciiWhitespace(tag.charAt(i + 1))) {
+          i++;
+        }
+      } else {
+        replaced.append(c);
+      }
+      i++;
+    }
+    return replaced.toString();
+  }
+
   @Override
   public void reset() throws IOException, UnsupportedOperationException {
     adSentenceStream.reset();
diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
index c78bf2b9a4..ab728f973b 100644
--- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
+++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
@@ -21,8 +21,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import opennlp.tools.commons.Internal;
 import opennlp.tools.formats.ad.ADSentenceStream.Sentence;
@@ -134,22 +132,15 @@ private boolean hasPunctuation(String text) {
     return false;
   }
 
-  // there are some different types of metadata depending on the corpus.
-  // TODO Merge these patterns
-  private static final Pattern META_1 = Pattern.compile("^(?:[a-zA-Z\\-]*(\\d+)).*?p=(\\d+).*");
-
   private void updateMeta() {
     if (this.sent != null) {
       String meta = this.sent.metadata();
-      Matcher m = META_1.matcher(meta);
-      int currentText;
-      int currentPara;
-      if (m.matches()) {
-        currentText = Integer.parseInt(m.group(1));
-        currentPara = Integer.parseInt(m.group(2));
-      } else {
+      int[] textAndPara = parseTextAndParagraph(meta);
+      if (textAndPara == null) {
         throw new RuntimeException("Invalid metadata: " + meta);
       }
+      int currentText = textAndPara[0];
+      int currentPara = textAndPara[1];
       isSamePara = isSameText = false;
       if (currentText == text)
         isSameText = true;
@@ -167,6 +158,66 @@ private void updateMeta() {
     }
   }
 
+  /**
+   * Parses the text and paragraph ids from sentence metadata, which differs between corpora:
+   * the text id is the ASCII digit run after any leading ASCII letters and hyphens, the
+   * paragraph id is the digit run after the first {@code p=} that at least one digit follows.
+   *
+   * @param meta The metadata.
+   * @return The text id and the paragraph id, or {@code null} if either is missing.
+   */
+  private int[] parseTextAndParagraph(String meta) {
+    int i = 0;
+    while (i < meta.length() && (isAsciiLetter(meta.charAt(i)) || meta.charAt(i) == '-')) {
+      i++;
+    }
+    int textStart = i;
+    while (i < meta.length() && isAsciiDigit(meta.charAt(i))) {
+      i++;
+    }
+    if (i == textStart) {
+      return null;
+    }
+    int text = Integer.parseInt(meta.substring(textStart, i));
+    int from = i;
+    while (from <= meta.length() - "p=".length()) {
+      int p = meta.indexOf("p=", from);
+      if (p == -1) {
+        return null;
+      }
+      int paraStart = p + 2;
+      int paraEnd = paraStart;
+      while (paraEnd < meta.length() && isAsciiDigit(meta.charAt(paraEnd))) {
+        paraEnd++;
+      }
+      if (paraEnd > paraStart) {
+        return new int[] {text, Integer.parseInt(meta.substring(paraStart, paraEnd))};
+      }
+      from = p + 1;
+    }
+    return null;
+  }
+
+  /**
+   * Tests for an ASCII letter.
+   *
+   * @param c The character.
+   * @return {@code true} for {@code a} to {@code z} or {@code A} to {@code Z}.
+   */
+  private boolean isAsciiLetter(char c) {
+    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
+  }
+
+  /**
+   * Tests for an ASCII digit.
+   *
+   * @param c The character.
+   * @return {@code true} for {@code 0} to {@code 9}.
+   */
+  private boolean isAsciiDigit(char c) {
+    return c >= '0' && c <= '9';
+  }
+
   @Override
   public void reset() throws IOException, UnsupportedOperationException {
     adSentenceStream.reset();
diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
index 1445f153b5..4bad4737c0 100644
--- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
+++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
@@ -33,6 +33,7 @@
 import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.Node;
 import opennlp.tools.util.FilterObjectStream;
 import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.StringUtil;
 
 /**
  * Stream filter which merges text lines into sentences, following the Arvores
@@ -69,9 +70,6 @@ public static class SentenceParser {
         .compile("^([=-]*)([^:=]+):([^\\(\\s]+)\\([\"'](.+)[\"']\\s*((?:<.+>)*)\\s*([^\\)]+)?\\)\\s+(.+)");
     private static final Pattern BIZARRE_LEAF_PATTERN = Pattern
         .compile("^([=-]*)([^:=]+=[^\\(\\s]+)\\(([\"'].+[\"'])?\\s*([^\\)]+)?\\)\\s+(.+)");
-    private static final Pattern PUNCTUATION_PATTERN = Pattern.compile("^(=*)(\\W+)$");
-    private static final Pattern PUNCTUATION_DOT_PATTERN = Pattern.compile("\\»\\s+\\.");
-    private static final Pattern PUNCTUATION_COMMA_PATTERN = Pattern.compile("\\»\\s+\\,");
 
     private String text,meta;
 
@@ -208,11 +206,82 @@ public Sentence parse(String sentenceString, int para, boolean isTitle, boolean
     }
 
     private String fixPunctuation(String text) {
-      text = PUNCTUATION_DOT_PATTERN.matcher(text).replaceAll("».");
-      text = PUNCTUATION_COMMA_PATTERN.matcher(text).replaceAll("»,");
+      text = replaceGuillemetPunctuation(text, '.', "».");
+      text = replaceGuillemetPunctuation(text, ',', "»,");
       return text;
     }
 
+    /**
+     * Removes the ASCII whitespace between a closing guillemet and a following punctuation
+     * character.
+     *
+     * @param text The text.
+     * @param punct The punctuation character.
+     * @param replacement The two characters to write in place of guillemet, whitespace, and
+     *                    punctuation.
+     * @return The text with those runs joined.
+     */
+    private String replaceGuillemetPunctuation(String text, char punct, String replacement) {
+      StringBuilder fixed = new StringBuilder(text.length());
+      int i = 0;
+      while (i < text.length()) {
+        char c = text.charAt(i);
+        if (c == '»' && i + 1 < text.length()) {
+          int j = i + 1;
+          while (j < text.length() && StringUtil.isAsciiWhitespace(text.charAt(j))) {
+            j++;
+          }
+          if (j > i + 1 && j < text.length() && text.charAt(j) == punct) {
+            fixed.append(replacement);
+            i = j + 1;
+            continue;
+          }
+        }
+        fixed.append(c);
+        i++;
+      }
+      return fixed.toString();
+    }
+
+    /**
+     * Parses a punctuation line: leading equals signs followed by one or more characters that
+     * are not ASCII letters, digits, or underscores. A line of equals signs only also matches,
+     * with the last one as lexeme.
+     *
+     * @param line The line.
+     * @return The level, as one more than the count of leading equals signs, and the lexeme,
+     *         or {@code null} if the line is not a punctuation line.
+     */
+    private String[] parsePunctuationLine(String line) {
+      if (line.isEmpty()) {
+        return null;
+      }
+      for (int i = 0; i < line.length(); i++) {
+        if (isAsciiWord(line.charAt(i))) {
+          return null;
+        }
+      }
+      int equals = 0;
+      while (equals < line.length() && line.charAt(equals) == '=') {
+        equals++;
+      }
+      if (equals == line.length()) {
+        return new String[] {String.valueOf(equals), "="};
+      }
+      return new String[] {String.valueOf(equals + 1), line.substring(equals)};
+    }
+
+    /**
+     * Tests for an ASCII letter, digit, or underscore.
+     *
+     * @param c The character.
+     * @return {@code true} for a word character.
+     */
+    private boolean isAsciiWord(char c) {
+      return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+          || (c >= '0' && c <= '9') || c == '_';
+    }
+
     /**
      * Parse a tree element from a AD line
      *
@@ -255,13 +324,11 @@ public TreeElement getElement(String line) {
         return leaf;
       }
 
-      Matcher punctuationMatcher = PUNCTUATION_PATTERN.matcher(line);
-      if (punctuationMatcher.matches()) {
-        int level = punctuationMatcher.group(1).length() + 1;
-        String lexeme = punctuationMatcher.group(2);
+      String[] punctuation = parsePunctuationLine(line);
+      if (punctuation != null) {
         Leaf leaf = new Leaf();
-        leaf.setLevel(level);
-        leaf.setLexeme(lexeme);
+        leaf.setLevel(Integer.parseInt(punctuation[0]));
+        leaf.setLexeme(punctuation[1]);
         return leaf;
       }
 
diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java
index 17c991e622..1ee1efcb44 100644
--- a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java
+++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluStream.java
@@ -27,9 +27,6 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
 import java.util.stream.Collectors;
 
 import opennlp.tools.util.InputStreamFactory;
@@ -45,8 +42,6 @@
 public class ConlluStream implements ObjectStream<ConlluSentence> {
   private final ObjectStream<String> sentenceStream;
 
-  private static final Pattern regex = Pattern.compile("text_([a-z]{2,3})");
-
   /**
    * Initializes a {@link ConlluStream}.
    *
@@ -155,7 +150,7 @@ private List<ConlluWordLine> postProcessContractions(List<ConlluWordLine> lines)
       index.put(line.getId(), i);
       if (line.getId().contains("-")) {
         List<String> expandedContractions = new ArrayList<>();
-        String[] ids = line.getId().split("-");
+        String[] ids = splitOnHyphen(line.getId());
         int start = Integer.parseInt(ids[0]);
         int end = Integer.parseInt(ids[1]);
         for (int j = start; j <= end; j++) {
@@ -230,15 +225,7 @@ private ConlluWordLine mergeAnnotation(ConlluWordLine contraction,
 
   private Map<Locale, String> addTextLang(String firstPart, String secondPart,
                                           Map<Locale, String> textLang) throws InvalidFormatException {
-    String lang = "";
-    try {
-      Matcher regexMatcher = regex.matcher(firstPart);
-      if (regexMatcher.find()) {
-        lang = regexMatcher.group(1);
-      }
-    } catch (PatternSyntaxException e) {
-      throw new InvalidFormatException(e);
-    }
+    String lang = extractTextLang(firstPart);
     if (!lang.isEmpty()) {
       textLang.put(Locale.of(lang), secondPart);
     }
@@ -248,6 +235,59 @@ private Map<Locale, String> addTextLang(String firstPart, String secondPart,
     return textLang;
   }
 
+  /**
+   * Splits a token id on hyphens with the result of {@code String.split("-")}: empty elements
+   * between consecutive hyphens are kept, trailing empty elements are dropped.
+   *
+   * @param id The token id.
+   * @return The elements in order.
+   */
+  private String[] splitOnHyphen(String id) {
+    if (id.isEmpty()) {
+      return new String[] {""};
+    }
+    List<String> parts = new ArrayList<>();
+    int start = 0;
+    for (int i = 0; i < id.length(); i++) {
+      if (id.charAt(i) == '-') {
+        parts.add(id.substring(start, i));
+        start = i + 1;
+      }
+    }
+    if (id.length() > start) {
+      parts.add(id.substring(start));
+    }
+    while (!parts.isEmpty() && parts.get(parts.size() - 1).isEmpty()) {
+      parts.remove(parts.size() - 1);
+    }
+    return parts.toArray(new String[0]);
+  }
+
+  /**
+   * Extracts the language code from a {@code text_xx} or {@code text_xxx} comment key: the two
+   * or three ASCII lowercase letters, preferring three, after the first {@code text_} that at
+   * least two follow.
+   *
+   * @param firstPart The comment key.
+   * @return The language code, or an empty string if there is none.
+   */
+  private String extractTextLang(String firstPart) {
+    int from = 0;
+    while ((from = firstPart.indexOf("text_", from)) != -1) {
+      int i = from + "text_".length();
+      int len = 0;
+      while (len < 3 && i + len < firstPart.length()
+          && firstPart.charAt(i + len) >= 'a' && firstPart.charAt(i + len) <= 'z') {
+        len++;
+      }
+      if (len >= 2) {
+        return firstPart.substring(i, i + len);
+      }
+      from++;
+    }
+    return "";
+  }
+
   @Override
   public void close() throws IOException {
     sentenceStream.close();
diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java
index 101b465259..e41a98dd38 100644
--- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java
+++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADNameSampleStreamTest.java
@@ -19,10 +19,16 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.stream.Stream;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
 
 import opennlp.tools.namefind.NameSample;
 import opennlp.tools.util.PlainTextByLineStream;
@@ -117,4 +123,79 @@ void testMissingRightContraction() {
     Assertions.assertEquals(new Span(5, 6, "person"), samples.get(7).getNames()[2]);
   }
 
+  private static Stream<Arguments> underscoreLexemes() {
+    return Stream.of(
+        Arguments.of("Rio_de_Janeiro", new String[] {"Rio", "de", "Janeiro"}),
+        Arguments.of("a__b", new String[] {"a", "b"}),
+        Arguments.of("_a", new String[] {"", "a"}),
+        Arguments.of("a_", new String[] {"a"}),
+        Arguments.of("__", new String[0]),
+        Arguments.of("_", new String[0]),
+        Arguments.of("\uD801\uDC12_\uD83D\uDE00", new String[] {"\uD801\uDC12", "\uD83D\uDE00"}),
+        Arguments.of("", new String[] {""}),
+        Arguments.of("casa", new String[] {"casa"}));
+  }
+
+  @ParameterizedTest
+  @MethodSource("underscoreLexemes")
+  void testSplitOnUnderscores(String lexeme, String[] expected) {
+    Assertions.assertArrayEquals(expected, ADNameSampleStream.splitOnUnderscores(lexeme));
+    Assertions.assertArrayEquals(lexeme.split("[_]+"), ADNameSampleStream.splitOnUnderscores(lexeme));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"casa", "São", "1990", "R2D2", "\uD801\uDC12\u0661"})
+  void testIsAlphaNumericAccepts(String token) {
+    Assertions.assertTrue(ADNameSampleStream.isAlphaNumeric(token));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"", "guarda-chuva", "R$", "a b", "\u00BD", "\uD83D\uDE00"})
+  void testIsAlphaNumericRejects(String token) {
+    Assertions.assertFalse(ADNameSampleStream.isAlphaNumeric(token));
+  }
+
+  private static Stream<Arguments> hyphenatedTokens() {
+    return Stream.of(
+        Arguments.of("guarda-", new String[] {"guarda", null, null}),
+        Arguments.of("a-", new String[] {"a", null, null}),
+        Arguments.of("-chuva", new String[] {null, "chuva", ""}),
+        Arguments.of("-chuva2!", new String[] {null, "chuva", "2!"}),
+        Arguments.of("guarda-chuva", new String[] {"guarda", "chuva", ""}),
+        Arguments.of("guarda-chuva-sol", new String[] {"guarda", "chuva", "-sol"}),
+        Arguments.of("São-Paulo", new String[] {"São", "Paulo", ""}),
+        // supplementary-plane letters are letters, a combining mark ends the letter run
+        Arguments.of("\uD801\uDC12-\uD801\uDC3A", new String[] {"\uD801\uDC12", "\uD801\uDC3A", ""}),
+        Arguments.of("e\u0301-a", new String[] {null, null, null}));
+  }
+
+  @ParameterizedTest
+  @MethodSource("hyphenatedTokens")
+  void testMatchHyphenatedToken(String token, String[] expected) {
+    String[] actual = ADNameSampleStream.matchHyphenatedToken(token);
+    if (expected[0] == null && expected[1] == null && expected[2] == null) {
+      Assertions.assertNull(actual);
+    } else {
+      Assertions.assertArrayEquals(expected, actual);
+    }
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"-", "--", "-1", "1-", "a1-b", "a-1", "a--b", "ab", "a -"})
+  void testMatchHyphenatedTokenRejects(String token) {
+    Assertions.assertNull(ADNameSampleStream.matchHyphenatedToken(token));
+  }
+
+  @ParameterizedTest
+  @CsvSource({"<NER:PROP>, PROP", "<PROP>, PROP", "<>, ''", "<NER:>, ''", "<a<b>, a<b",
+      "<NER:NER:X>, NER:X", "<ner:PROP>, ner:PROP", "<\uD83D\uDE00>, \uD83D\uDE00"})
+  void testTagContent(String tag, String expected) {
+    Assertions.assertEquals(expected, ADNameSampleStream.tagContent(tag));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"", "<", ">", "PROP", "<PROP", "PROP>"})
+  void testTagContentRejects(String tag) {
+    Assertions.assertNull(ADNameSampleStream.tagContent(tag));
+  }
 }
diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java
index cbbae22012..5fc873a88b 100644
--- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java
+++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADPOSSampleStreamTest.java
@@ -19,10 +19,14 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.stream.Stream;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 
 import opennlp.tools.postag.POSSample;
 import opennlp.tools.util.PlainTextByLineStream;
@@ -108,4 +112,23 @@ void testIncludeFeats() throws IOException {
     }
   }
 
+  private static Stream<Arguments> tags() {
+    return Stream.of(
+        Arguments.of("v-fin", "v-fin"),
+        Arguments.of("PR 3S IND", "PR=3S=IND"),
+        Arguments.of("PR  \t3S", "PR=3S"),
+        Arguments.of(" PR 3S ", "=PR=3S="),
+        Arguments.of(" ", "="),
+        Arguments.of("", ""),
+        Arguments.of("PR\u00A03S", "PR\u00A03S"),
+        Arguments.of("\r\n\u000B\f", "="),
+        Arguments.of("\uD83D\uDE00 x", "\uD83D\uDE00=x"));
+  }
+
+  @ParameterizedTest
+  @MethodSource("tags")
+  void testReplaceWhitespaceWithEquals(String tag, String expected) {
+    Assertions.assertEquals(expected, ADPOSSampleStream.replaceWhitespaceWithEquals(tag));
+  }
+
 }
diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java
index 58f45635af..2a32e7bb98 100644
--- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java
+++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceSampleStreamTest.java
@@ -19,12 +19,15 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Iterator;
+import java.util.List;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.ObjectStream;
 import opennlp.tools.util.PlainTextByLineStream;
 import opennlp.tools.util.Span;
 
@@ -59,4 +62,28 @@ void testSentences() {
     Assertions.assertEquals(new Span(120, 180), samples.get(0).getSentences()[1]);
   }
 
+  @Test
+  void testInvalidMetadataIsRejected() throws IOException {
+    // the second sentence id "AX" has no digits, so its metadata cannot be parsed
+    List<String> lines = List.of(
+        "<s>",
+        "SOURCE: src",
+        "1001 Hello world .",
+        "</s>",
+        "<s>",
+        "SOURCE: src",
+        "AX Hi there .",
+        "</s>");
+    Iterator<String> iterator = lines.iterator();
+    ObjectStream<String> lineStream = new ObjectStream<>() {
+      @Override
+      public String read() {
+        return iterator.hasNext() ? iterator.next() : null;
+      }
+    };
+    try (ADSentenceSampleStream stream = new ADSentenceSampleStream(lineStream, true)) {
+      Assertions.assertThrows(RuntimeException.class, stream::read);
+    }
+  }
+
 }
diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java
new file mode 100644
index 0000000000..1baf781126
--- /dev/null
+++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceStreamTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.formats.ad;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.formats.ad.ADSentenceStream.Sentence;
+import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser;
+import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.Leaf;
+import opennlp.tools.formats.ad.ADSentenceStream.SentenceParser.TreeElement;
+
+public class ADSentenceStreamTest {
+
+  @Test
+  void testPunctuationLeaf() {
+    SentenceParser parser = new SentenceParser();
+
+    Leaf leaf = (Leaf) parser.getElement("==,");
+    Assertions.assertEquals(3, leaf.getLevel());
+    Assertions.assertEquals(",", leaf.getLexeme());
+
+    leaf = (Leaf) parser.getElement(".");
+    Assertions.assertEquals(1, leaf.getLevel());
+    Assertions.assertEquals(".", leaf.getLexeme());
+
+    // a line of only equals signs matches, with the last one as lexeme
+    leaf = (Leaf) parser.getElement("===");
+    Assertions.assertEquals(3, leaf.getLevel());
+    Assertions.assertEquals("=", leaf.getLexeme());
+
+    // non-word characters other than equals make up the lexeme
+    leaf = (Leaf) parser.getElement("=!?");
+    Assertions.assertEquals(2, leaf.getLevel());
+    Assertions.assertEquals("!?", leaf.getLexeme());
+
+    // a word character excludes the punctuation parse, the line is treated
+    // as a bizarre leaf instead
+    TreeElement element = parser.getElement("=ab");
+    Assertions.assertTrue(element.isLeaf());
+    leaf = (Leaf) element;
+    Assertions.assertEquals(2, leaf.getLevel());
+    Assertions.assertEquals("b", leaf.getLexeme());
+  }
+
+  @Test
+  void testFixPunctuation() {
+    SentenceParser parser = new SentenceParser();
+
+    Sentence sentence = parser.parse(
+        "<s>\nSOURCE: src\n1001 Olá mundo » .\n</s>\n", 1, false, false);
+    Assertions.assertEquals("Olá mundo ».", sentence.text());
+
+    sentence = parser.parse(
+        "<s>\nSOURCE: src\n1001 Olá » , tudo bem » .\n</s>\n", 1, false, false);
+    Assertions.assertEquals("Olá », tudo bem ».", sentence.text());
+
+    // without whitespace between » and the punctuation nothing is replaced
+    sentence = parser.parse(
+        "<s>\nSOURCE: src\n1001 Olá mundo ».\n</s>\n", 1, false, false);
+    Assertions.assertEquals("Olá mundo ».", sentence.text());
+  }
+}
diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluStreamTest.java
index 5716a84ab6..afc11909f7 100644
--- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluStreamTest.java
+++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluStreamTest.java
@@ -17,7 +17,9 @@
 
 package opennlp.tools.formats.conllu;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
@@ -28,6 +30,8 @@
 import org.junit.jupiter.api.Test;
 
 import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.InputStreamFactory;
+import opennlp.tools.util.InvalidFormatException;
 import opennlp.tools.util.ObjectStream;
 
 public class ConlluStreamTest extends AbstractConlluSampleStreamTest<SentenceSample> {
@@ -106,4 +110,35 @@ void testOptionalComments() throws IOException {
       Assertions.assertNull(stream.read(), "Stream must be exhausted");
     }
   }
+
+  @Test
+  void testContractionIdsAreMerged() throws IOException {
+    try (ObjectStream<ConlluSentence> stream = getStream("es-ud-sample.conllu")) {
+      ConlluSentence sent1 = stream.read();
+
+      Assertions.assertEquals(55, sent1.getWordLines().size());
+      Assertions.assertEquals("1-3", sent1.getWordLines().get(0).getId());
+      Assertions.assertEquals("Digámoslo", sent1.getWordLines().get(0).getForm());
+      Assertions.assertEquals("15-16", sent1.getWordLines().get(12).getId());
+      for (ConlluWordLine wordLine : sent1.getWordLines()) {
+        Assertions.assertFalse(wordLine.getId().equals("1")
+            || wordLine.getId().equals("2") || wordLine.getId().equals("3")
+            || wordLine.getId().equals("15") || wordLine.getId().equals("16"),
+            "Expanded contraction parts must be removed");
+      }
+    }
+  }
+
+  @Test
+  void testInvalidTextLangCodeIsRejected() throws IOException {
+    // "text_e" has a single lowercase letter, so no language code can be extracted
+    InputStreamFactory in = () -> new ByteArrayInputStream(
+        ("# text_e = Bonjour\n"
+            + "1\tBonjour\tbonjour\tINTJ\t_\t_\t0\troot\t_\t_\n")
+            .getBytes(StandardCharsets.UTF_8));
+
+    try (ObjectStream<ConlluSentence> stream = new ConlluStream(in)) {
+      Assertions.assertThrows(InvalidFormatException.class, stream::read);
+    }
+  }
 }
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java
index 04007f9109..e8639af23e 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java
@@ -19,7 +19,8 @@
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.regex.Pattern;
+
+import opennlp.tools.util.StringUtil;
 
 /**
  * Simple feature generator for learning statistical lemmatizers.
@@ -34,9 +35,6 @@ public class DefaultLemmatizerContextGenerator implements LemmatizerContextGener
   private static final int PREFIX_LENGTH = 5;
   private static final int SUFFIX_LENGTH = 7;
 
-  private static final Pattern PATTERN_HAS_CAP = Pattern.compile("[A-Z]");
-  private static final Pattern PATTERN_HAS_NUM = Pattern.compile("[0-9]");
-
   public DefaultLemmatizerContextGenerator() {
   }
 
@@ -105,11 +103,11 @@ public String[] getContext(int index, String[] toks, String[] tags, String[] pre
       features.add("h");
     }
 
-    if (PATTERN_HAS_CAP.matcher(lex).find()) {
+    if (StringUtil.containsAsciiUpperCase(lex)) {
       features.add("c");
     }
 
-    if (PATTERN_HAS_NUM.matcher(lex).find()) {
+    if (StringUtil.containsAsciiDigit(lex)) {
       features.add("d");
     }
 
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/BioCodec.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/BioCodec.java
index f7d6c5e852..563c9bd55f 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/BioCodec.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/BioCodec.java
@@ -20,8 +20,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import opennlp.tools.util.SequenceCodec;
 import opennlp.tools.util.Span;
@@ -47,17 +45,37 @@ public class BioCodec implements SequenceCodec<String> {
   public static final String CONTINUE = "cont";
   public static final String OTHER = "other";
 
-  private static final Pattern TYPED_OUTCOME_PATTERN = Pattern.compile("(.+)-\\w+");
-
   static String extractNameType(String outcome) {
-    Matcher matcher = TYPED_OUTCOME_PATTERN.matcher(outcome);
-    if (matcher.matches()) {
-      return matcher.group(1);
+    int separator = outcome.lastIndexOf('-');
+    if (separator > 0 && isWordChars(outcome, separator + 1)) {
+      return outcome.substring(0, separator);
     }
 
     return null;
   }
 
+  /**
+   * Tests whether the rest of an outcome is a non-empty run of ASCII letters, digits, or
+   * underscores.
+   *
+   * @param outcome The outcome label.
+   * @param from The offset the run starts at.
+   * @return {@code true} if at least one character follows and all are word characters.
+   */
+  private static boolean isWordChars(String outcome, int from) {
+    if (from >= outcome.length()) {
+      return false;
+    }
+    for (int i = from; i < outcome.length(); i++) {
+      char c = outcome.charAt(i);
+      if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+          || (c >= '0' && c <= '9') || c == '_')) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   @Override
   public Span[] decode(List<String> c) {
     int start = -1;
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderME.java
index f1dec349c1..b8fa9ef937 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderME.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/namefind/NameFinderME.java
@@ -24,8 +24,6 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import opennlp.tools.commons.ThreadSafe;
 import opennlp.tools.ml.BeamSearch;
@@ -66,7 +64,6 @@ public class NameFinderME implements TokenNameFinder, Probabilistic {
 
   private static final String[][] EMPTY = new String[0][0];
   public static final int DEFAULT_BEAM_SIZE = 3;
-  private static final Pattern typedOutcomePattern = Pattern.compile("(.+)-\\w+");
 
   public static final String START = "start";
   public static final String CONTINUE = "cont";
@@ -327,12 +324,7 @@ else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) {
    * @return The name type, or {@code null} if not set.
    */
   static String extractNameType(String outcome) {
-    Matcher matcher = typedOutcomePattern.matcher(outcome);
-    if (matcher.matches()) {
-      return matcher.group(1);
-    }
-
-    return null;
+    return BioCodec.extractNameType(outcome);
   }
 
   /**
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java
index 93c256c60f..1060b990ad 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java
@@ -20,10 +20,10 @@
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.regex.Pattern;
 
 import opennlp.tools.dictionary.Dictionary;
 import opennlp.tools.util.StringList;
+import opennlp.tools.util.StringUtil;
 
 /**
  * A default {@link POSContextGenerator context generator} for a {@link POSTagger}.
@@ -38,9 +38,6 @@ public class DefaultPOSContextGenerator implements POSContextGenerator {
   private static final int PREFIX_LENGTH = 4;
   private static final int SUFFIX_LENGTH = 4;
 
-  private static final Pattern hasCap = Pattern.compile("[A-Z]");
-  private static final Pattern hasNum = Pattern.compile("[0-9]");
-
   private final Dictionary dict;
 
   /**
@@ -165,11 +162,11 @@ public String[] getContext(int index, Object[] tokens, String[] tags) {
         e.add("h");
       }
 
-      if (hasCap.matcher(lex).find()) {
+      if (StringUtil.containsAsciiUpperCase(lex)) {
         e.add("c");
       }
 
-      if (hasNum.matcher(lex).find()) {
+      if (StringUtil.containsAsciiDigit(lex)) {
         e.add("d");
       }
     }
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/AlphaNumericCheck.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/AlphaNumericCheck.java
new file mode 100644
index 0000000000..d9bbf5cbab
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/AlphaNumericCheck.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.tokenize;
+
+import java.util.BitSet;
+import java.util.regex.Pattern;
+
+/**
+ * Decides whether a token is alphanumeric under a tokenizer model's alphanumeric
+ * {@link Pattern}. A pattern of the shape {@code ^[...]+$} whose class holds only literal
+ * characters and simple ranges, which covers every built-in language default, is evaluated
+ * as a character set lookup. Any other pattern is evaluated by the regular expression engine,
+ * so the result is the same as {@code pattern.matcher(token).matches()} in both cases.
+ */
+final class AlphaNumericCheck {
+
+  private static final String CLASS_PREFIX = "^[";
+  private static final String CLASS_SUFFIX = "]+$";
+
+  private final BitSet characters;
+  private final Pattern pattern;
+
+  private AlphaNumericCheck(BitSet characters, Pattern pattern) {
+    this.characters = characters;
+    this.pattern = pattern;
+  }
+
+  /**
+   * Creates the check for a pattern.
+   *
+   * @param pattern The alphanumeric pattern. Must not be {@code null}.
+   * @return A check that accepts exactly the tokens the pattern matches as a whole.
+   * @throws IllegalArgumentException If {@code pattern} is {@code null}.
+   */
+  static AlphaNumericCheck of(Pattern pattern) {
+    if (pattern == null) {
+      throw new IllegalArgumentException("pattern must not be null");
+    }
+    if (pattern.flags() != 0) {
+      return new AlphaNumericCheck(null, pattern);
+    }
+    final BitSet characters = parseCharacterClass(pattern.pattern());
+    return new AlphaNumericCheck(characters, characters == null ? pattern : null);
+  }
+
+  /**
+   * Tests a token.
+   *
+   * @param token The token.
+   * @return {@code true} if the whole token matches the pattern.
+   */
+  boolean test(CharSequence token) {
+    if (characters == null) {
+      return pattern.matcher(token).matches();
+    }
+    if (token.isEmpty()) {
+      return false;
+    }
+    for (int i = 0; i < token.length(); i++) {
+      if (!characters.get(token.charAt(i))) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Tells whether the check runs as a character set lookup.
+   *
+   * @return {@code true} for a set lookup, {@code false} when the pattern is evaluated as
+   *         a regular expression.
+   */
+  boolean isCharacterSet() {
+    return characters != null;
+  }
+
+  /**
+   * Reads a pattern of the shape {@code ^[...]+$} into the set of characters its class accepts.
+   * Inside the class only literal characters and ranges written as {@code x-y} are understood; a
+   * hyphen in first or last position is literal. Escapes, negation, nested classes,
+   * intersections, and anything outside the Basic Multilingual Plane make the pattern
+   * ineligible.
+   *
+   * @param regex The pattern text.
+   * @return The accepted characters, or {@code null} if the pattern is not of that shape.
+   */
+  private static BitSet parseCharacterClass(String regex) {
+    if (!regex.startsWith(CLASS_PREFIX) || !regex.endsWith(CLASS_SUFFIX)
+        || regex.length() <= CLASS_PREFIX.length() + CLASS_SUFFIX.length()) {
+      return null;
+    }
+    final String body = regex.substring(CLASS_PREFIX.length(), regex.length() - CLASS_SUFFIX.length());
+    final BitSet characters = new BitSet();
+    int i = 0;
+    while (i < body.length()) {
+      final char c = body.charAt(i);
+      if (!isLiteral(c)) {
+        return null;
+      }
+      if (i + 2 < body.length() && body.charAt(i + 1) == '-') {
+        final char to = body.charAt(i + 2);
+        if (!isLiteral(to) || to < c) {
+          return null;
+        }
+        characters.set(c, to + 1);
+        i += 3;
+      } else {
+        characters.set(c);
+        i++;
+      }
+    }
+    return characters;
+  }
+
+  /**
+   * Tests whether a character stands for itself inside a character class.
+   *
+   * @param c The character.
+   * @return {@code false} for class syntax and for surrogates, {@code true} otherwise.
+   */
+  private static boolean isLiteral(char c) {
+    return c != '[' && c != ']' && c != '\\' && c != '^' && c != '&' && !Character.isSurrogate(c);
+  }
+}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokSpanEventStream.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokSpanEventStream.java
index 36f4a98de4..a84120cdb3 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokSpanEventStream.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokSpanEventStream.java
@@ -44,7 +44,7 @@ public class TokSpanEventStream extends AbstractEventStream<TokenSample> {
 
   private final boolean skipAlphaNumerics;
 
-  private final Pattern alphaNumeric;
+  private final AlphaNumericCheck alphaNumeric;
 
   /**
    * Initializes a new event stream based on the data stream using a {@link TokenContextGenerator}.
@@ -59,7 +59,7 @@ public class TokSpanEventStream extends AbstractEventStream<TokenSample> {
   public TokSpanEventStream(ObjectStream<TokenSample> tokenSamples, boolean skipAlphaNumerics,
                             Pattern alphaNumeric, TokenContextGenerator cg) {
     super(tokenSamples);
-    this.alphaNumeric = alphaNumeric;
+    this.alphaNumeric = AlphaNumericCheck.of(alphaNumeric);
     this.skipAlphaNumerics = skipAlphaNumerics;
     this.cg = cg;
   }
@@ -119,7 +119,7 @@ protected Iterator<Event> createEvents(TokenSample tokenSample) {
         //adjust cSpan to text offsets
         cSpan = new Span(cSpan.getStart() + start, cSpan.getEnd() + start);
         //should we skip this token
-        if (ctok.length() > 1 && (!skipAlphaNumerics || !alphaNumeric.matcher(ctok).matches())) {
+        if (ctok.length() > 1 && (!skipAlphaNumerics || !alphaNumeric.test(ctok))) {
 
           //find offsets of annotated tokens inside of candidate tokens
           boolean foundTrainingTokens = false;
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerME.java
index de9ee4ab67..f42df609e6 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerME.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/TokenizerME.java
@@ -22,7 +22,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.regex.Pattern;
 
 import opennlp.tools.commons.ThreadSafe;
 import opennlp.tools.dictionary.Dictionary;
@@ -93,7 +92,7 @@ public class TokenizerME extends AbstractTokenizer implements Probabilistic {
    */
   public static final String NO_SPLIT = "F";
 
-  private final Pattern alphanumeric;
+  private final AlphaNumericCheck alphanumeric;
 
   /*
    * The maximum entropy model to use to evaluate contexts.
@@ -156,7 +155,7 @@ public TokenizerME(TokenizerModel model, Dictionary abbDict) {
     this.abbDict = abbDict;
     TokenizerFactory factory = model.getFactory();
     this.cg = factory.getContextGenerator();
-    this.alphanumeric = factory.getAlphaNumericPattern();
+    this.alphanumeric = AlphaNumericCheck.of(factory.getAlphaNumericPattern());
     this.useAlphaNumericOptimization = factory.isUseAlphaNumericOptimization();
   }
 
@@ -203,7 +202,7 @@ public Span[] tokenizePos(String d) {
       if (tok.length() < 2) {
         localTokens.add(s);
         localProbs.add(1d);
-      } else if (useAlphaNumericOptimization() && alphanumeric.matcher(tok).matches()) {
+      } else if (useAlphaNumericOptimization() && alphanumeric.test(tok)) {
         localTokens.add(s);
         localProbs.add(1d);
       } else {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java
index 951fca5b5d..ab0d7add09 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lang/en/TokenSampleStream.java
@@ -24,13 +24,13 @@
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.regex.Pattern;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import opennlp.tools.tokenize.TokenSample;
 import opennlp.tools.util.Span;
+import opennlp.tools.util.StringUtil;
 
 /**
  * Class which produces an Iterator&lt;TokenSample&gt; from a file of space delimited token.
@@ -42,7 +42,6 @@ public class TokenSampleStream implements Iterator<TokenSample> {
   private static final Logger logger = LoggerFactory.getLogger(TokenSampleStream.class);
   private final BufferedReader in;
   private String line;
-  private final Pattern alphaNumeric = Pattern.compile("[A-Za-z0-9]");
   private boolean evenq = true;
 
   public TokenSampleStream(InputStream is) throws IOException {
@@ -55,7 +54,7 @@ public boolean hasNext() {
   }
 
   public TokenSample next() {
-    String[] tokens = line.split("\\s+");
+    String[] tokens = StringUtil.splitOnAsciiWhitespace(line);
     if (tokens.length == 0) {
       evenq = true;
     }
@@ -73,7 +72,7 @@ public TokenSample next() {
         default -> token;
       };
       if (sb.length() != 0) {
-        if (!alphaNumeric.matcher(token).find() || token.startsWith("'") || token.equalsIgnoreCase("n't")) {
+        if (!containsAsciiAlphaNum(token) || token.startsWith("'") || token.equalsIgnoreCase("n't")) {
           if ((token.equals("``") || token.equals("--") || token.equals("$") ||
               token.equals("(")  || token.equals("&")  || token.equals("#") ||
               (token.equals("\"") && (evenq && ti != tokens.length - 1)))
@@ -113,6 +112,23 @@ public void remove() {
     throw new UnsupportedOperationException();
   }
 
+
+  /**
+   * Tests whether a token contains an ASCII letter or digit.
+   *
+   * @param token The token.
+   * @return {@code true} if one is present.
+   */
+  private boolean containsAsciiAlphaNum(String token) {
+    for (int i = 0; i < token.length(); i++) {
+      char c = token.charAt(i);
+      if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   private static void usage() {
     logger.info("TokenSampleStream [-spans] < in");
     logger.info("Where in is a space delimited list of tokens.");
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java
index 0f6f6800d0..195e932447 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java
@@ -40,8 +40,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -256,13 +254,23 @@ private static String downloadChecksumFile(String sha512, Path model) throws IOE
   /**
    * Extracts the hash from the content of a checksum file, which holds the hash followed by the
    * name of the file it applies to.
+   *
+   * @param checksumFileContent The file content.
+   * @return The hash, or {@code null} if the content is {@code null} or blank.
    */
-  private static String parseChecksum(String checksumFileContent) {
+  static String parseChecksum(String checksumFileContent) {
     if (checksumFileContent == null) {
       return null;
     }
     final String trimmed = checksumFileContent.trim();
-    return trimmed.isEmpty() ? null : trimmed.split("\\s")[0];
+    if (trimmed.isEmpty()) {
+      return null;
+    }
+    int end = 0;
+    while (end < trimmed.length() && !StringUtil.isAsciiWhitespace(trimmed.charAt(end))) {
+      end++;
+    }
+    return trimmed.substring(0, end);
   }
 
   private static void verifyChecksum(Path model, String expectedChecksum) throws IOException {
@@ -324,7 +332,6 @@ private static Path getDownloadHome() {
   @Internal
   static class DownloadParser {
 
-    private static final Pattern LINK_PATTERN = Pattern.compile("<a href=\\\"(.*?)\\\">(.*?)</a>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
     private final URL indexUrl;
 
     DownloadParser(URL indexUrl) {
@@ -334,14 +341,60 @@ static class DownloadParser {
 
     Map<String, Map<ModelType, URL>> getAvailableModels()
         throws MalformedURLException, URISyntaxException {
-      final Matcher matcher = LINK_PATTERN.matcher(fetchPageIndex());
+      return toMap(extractLinks(fetchPageIndex()));
+    }
 
+    /**
+     * Collects the href values of the anchor elements in an index page. The tag name and
+     * attribute are matched ignoring ASCII case, a value ends at the first {@code ">}, a link
+     * ends at the first {@code </a>}, and both may span lines. An anchor without a closing tag
+     * is skipped.
+     *
+     * @param page The page content.
+     * @return The href values in order.
+     */
+    static List<String> extractLinks(String page) {
       final List<String> links = new ArrayList<>();
-      while (matcher.find()) {
-        links.add(matcher.group(1));
+      int from = 0;
+      while ((from = indexOfIgnoreCase(page, "<a href=\"", from)) != -1) {
+        final int valueStart = from + "<a href=\"".length();
+        final int valueEnd = page.indexOf("\">", valueStart);
+        if (valueEnd != -1) {
+          final int close = indexOfIgnoreCase(page, "</a>", valueEnd + 2);
+          if (close != -1) {
+            links.add(page.substring(valueStart, valueEnd));
+            from = close + "</a>".length();
+            continue;
+          }
+        }
+        from++;
       }
+      return links;
+    }
 
-      return toMap(links);
+    /**
+     * Finds a lowercase ASCII literal, ignoring the case of ASCII letters in the text.
+     *
+     * @param text The text.
+     * @param literal The lowercase literal.
+     * @param from The start offset.
+     * @return The first match offset, or {@code -1}.
+     */
+    private static int indexOfIgnoreCase(String text, String literal, int from) {
+      outer:
+      for (int i = from; i + literal.length() <= text.length(); i++) {
+        for (int j = 0; j < literal.length(); j++) {
+          char c = text.charAt(i + j);
+          if (c >= 'A' && c <= 'Z') {
+            c += 'a' - 'A';
+          }
+          if (c != literal.charAt(j)) {
+            continue outer;
+          }
+        }
+        return i;
+      }
+      return -1;
     }
 
     private Map<String, Map<ModelType, URL>> toMap(List<String> links)
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/BrownCluster.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/BrownCluster.java
index 221816b49a..d40044ab99 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/BrownCluster.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/BrownCluster.java
@@ -26,9 +26,10 @@
 import java.io.OutputStreamWriter;
 import java.io.Writer;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
-import java.util.regex.Pattern;
 
 import opennlp.tools.util.model.ArtifactSerializer;
 import opennlp.tools.util.model.SerializableArtifact;
@@ -47,8 +48,6 @@
  */
 public class BrownCluster implements SerializableArtifact {
 
-  private static final Pattern tabPattern = Pattern.compile("\t");
-
   public static class BrownClusterSerializer implements ArtifactSerializer<BrownCluster> {
 
     @Override
@@ -82,7 +81,7 @@ public BrownCluster(InputStream in) throws IOException {
 
       String line;
       while ((line = breader.readLine()) != null) {
-        String[] lineArray = tabPattern.split(line);
+        String[] lineArray = splitTabs(line);
         if (lineArray.length == 3) {
           int freq = Integer.parseInt(lineArray[2]);
           if (freq > 5 ) {
@@ -96,6 +95,31 @@ else if (lineArray.length == 2) {
     }
   }
 
+  /**
+   * Splits on tabs with the result of {@code String.split("\\t")}: trailing empty fields are
+   * dropped and an empty line yields a single empty field.
+   *
+   * @param line The line.
+   * @return The fields in order.
+   */
+  private String[] splitTabs(String line) {
+    if (line.isEmpty()) {
+      return new String[] {""};
+    }
+    List<String> fields = new ArrayList<>();
+    int start = 0;
+    int separator;
+    while ((separator = line.indexOf('\t', start)) != -1) {
+      fields.add(line.substring(start, separator));
+      start = separator + 1;
+    }
+    fields.add(line.substring(start));
+    while (!fields.isEmpty() && fields.get(fields.size() - 1).isEmpty()) {
+      fields.remove(fields.size() - 1);
+    }
+    return fields.toArray(new String[0]);
+  }
+
   /**
    * Check if a token is in the Brown:paths, token map.
    * 
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java
index 2237302191..b585f207d6 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorUtil.java
@@ -18,15 +18,11 @@
 
 package opennlp.tools.util.featuregen;
 
-import java.util.regex.Pattern;
-
 /**
  * This class provide common utilities for feature generation.
  */
 public class FeatureGeneratorUtil {
 
-  private static final Pattern capPeriod = Pattern.compile("^[A-ZÄÖÜ]\\.$");
-
   /**
    * Generates a class name for the specified token.
    * The classes are as follows where the first matching class is used:
@@ -98,7 +94,7 @@ else if (pattern.isAllCapitalLetter()) {
         feat = "ac";
       }
     }
-    else if (capPeriod.matcher(token).find()) {
+    else if (isCapPeriod(token)) {
       feat = "cp";
     }
     else if (pattern.isInitialCapitalLetter()) {
@@ -110,4 +106,24 @@ else if (pattern.isInitialCapitalLetter()) {
 
     return (feat);
   }
+
+  /**
+   * Tests for a single capital followed by a period.
+   *
+   * @param token The token.
+   * @return {@code true} for exactly that shape.
+   */
+  private static boolean isCapPeriod(String token) {
+    return token.length() == 2 && isCapPeriodStart(token.charAt(0)) && token.charAt(1) == '.';
+  }
+
+  /**
+   * Tests for an ASCII capital or a German umlaut capital.
+   *
+   * @param c The character.
+   * @return {@code true} for one of those capitals.
+   */
+  private static boolean isCapPeriodStart(char c) {
+    return (c >= 'A' && c <= 'Z') || c == 'Ä' || c == 'Ö' || c == 'Ü';
+  }
 }
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java
index 8596244de7..6f71abaa5a 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java
@@ -19,7 +19,6 @@
 package opennlp.tools.util.featuregen;
 
 import java.util.List;
-import java.util.regex.Pattern;
 
 import opennlp.tools.tokenize.SimpleTokenizer;
 import opennlp.tools.tokenize.Tokenizer;
@@ -38,7 +37,6 @@ public class TokenPatternFeatureGenerator implements AdaptiveFeatureGenerator {
   private static final String SUB_TOKEN_PART2_PREFIX = "pt2=" ;
   private static final String SUB_TOKEN_PART3_PREFIX = "pt3=" ;
 
-  private final Pattern noLetters = Pattern.compile("[^a-zA-Z]");
   private final Tokenizer tokenizer;
 
   /**
@@ -87,11 +85,27 @@ public void createFeatures(List<String> feats, String[] toks, int index, String[
 
       pattern.append(FeatureGeneratorUtil.tokenFeature(tokenized[i]));
 
-      if (!noLetters.matcher(tokenized[i]).find()) {
+      if (!containsNonLetter(tokenized[i])) {
         feats.add(SUB_TOKEN_PREFIX + StringUtil.toLowerCase(tokenized[i]));
       }
     }
 
     feats.add("pta=" + pattern);
   }
+
+  /**
+   * Tests whether a token contains a character outside the ASCII letters.
+   *
+   * @param token The token.
+   * @return {@code true} if one is present.
+   */
+  private boolean containsNonLetter(String token) {
+    for (int i = 0; i < token.length(); i++) {
+      char c = token.charAt(i);
+      if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'))) {
+        return true;
+      }
+    }
+    return false;
+  }
 }
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java
index d2b75335ae..95dc047d85 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java
@@ -16,8 +16,6 @@
  */
 package opennlp.tools.util.normalizer;
 
-import java.util.regex.Pattern;
-
 /**
  * A {@link EmojiCharSequenceNormalizer} implementation that normalizes text
  * in terms of emojis. Every encounter will be replaced by a whitespace.
@@ -36,15 +34,44 @@ public static EmojiCharSequenceNormalizer getInstance() {
     return INSTANCE;
   }
 
-  private static final Pattern EMOJI_REGEX =
-      Pattern.compile("[\\uD83C-\\uDBFF\\uDC00-\\uDFFF]+");
+  /**
+   * The lowest code point that is replaced: the first high surrogate of the emoji planes.
+   * Lone surrogates and every BMP character from here up count as well.
+   */
+  private static final int LOWER_CODE_POINT = 0xD83C;
+
+  /** The highest code point that is replaced. */
+  private static final int UPPER_CODE_POINT = 0x10FC00;
 
-  /** {@inheritDoc} */
+  /**
+   * {@inheritDoc}
+   * Every maximal run of code points in {@code [U+D83C, U+10FC00]} becomes one space.
+   */
   @Override
   public CharSequence normalize (CharSequence text) {
     if (text == null) {
       throw new IllegalArgumentException("The text must not be null.");
     }
-    return EMOJI_REGEX.matcher(text).replaceAll(" ");
+    StringBuilder normalized = new StringBuilder(text.length());
+    int i = 0;
+    while (i < text.length()) {
+      int cp = Character.codePointAt(text, i);
+      if (cp >= LOWER_CODE_POINT && cp <= UPPER_CODE_POINT) {
+        i += Character.charCount(cp);
+        while (i < text.length()) {
+          int next = Character.codePointAt(text, i);
+          if (next < LOWER_CODE_POINT || next > UPPER_CODE_POINT) {
+            break;
+          }
+          i += Character.charCount(next);
+        }
+        normalized.append(' ');
+      }
+      else {
+        normalized.appendCodePoint(cp);
+        i += Character.charCount(cp);
+      }
+    }
+    return normalized.toString();
   }
 }
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGeneratorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGeneratorTest.java
new file mode 100644
index 0000000000..ea0743a91d
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGeneratorTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.lemmatizer;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+public class DefaultLemmatizerContextGeneratorTest {
+
+  private static final DefaultLemmatizerContextGenerator GENERATOR =
+      new DefaultLemmatizerContextGenerator();
+
+  @ParameterizedTest
+  @CsvSource({
+      "Token9, true, true",
+      "token, false, false",
+      "TOKEN, true, false",
+      "1990, false, true",
+      // accented capitals and non-ASCII digits are not matched
+      "Étudiant, false, false",
+      "x٥, false, false",
+      "well-known, false, false"})
+  void testCapitalAndDigitFeatures(String token, boolean capital, boolean digit) {
+    List<String> features = Arrays.asList(GENERATOR.getContext(0,
+        new String[] {token}, new String[] {"tag"}, null));
+    Assertions.assertEquals(capital, features.contains("c"), "capital feature of " + token);
+    Assertions.assertEquals(digit, features.contains("d"), "digit feature of " + token);
+  }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/BioCodecTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/BioCodecTest.java
index 55334fd190..641e785881 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/BioCodecTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/BioCodecTest.java
@@ -23,6 +23,9 @@
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
 
 import opennlp.tools.util.Span;
 
@@ -262,4 +265,17 @@ void testCompatibilityRepeated() {
         new String[] {A_START, A_START, A_CONTINUE, A_CONTINUE, B_START, B_START, OTHER, OTHER}));
   }
 
+  @ParameterizedTest
+  @CsvSource({"atype-start, atype", "a-b-start, a-b", "type_1-cont, type_1", "Type9-X_1, Type9"})
+  void testExtractNameType(String outcome, String type) {
+    Assertions.assertEquals(type, BioCodec.extractNameType(outcome));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"start", "other", "-start", "atype-", "atype-st.art", "atype-st art",
+      "atype-stärt"})
+  void testExtractNameTypeWithoutType(String outcome) {
+    Assertions.assertNull(BioCodec.extractNameType(outcome));
+  }
+
 }
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
index 3c8c8e6bc1..7266f3008f 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
@@ -22,6 +22,9 @@
 import java.util.Collections;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
 
 import opennlp.tools.util.MockInputStreamFactory;
 import opennlp.tools.util.ObjectStream;
@@ -32,6 +35,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
@@ -325,4 +329,16 @@ void testNameFinderWithMultipleTypes() throws Exception {
     assertEquals("organization", names2[1].getType());
   }
 
+  @ParameterizedTest
+  @CsvSource({"atype-start, atype", "a-b-start, a-b", "type_1-cont, type_1"})
+  void testExtractNameType(String outcome, String type) {
+    assertEquals(type, NameFinderME.extractNameType(outcome));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"start", "other", "-start", "atype-", "atype-st.art"})
+  void testExtractNameTypeWithoutType(String outcome) {
+    assertNull(NameFinderME.extractNameType(outcome));
+  }
+
 }
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/DefaultPOSContextGeneratorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/DefaultPOSContextGeneratorTest.java
index 9e84538b3e..3f728182b2 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/DefaultPOSContextGeneratorTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/postag/DefaultPOSContextGeneratorTest.java
@@ -102,6 +102,23 @@ void noDictionaryMatch() {
         + Arrays.toString(actual));
   }
 
+  @Test
+  void capitalAndDigitFeatures() {
+    DefaultPOSContextGenerator generator = new DefaultPOSContextGenerator(null);
+
+    // accept sides: ASCII capital letter and ASCII digit
+    final String[] withCapAndNum = generator.getContext(0,
+        new Object[] {"Token9"}, new String[] {"tag"});
+    Assertions.assertTrue(Arrays.asList(withCapAndNum).contains("c"));
+    Assertions.assertTrue(Arrays.asList(withCapAndNum).contains("d"));
+
+    // reject sides: accented capitals and non-ASCII digits are not matched
+    final String[] accented = generator.getContext(0,
+        new Object[] {"Étudiant٥"}, new String[] {"tag"});
+    Assertions.assertFalse(Arrays.asList(accented).contains("c"));
+    Assertions.assertFalse(Arrays.asList(accented).contains("d"));
+  }
+
   @Test
   void dictionaryMatch() {
     int indexWithDictionaryMatch = 2;
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/AlphaNumericCheckTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/AlphaNumericCheckTest.java
new file mode 100644
index 0000000000..d1b2f5e68e
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/AlphaNumericCheckTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.tokenize;
+
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.tokenize.lang.Factory;
+
+public class AlphaNumericCheckTest {
+
+  private static final List<String> LANGUAGES =
+      List.of("en", "es", "it", "pt", "ca", "pl", "de", "fr", "nl", "xx");
+
+  private static final List<String> TOKENS = List.of(
+      "", "a", "Z", "0", "abc123", "Straße", "Café", "señor", "łódź", "ijs", "Ÿ", "-", "a-b",
+      "a b", "a\nb", "\n", "abc\n", "ñ", "Ç", "ß", "é", " ", "١٢٣", "Ａ", "𐐒", "😀a",
+      "aé", "AB.", "x_y", "[", "]", "\\", "^", "&", "$");
+
+  private static Stream<Arguments> builtInPatternsAndTokens() {
+    return LANGUAGES.stream().flatMap(language -> {
+      Pattern pattern = new Factory().getAlphanumeric(language);
+      return TOKENS.stream().map(token -> Arguments.of(language, pattern, token));
+    });
+  }
+
+  @ParameterizedTest(name = "{0}: \"{2}\"")
+  @MethodSource("builtInPatternsAndTokens")
+  void testBuiltInPatternsAgreeWithRegex(String language, Pattern pattern, String token) {
+    AlphaNumericCheck check = AlphaNumericCheck.of(pattern);
+    Assertions.assertTrue(check.isCharacterSet(), language + " default runs as a set lookup");
+    Assertions.assertEquals(pattern.matcher(token).matches(), check.test(token));
+  }
+
+  private static Stream<Arguments> customPatternsAndTokens() {
+    List<String> patterns = List.of(
+        "^[a-z-]+$", "^[-a-z]+$", "^[a-c1-3]+$", "^[a]+$", "^[a-zA-Z0-9_]+$",
+        "^[\\p{L}]+$", "^[^a-z]+$", "^[a-z&&[^b]]+$", "^[\\d]+$", "^[a-z]+$|^[0-9]+$",
+        "^[a-z]*$", "[a-z]+", "^(?i)[a-z]+$", "^[a-z]+\\d$", "^[ab\\]]+$", "^[a-]+$",
+        "^[😀]+$");
+    return patterns.stream().flatMap(regex -> {
+      Pattern pattern = Pattern.compile(regex);
+      return TOKENS.stream().map(token -> Arguments.of(regex, pattern, token));
+    });
+  }
+
+  @ParameterizedTest(name = "{0}: \"{2}\"")
+  @MethodSource("customPatternsAndTokens")
+  void testCustomPatternsAgreeWithRegex(String regex, Pattern pattern, String token) {
+    Assertions.assertEquals(pattern.matcher(token).matches(), AlphaNumericCheck.of(pattern).test(token));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"^[A-Za-z0-9]+$", "^[a-z-]+$", "^[-a-z]+$", "^[a-]+$", "^[a-c1-3]+$",
+      "^[a]+$", "^[a-zA-Z0-9_]+$", "^[0-9a-záãâàéêíóõôúüçA-ZÁÃÂÀÉÊÍÓÕÔÚÜÇ]+$"})
+  void testEligiblePatternsRunAsSetLookup(String regex) {
+    Assertions.assertTrue(AlphaNumericCheck.of(Pattern.compile(regex)).isCharacterSet());
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"^[\\p{L}]+$", "^[^a-z]+$", "^[a-z&&[^b]]+$", "^[\\d]+$",
+      "^[a-z]+$|^[0-9]+$", "^[a-z]*$", "[a-z]+", "^(?i)[a-z]+$", "^[a-z]+\\d$", "^[ab\\]]+$",
+      "^[😀]+$"})
+  void testOtherPatternsFallBackToRegex(String regex) {
+    Assertions.assertFalse(AlphaNumericCheck.of(Pattern.compile(regex)).isCharacterSet());
+  }
+
+  @Test
+  void testFlagsForceRegex() {
+    Pattern pattern = Pattern.compile("^[a-z]+$", Pattern.CASE_INSENSITIVE);
+    AlphaNumericCheck check = AlphaNumericCheck.of(pattern);
+    Assertions.assertFalse(check.isCharacterSet());
+    Assertions.assertTrue(check.test("ABC"));
+  }
+
+  @Test
+  void testNullPatternIsRejected() {
+    Assertions.assertThrows(IllegalArgumentException.class, () -> AlphaNumericCheck.of(null));
+  }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lang/en/TokenSampleStreamTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lang/en/TokenSampleStreamTest.java
new file mode 100644
index 0000000000..f63d1d1d3d
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lang/en/TokenSampleStreamTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.tokenize.lang.en;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.Span;
+
+public class TokenSampleStreamTest {
+
+  private static TokenSampleStream stream(String text) throws IOException {
+    return new TokenSampleStream(new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)));
+  }
+
+  @Test
+  void testReadsTokensAndSpans() throws IOException {
+    TokenSampleStream stream = stream("The dog 's -LRB- big -RRB- .\n");
+    Assertions.assertTrue(stream.hasNext());
+    TokenSample sample = stream.next();
+    Assertions.assertEquals("The dog's ( big).", sample.getText());
+    Assertions.assertArrayEquals(new Span[] {new Span(0, 3), new Span(4, 7), new Span(7, 9),
+        new Span(10, 11), new Span(12, 15), new Span(15, 16), new Span(16, 17)},
+        sample.getTokenSpans());
+    Assertions.assertFalse(stream.hasNext());
+  }
+
+  @Test
+  void testCollapsesWhitespaceRuns() throws IOException {
+    TokenSample sample = stream("a \t b\n").next();
+    Assertions.assertEquals("a b", sample.getText());
+    Assertions.assertEquals(2, sample.getTokenSpans().length);
+  }
+
+  @Test
+  void testLeadingWhitespaceYieldsEmptyFirstToken() throws IOException {
+    // a leading run gives one empty token, as String.split("\\s+") does
+    TokenSample sample = stream("  a\n").next();
+    Assertions.assertEquals("a", sample.getText());
+    Assertions.assertArrayEquals(new Span[] {new Span(0, 0), new Span(0, 1)},
+        sample.getTokenSpans());
+  }
+
+  @Test
+  void testWhitespaceOnlyLineHasNoTokens() throws IOException {
+    TokenSample sample = stream("   \n").next();
+    Assertions.assertEquals("", sample.getText());
+    Assertions.assertEquals(0, sample.getTokenSpans().length);
+  }
+
+  @Test
+  void testNonAsciiWhitespaceIsNotASeparator() throws IOException {
+    // no-break space is not in the ASCII whitespace set
+    TokenSample sample = stream("a b c\n").next();
+    Assertions.assertEquals(2, sample.getTokenSpans().length);
+    Assertions.assertEquals("a b", sample.getText().substring(0, 3));
+  }
+
+  @Test
+  void testQuoteAndPunctuationAttachment() throws IOException {
+    // a token without an ASCII letter or digit attaches to the previous token
+    TokenSample sample = stream("Hello , world !\n").next();
+    Assertions.assertEquals("Hello, world!", sample.getText());
+    Assertions.assertArrayEquals(new Span[] {new Span(0, 5), new Span(5, 6), new Span(7, 12),
+        new Span(12, 13)}, sample.getTokenSpans());
+
+    // a token holding a digit is a word and gets a space in front
+    sample = stream("Room 101 .\n").next();
+    Assertions.assertEquals("Room 101.", sample.getText());
+  }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/BrownClusterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/BrownClusterTest.java
new file mode 100644
index 0000000000..4fdb46295f
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/BrownClusterTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util.featuregen;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class BrownClusterTest {
+
+  private static BrownCluster cluster(String lexicon) throws IOException {
+    return new BrownCluster(new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8)));
+  }
+
+  @Test
+  void testThreeColumnLinesNeedFrequencyAboveFive() throws IOException {
+    BrownCluster cluster = cluster("0101\tthe\t10\n0110\trare\t5\n");
+    Assertions.assertEquals("0101", cluster.lookupToken("the"));
+    Assertions.assertNull(cluster.lookupToken("rare"));
+  }
+
+  @Test
+  void testTwoColumnLinesMapFirstFieldToSecond() throws IOException {
+    BrownCluster cluster = cluster("dog\t0111\n");
+    Assertions.assertEquals("0111", cluster.lookupToken("dog"));
+  }
+
+  @Test
+  void testTrailingTabsAreDropped() throws IOException {
+    // trailing empty fields do not count, as with String.split("\t")
+    BrownCluster cluster = cluster("cat\t0100\t\t\n0011\tbird\t7\t\n");
+    Assertions.assertEquals("0100", cluster.lookupToken("cat"));
+    Assertions.assertEquals("0011", cluster.lookupToken("bird"));
+  }
+
+  @Test
+  void testLinesWithOtherFieldCountsAreIgnored() throws IOException {
+    BrownCluster cluster = cluster("\n\t\nsingle\na\tb\tc\td\n0101\tfish\t10\n");
+    Assertions.assertNull(cluster.lookupToken("single"));
+    Assertions.assertNull(cluster.lookupToken("a"));
+    Assertions.assertNull(cluster.lookupToken(""));
+    Assertions.assertEquals("0101", cluster.lookupToken("fish"));
+  }
+
+  @Test
+  void testSpacesAreNotSeparators() throws IOException {
+    BrownCluster cluster = cluster("0101 the 10\n");
+    Assertions.assertNull(cluster.lookupToken("the"));
+  }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java
index cd35f092ad..6aa668c807 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java
@@ -19,6 +19,8 @@
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
 
 public class FeatureGeneratorUtilTest {
 
@@ -70,6 +72,14 @@ void testGerman() {
     Assertions.assertEquals("sc", FeatureGeneratorUtil.tokenFeature("Ü"));
   }
 
+  @ParameterizedTest
+  @CsvSource({"A., cp", "Z., cp", "Ä., cp", "Ö., cp", "Ü., cp",
+      // lower case initial, other capital, other second character, longer tokens
+      "a., other", "É., ic", "'A,', ic", "Ab., ic", "AB., ic"})
+  void testCapPeriod(String token, String feature) {
+    Assertions.assertEquals(feature, FeatureGeneratorUtil.tokenFeature(token));
+  }
+
   @Test
   void testJapanese() {
     // Hiragana
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/TokenPatternFeatureGeneratorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/TokenPatternFeatureGeneratorTest.java
index 28e8e8c9d1..47c96696a2 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/TokenPatternFeatureGeneratorTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/TokenPatternFeatureGeneratorTest.java
@@ -71,4 +71,19 @@ void testSentence() {
     Assertions.assertEquals("st=sentence", features.get(12));
     Assertions.assertEquals("pta=iclclclclc", features.get(13));
   }
+
+  @Test
+  void testSkipsNonLetterSubTokens() {
+
+    String[] testSentence = new String[] {"well-known"};
+    final int testTokenIndex = 0;
+
+    AdaptiveFeatureGenerator generator = new TokenPatternFeatureGenerator();
+
+    generator.createFeatures(features, testSentence, testTokenIndex, null);
+    // the hyphen sub-token must not produce an "st=" feature
+    Assertions.assertFalse(features.contains("st=-"));
+    Assertions.assertTrue(features.contains("st=well"));
+    Assertions.assertTrue(features.contains("st=known"));
+  }
 }
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java
index ac04d7d4ef..0365d00303 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java
@@ -21,6 +21,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 
@@ -73,6 +74,37 @@ void testNullUrl() {
     );
   }
 
+  private static Stream<Arguments> indexPages() {
+    return Stream.of(
+        Arguments.of("<a href=\"a.bin\">x</a><a href=\"b.bin\">y</a>", List.of("a.bin", "b.bin")),
+        // tag and attribute names ignore case
+        Arguments.of("<A HREF=\"c.bin\">z</A>", List.of("c.bin")),
+        Arguments.of("<a href=\"a.bin\">x</a><A hRef=\"b.bin\">y</A>", List.of("a.bin", "b.bin")),
+        // an anchor without closing tag is skipped, an earlier one is still found
+        Arguments.of("<a href=\"d.bin\">x", List.of()),
+        Arguments.of("<a href=\"a.bin\">x</a><a href=\"d.bin\">y", List.of("a.bin")),
+        // the first "</a>" closes the link, so nested anchor markup is swallowed
+        Arguments.of("<a href=\"d.bin\">x <a href=\"e.bin\">y</a>", List.of("d.bin")),
+        // the href value ends at the first "\">", so it may hold other markup
+        Arguments.of("<a href=\"f<b>g.bin\">f</a>", List.of("f<b>g.bin")),
+        // values and link text may span lines
+        Arguments.of("<a href=\"a.bin\">x</a> <a href=\"h\ni.bin\">y\nz</a>", List.of("a.bin", "h\ni.bin")),
+        Arguments.of("no links here", List.of()),
+        Arguments.of("", List.of()),
+        // an empty href value is kept, a single-quoted one is not an anchor
+        Arguments.of("<a href=\"\"></a>", List.of("")),
+        Arguments.of("<a href='a.bin'>x</a>", List.of()),
+        // the closing tag is matched case-insensitively and only as "</a>"
+        Arguments.of("<a href=\"a.bin\">x</ A><a href=\"b.bin\">y</A >z</a>", List.of("a.bin")),
+        Arguments.of("<a href=\"\uD83D\uDE00.bin\">x</a>", List.of("\uD83D\uDE00.bin")));
+  }
+
+  @ParameterizedTest
+  @MethodSource("indexPages")
+  void testExtractLinks(String page, List<String> expected) {
+    assertEquals(expected, DownloadUtil.DownloadParser.extractLinks(page));
+  }
+
   @Test
   void testInvalidUrl() {
     try {
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
index f92ada9c99..299487596c 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
@@ -36,6 +36,7 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -65,6 +66,31 @@ public void testDownloadModelByURL(String language, URL url) throws IOException
     assertTrue(model.isLoadedFromSerialized());
   }
 
+  @ParameterizedTest
+  @MethodSource("checksumFiles")
+  void testParseChecksum(String content, String expected) {
+    assertEquals(expected, DownloadUtil.parseChecksum(content));
+  }
+
+  private static Stream<Arguments> checksumFiles() {
+    return Stream.of(
+        Arguments.of("abc123  model.bin", "abc123"),
+        Arguments.of("abc123\tmodel.bin\n", "abc123"),
+        Arguments.of("  abc123 model.bin", "abc123"),
+        Arguments.of("abc123", "abc123"),
+        Arguments.of("abc123 *model.bin\r\n", "abc123"),
+        Arguments.of("abc123 model.bin\ndef456 other.bin\n", "abc123"),
+        Arguments.of("abc123\u000Bmodel.bin", "abc123"),
+        Arguments.of("abc123\u00A0model.bin", "abc123\u00A0model.bin"));
+  }
+
+  @ParameterizedTest
+  @NullAndEmptySource
+  @ValueSource(strings = {" ", "\t\n"})
+  void testParseChecksumOfBlankContent(String content) {
+    assertNull(DownloadUtil.parseChecksum(content));
+  }
+
   @Test
   @EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
   public void testExistsModel() throws IOException {
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java
index 1951c114a0..353ecfbd8c 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java
@@ -28,6 +28,7 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
 
@@ -185,6 +186,95 @@ void testIsUnicodeWhitespaceDelegates() {
     Assertions.assertFalse(StringUtil.isUnicodeWhitespace(0x200B));
   }
 
+  // -------------------------------------------------------------------------
+  // isAsciiWhitespace
+  // -------------------------------------------------------------------------
+
+  @ParameterizedTest
+  @ValueSource(chars = {' ', '\t', '\n', '\u000B', '\f', '\r'})
+  void testIsAsciiWhitespaceAccepts(char c) {
+    Assertions.assertTrue(StringUtil.isAsciiWhitespace(c));
+  }
+
+  @ParameterizedTest
+  @ValueSource(chars = {'a', '0', '_', '\u0000', '\u001C', '\u0085', '\u00A0', '\u2003', '\u3000'})
+  void testIsAsciiWhitespaceRejects(char c) {
+    Assertions.assertFalse(StringUtil.isAsciiWhitespace(c));
+  }
+
+  // -------------------------------------------------------------------------
+  // splitOnAsciiWhitespace
+  // -------------------------------------------------------------------------
+
+  @Test
+  void testSplitOnAsciiWhitespaceNullThrows() {
+    Assertions.assertThrows(IllegalArgumentException.class,
+        () -> StringUtil.splitOnAsciiWhitespace(null));
+  }
+
+  private static Stream<Arguments> asciiSplits() {
+    return Stream.of(
+        Arguments.of("", new String[] {""}),
+        Arguments.of("   ", new String[0]),
+        Arguments.of("\t\n\r", new String[0]),
+        Arguments.of("a", new String[] {"a"}),
+        Arguments.of("a b", new String[] {"a", "b"}),
+        Arguments.of("hello   world", new String[] {"hello", "world"}),
+        Arguments.of("a\t\u000B\fb", new String[] {"a", "b"}),
+        Arguments.of("  a", new String[] {"", "a"}),
+        Arguments.of("a  ", new String[] {"a"}),
+        Arguments.of("  a\tb  ", new String[] {"", "a", "b"}),
+        Arguments.of("a\u00A0b", new String[] {"a\u00A0b"}),
+        Arguments.of("a\u2003b c", new String[] {"a\u2003b", "c"}),
+        Arguments.of("\uD801\uDC12 \uD83D\uDE00", new String[] {"\uD801\uDC12", "\uD83D\uDE00"}),
+        Arguments.of(" \r\n ", new String[0]));
+  }
+
+  @ParameterizedTest
+  @MethodSource("asciiSplits")
+  void testSplitOnAsciiWhitespaceMatchesStringSplit(String input, String[] expected) {
+    Assertions.assertArrayEquals(expected, StringUtil.splitOnAsciiWhitespace(input));
+    Assertions.assertArrayEquals(input.split("\\s+"), StringUtil.splitOnAsciiWhitespace(input));
+  }
+
+  // -------------------------------------------------------------------------
+  // containsAsciiUpperCase, containsAsciiDigit
+  // -------------------------------------------------------------------------
+
+  @Test
+  void testContainsAsciiNullThrows() {
+    Assertions.assertThrows(IllegalArgumentException.class,
+        () -> StringUtil.containsAsciiUpperCase(null));
+    Assertions.assertThrows(IllegalArgumentException.class,
+        () -> StringUtil.containsAsciiDigit(null));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"A", "Z", "aBc", "token-X", "1A", "Q\uD801\uDC12"})
+  void testContainsAsciiUpperCaseAccepts(String input) {
+    Assertions.assertTrue(StringUtil.containsAsciiUpperCase(input));
+    Assertions.assertTrue(StringUtil.containsAsciiUpperCase(new StringBuilder(input)));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"", "abc", "123", "Étudiant", "ÄÖÜ", "\uFF21", "\uD801\uDC12", "_-."})
+  void testContainsAsciiUpperCaseRejects(String input) {
+    Assertions.assertFalse(StringUtil.containsAsciiUpperCase(input));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"0", "9", "a1b", "x-2", "9A", "\uD801\uDC120"})
+  void testContainsAsciiDigitAccepts(String input) {
+    Assertions.assertTrue(StringUtil.containsAsciiDigit(input));
+    Assertions.assertTrue(StringUtil.containsAsciiDigit(new StringBuilder(input)));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"", "abc", "ABC", "\u0661", "\uFF11", "\u00BD", "\uD835\uDFCE", "_-."})
+  void testContainsAsciiDigitRejects(String input) {
+    Assertions.assertFalse(StringUtil.containsAsciiDigit(input));
+  }
+
   // -------------------------------------------------------------------------
   // splitOnUnicodeWhitespace
   // -------------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java
index f63d2d1f9e..b965cf1536 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizerTest.java
@@ -40,4 +40,29 @@ void normalizeEmoji() {
         "Any funny text goes here    ", normalizer.normalize(s));
   }
 
+  @Test
+  void normalizeUnpairedSurrogates() {
+    // a lone high surrogate and a lone low surrogate are matched individually
+    String s = "a" + '\uD83C' + "b" + '\uDC00' + "c";
+    Assertions.assertEquals("a b c", normalizer.normalize(s));
+
+    // adjacent surrogates, paired or not, collapse into a single space
+    StringBuilder sb = new StringBuilder();
+    sb.append("x").append('\uD83C').append('\uDC00').append("y");
+    Assertions.assertEquals("x y", normalizer.normalize(sb));
+  }
+
+  @Test
+  void normalizeMatchesCodePointsNotOnlyEmoji() {
+    // the matched code point range is [U+D83C, U+10FC00], so BMP characters
+    // from U+D83C up are replaced as well
+    Assertions.assertEquals("a b", normalizer.normalize("a" + '\uE000' + "b"));
+
+    // supplementary code points beyond U+10FC00 are kept verbatim
+    StringBuilder sb = new StringBuilder();
+    sb.append("a").appendCodePoint(0x10FFFF).append("b");
+    Assertions.assertEquals("a" + new String(Character.toChars(0x10FFFF)) + "b",
+        normalizer.normalize(sb));
+  }
+
 }