OPENNLP-1885: Add subword API and WordPiece encoder, remove BertTokenizer

opennlp-api/src/main/java/opennlp/tools/tokenize/BertNormalization.java

@@ -19,11 +19,14 @@
 
 /**
  * Character classifications and text transforms of the reference BERT
- * {@code BasicTokenizer}, shared by {@link BertTokenizer} and
+ * {@code BasicTokenizer}, shared by {@link WordpieceEncoder} and
  * {@link WordpieceTokenizer}.
  */
 final class BertNormalization {
 
+  /** Default maximum word length used by BERT wordpiece tokenizers. */
+  static final int DEFAULT_MAX_WORD_CODE_POINTS = 100;
+
   private BertNormalization() {
   }
 
@@ -70,6 +73,12 @@ static boolean isWhitespace(int codePoint) {
     return Character.getType(codePoint) == Character.SPACE_SEPARATOR;
   }
 
+  /** Tests whether a code point is a Unicode line or paragraph separator. */
+  static boolean isLineOrParagraphSeparator(int codePoint) {
+    final int type = Character.getType(codePoint);
+    return type == Character.LINE_SEPARATOR || type == Character.PARAGRAPH_SEPARATOR;
+  }
+
   /**
    * A punctuation character in the BERT sense: any non-alphanumeric ASCII
    * character that is not whitespace, or any Unicode punctuation category.

opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java

@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.tokenize;
+
+/**
+ * One subword unit produced by a {@link SubwordTokenizer}, including the model representation
+ * and source range.
+ *
+ * <p>The piece string is in the tokenizer's normalized form and need not equal the input.
+ * {@code start} and {@code end} are UTF-16 offsets into the original text, so the
+ * surface associated with this piece is {@code text.subSequence(start, end)}. A span can include
+ * adjacent source characters when normalization reorders characters. Pieces without source text,
+ * such as control symbols, report an empty span with {@code start == end}.</p>
+ *
+ * @param piece The piece in the vocabulary's normalized form; must not be {@code null} or empty.
+ * @param id    The non-negative vocabulary id of the piece.
+ * @param start The inclusive start offset in the original text.
+ * @param end   The exclusive end offset in the original text; at least {@code start}.
+ * @since 3.0.0
+ */
+public record SubwordPiece(String piece, int id, int start, int end) {
+
+  /**
+   * Instantiates a {@link SubwordPiece}.
+   *
+   * @throws IllegalArgumentException Thrown if {@code piece} is {@code null} or empty,
+   *     {@code id} is negative, or the span is negative or inverted.
+   */
+  public SubwordPiece {
+    if (piece == null) {
+      throw new IllegalArgumentException("piece must not be null");
+    }
+    if (piece.isEmpty()) {
+      throw new IllegalArgumentException("piece must not be empty");
+    }
+    if (id < 0) {
+      throw new IllegalArgumentException("id must not be negative");
+    }
+    if (start < 0) {
+      throw new IllegalArgumentException("start must not be negative");
+    }
+    if (end < start) {
+      throw new IllegalArgumentException("end must be at least start");
+    }
+  }
+
+}

opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.tokenize;
+
+import java.util.List;
+
+/**
+ * Splits text into subword units from a fixed model vocabulary, reporting the model id and
+ * original-text span for each unit.
+ *
+ * <p>Segmentation follows model entries, not linguistic token boundaries. Each piece is in the
+ * model's normalized form and need not equal the input. Offsets in each {@link SubwordPiece}
+ * refer to the original input text.</p>
+ *
+ * <p>An implementation may include model control pieces with empty source spans. Their presence
+ * and placement are part of that tokenizer's contract, not this interface.</p>
+ *
+ * <p>Thread safety is implementation specific.</p>
+ *
+ * @since 3.0.0
+ */
+public interface SubwordTokenizer {
+
+  /**
+   * Encodes text into subword pieces.
+   *
+   * @param text The text to encode; must not be {@code null}.
+   * @return The pieces in model order; may be empty.
+   * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+   */
+  List<SubwordPiece> encode(CharSequence text);
+
+  /**
+   * Encodes text into vocabulary ids.
+   *
+   * @param text The text to encode; must not be {@code null}.
+   * @return The ids from {@link #encode(CharSequence)}, in the same order.
+   * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+   */
+  default int[] encodeToIds(CharSequence text) {
+    final List<SubwordPiece> pieces = encode(text);
+    final int[] ids = new int[pieces.size()];
+    for (int i = 0; i < ids.length; i++) {
+      ids[i] = pieces.get(i).id();
+    }
+    return ids;
+  }
+
+  /**
+   * Encodes text into piece strings in the vocabulary's normalized form.
+   *
+   * @param text The text to encode; must not be {@code null}.
+   * @return The piece strings from {@link #encode(CharSequence)}, in the same order.
+   * @throws IllegalArgumentException Thrown if {@code text} is {@code null}.
+   */
+  default String[] encodeToPieces(CharSequence text) {
+    final List<SubwordPiece> pieces = encode(text);
+    final String[] out = new String[pieces.size()];
+    for (int i = 0; i < out.length; i++) {
+      out[i] = pieces.get(i).piece();
+    }
+    return out;
+  }
+}