OPENNLP-1877: Static embedding engine for modern distilled embedding tables (opennlp-embeddings)

.github/workflows/allowlist-check.yml

@@ -32,6 +32,13 @@ on:
     paths:
       - ".github/**"
 
+concurrency:
+  # A new commit on a pull request makes the run in flight obsolete, so cancel it
+  # instead of paying for a result nobody will read. Pushes to a branch group by ref
+  # and are never cancelled, so every commit on main keeps a full build.
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 permissions:
   contents: read
 

.github/workflows/license.yml

@@ -28,7 +28,7 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up JDK 21
-        uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
+        uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
         with:
           java-version: '21'
           distribution: 'temurin'

.github/workflows/maven.yml

@@ -24,6 +24,13 @@ on:
     branches:
       - main
 
+concurrency:
+  # A new commit on a pull request makes the run in flight obsolete, so cancel it
+  # instead of paying for a result nobody will read. Pushes to a branch group by ref
+  # and are never cancelled, so every commit on main keeps a full build.
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   build:
     runs-on: ${{ matrix.os }}
@@ -49,7 +56,7 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-maven-
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
+      uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
       with:
         distribution: temurin
         java-version: ${{ matrix.java }}
@@ -89,7 +96,7 @@ jobs:
           sudo apt-get install -y bats
 
       - name: Set up JDK 21
-        uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
+        uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
         with:
           distribution: temurin
           java-version: 21
@@ -148,7 +155,7 @@ jobs:
           brew install bats-core
 
       - name: Set up JDK 21
-        uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
+        uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
         with:
           distribution: temurin
           java-version: 21
@@ -199,7 +206,7 @@ jobs:
         shell: pwsh
 
       - name: Set up JDK 21
-        uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
+        uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
         with:
           distribution: temurin
           java-version: 21

.github/workflows/publish-snapshots.yml

@@ -41,7 +41,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
       - name: Setup Java
-        uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
+        uses: actions/setup-java@dd06d9cba3e5552c54d9f8ea23572deb30010f7c # v6.0.0
         with:
           distribution: temurin
           java-version: 21

.mvn/wrapper/maven-wrapper.properties

@@ -1,19 +1,3 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-wrapperVersion=3.3.2
+wrapperVersion=3.3.4
 distributionType=only-script
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.6/apache-maven-3.9.6-bin.zip
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.16/apache-maven-3.9.16-bin.zip

NOTICE

@@ -162,10 +162,10 @@ List of third-party dependencies grouped by their license type.
 
     MIT License
 
-        * onnx-runtime (com.microsoft.onnxruntime:onnxruntime_gpu:1.28.0 - https://microsoft.github.io/onnxruntime/)
-        * onnx-runtime (com.microsoft.onnxruntime:onnxruntime:1.28.0 - https://microsoft.github.io/onnxruntime/)
+        * onnx-runtime (com.microsoft.onnxruntime:onnxruntime_gpu:1.29.0 - https://microsoft.github.io/onnxruntime/)
+        * onnx-runtime (com.microsoft.onnxruntime:onnxruntime:1.29.0 - https://microsoft.github.io/onnxruntime/)
         * SLF4J API Module (org.slf4j:slf4j-api:2.0.18 - http://www.slf4j.org)
 
     The MIT License (MIT)
 
-        * ClassGraph (io.github.classgraph:classgraph:4.8.186 - https://github.com/classgraph/classgraph)
+        * ClassGraph (io.github.classgraph:classgraph:4.8.194 - https://github.com/classgraph/classgraph)

SECURITY.md

@@ -148,9 +148,14 @@ JVM arguments, or the `OPENNLP_MAX_ENTRIES` setting.
 CDN (`https://dlcdn.apache.org/opennlp/`) and verifies each download against the
 published SHA-512 checksum before use. A mismatch fails the load.
 
-Note that verification happens at **download** time. A model already present in
-the local download cache is currently not re-verified when it is loaded again, so
-treat the cache directory (`~/.opennlp` by default) as trusted storage and protect
+The published checksum is stored next to the model in the download cache, and a
+model already present in that cache is re-verified against it on every load. This
+check is performed locally and does not contact the CDN again. A cache entry
+written by an OpenNLP version that predates this behaviour has no stored checksum;
+for those the published checksum is fetched once and then stored, and if it cannot
+be retrieved the model is loaded and a warning is logged.
+
+Treat the cache directory (`~/.opennlp` by default) as trusted storage and protect
 it accordingly.
 
 The base URL can be overridden with the `OPENNLP_DOWNLOAD_BASE_URL` system

dev/README-hunspell-dictionaries.md

@@ -0,0 +1,108 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+# Hunspell dictionaries for the affix stemmer
+
+The Hunspell stemmer (`opennlp.tools.stemmer.hunspell`) reads a user-supplied
+`.dic` word list and its `.aff` affix file. Apache OpenNLP bundles no dictionary
+data. The dictionary's readme states its license.
+
+## Where dictionaries come from
+
+The LibreOffice project maintains Hunspell dictionaries by language at
+`github.com/LibreOffice/dictionaries`. Each dictionary has a separate license.
+For example, SCOWL is the source for the `en_US` dictionary, with terms in
+`README_en_US.txt`. Other sources can be used when the `.aff` and `.dic` files
+follow the Hunspell format.
+
+OpenNLP does not ship a URL catalog. Applications that manage downloads can keep a
+properties file with an entry id followed by `.url`, `.sha512`, and optionally
+`.filename` keys. Use a URL for a stable release or commit.
+
+## Option A: application catalog
+
+Catalog downloads stay inactive until you set `-Dopennlp.download.remote=true`.
+
+```java
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import opennlp.tools.stemmer.hunspell.HunspellDictionaryDownload;
+import opennlp.tools.util.DictionaryCatalog;
+
+// JVM flag: -Dopennlp.download.remote=true
+try (InputStream in = Files.newInputStream(Path.of("dictionary-catalog.properties"))) {
+  DictionaryCatalog catalog = DictionaryCatalog.load(in);
+  HunspellDictionaryDownload.downloadFromCatalog(
+      catalog, "en_US", Path.of("/tmp/hunspell-en_US"));
+}
+```
+
+For `en_US`, the catalog ids are `hunspell.en_US.aff`, `hunspell.en_US.dic`, and
+optionally `hunspell.en_US.readme`. A complete catalog example lives at
+`opennlp-core/opennlp-runtime/src/test/resources/opennlp/tools/util/dictionary-catalog.properties`.
+The download test uses local file URLs to exercise this flow without network access.
+
+## Option B: your own files
+
+Fetch `.aff` / `.dic` (and the license readme) with any tool, or with
+`ResourceInstaller.install(uri, directory, sha512)`, then load them:
+
+```java
+import java.nio.file.Path;
+import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.stemmer.hunspell.HunspellDictionary;
+import opennlp.tools.stemmer.hunspell.HunspellStemmerFactory;
+
+HunspellDictionary dictionary = HunspellDictionary.load(
+    Path.of("/tmp/hunspell-en_US/en_US.aff"),
+    Path.of("/tmp/hunspell-en_US/en_US.dic"));
+HunspellStemmerFactory factory = new HunspellStemmerFactory(dictionary);
+
+Stemmer stemmer = factory.newStemmer();
+CharSequence stem = stemmer.stem("workers");
+```
+
+The result depends on the loaded dictionary. The in-tree manual example uses a
+small dictionary and checks that `workers` stems to `worker`.
+
+The dictionary is immutable and safe to share between threads. The factory creates a
+new stemmer for each call, so each thread can use its own instance. A dictionary that
+declares a non-UTF-8 encoding through the `SET` directive in its `.aff` file is decoded
+accordingly; no conversion is required.
+
+## Testing against real dictionaries
+
+The in-tree tests use project-authored fixtures only. An opt-in test class, `HunspellRealDictionaryTest`, also checks everyday morphology with the LibreOffice `en_US`, `de_DE_frami`, and `hu_HU` dictionaries. Point it at one directory containing all listed `<name>.aff` and `<name>.dic` files. A missing dictionary skips the associated test; a dictionary that cannot be loaded fails it.
+
+```
+./mvnw test -pl opennlp-core/opennlp-runtime -am \
+    -Dtest=HunspellRealDictionaryTest -Dsurefire.failIfNoSpecifiedTests=false \
+    -Dopennlp.hunspell.dict.dir=/tmp/hunspell-dicts
+```
+
+## What the engine supports
+
+The engine applies `PFX` and `SFX` rules with strip strings and character-class conditions. It supports a prefix and suffix cross-product, a double suffix sequence connected by continuation classes, identity rules in continuation paths, file-wide `FLAG` modes, file-wide `AF` aliases, and the `SET` encoding declaration. Numeric flags range from 1 through 65000.
+
+Compound decomposition supports `COMPOUNDFLAG`, `COMPOUNDBEGIN`, `COMPOUNDMIDDLE`, `COMPOUNDEND`, `COMPOUNDMIN`, `COMPOUNDWORDMAX`, `COMPOUNDPERMITFLAG`, `COMPOUNDFORBIDFLAG`, `CHECKCOMPOUNDDUP`, `CHECKCOMPOUNDCASE`, and `CHECKCOMPOUNDTRIPLE`. Compound boundaries and minimum lengths use Unicode code points. `NEEDAFFIX` (also named `PSEUDOROOT`), `ONLYINCOMPOUND`, `FORBIDDENWORD`, `CIRCUMFIX`, and `FULLSTRIP` control whether an analysis is accepted.
+
+Other directives are skipped. Their conversion, suggestion, or advanced compound behavior is not applied by this affix stemmer. Comments and unused metadata may contain legacy-encoded bytes even when the file uses UTF-8. Parsed rules and dictionary text are decoded strictly. Default and `long` flag modes preserve raw one-byte flag values used by published UTF-8 dictionaries. Invalid rule counts, aliases, flags, and compound limits fail during loading. Each affix or dictionary stream is rejected when it exceeds `HunspellDictionary.MAX_STREAM_BYTES` (64 MiB).
+
+Skipped directives include `ICONV`, `OCONV`, `COMPLEXPREFIXES`, `COMPOUNDRULE`,
+`IGNORE`, and `KEEPCASE`. Loading a dictionary does not apply these rules;
+results can differ from Hunspell for words that need them.

dev/README-mecab-dictionaries.md

@@ -0,0 +1,126 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+# CJK dictionaries for the lattice tokenizer
+
+The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and Korean over a MeCab-format dictionary, and the unigram segmenter handles Chinese over a plain word-frequency lexicon. Apache OpenNLP bundles no dictionary data: you download a dictionary from the project of your choice, and each dictionary contains its own license. Read the license file inside the archive before use.
+
+## Known MeCab-format dictionary projects
+
+| Catalog id | Dictionary | Language | Encoding |
+|---|---|---|---|
+| `mecab.ipadic` | IPADIC 2.7.0 | Japanese | EUC-JP |
+| `mecab.ko-dic` | mecab-ko-dic 2.1.1 | Korean | UTF-8 |
+
+Example download URLs and SHA-512 digests for those ids live in the test resource
+`opennlp-core/opennlp-runtime/src/test/resources/opennlp/tools/util/dictionary-catalog.properties`. Both archives are
+gzip-compressed tars; `MecabDictionaryInstaller` reads the ustar, pax, and GNU
+formats through `ResourceInstaller`.
+
+The installer extracts only the dictionary payload: the `*.csv` and `*.def` files a
+`MecabDictionary` reads, plus the `dicrc` configuration file the distributions ship
+alongside them. `ResourceInstaller` rejects paths outside the staging directory,
+then `MecabDictionaryInstaller` flattens the selected files into the target. The
+returned value is the number of dictionary files installed.
+
+## Option A: opt-in catalog install
+
+Applications supply the catalog. Catalog URLs are inactive until you set
+`-Dopennlp.download.remote=true` or the equivalent system property in code.
+
+```java
+import java.nio.file.Path;
+import opennlp.tools.tokenize.lattice.MecabDictionaryInstaller;
+import opennlp.tools.util.DictionaryCatalog;
+
+// JVM flag: -Dopennlp.download.remote=true
+DictionaryCatalog catalog = DictionaryCatalog.load(catalogProperties);
+int files = MecabDictionaryInstaller.installFromCatalog(
+    catalog, "mecab.ipadic", Path.of("ipadic"));
+```
+
+## Option B: your own URL and digest
+
+```java
+import java.net.URI;
+import java.nio.file.Path;
+import opennlp.tools.tokenize.lattice.MecabDictionaryInstaller;
+
+String expectedSha512 = "..."; // the 128-hex SHA-512 of the archive
+int files = MecabDictionaryInstaller.install(
+    URI.create("https://example.example/dict.tar.gz"),
+    Path.of("dict"),
+    expectedSha512);
+```
+
+A local `file:` URI may omit the digest:
+`MecabDictionaryInstaller.install(localArchive.toUri(), targetDirectory)`.
+HTTP and HTTPS sources require a digest. Other URI schemes are rejected.
+
+## Size budgets for larger dictionaries
+
+Fetching and unpacking go through `ResourceInstaller` and are bounded so a crafted
+archive cannot fill the disk: by default one download is capped at 1 GiB, the
+unpacked payload at 4 GiB, and the archive at 100000 entries. IPADIC and
+mecab-ko-dic fit comfortably. For larger dictionaries, such as UniDic, raise the
+limits at JVM startup:
+
+```bash
+-Dopennlp.download.max.bytes=4294967296 \
+-Dopennlp.install.max.total.bytes=8589934592 \
+-Dopennlp.install.max.entries=200000
+```
+
+Missing, invalid, and nonpositive property values use the default limits.
+
+## Load and tokenize
+
+`MecabDictionary.load(Path)` assumes UTF-8. IPADIC needs the two-argument overload:
+
+```java
+import java.nio.charset.Charset;
+import java.nio.file.Path;
+import opennlp.tools.tokenize.lattice.LatticeTokenizer;
+import opennlp.tools.tokenize.lattice.MecabDictionary;
+
+MecabDictionary dictionary =
+    MecabDictionary.load(Path.of("ipadic"), Charset.forName("EUC-JP"));
+LatticeTokenizer tokenizer = new LatticeTokenizer(dictionary);
+// "Tokyo-to ni iku" (go to the Tokyo metropolis), escaped to keep this file ASCII
+String[] tokens = tokenizer.tokenize("\u6771\u4EAC\u90FD\u306B\u884C\u304F");
+```
+
+For a UTF-8 dictionary such as mecab-ko-dic, `MecabDictionary.load(Path.of("ko-dic"))`
+is enough. Loaded dictionaries and tokenizers are immutable and safe to share between
+threads, so load once and reuse.
+
+## Chinese: the unigram segmenter needs only a frequency lexicon
+
+`opennlp.tools.tokenize.lattice.UnigramSegmenter` does not use MeCab dictionaries. It
+loads a plain text lexicon, one entry per line: the word, its count, and optionally a
+tag, separated by whitespace. Any word-frequency list you have the rights to use works:
+
+```java
+import java.nio.file.Path;
+import opennlp.tools.tokenize.lattice.UnigramSegmenter;
+
+UnigramSegmenter segmenter = UnigramSegmenter.load(Path.of("words.txt"));
+// "wo laidao Beijing Tian'anmen" (I arrive at Beijing Tiananmen), escaped as above
+String[] tokens = segmenter.tokenize("\u6211\u6765\u5230\u5317\u4EAC\u5929\u5B89\u95E8");
+```
+
+The lexicon archive includes its license; OpenNLP bundles no data.