OPENNLP-1880: Add a Lexical knowledge base seam with WN-LMF and WNDB readers and a Morphy lemmatizer

dev/test-omw-wordnets.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+readonly RELEASE_URL="https://github.com/omwn/omw-data/releases/download/v2.0"
+readonly IT_SHA512="d0ed09eaa6617509a7c8a1162e92d8085570328f5f773a108a168d7c517b02c8a1a79fe81297b7e94722b480b22f44f34419c46675f0c3e3af253504c2c5b380"
+readonly ES_SHA512="86851763f10cf9ba1c5ea42e8c09bcbff7954aea22c62fbd229bcf546f236de4d67251582ecfd7956e8dae22e975fdf1d5bfa3fd7e2fbf4d901751c89bc0ca66"
+readonly SV_SHA512="897a79c6a6ec43c10024c6ee55aac886bd27182e28127ef1248cc8748c1189e573b05b0b2cd9ada91756f0527b25cbac9a49b88a1f13a897f5442da7f0656c13"
+
+fixture_dir=$(mktemp -d "${TMPDIR:-/tmp}/opennlp-omw.XXXXXXXX")
+trap 'rm -rf -- "$fixture_dir"' EXIT
+
+fetch() {
+  local language=$1
+  local expected=$2
+  local archive="$fixture_dir/omw-$language-2.0.tar.xz"
+  curl --fail --location --silent --show-error \
+    --connect-timeout 15 --max-time 120 \
+    --output "$archive" "$RELEASE_URL/omw-$language-2.0.tar.xz"
+  local actual
+  actual=$(sha512sum "$archive" | cut -d' ' -f1)
+  if [[ "$actual" != "$expected" ]]; then
+    echo "SHA-512 mismatch for omw-$language-2.0.tar.xz" >&2
+    exit 1
+  fi
+  tar -xJf "$archive" -C "$fixture_dir"
+}
+
+fetch it "$IT_SHA512"
+fetch es "$ES_SHA512"
+fetch sv "$SV_SHA512"
+
+repo_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+cd "$repo_dir"
+./mvnw -pl opennlp-extensions/opennlp-wordnet -am \
+  -Dopennlp.forkCount=1 -Drat.skip=true \
+  -Dtest=WnLmfOmwIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false \
+  -Dopennlp.wordnet.omwDir="$fixture_dir" test

opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.wordnet;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Lemma and synset lookup over a loaded lexical-semantic resource in the WordNet family. Synset
+ * identifiers are opaque and source-qualified (see {@link Synset#id()}). Lookups return matches
+ * in the source's sense order. Lookup results are non-{@code null} lists.
+ *
+ * <p>How a queried lemma is matched against the source's written forms is implementation
+ * specific and documented there. Returned {@link Synset#lemmas() lemmas} preserve the source's
+ * written forms, with spaces in multiword lemmas.</p>
+ *
+ * <p>Thread safety is implementation specific.</p>
+ *
+ * @since 3.0.0
+ */
+public interface LexicalKnowledgeBase {
+
+  /**
+   * Finds the synsets containing a lemma with a part of speech, in the source's sense order
+   * (the most salient sense first when the source ranks senses).
+   *
+   * @param lemma The lemma to look up. Must not be {@code null}.
+   * @param pos   The part of speech to look it up as. Must not be {@code null}.
+   * @return The matching synsets, never {@code null}; empty when the lexicon does not contain
+   *     the lemma with that part of speech.
+   * @throws IllegalArgumentException Thrown if {@code lemma} or {@code pos} is {@code null}.
+   */
+  List<Synset> lookup(String lemma, WordNetPOS pos);
+
+  /**
+   * Finds a synset by its opaque identifier.
+   *
+   * @param synsetId A synset identifier returned by this lexicon. Must not be {@code null}.
+   * @return The synset, or empty when this lexicon has no synset with that identifier.
+   * @throws IllegalArgumentException Thrown if {@code synsetId} is {@code null}.
+   */
+  Optional<Synset> synset(String synsetId);
+
+  /**
+   * Navigates one typed relation from a synset.
+   *
+   * @param synsetId The source synset identifier. Must not be {@code null}.
+   * @param relation The relation type to follow. Must not be {@code null}.
+   * @return The target synset ids in source order, never {@code null}; empty when the synset is
+   *     unknown or has no relation of that type.
+   * @throws IllegalArgumentException Thrown if {@code synsetId} or {@code relation} is
+   *     {@code null}.
+   */
+  default List<String> related(String synsetId, WordNetRelation relation) {
+    if (synsetId == null) {
+      throw new IllegalArgumentException("synsetId must not be null");
+    }
+    if (relation == null) {
+      throw new IllegalArgumentException("relation must not be null");
+    }
+    return synset(synsetId).map(s -> s.related(relation)).orElse(List.of());
+  }
+
+  /**
+   * Tests whether the lexicon contains a lemma with a part of speech. The default implementation
+   * delegates to {@link #lookup(String, WordNetPOS)}.
+   *
+   * @param lemma The lemma to test. Must not be {@code null}.
+   * @param pos   The part of speech to test it as. Must not be {@code null}.
+   * @return {@code true} if the lexicon contains the lemma with that part of speech.
+   * @throws IllegalArgumentException Thrown if {@code lemma} or {@code pos} is {@code null}.
+   */
+  default boolean contains(String lemma, WordNetPOS pos) {
+    if (lemma == null) {
+      throw new IllegalArgumentException("lemma must not be null");
+    }
+    if (pos == null) {
+      throw new IllegalArgumentException("pos must not be null");
+    }
+    return !lookup(lemma, pos).isEmpty();
+  }
+}

opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java

@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.wordnet;
+
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+
+import opennlp.tools.commons.ThreadSafe;
+
+/**
+ * One synonym set: a single lexicalized concept with its member lemmas, gloss, and typed
+ * relations to other synsets.
+ *
+ * <p>The {@link #id() id} is an opaque, source-qualified string created by the reader that
+ * produced the synset; consumers must not parse it, only pass it back to
+ * {@link LexicalKnowledgeBase#synset(String)} and compare it for equality. Relations map each
+ * {@link WordNetRelation} present on this synset to the target synset ids in source order.</p>
+ *
+ * <p>Instances are immutable and thread-safe: the list and map components are defensively
+ * copied to immutable views at construction.</p>
+ *
+ * @param id        The opaque, source-qualified synset identifier. Must not be {@code null} or
+ *                  empty.
+ * @param pos       The part of speech. Must not be {@code null}.
+ * @param lemmas    The member lemmas in source order, human-readable (multiword lemmas use
+ *                  spaces, not the underscores some formats store). Must not be {@code null} and
+ *                  must not contain {@code null} or empty elements. May be empty for an
+ *                  unlexicalized concept retained as a relation target.
+ * @param gloss     The definition text, possibly empty when the source has none. Must not be
+ *                  {@code null}.
+ * @param relations The typed relations, each mapping to the target synset ids in source order.
+ *                  Must not be {@code null}; keys must not be {@code null}; each value must be
+ *                  a non-empty list of non-{@code null}, non-empty target ids.
+ * @since 3.0.0
+ */
+@ThreadSafe
+public record Synset(
+    String id,
+    WordNetPOS pos,
+    List<String> lemmas,
+    String gloss,
+    Map<WordNetRelation, List<String>> relations) {
+
+  private static final String RELATION_PREFIX = "Relation ";
+
+  /**
+   * Creates a synset.
+   *
+   * @throws IllegalArgumentException Thrown if any component violates its documented constraint.
+   */
+  public Synset {
+    if (id == null || id.isEmpty()) {
+      throw new IllegalArgumentException("id must not be null or empty");
+    }
+    if (pos == null) {
+      throw new IllegalArgumentException("pos must not be null");
+    }
+    if (lemmas == null) {
+      throw new IllegalArgumentException("lemmas must not be null for synset " + id);
+    }
+    for (final String lemma : lemmas) {
+      if (lemma == null || lemma.isEmpty()) {
+        throw new IllegalArgumentException(
+            "lemmas must not contain a null or empty element for synset " + id);
+      }
+    }
+    if (gloss == null) {
+      throw new IllegalArgumentException("gloss must not be null for synset " + id);
+    }
+    if (relations == null) {
+      throw new IllegalArgumentException("relations must not be null for synset " + id);
+    }
+    final Map<WordNetRelation, List<String>> copiedRelations =
+        new EnumMap<>(WordNetRelation.class);
+    for (final Map.Entry<WordNetRelation, List<String>> relation : relations.entrySet()) {
+      if (relation.getKey() == null) {
+        throw new IllegalArgumentException("relations must not contain a null key for synset " + id);
+      }
+      final List<String> targets = relation.getValue();
+      if (targets == null || targets.isEmpty()) {
+        throw new IllegalArgumentException(RELATION_PREFIX + relation.getKey()
+            + " must map to a non-empty target list for synset " + id);
+      }
+      for (final String target : targets) {
+        if (target == null || target.isEmpty()) {
+          throw new IllegalArgumentException(RELATION_PREFIX + relation.getKey()
+              + " must not contain a null or empty target id for synset " + id);
+        }
+      }
+      copiedRelations.put(relation.getKey(), List.copyOf(targets));
+    }
+    lemmas = List.copyOf(lemmas);
+    relations = Collections.unmodifiableMap(copiedRelations);
+  }
+
+  /**
+   * Finds the target synset ids of one relation type.
+   *
+   * @param relation The relation type. Must not be {@code null}.
+   * @return The target synset ids in source order, never {@code null}; empty when this synset
+   *     has no relation of that type.
+   * @throws IllegalArgumentException Thrown if {@code relation} is {@code null}.
+   */
+  public List<String> related(WordNetRelation relation) {
+    if (relation == null) {
+      throw new IllegalArgumentException("relation must not be null");
+    }
+    final List<String> targets = relations.get(relation);
+    return targets == null ? List.of() : targets;
+  }
+}

opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPOS.java

@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.wordnet;
+
+/**
+ * The four parts of speech a wordnet-style lexicon distinguishes.
+ *
+ * <p>The enum does not expose the single-letter codes used by file formats. Each reader maps its
+ * format's codes to these values. Adjective satellites normalize to
+ * {@link #ADJECTIVE}, with the cluster structure preserved through
+ * {@link WordNetRelation#SIMILAR_TO}.</p>
+ *
+ * @since 3.0.0
+ */
+public enum WordNetPOS {
+
+  /** Nouns. */
+  NOUN,
+
+  /** Verbs. */
+  VERB,
+
+  /** Adjectives, including adjective satellites. */
+  ADJECTIVE,
+
+  /** Adverbs. */
+  ADVERB
+}