OPENNLP-1886: UniNE light and minimal stemmer tiers with vocabulary parity fixtures

LICENSE

@@ -265,6 +265,40 @@ distributed under the BSD license:
 	OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+The following license applies to the UniNE light and minimal stemmer
+algorithms and their CLEF-derived parity fixtures in opennlp-core/opennlp-runtime,
+under opennlp.tools.stemmer.light. The English minimal and Spanish minimal
+stemmers and fixtures are excluded. The UniNE code is adapted from Apache
+Lucene, which based it on code by Jacques Savoy
+(http://members.unine.ch/jacques.savoy/clef/index.html), distributed
+under the BSD license:
+
+	Copyright (c) 2005, Jacques Savoy
+	All rights reserved.
+
+	Redistribution and use in source and binary forms, with or without
+	modification, are permitted provided that the following conditions are met:
+
+	Redistributions of source code must retain the above copyright notice, this
+	list of conditions and the following disclaimer. Redistributions in binary
+	form must reproduce the above copyright notice, this list of conditions and
+	the following disclaimer in the documentation and/or other materials
+	provided with the distribution. Neither the name of the author nor the names
+	of its contributors may be used to endorse or promote products derived from
+	this software without specific prior written permission.
+
+	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+	AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+	IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+	ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+	LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+	CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+	SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+	INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+	CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+	ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+	POSSIBILITY OF SUCH DAMAGE.
+
 The following license applies to the Wordpiece tokenizer implementation:
 
     The MIT License (MIT)

opennlp-core/opennlp-runtime/pom.xml

@@ -1,4 +1,22 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
@@ -116,6 +134,11 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-failsafe-plugin</artifactId>
+        <configuration>
+          <systemPropertyVariables>
+            <opennlp.runtimeJar>${project.build.directory}/${project.build.finalName}.jar</opennlp.runtimeJar>
+          </systemPropertyVariables>
+        </configuration>
       </plugin>
 
     </plugins>
@@ -187,4 +210,4 @@
     </profile>
   </profiles>
 
-</project>
+</project>

opennlp-core/opennlp-runtime/src/main/appended-resources/META-INF/LICENSE

@@ -0,0 +1,31 @@
+The light and minimal stemmers in opennlp.tools.stemmer.light, except
+EnglishMinimalStemmer and SpanishMinimalStemmer, include code by Jacques Savoy
+under the BSD license below. Adapted from Apache Lucene at revision
+4965e8d4d960445a0522fae512c60c6d8f11fc29:
+https://github.com/apache/lucene/tree/4965e8d4d960445a0522fae512c60c6d8f11fc29/lucene/analysis/common
+
+Copyright (c) 2005, Jacques Savoy
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer. Redistributions in binary
+form must reproduce the above copyright notice, this list of conditions and
+the following disclaimer in the documentation and/or other materials
+provided with the distribution. Neither the name of the author nor the names
+of its contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/AbstractCharArrayStemmer.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.stemmer.light;
+
+import opennlp.tools.stemmer.Stemmer;
+
+/**
+ * Base class of the light and minimal stemmers. It implements the public
+ * {@link Stemmer#stem(CharSequence)} contract once: the word is validated, copied into a
+ * {@code char[]} buffer, stemmed in place by the concrete algorithm via {@link #stem(char[], int)},
+ * and returned as a {@link String} of the resulting length.
+ */
+abstract class AbstractCharArrayStemmer implements Stemmer {
+
+  /** {@inheritDoc} */
+  @Override
+  public CharSequence stem(CharSequence word) {
+    if (word == null) {
+      throw new IllegalArgumentException("word must not be null");
+    }
+    final int length = word.length();
+    final char[] buffer = new char[length];
+    for (int i = 0; i < length; i++) {
+      buffer[i] = word.charAt(i);
+    }
+    final int stemmedLength = stem(buffer, length);
+    return new String(buffer, 0, stemmedLength);
+  }
+
+  /**
+   * Stems the buffer prefix in place.
+   *
+   * @param s   The buffer holding the word; the algorithm may overwrite characters in place.
+   * @param len The filled length of the buffer.
+   * @return The length of the stem within {@code s}.
+   */
+  abstract int stem(char[] s, int len);
+}

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/EnglishMinimalStemmer.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.stemmer.light;
+
+import opennlp.tools.commons.ThreadSafe;
+import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.stemmer.StemmerFactory;
+
+/**
+ * Minimal plural stemmer for English.
+ *
+ * <p>This stemmer implements the "S-Stemmer" from
+ * <a href="https://doi.org/10.1002/(SICI)1097-4571(199101)42:1%3C7::AID-ASI2%3E3.0.CO;2-P">
+ * <i>How Effective Is Suffixing?</i></a> by Donna Harman (JASIS 42(1), 1991).
+ *
+ * <p>Based on Apache Lucene's implementation. Instances are stateless, thread-safe, and implement
+ * {@link StemmerFactory}. Input must use lowercase NFC; the stemmer does not apply case folding or
+ * Unicode normalization.</p>
+ *
+ * @see <a href="https://github.com/apache/lucene/blob/4965e8d4d960445a0522fae512c60c6d8f11fc29/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/EnglishMinimalStemmer.java">
+ *     Apache Lucene EnglishMinimalStemmer</a>
+ * @since 3.0.0
+ */
+@ThreadSafe
+public final class EnglishMinimalStemmer extends AbstractCharArrayStemmer
+    implements StemmerFactory {
+  /** {@inheritDoc} */
+  @Override
+  public Stemmer newStemmer() {
+    return new EnglishMinimalStemmer();
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  @SuppressWarnings("fallthrough")
+  int stem(char[] s, int len) {
+    if (len < 3 || s[len - 1] != 's') return len;
+
+    switch (s[len - 2]) {
+      case 'u':
+      case 's':
+        return len;
+      case 'e':
+        if (len > 3 && s[len - 3] == 'i' && s[len - 4] != 'a' && s[len - 4] != 'e') {
+          s[len - 3] = 'y';
+          return len - 2;
+        }
+        if (s[len - 3] == 'i' || s[len - 3] == 'a' || s[len - 3] == 'o' || s[len - 3] == 'e')
+          return len; /* intentional fallthrough */
+      default:
+        return len - 1;
+    }
+  }
+}