diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java new file mode 100644 index 0000000000..1ab520f5fb --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyArc.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import opennlp.tools.util.StringUtil; + +/** + * One labeled edge of a {@link DependencyGraph}: the token at {@code dependent} is governed + * by the token at {@code head} under the given {@code relation}. + * + *

Indices are zero-based positions in the token array the graph was built over. A + * {@code head} of {@link #ROOT_HEAD} marks the dependent as the sentence root, which is + * attached to the artificial root node rather than to another token.

+ * + * @param head The zero-based index of the governing token, or {@link #ROOT_HEAD} when the + * dependent is the sentence root. + * @param dependent The zero-based index of the governed token. + * @param relation The dependency relation label, for example {@code nsubj}. + * + * @since 3.0.0 + */ +public record DependencyArc(int head, int dependent, String relation) { + + /** + * The {@code head} value marking an arc from the artificial root node. + */ + public static final int ROOT_HEAD = -1; + + /** + * Validates the arc invariants. + * + * @throws IllegalArgumentException Thrown if {@code dependent} is negative, {@code head} + * is less than {@link #ROOT_HEAD}, the arc is a self-loop, or {@code relation} + * is {@code null} or blank. + */ + public DependencyArc { + if (dependent < 0) { + throw new IllegalArgumentException("dependent must not be negative: " + dependent); + } + if (head < ROOT_HEAD) { + throw new IllegalArgumentException("head must be a token index or ROOT_HEAD: " + head); + } + if (head == dependent) { + throw new IllegalArgumentException("arc must not be a self-loop: " + head); + } + if (relation == null || StringUtil.isBlank(relation)) { + throw new IllegalArgumentException("relation must not be null or blank"); + } + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java new file mode 100644 index 0000000000..96a58938f4 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyGraph.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.StringUtil; + +/** + * An immutable dependency tree over one sentence: for every token, the index of its head + * and the label of the relation to that head. + * + *

Token indices are zero-based positions in the sentence the graph was built for. + * Exactly one token carries the head value {@link DependencyArc#ROOT_HEAD}, marking it as + * the sentence root. Instances are immutable and safe to share between threads.

+ * + * @since 3.0.0 + */ +@ThreadSafe +public final class DependencyGraph { + + /** Traversal state of a token whose head chain has not been followed yet. */ + private static final byte UNVISITED = 0; + + /** Traversal state of a token on the head chain currently being followed. */ + private static final byte VISITING = 1; + + /** Traversal state of a token whose head chain is known to reach the root. */ + private static final byte VISITED = 2; + + private final int[] heads; + private final String[] relations; + + /** + * Wraps already validated arrays; instances are created through {@link #of}. + * + * @param heads The validated head array, owned by the new instance. + * @param relations The validated relation array, owned by the new instance. + */ + private DependencyGraph(int[] heads, String[] relations) { + this.heads = heads; + this.relations = relations; + } + + /** + * Creates a {@link DependencyGraph} from parallel head and relation arrays. + * + * @param heads For each token, the zero-based index of its head token, or + * {@link DependencyArc#ROOT_HEAD} for the sentence root. Must not be + * {@code null} or empty, every value must be a valid token index or + * {@link DependencyArc#ROOT_HEAD}, no token may head itself, and exactly + * one token must be the root. + * @param relations For each token, the label of the relation to its head. Must not be + * {@code null}, must have the same length as {@code heads}, and no + * entry may be {@code null} or blank. + * @return A validated {@link DependencyGraph}. Never {@code null}. + * @throws IllegalArgumentException Thrown if any of the above constraints is violated. + */ + public static DependencyGraph of(int[] heads, String[] relations) { + if (heads == null || relations == null) { + throw new IllegalArgumentException("heads and relations must not be null"); + } + if (heads.length == 0) { + throw new IllegalArgumentException("a dependency graph needs at least one token"); + } + if (heads.length != relations.length) { + throw new IllegalArgumentException("heads and relations must have the same length: " + + heads.length + " != " + relations.length); + } + int roots = 0; + for (int i = 0; i < heads.length; i++) { + if (heads[i] == DependencyArc.ROOT_HEAD) { + roots++; + } else if (heads[i] < 0 || heads[i] >= heads.length) { + throw new IllegalArgumentException("head of token " + i + + " is out of range: " + heads[i]); + } else if (heads[i] == i) { + throw new IllegalArgumentException("token " + i + " must not head itself"); + } + if (relations[i] == null || StringUtil.isBlank(relations[i])) { + throw new IllegalArgumentException("relation of token " + i + " must not be blank"); + } + } + if (roots != 1) { + throw new IllegalArgumentException("expected exactly one root, found " + roots); + } + checkAcyclic(heads); + return new DependencyGraph(heads.clone(), relations.clone()); + } + + /** + * Rejects a cycle that is disconnected from the single root by following every token's + * head chain until it reaches the root or a token already known to reach it. + * + * @param heads The head array, already checked for range, self-heads, and root count. + * @throws IllegalArgumentException Thrown if a head chain returns to a token on that + * same chain. + */ + private static void checkAcyclic(int[] heads) { + final byte[] states = new byte[heads.length]; + for (int start = 0; start < heads.length; start++) { + int current = start; + while (current != DependencyArc.ROOT_HEAD && states[current] == UNVISITED) { + states[current] = VISITING; + current = heads[current]; + } + if (current != DependencyArc.ROOT_HEAD && states[current] == VISITING) { + throw new IllegalArgumentException( + "dependency graph contains a cycle at token " + current); + } + current = start; + while (current != DependencyArc.ROOT_HEAD && states[current] == VISITING) { + states[current] = VISITED; + current = heads[current]; + } + } + } + + /** + * @return The number of tokens the graph spans. + */ + public int size() { + return heads.length; + } + + /** + * Retrieves the head of a token. + * + * @param index The zero-based token index. Must be within {@code [0, size())}. + * @return The zero-based index of the head token, or {@link DependencyArc#ROOT_HEAD} + * when the token is the sentence root. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int headOf(int index) { + checkIndex(index); + return heads[index]; + } + + /** + * Retrieves the relation label of a token. + * + * @param index The zero-based token index. Must be within {@code [0, size())}. + * @return The label of the relation between the token and its head. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public String relationOf(int index) { + checkIndex(index); + return relations[index]; + } + + /** + * @return The zero-based index of the sentence root token. + * @throws IllegalStateException Thrown if no token carries {@link DependencyArc#ROOT_HEAD}, + * which {@link #of} rules out. + */ + public int root() { + for (int i = 0; i < heads.length; i++) { + if (heads[i] == DependencyArc.ROOT_HEAD) { + return i; + } + } + throw new IllegalStateException("graph invariant violated: no root present"); + } + + /** + * @return All arcs of the graph in token order, one per token. Never {@code null}. + */ + public List arcs() { + final List arcs = new ArrayList<>(heads.length); + for (int i = 0; i < heads.length; i++) { + arcs.add(new DependencyArc(heads[i], i, relations[i])); + } + return Collections.unmodifiableList(arcs); + } + + /** + * Rejects a token index outside {@code [0, size())}. + * + * @param index The zero-based token index to check. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + private void checkIndex(int index) { + if (index < 0 || index >= heads.length) { + throw new IllegalArgumentException("token index out of range: " + index + + ", size: " + heads.length); + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DependencyGraph other)) { + return false; + } + return Arrays.equals(heads, other.heads) && Arrays.equals(relations, other.relations); + } + + @Override + public int hashCode() { + return 31 * Arrays.hashCode(heads) + Arrays.hashCode(relations); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < heads.length; i++) { + if (i > 0) { + sb.append(' '); + } + sb.append(i).append("<-").append(heads[i]).append(':').append(relations[i]); + } + return sb.toString(); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java new file mode 100644 index 0000000000..a41fa38301 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencyParser.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * The interface for dependency parsers, which assign every token of a sentence a syntactic + * head and a relation label, forming a single-rooted tree over the sentence. + * + *

Dependency parsing complements the constituency {@link opennlp.tools.parser.Parser}: + * where a constituency parse groups tokens into nested phrases, a dependency parse links + * each token directly to the token it modifies. The result is a {@link DependencyGraph} + * whose indices refer back to the input token array, so spans computed for those tokens + * remain valid for the parse.

+ * + *

Thread safety is implementation specific.

+ * + * @see DependencyGraph + * @since 3.0.0 + */ +public interface DependencyParser { + + /** + * Parses a sentence into a {@link DependencyGraph}. + * + * @param tokens The input tokens. Must not be {@code null}, must contain + * at least one token, and must not contain {@code null} entries. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null}, must have the same length as {@code tokens}, and must not + * contain {@code null} entries. + * @return A {@link DependencyGraph} over the given tokens. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code tokens} or {@code tags} is + * {@code null}, empty, or of mismatched length. + */ + DependencyGraph parse(String[] tokens, String[] tags); +} diff --git a/opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java new file mode 100644 index 0000000000..3f781316cb --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/depparse/DependencySample.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.Arrays; +import java.util.Objects; + +import opennlp.tools.commons.ThreadSafe; + +/** + * One dependency-annotated sentence: tokens, their part-of-speech tags, and the gold + * {@link DependencyGraph} over them. Used for training and evaluating a + * {@link DependencyParser}. + * + *

Instances are immutable and safe to share between threads.

+ * + * @since 3.0.0 + */ +@ThreadSafe +public class DependencySample { + + private final String[] tokens; + private final String[] tags; + private final DependencyGraph graph; + + /** + * Initializes a {@link DependencySample}. + * + * @param tokens The input tokens. Must not be {@code null} or empty and must + * not contain {@code null} entries. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null}, must have the same length as {@code tokens}, and must not + * contain {@code null} entries. + * @param graph The dependency graph over the tokens. Must not be {@code null} and its + * {@link DependencyGraph#size()} must equal the number of tokens. + * @throws IllegalArgumentException Thrown if any parameter is {@code null} or the + * lengths disagree. + */ + public DependencySample(String[] tokens, String[] tags, DependencyGraph graph) { + if (graph == null) { + throw new IllegalArgumentException("graph must not be null"); + } + checkTokensAndTags(tokens, tags); + if (tokens.length != graph.size()) { + throw new IllegalArgumentException("tokens, tags and graph must agree in length: " + + tokens.length + ", " + tags.length + ", " + graph.size()); + } + this.tokens = tokens.clone(); + this.tags = tags.clone(); + this.graph = graph; + } + + /** + * Validates token and tag arrays shared by samples and parser entry points. + * + * @param tokens The token array. + * @param tags The aligned tag array. + * @throws IllegalArgumentException Thrown if an array is null or empty, the lengths + * do not match, or an entry is null. + */ + static void checkTokensAndTags(String[] tokens, String[] tags) { + if (tokens == null || tags == null) { + throw new IllegalArgumentException("tokens and tags must not be null"); + } + if (tokens.length == 0) { + throw new IllegalArgumentException("tokens must not be empty"); + } + if (tokens.length != tags.length) { + throw new IllegalArgumentException("tokens and tags must have the same length: " + + tokens.length + " != " + tags.length); + } + for (int i = 0; i < tokens.length; i++) { + if (tokens[i] == null) { + throw new IllegalArgumentException("token must not be null at index " + i); + } + if (tags[i] == null) { + throw new IllegalArgumentException("tag must not be null at index " + i); + } + } + } + + /** + * @return The tokens of the sentence. Never {@code null}. + */ + public String[] getTokens() { + return tokens.clone(); + } + + /** + * @return The part-of-speech tags aligned with the tokens. Never {@code null}. + */ + public String[] getTags() { + return tags.clone(); + } + + /** + * @return The dependency graph over the tokens. Never {@code null}. + */ + public DependencyGraph getGraph() { + return graph; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DependencySample other)) { + return false; + } + return Arrays.equals(tokens, other.tokens) && Arrays.equals(tags, other.tags) + && graph.equals(other.graph); + } + + @Override + public int hashCode() { + return Objects.hash(Arrays.hashCode(tokens), Arrays.hashCode(tags), graph); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < tokens.length; i++) { + sb.append(i + 1).append('\t').append(tokens[i]).append('\t').append(tags[i]) + .append('\t').append(graph.headOf(i) + 1).append('\t').append(graph.relationOf(i)) + .append(System.lineSeparator()); + } + return sb.toString(); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java new file mode 100644 index 0000000000..b4428a2e8c --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencyGraphTest.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the invariants and accessors of {@link DependencyGraph} and {@link DependencyArc}. + */ +public class DependencyGraphTest { + + /** The three-token graph shared by the accessor tests. */ + private static DependencyGraph sample() { + return DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}); + } + + @Test + void testAccessors() { + final DependencyGraph graph = sample(); + assertEquals(3, graph.size()); + assertEquals(1, graph.headOf(0)); + assertEquals(2, graph.headOf(1)); + assertEquals(DependencyArc.ROOT_HEAD, graph.headOf(2)); + assertEquals("nsubj", graph.relationOf(1)); + assertEquals(2, graph.root()); + } + + @Test + void testArcsAreInTokenOrder() { + final List arcs = sample().arcs(); + assertEquals(3, arcs.size()); + assertEquals(new DependencyArc(1, 0, "det"), arcs.get(0)); + assertEquals(new DependencyArc(2, 1, "nsubj"), arcs.get(1)); + assertEquals(new DependencyArc(DependencyArc.ROOT_HEAD, 2, "root"), arcs.get(2)); + } + + @Test + void testEqualsAndHashCode() { + assertEquals(sample(), sample()); + assertEquals(sample().hashCode(), sample().hashCode()); + assertNotEquals(sample(), DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"amod", "nsubj", "root"})); + } + + @Test + void testInputArraysAreCopied() { + final int[] heads = {1, -1}; + final String[] relations = {"nsubj", "root"}; + final DependencyGraph graph = DependencyGraph.of(heads, relations); + heads[0] = 0; + relations[0] = "det"; + assertEquals(1, graph.headOf(0)); + assertEquals("nsubj", graph.relationOf(0)); + } + + @Test + void testNullArraysThrow() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(null, new String[] {"root"})); + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1}, null)); + } + + @Test + void testEmptyGraphThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[0], new String[0])); + } + + @Test + void testLengthMismatchThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1}, new String[] {"root", "nsubj"})); + } + + @Test + void testRootCountIsEnforced() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {1, 0}, new String[] {"a", "b"})); + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1, -1}, new String[] {"root", "root"})); + } + + @Test + void testDisconnectedCycleThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-1, 2, 1}, + new String[] {"root", "dep", "dep"})); + } + + @Test + void testSelfHeadThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {0, -1}, new String[] {"a", "root"})); + } + + @Test + void testOutOfRangeHeadThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {2, -1}, new String[] {"a", "root"})); + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {-3, -1}, new String[] {"a", "root"})); + } + + @Test + void testBlankRelationThrows() { + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {1, -1}, new String[] {" ", "root"})); + // blankness follows the toolkit whitespace definition, which covers the no-break + // space U+00A0 that the JDK predicate leaves out + assertThrows(IllegalArgumentException.class, + () -> DependencyGraph.of(new int[] {1, -1}, new String[] {"\u00A0", "root"})); + // and a label that only looks unusual is still content + assertEquals("nmod:poss", DependencyGraph.of(new int[] {1, -1}, + new String[] {"nmod:poss", "root"}).relationOf(0)); + } + + @Test + void testIndexBoundsThrow() { + final DependencyGraph graph = sample(); + assertThrows(IllegalArgumentException.class, () -> graph.headOf(-1)); + assertThrows(IllegalArgumentException.class, () -> graph.relationOf(3)); + } + + @Test + void testArcValidation() { + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(0, 0, "root")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, -1, "det")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(-2, 0, "det")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, " ")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, "\u00A0")); + assertThrows(IllegalArgumentException.class, () -> new DependencyArc(1, 0, null)); + assertEquals("det", new DependencyArc(1, 0, "det").relation()); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java new file mode 100644 index 0000000000..fb02bd5ee3 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/depparse/DependencySampleTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the invariants of {@link DependencySample}. + */ +public class DependencySampleTest { + + private static final String[] TOKENS = {"the", "dog", "barks"}; + private static final String[] TAGS = {"DT", "NN", "VBZ"}; + + /** The graph matching {@link #TOKENS} and {@link #TAGS}. */ + private static DependencyGraph graph() { + return DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}); + } + + @Test + void testAccessors() { + final DependencySample sample = new DependencySample(TOKENS, TAGS, graph()); + assertArrayEquals(TOKENS, sample.getTokens()); + assertArrayEquals(TAGS, sample.getTags()); + assertEquals(graph(), sample.getGraph()); + } + + @Test + void testEquals() { + assertEquals(new DependencySample(TOKENS, TAGS, graph()), + new DependencySample(TOKENS, TAGS, graph())); + } + + @Test + void testNullArgumentsThrow() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(null, TAGS, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, null, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, TAGS, null)); + } + + @Test + void testNullTokenOrTagThrows() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(new String[] {"the", null, "barks"}, TAGS, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, new String[] {"DT", null, "VBZ"}, graph())); + } + + @Test + void testLengthMismatchThrows() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(new String[] {"one"}, TAGS, graph())); + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(TOKENS, new String[] {"DT"}, graph())); + } + + @Test + void testEmptySampleThrows() { + assertThrows(IllegalArgumentException.class, + () -> new DependencySample(new String[0], new String[0], graph())); + } + + @Test + void testInputArraysAreCopied() { + final String[] tokens = TOKENS.clone(); + final DependencySample sample = new DependencySample(tokens, TAGS, graph()); + tokens[0] = "a"; + assertEquals("the", sample.getTokens()[0]); + } +} diff --git a/opennlp-core/opennlp-formats/dev/README-ud-treebanks.md b/opennlp-core/opennlp-formats/dev/README-ud-treebanks.md new file mode 100644 index 0000000000..9bc2aebd16 --- /dev/null +++ b/opennlp-core/opennlp-formats/dev/README-ud-treebanks.md @@ -0,0 +1,73 @@ + + +# Universal Dependencies treebanks for the dependency parser evaluation + +The dependency parser's unit tests are fully self-contained, but its accuracy evaluation runs against real Universal Dependencies treebanks that the user provides. Apache OpenNLP bundles no treebank data, distributes none, and ships no models trained on it; the treebanks are used to reproduce accuracy numbers on your own machine. + +## Getting a treebank + +Every UD treebank lives in its own repository under `github.com/UniversalDependencies`, with its splits named `-ud-train.conllu`, `-dev`, and `-test`. Pass the helper a full commit SHA so later runs use the same data. This example selects the official `r2.18` commit of `UD_English-EWT`: + +``` +./download-ud-treebank.sh \ + UD_English-EWT \ + b7711cce01cdd4f5fcc0a8199b8a50d951b16c0c \ + /tmp/ud-ewt +``` + +produces `/tmp/ud-ewt/train.conllu` and `/tmp/ud-ewt/test.conllu`. Any treebank that publishes both splits works the same way. The helper is for experiments of your own with the training and evaluation API; the pinned evaluation below reads the Universal Dependencies 2.0 release layout of the shared evaluation data instead. + +## Running the accuracy evaluation + +`UniversalDependencyParserEval` in `opennlp-eval-tests` extends `AbstractEvalTest` and reads the Universal Dependencies 2.0 treebanks under `ud20/` in `OPENNLP_DATA_DIR`, the shared `opennlp-data.zip` archive all evaluations use. It verifies the MD5 digest of every split it reads, trains each parser from scratch on a training split, and asserts the exact scores on sentences the parser has not seen: + +| Test | Parser | Treebank | Held-out data | +|---|---|---|---| +| `crossValidateTransitionParserEnglish` | transition | `UD_English` | 5-fold cross validation of the training split | +| `trainAndEvalTransitionParserEnglish` | transition | `UD_English` | development split | +| `trainAndEvalTransitionParserGerman` | transition | `UD_German` | development split | +| `trainAndEvalTransitionParserSpanishAncora` | transition | `UD_Spanish-AnCora` | development split | +| `trainAndEvalTransitionParserFrench` | transition | `UD_French` | development split | +| `trainAndEvalFeedforwardParserEnglish` | feedforward | `UD_English` | development split | +| `trainAndEvalFeedforwardParserSpanishAncora` | feedforward | `UD_Spanish-AnCora` | development split | + +The transition parser is the maximum-entropy arc-standard parser trained with a feature cutoff of 5; the feedforward parser is trained with `FeedforwardDependencyTrainer.Settings.defaults()` and decodes greedily. Each test pins four numbers: the unlabeled attachment score (UAS, the fraction of tokens with the correct head), the labeled attachment score (LAS, the fraction with the correct head and relation label), and both again over the tokens not tagged `PUNCT`, the customary reporting convention for Universal Dependencies. The cross validation trains five parsers, each on four fifths of the English training split, and scores each on the remaining fifth, so every training sentence is scored once by a parser that did not see it. The pinned values, rounded to the four decimals the tests assert with `ACCURACY_DELTA`; the token column counts every scored token, and the two rightmost columns leave out the tokens tagged `PUNCT`: + +| Test | Tokens | UAS | LAS | UAS no punct | LAS no punct | +|---|---|---|---|---|---| +| `crossValidateTransitionParserEnglish` | 204,585 | 0.8205 | 0.7885 | 0.8422 | 0.8065 | +| `trainAndEvalTransitionParserEnglish` | 25,148 | 0.8182 | 0.7861 | 0.8372 | 0.8015 | +| `trainAndEvalTransitionParserGerman` | 12,348 | 0.7843 | 0.7306 | 0.8030 | 0.7414 | +| `trainAndEvalTransitionParserSpanishAncora` | 52,336 | 0.8355 | 0.7914 | 0.8599 | 0.8099 | +| `trainAndEvalTransitionParserFrench` | 35,766 | 0.8501 | 0.8192 | 0.8795 | 0.8450 | +| `trainAndEvalFeedforwardParserEnglish` | 25,148 | 0.8413 | 0.8174 | 0.8540 | 0.8273 | +| `trainAndEvalFeedforwardParserSpanishAncora` | 52,336 | 0.8617 | 0.8279 | 0.8780 | 0.8396 | + +``` +./mvnw test -pl opennlp-eval-tests -am -Peval-tests \ + -Dtest=UniversalDependencyParserEval -Dsurefire.failIfNoSpecifiedTests=false \ + -Dopennlp.forkCount=1 -DOPENNLP_DATA_DIR=/path/to/opennlp-data +``` + +A plain build needs no network access or external data; the evaluation runs only under the profile. The whole class takes about 46 minutes on one core: the four transition-parser development runs take one and a half to three minutes each, the five-fold cross validation about six minutes, the feedforward run on English about eleven minutes and on Spanish about 22 minutes. + +The scores use the treebank's segmentation, tokens, and universal part-of-speech tags. They measure dependency parsing by itself, not the errors of an upstream text pipeline. The reader retains the syntactic lines of multiword tokens and skips trees with a placeholder in the head or relation column. The arc-standard trainers skip non-projective trees because that transition system cannot derive them. + +## Licensing + +Each treebank carries its own license, stated in its repository README, and downloading one means accepting those terms yourself. The annotations of `UD_English-EWT`, for example, are licensed under CC BY-SA 4.0. The project's handling: treebanks are benchmark inputs on the user's machine only; no treebank data enters the source tree or any release artifact, and the project publishes no models trained on share-alike data. If you train and distribute your own model from a treebank, checking that treebank's terms is your responsibility. diff --git a/opennlp-core/opennlp-formats/dev/download-ud-treebank.sh b/opennlp-core/opennlp-formats/dev/download-ud-treebank.sh new file mode 100755 index 0000000000..8cf55cff63 --- /dev/null +++ b/opennlp-core/opennlp-formats/dev/download-ud-treebank.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fetches one Universal Dependencies treebank at a pinned commit and copies its train and +# test splits to /train.conllu and /test.conllu. See +# README-ud-treebanks.md in this directory for the evaluation command and the licensing +# notes; each treebank carries its own license, which you accept by downloading it. +# Nothing is bundled with Apache OpenNLP. + +set -euo pipefail + +usage() { + echo "usage: $0 " >&2 + echo "" >&2 + echo " treebank-repository a repository name under github.com/UniversalDependencies," >&2 + echo " for example UD_English-EWT" >&2 + echo " commit the full lowercase commit SHA to download" >&2 + echo " target-dir where train.conllu and test.conllu are placed" >&2 + exit 2 +} + +[ $# -ne 3 ] && usage +treebank="$1" +commit="$2" +target="$3" + +commit_length=40 +commit_error="commit must be a full ${commit_length}-character lowercase SHA" +if [ "${#commit}" -ne "${commit_length}" ]; then + echo "${commit_error}" >&2 + exit 2 +fi +case "${commit}" in + *[!0-9a-f]*) + echo "${commit_error}" >&2 + exit 2 + ;; +esac + +# Fetch only the selected commit into a temporary checkout. Only the .conllu files +# are copied to the target directory. +checkout="$(mktemp -d)" +trap 'rm -rf "${checkout}"' EXIT +git init --quiet "${checkout}/${treebank}" +git -C "${checkout}/${treebank}" remote add origin \ + "https://github.com/UniversalDependencies/${treebank}.git" +echo "fetching ${treebank} at ${commit}" +git -C "${checkout}/${treebank}" fetch --quiet --depth 1 origin "${commit}" +actual_commit="$(git -C "${checkout}/${treebank}" rev-parse FETCH_HEAD)" +if [ "${actual_commit}" != "${commit}" ]; then + echo "fetched ${actual_commit}, expected ${commit}" >&2 + exit 1 +fi +git -C "${checkout}/${treebank}" checkout --quiet --detach FETCH_HEAD + +mkdir -p "${target}" +for split in train test; do + # UD names its files -ud-.conllu; the code prefix varies per + # treebank, so match on the stable -ud- suffix. + found="" + for f in "${checkout}/${treebank}/"*"-ud-${split}.conllu"; do + [ -e "$f" ] && found="$f" && break + done + if [ -z "${found}" ]; then + echo "no *-ud-${split}.conllu in ${treebank}; the treebank may not publish" >&2 + echo "that split (some hide test data or ship dev only)" >&2 + exit 1 + fi + cp "${found}" "${target}/${split}.conllu" + echo "wrote ${target}/${split}.conllu ($(wc -l < "${target}/${split}.conllu") lines)" +done + +echo "" +echo "see README-ud-treebanks.md for running UniversalDependencyParserEval on the data" diff --git a/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java new file mode 100644 index 0000000000..ef6bdad8c5 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/ConlluDependencySampleStream.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.depparse.DependencyGraph; +import opennlp.tools.depparse.DependencySample; +import opennlp.tools.util.InputStreamFactory; +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.StringUtil; + +/** + * Reads {@link DependencySample samples} directly from + * CoNLL-U content, mapping + * the {@code HEAD} and {@code DEPREL} columns of the basic dependency annotation. + * + *

This reader does not use {@link ConlluStream}, which merges multiword token ranges + * with their syntactic words for token and lemma samples. Dependency samples instead + * omit range lines and empty nodes while retaining the syntactic word rows. Sentences + * with incomplete or invalid basic dependencies are skipped and counted.

+ * + * @since 3.0.0 + */ +public class ConlluDependencySampleStream implements ObjectStream { + + private static final Logger logger = + LoggerFactory.getLogger(ConlluDependencySampleStream.class); + + /** The number of tab-separated columns of a CoNLL-U word line. */ + private static final int COLUMNS = 10; + + /** The column holding the word index, a multiword token range, or an empty node id. */ + private static final int ID = 0; + + /** The column holding the word form. */ + private static final int FORM = 1; + + /** The column holding the universal part-of-speech tag. */ + private static final int UPOS = 3; + + /** The column holding the language-specific part-of-speech tag. */ + private static final int XPOS = 4; + + /** The column holding the one-based index of the head, {@code 0} for the root. */ + private static final int HEAD = 6; + + /** The column holding the relation label to the head. */ + private static final int DEPREL = 7; + + /** The CoNLL-U placeholder of a missing value. */ + private static final String PLACEHOLDER = "_"; + + /** The byte order mark some editors prepend to UTF-8 content. */ + private static final char BOM = '\ufeff'; + + /** The first character of a comment line. */ + private static final char COMMENT = '#'; + + private final InputStreamFactory in; + private final int tagColumn; + + private BufferedReader reader; + private boolean firstLine = true; + private int skipped; + + /** + * Initializes the stream. + * + * @param in The CoNLL-U content. Must not be {@code null}. + * @param tagset The tagset whose part-of-speech column feeds the sample tags. Must + * not be {@code null}. + * @throws IOException Thrown if opening the content fails. + * @throws IllegalArgumentException Thrown if any parameter is {@code null}. + */ + public ConlluDependencySampleStream(InputStreamFactory in, ConlluTagset tagset) + throws IOException { + if (in == null) { + throw new IllegalArgumentException("in must not be null"); + } + if (tagset == null) { + throw new IllegalArgumentException("tagset must not be null"); + } + this.in = in; + this.tagColumn = tagset == ConlluTagset.U ? UPOS : XPOS; + this.reader = open(); + } + + /** + * {@inheritDoc} + * Sentences without a usable basic dependency annotation are skipped, and their count + * is logged once the content is exhausted. + */ + @Override + public DependencySample read() throws IOException { + List words; + while (!(words = nextSentence()).isEmpty()) { + final DependencySample sample = convert(words); + if (sample != null) { + return sample; + } + skipped++; + } + if (skipped > 0) { + logger.warn("Skipped {} sentence(s) without a complete basic dependency annotation.", + skipped); + skipped = 0; + } + return null; + } + + /** + * Reads the syntactic word lines of the next sentence: comments, multiword token + * ranges, and empty nodes are dropped; an empty list means the end of the content. + * + *

Sentences are separated by any line {@link StringUtil#isBlank(CharSequence)} + * accepts, so a separator carrying a stray no-break space still separates rather than + * reaching the word line parser.

+ * + * @return The word lines of the next sentence, or an empty list at the end of the + * content. Never {@code null}. + * @throws IOException Thrown if reading fails. + * @throws InvalidFormatException Thrown if a word line does not have the expected column count. + */ + private List nextSentence() throws IOException { + final List words = new ArrayList<>(); + String line; + while ((line = reader.readLine()) != null) { + if (firstLine) { + firstLine = false; + if (!line.isEmpty() && line.charAt(0) == BOM) { + line = line.substring(1); + } + } + if (StringUtil.isBlank(line)) { + if (!words.isEmpty()) { + return words; + } + continue; + } + if (line.charAt(0) == COMMENT) { + continue; + } + final String[] fields = splitFields(line); + if (fields.length != COLUMNS) { + throw new InvalidFormatException("CoNLL-U word line has " + fields.length + + " columns, expected " + COLUMNS + ": " + line); + } + final String id = fields[ID]; + if (id.indexOf('-') < 0 && id.indexOf('.') < 0) { + words.add(fields); + } + } + return words; + } + + /** + * Splits a CoNLL-U word line into its tab-delimited fields, retaining empty fields. + * + * @param line The line to split. + * @return The fields in source order. Never {@code null}. + */ + private String[] splitFields(String line) { + final List fields = new ArrayList<>(); + int fieldStart = 0; + for (int i = 0; i < line.length(); i++) { + if (line.charAt(i) == '\t') { + fields.add(line.substring(fieldStart, i)); + fieldStart = i + 1; + } + } + fields.add(line.substring(fieldStart)); + return fields.toArray(String[]::new); + } + + /** + * Converts one sentence into a sample. + * + * @param words The word lines of the sentence. + * @return The converted sample, or {@code null} when the sentence's annotation is + * unusable, for example an underscore head or relation or a graph that is not + * a tree. + */ + private DependencySample convert(List words) { + final int n = words.size(); + final String[] tokens = new String[n]; + final String[] tags = new String[n]; + final int[] heads = new int[n]; + final String[] relations = new String[n]; + for (int i = 0; i < n; i++) { + final String[] word = words.get(i); + if (!Integer.toString(i + 1).equals(word[ID])) { + return null; + } + tokens[i] = word[FORM]; + tags[i] = word[tagColumn]; + if (PLACEHOLDER.equals(word[DEPREL])) { + return null; + } + relations[i] = word[DEPREL]; + try { + heads[i] = Integer.parseInt(word[HEAD]) - 1; + } catch (NumberFormatException e) { + return null; + } + } + try { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } catch (IllegalArgumentException e) { + return null; + } + } + + /** + * {@inheritDoc} + * Reopens the content through the {@link InputStreamFactory}, which must therefore + * produce a fresh stream on every call. + */ + @Override + public void reset() throws IOException, UnsupportedOperationException { + reader.close(); + reader = open(); + firstLine = true; + skipped = 0; + } + + /** {@inheritDoc} */ + @Override + public void close() throws IOException { + reader.close(); + } + + /** + * Opens a fresh UTF-8 reader over the content. + * + * @return A reader positioned at the start of the content. Never {@code null}. + * @throws IOException Thrown if opening the content fails. + */ + private BufferedReader open() throws IOException { + return new BufferedReader( + new InputStreamReader(in.createInputStream(), StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT))); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java new file mode 100644 index 0000000000..510d60c1d7 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencyParserUsageTest.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.depparse.DependencyArc; +import opennlp.tools.depparse.DependencyEvaluator; +import opennlp.tools.depparse.DependencyGraph; +import opennlp.tools.depparse.DependencyModel; +import opennlp.tools.depparse.DependencyParser; +import opennlp.tools.depparse.DependencyParserME; +import opennlp.tools.depparse.DependencySample; +import opennlp.tools.depparse.FeedforwardDependencyModel; +import opennlp.tools.depparse.FeedforwardDependencyParser; +import opennlp.tools.depparse.FeedforwardDependencyTrainer; +import opennlp.tools.util.InputStreamFactory; +import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Demonstrates the full dependency parsing workflow on a self-contained fixture: read + * gold sentences from CoNLL-U content, train a {@link DependencyParserME}, parse a + * sentence, inspect the resulting {@link DependencyGraph}, and persist the model. + * + *

The fixture contains four sentences and needs no external data. Repetition makes + * the expected training results deterministic.

+ */ +public class ConlluDependencyParserUsageTest { + + /** + * Joins the ten CoNLL-U columns of one word line with tabs. + * + * @param fields The column values; exactly ten are expected by the format. + * @return The joined word line. Never {@code null}. + */ + private static String line(String... fields) { + return String.join("\t", fields); + } + + /** + * The training fixture: four gold sentences in CoNLL-U form. The {@code HEAD} column + * is one-based with {@code 0} marking the root; the reader converts it to the + * zero-based convention of {@link DependencyGraph}. + */ + private static final String CONLLU = String.join("\n", + "# text = the dog barks", + line("1", "the", "the", "DET", "DT", "_", "2", "det", "_", "_"), + line("2", "dog", "dog", "NOUN", "NN", "_", "3", "nsubj", "_", "_"), + line("3", "barks", "bark", "VERB", "VBZ", "_", "0", "root", "_", "_"), + "", + "# text = dogs bark", + line("1", "dogs", "dog", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "", + "# text = she eats fish", + line("1", "she", "she", "PRON", "PRP", "_", "2", "nsubj", "_", "_"), + line("2", "eats", "eat", "VERB", "VBZ", "_", "0", "root", "_", "_"), + line("3", "fish", "fish", "NOUN", "NN", "_", "2", "obj", "_", "_"), + "", + "# text = Alice sent Bob a message", + line("1", "Alice", "Alice", "PROPN", "NNP", "_", "2", "nsubj", "_", "_"), + line("2", "sent", "send", "VERB", "VBD", "_", "0", "root", "_", "_"), + line("3", "Bob", "Bob", "PROPN", "NNP", "_", "2", "iobj", "_", "_"), + line("4", "a", "a", "DET", "DT", "_", "5", "det", "_", "_"), + line("5", "message", "message", "NOUN", "NN", "_", "2", "obj", "_", "_"), + line("6", ".", ".", "PUNCT", ".", "_", "2", "punct", "_", "_"), + "") + "\n"; + + /** How often the fixture is repeated so the trainer sees enough evidence per feature. */ + private static final int REPETITIONS = 40; + + /** The tokens of the four fixture sentences. */ + private static final long FIXTURE_WORDS = 14; + + /** The beam size of the manual's feedforward example. */ + private static final int BEAM_SIZE = 4; + + private static final String[] ALICE_TOKENS = {"Alice", "sent", "Bob", "a", "message", "."}; + private static final String[] ALICE_TAGS = {"PROPN", "VERB", "PROPN", "DET", "NOUN", "PUNCT"}; + private static final DependencyGraph ALICE_GRAPH = DependencyGraph.of( + new int[] {1, -1, 1, 4, 1, 1}, + new String[] {"nsubj", "root", "iobj", "det", "obj", "punct"}); + + private static DependencyModel model; + private static DependencyParserME parser; + + /** + * Reads the fixture sentences through the CoNLL-U reader. + * + * @return One sample per fixture sentence, in file order. Never {@code null}. + * @throws IOException Thrown if reading the in-memory content fails. + */ + private static List readFixture() throws IOException { + final InputStreamFactory in = + () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8)); + final List samples = new ArrayList<>(); + try (ConlluDependencySampleStream stream = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + DependencySample sample; + while ((sample = stream.read()) != null) { + samples.add(sample); + } + } + return samples; + } + + /** + * Trains the parser once for all tests: read the fixture, repeat it for evidence, + * and hand the samples to the trainer. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ + @BeforeAll + static void trainParser() throws IOException { + final List fixture = readFixture(); + final List trainingSamples = new ArrayList<>(); + for (int i = 0; i < REPETITIONS; i++) { + trainingSamples.addAll(fixture); + } + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + model = DependencyParserME.train("eng", + ObjectStreamUtils.createObjectStream(trainingSamples), parameters); + parser = new DependencyParserME(model); + } + + @Test + void testReaderDeliversTheGoldAnnotation() throws IOException { + final List fixture = readFixture(); + assertEquals(4, fixture.size()); + final DependencySample first = fixture.get(0); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), first.getGraph()); + assertEquals("NOUN", first.getTags()[1]); + } + + @Test + void testParseAssignsHeadsAndRelations() { + // Parsing takes the tokens and their part-of-speech tags; the result names, for + // every token, its head token and the relation between the two. + final DependencyGraph parse = parser.parse( + new String[] {"the", "dog", "barks"}, new String[] {"DET", "NOUN", "VERB"}); + assertEquals(1, parse.headOf(0)); + assertEquals("det", parse.relationOf(0)); + assertEquals(2, parse.headOf(1)); + assertEquals("nsubj", parse.relationOf(1)); + assertEquals(DependencyArc.ROOT_HEAD, parse.headOf(2)); + assertEquals("root", parse.relationOf(2)); + assertEquals(2, parse.root()); + } + + @Test + void testParseAliceSentence() { + assertEquals(ALICE_GRAPH, parser.parse(ALICE_TOKENS, ALICE_TAGS)); + } + + @Test + void testArcsNameDependentHeadAndRelation() { + // The manual's loop over the arcs of a parse, collecting instead of printing. + final String[] tokens = ALICE_TOKENS; + final String[] tags = ALICE_TAGS; + final DependencyGraph graph = parser.parse(tokens, tags); + final List lines = new ArrayList<>(); + for (DependencyArc arc : graph.arcs()) { + final String dependent = tokens[arc.dependent()]; + final String head = arc.head() == DependencyArc.ROOT_HEAD + ? "ROOT" : tokens[arc.head()]; + lines.add(dependent + " -> " + head + " (" + arc.relation() + ")"); + } + assertEquals(List.of( + "Alice -> sent (nsubj)", + "sent -> ROOT (root)", + "Bob -> sent (iobj)", + "a -> message (det)", + "message -> sent (obj)", + ". -> sent (punct)"), lines); + } + + @Test + void testFeedforwardTrainSaveReloadAndParse(@TempDir Path dir) throws IOException { + // The manual's feedforward example: train from the CoNLL-U reader with the default + // settings, save the model, load it back, and parse with a beam of four. The parse + // of the reloaded model must equal the parse of the model in memory. + final Path treebank = dir.resolve("train.conllu"); + Files.writeString(treebank, repeat(CONLLU, REPETITIONS), StandardCharsets.UTF_8); + final Path modelFile = dir.resolve("en-depparse-ff.bin"); + final InputStreamFactory trainingData = () -> Files.newInputStream(treebank); + + FeedforwardDependencyModel model; + try (ObjectStream samples = + new ConlluDependencySampleStream(trainingData, ConlluTagset.U)) { + model = FeedforwardDependencyTrainer.train( + samples, FeedforwardDependencyTrainer.Settings.defaults()); + } + try (OutputStream out = Files.newOutputStream(modelFile)) { + model.serialize(out); + } + + DependencyParser reloaded = new FeedforwardDependencyParser( + FeedforwardDependencyModel.load(modelFile), BEAM_SIZE); + final DependencyGraph expected = + new FeedforwardDependencyParser(model, BEAM_SIZE).parse(ALICE_TOKENS, ALICE_TAGS); + final DependencyGraph parse = reloaded.parse(ALICE_TOKENS, ALICE_TAGS); + assertEquals(expected, parse); + assertEquals(ALICE_TOKENS.length, parse.size()); + assertEquals(ALICE_GRAPH.root(), parse.root()); + } + + /** + * Repeats CoNLL-U content so a small fixture gives the trainers enough evidence. + * + * @param conllu The content to repeat; sentences are separated by a blank line. + * @param times How often to repeat it. + * @return The repeated content. Never {@code null}. + */ + private static String repeat(String conllu, int times) { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < times; i++) { + sb.append(conllu).append("\n\n"); + } + return sb.toString(); + } + + @Test + void testEvaluatorScoresTheParserAgainstGoldSamples() throws IOException { + // The evaluator parses each gold sentence and accumulates the two standard scores; + // the expected score is exact because the parser is evaluated on its training data. + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(readFixture())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + assertEquals(FIXTURE_WORDS, evaluator.getWordCount()); + } + + @Test + void testPersistedModelParsesLikeTheOriginal(@TempDir Path dir) throws IOException { + // A trained model is saved to a file and loaded back like any other tool model; the + // reloaded parser must produce the exact same parse as the original. + final Path file = dir.resolve("en-depparse.bin"); + model.serialize(file); + final DependencyParserME reloaded = new DependencyParserME(new DependencyModel(file)); + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), + reloaded.parse(new String[] {"she", "eats", "fish"}, + new String[] {"PRON", "VERB", "NOUN"})); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java new file mode 100644 index 0000000000..5ae6e73d88 --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/ConlluDependencySampleStreamTest.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.depparse.DependencyArc; +import opennlp.tools.depparse.DependencySample; +import opennlp.tools.util.InputStreamFactory; +import opennlp.tools.util.InvalidFormatException; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests that the raw reader maps the basic dependency columns, keeps the syntactic + * words of multiword tokens while dropping the range line itself, and skips sentences + * without a usable annotation. + */ +public class ConlluDependencySampleStreamTest { + + /** Joins the ten CoNLL-U columns of one word line with tabs. */ + private static String line(String... fields) { + return String.join("\t", fields); + } + + private static final String CONLLU = String.join("\n", + "# sent_id = test-1", + "# text = He bought the bonds", + line("1", "He", "he", "PRON", "PRP", "_", "2", "nsubj", "_", "_"), + line("2", "bought", "buy", "VERB", "VBD", "_", "0", "root", "_", "_"), + line("3", "the", "the", "DET", "DT", "_", "4", "det", "_", "_"), + line("4", "bonds", "bond", "NOUN", "NNS", "_", "2", "obj", "_", "_"), + "", + "# sent_id = test-2", + "# text = Broken", + line("1", "Broken", "broken", "ADJ", "JJ", "_", "_", "_", "_", "_"), + "", + "# sent_id = test-3", + "# text = im Haus", + line("1-2", "im", "_", "_", "_", "_", "_", "_", "_", "_"), + line("1", "in", "in", "ADP", "APPR", "_", "2", "case", "_", "_"), + line("2", "Haus", "Haus", "NOUN", "NN", "_", "0", "root", "_", "_"), + "", + "# sent_id = test-4", + "# text = Dogs bark", + line("1", "Dogs", "dog", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "") + "\n"; + + /** An in-memory factory over the shared fixture. */ + private static InputStreamFactory factory() { + return () -> new ByteArrayInputStream(CONLLU.getBytes(StandardCharsets.UTF_8)); + } + + /** A stream over the shared fixture using the universal tagset. */ + private static ConlluDependencySampleStream stream() throws IOException { + return new ConlluDependencySampleStream(factory(), ConlluTagset.U); + } + + @Test + void testReadsSamplesKeepsContractionsAndSkipsUnusableSentences() throws IOException { + try (ConlluDependencySampleStream samples = stream()) { + final DependencySample first = samples.read(); + assertNotNull(first); + assertArrayEquals(new String[] {"He", "bought", "the", "bonds"}, first.getTokens()); + assertArrayEquals(new String[] {"PRON", "VERB", "DET", "NOUN"}, first.getTags()); + assertEquals(1, first.getGraph().headOf(0)); + assertEquals(DependencyArc.ROOT_HEAD, first.getGraph().headOf(1)); + assertEquals(3, first.getGraph().headOf(2)); + assertEquals("obj", first.getGraph().relationOf(3)); + + // the underscore-head sentence is skipped; the contraction sentence is kept, + // with the range line dropped and its syntactic words intact + final DependencySample second = samples.read(); + assertNotNull(second); + assertArrayEquals(new String[] {"in", "Haus"}, second.getTokens()); + assertEquals(1, second.getGraph().headOf(0)); + assertEquals("case", second.getGraph().relationOf(0)); + + final DependencySample third = samples.read(); + assertNotNull(third); + assertArrayEquals(new String[] {"Dogs", "bark"}, third.getTokens()); + + assertNull(samples.read()); + } + } + + @Test + void testResetRestartsTheStream() throws IOException { + try (ConlluDependencySampleStream samples = stream()) { + assertNotNull(samples.read()); + samples.reset(); + final DependencySample first = samples.read(); + assertNotNull(first); + assertArrayEquals(new String[] {"He", "bought", "the", "bonds"}, first.getTokens()); + } + } + + @Test + void testXposTagsetSelectsTheOtherColumn() throws IOException { + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(factory(), ConlluTagset.X)) { + assertArrayEquals(new String[] {"PRP", "VBD", "DT", "NNS"}, + samples.read().getTags()); + } + } + + @Test + void testMalformedLineIsRejected() { + final InputStreamFactory bad = () -> new ByteArrayInputStream( + "1\ttoo\tfew\tcolumns\n".getBytes(StandardCharsets.UTF_8)); + assertThrows(InvalidFormatException.class, + () -> new ConlluDependencySampleStream(bad, ConlluTagset.U).read()); + } + + @Test + void testExtraColumnIsRejected() { + final InputStreamFactory bad = () -> new ByteArrayInputStream( + (line("1", "word", "word", "NOUN", "NN", "_", "0", "root", "_", "_", + "extra") + "\n").getBytes(StandardCharsets.UTF_8)); + assertThrows(InvalidFormatException.class, + () -> new ConlluDependencySampleStream(bad, ConlluTagset.U).read()); + } + + @Test + void testUtf8BomIsAccepted() throws IOException { + final String content = "\ufeff" + line("1", "Word", "word", "NOUN", "NN", "_", + "0", "root", "_", "_") + "\n"; + final InputStreamFactory in = () -> new ByteArrayInputStream( + content.getBytes(StandardCharsets.UTF_8)); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + final DependencySample sample = samples.read(); + assertNotNull(sample); + assertArrayEquals(new String[] {"Word"}, sample.getTokens()); + } + } + + @Test + void testSemanticallyInvalidAnnotationIsSkippedNotFatal() throws IOException { + // Structurally well-formed lines whose annotation cannot form a valid tree, here an + // out-of-range head and a rootless cycle, skip the sentence instead of failing, so + // one broken sentence cannot abort reading a large treebank. + final String content = String.join("\n", + line("1", "far", "far", "ADV", "RB", "_", "5", "advmod", "_", "_"), + line("2", "off", "off", "ADP", "RP", "_", "0", "root", "_", "_"), + "", + line("1", "loop", "loop", "NOUN", "NN", "_", "2", "dep", "_", "_"), + line("2", "back", "back", "ADV", "RB", "_", "1", "dep", "_", "_"), + "", + line("1", "Fine", "fine", "ADJ", "JJ", "_", "0", "root", "_", "_"), + "") + "\n"; + final InputStreamFactory in = + () -> new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + final DependencySample onlyValid = samples.read(); + assertNotNull(onlyValid); + assertArrayEquals(new String[] {"Fine"}, onlyValid.getTokens()); + assertEquals(DependencyArc.ROOT_HEAD, onlyValid.getGraph().headOf(0)); + assertNull(samples.read()); + } + } + + @Test + void testNonSequentialWordIdsAreSkipped() throws IOException { + final String content = line("2", "Dogs", "dog", "NOUN", "NNS", "_", "0", + "root", "_", "_") + "\n"; + final InputStreamFactory in = () -> new ByteArrayInputStream( + content.getBytes(StandardCharsets.UTF_8)); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + assertNull(samples.read()); + } + } + + /** + * Verifies that a word whose relation is the underscore placeholder makes the + * sentence incomplete: it is skipped like an underscore head, and reading continues + * with the next sentence. + * + * @throws IOException Thrown if reading fails. + */ + @Test + void testUnderscoreRelationIsSkipped() throws IOException { + final String content = String.join("\n", + line("1", "Dogs", "dog", "NOUN", "NNS", "_", "2", "_", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "", + line("1", "Cats", "cat", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "purr", "purr", "VERB", "VBP", "_", "0", "root", "_", "_"), + "") + "\n"; + final InputStreamFactory in = () -> new ByteArrayInputStream( + content.getBytes(StandardCharsets.UTF_8)); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + final DependencySample sample = samples.read(); + assertNotNull(sample); + assertArrayEquals(new String[] {"Cats", "purr"}, sample.getTokens()); + assertEquals("nsubj", sample.getGraph().relationOf(0)); + assertNull(samples.read()); + } + } + + @Test + void testMalformedUtf8Throws() throws IOException { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + bytes.writeBytes("1\t".getBytes(StandardCharsets.UTF_8)); + bytes.write(0xc3); + bytes.writeBytes("\t_\tNOUN\tNNS\t_\t0\troot\t_\t_\n" + .getBytes(StandardCharsets.UTF_8)); + final InputStreamFactory in = () -> new ByteArrayInputStream(bytes.toByteArray()); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + assertThrows(IOException.class, samples::read); + } + } + + @Test + void testSeparatorLineOfNonBreakingSpaceSeparatesSentences() throws IOException { + // A separator line carrying a stray no-break space is still a separator: OpenNLP + // counts U+00A0 as whitespace, so such a line must not reach the word-line parser + // and abort the stream. + final String content = String.join("\n", + line("1", "Dogs", "dog", "NOUN", "NNS", "_", "2", "nsubj", "_", "_"), + line("2", "bark", "bark", "VERB", "VBP", "_", "0", "root", "_", "_"), + "\u00A0", + line("1", "Fine", "fine", "ADJ", "JJ", "_", "0", "root", "_", "_"), + "") + "\n"; + final InputStreamFactory in = + () -> new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + final DependencySample first = samples.read(); + assertNotNull(first); + assertArrayEquals(new String[] {"Dogs", "bark"}, first.getTokens()); + final DependencySample second = samples.read(); + assertNotNull(second); + assertArrayEquals(new String[] {"Fine"}, second.getTokens()); + assertNull(samples.read()); + } + } + + @Test + void testEmptyContentYieldsNoSample() throws IOException { + final InputStreamFactory in = () -> new ByteArrayInputStream(new byte[0]); + try (ConlluDependencySampleStream samples = + new ConlluDependencySampleStream(in, ConlluTagset.U)) { + assertNull(samples.read()); + } + } + + @Test + void testValidation() { + assertThrows(IllegalArgumentException.class, + () -> new ConlluDependencySampleStream(null, ConlluTagset.U)); + assertThrows(IllegalArgumentException.class, + () -> new ConlluDependencySampleStream(factory(), null)); + } +} diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/UdTreebankDownloadScriptTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/UdTreebankDownloadScriptTest.java new file mode 100644 index 0000000000..ea1eb9a06f --- /dev/null +++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/UdTreebankDownloadScriptTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats.conllu; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Checks the argument handling of {@code dev/download-ud-treebank.sh} without network + * access: a stub {@code git} on the {@code PATH} fails loudly if the script ever reaches it. + */ +@EnabledOnOs({OS.LINUX, OS.MAC}) +public class UdTreebankDownloadScriptTest { + + @Test + void testRequiresACommitForReproducibleInput(@TempDir Path tempDir) + throws IOException, InterruptedException { + final Path fakeBin = Files.createDirectory(tempDir.resolve("bin")); + final Path fakeGit = fakeBin.resolve("git"); + Files.writeString(fakeGit, "#!/usr/bin/env bash\nexit 99\n", StandardCharsets.UTF_8); + Files.setPosixFilePermissions(fakeGit, PosixFilePermissions.fromString("rwxr-xr-x")); + + final ProcessBuilder processBuilder = new ProcessBuilder("bash", + Path.of("dev", "download-ud-treebank.sh").toString(), + "UD_English-EWT", tempDir.resolve("treebank").toString()); + processBuilder.redirectErrorStream(true); + processBuilder.environment().put("PATH", fakeBin + ":" + + processBuilder.environment().get("PATH")); + final Process process = processBuilder.start(); + final String output = new String(process.getInputStream().readAllBytes(), + StandardCharsets.UTF_8); + + assertEquals(2, process.waitFor(), output); + assertTrue(output.contains(""), output); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java new file mode 100644 index 0000000000..ff94efd0c7 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardOracle.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; + + +/** + * The static oracle for the arc-standard system: derives the transition sequence that + * reproduces a gold {@link DependencyGraph}. + * + *

An arc is only created once all dependents of the token being attached have been + * collected, which is the arc-standard correctness condition. The oracle is defined for + * projective trees only; a non-projective gold graph has no arc-standard derivation and + * is rejected.

+ * + * @since 3.0.0 + */ +final class ArcStandardOracle { + + /** Prevents construction of this utility class. */ + private ArcStandardOracle() { + } + + /** + * Derives the gold transition sequence for a graph. + * + * @param gold The gold dependency graph. Must not be {@code null} and must be + * projective. + * @return The transitions that rebuild {@code gold} from the start configuration, in + * order. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code gold} is {@code null} or not + * projective. + */ + static List transitions(DependencyGraph gold) { + if (gold == null) { + throw new IllegalArgumentException("gold must not be null"); + } + final int n = gold.size(); + final int[] goldDependents = new int[n]; + for (int i = 0; i < n; i++) { + final int head = gold.headOf(i); + if (head >= 0) { + goldDependents[head]++; + } + } + + final ArcStandardState state = new ArcStandardState(n); + final List transitions = new ArrayList<>(2 * n); + while (!state.isTerminal()) { + final Transition next = nextTransition(gold, goldDependents, state); + if (next == null) { + throw new IllegalArgumentException( + "gold graph has no arc-standard derivation (non-projective): " + gold); + } + state.apply(next); + transitions.add(next); + } + return transitions; + } + + /** + * Picks the gold transition for the current configuration. + * + * @param gold The gold graph being derived. + * @param goldDependents The gold dependent count per token, indexed by head. + * @param state The current configuration. + * @return The next gold transition, or {@code null} when the configuration is stuck, + * which only happens for non-projective input. + */ + private static Transition nextTransition(DependencyGraph gold, int[] goldDependents, + ArcStandardState state) { + final int s0 = state.stack(0); + final int s1 = state.stack(1); + if (s1 >= 0 && gold.headOf(s1) == s0) { + final Transition leftArc = Transition.leftArc(gold.relationOf(s1)); + if (state.canApply(leftArc)) { + return leftArc; + } + } + if (s0 >= 0 && s1 != ArcStandardState.NONE && gold.headOf(s0) == s1 + && state.assignedDependents(s0) == goldDependents[s0]) { + final Transition rightArc = Transition.rightArc(gold.relationOf(s0)); + if (state.canApply(rightArc)) { + return rightArc; + } + } + if (state.canApply(Transition.SHIFT)) { + return Transition.SHIFT; + } + return null; + } + /** + * Tests whether a gold graph is projective: no pair of arcs crosses when the arcs are + * placed above the token sequence. The projective graphs are the ones with an + * arc-standard derivation, so callers apply this test before requesting + * {@link #transitions}. + * + * @param gold The gold graph. Must not be {@code null}. + * @return {@code true} if no pair of arcs crosses. + * @throws IllegalArgumentException Thrown if {@code gold} is {@code null}. + */ + static boolean isProjective(DependencyGraph gold) { + if (gold == null) { + throw new IllegalArgumentException("gold must not be null"); + } + for (int first = 0; first < gold.size(); first++) { + final int firstLow = Math.min(first, gold.headOf(first)); + final int firstHigh = Math.max(first, gold.headOf(first)); + for (int other = first + 1; other < gold.size(); other++) { + final int otherLow = Math.min(other, gold.headOf(other)); + final int otherHigh = Math.max(other, gold.headOf(other)); + if ((firstLow < otherLow && otherLow < firstHigh && firstHigh < otherHigh) + || (otherLow < firstLow && firstLow < otherHigh && otherHigh < firstHigh)) { + return false; + } + } + } + return true; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java new file mode 100644 index 0000000000..019539fe72 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ArcStandardState.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.Arrays; + +/** + * The mutable configuration of an arc-standard parse: a stack, a buffer of remaining + * tokens, and the arcs assigned so far. The arc-standard transition system is described + * in Nivre (2004). + * + *

The stack bottom holds the artificial root, exposed as {@link #ROOT}. Positions that + * do not exist, such as the second stack element in the initial configuration, are exposed + * as {@link #NONE}. A right arc from the artificial root is only applicable once the buffer + * is empty and the root is the only other stack element, which guarantees every completed + * parse has exactly one sentence root.

+ * + *

Instances are confined to a single parse and must not be shared between threads.

+ * + * @since 3.0.0 + */ +final class ArcStandardState { + + /** The stack value representing the artificial root node. */ + public static final int ROOT = -1; + + /** The value returned for stack or buffer positions that do not exist. */ + public static final int NONE = -2; + + private final int tokenCount; + private final int[] stack; + private final int[] heads; + private final String[] relations; + private final int[] assignedDependents; + private final int[] leftmostDependents; + private final int[] rightmostDependents; + + private int top; + private int bufferFront; + + /** + * Initializes the start configuration for a sentence: the artificial root on the stack + * and every token in the buffer. + * + * @param tokenCount The number of tokens in the sentence. Must be greater than zero. + * @throws IllegalArgumentException Thrown if {@code tokenCount} is not positive. + */ + ArcStandardState(int tokenCount) { + if (tokenCount <= 0) { + throw new IllegalArgumentException("tokenCount must be positive: " + tokenCount); + } + this.tokenCount = tokenCount; + this.stack = new int[tokenCount + 1]; + this.stack[0] = ROOT; + this.top = 0; + this.bufferFront = 0; + this.heads = new int[tokenCount]; + this.relations = new String[tokenCount]; + this.assignedDependents = new int[tokenCount]; + this.leftmostDependents = new int[tokenCount]; + this.rightmostDependents = new int[tokenCount]; + Arrays.fill(this.leftmostDependents, NONE); + Arrays.fill(this.rightmostDependents, NONE); + } + + /** + * Deep-copies {@code source}; used only by {@link #copy()}. + * + * @param source The state to copy. + */ + private ArcStandardState(ArcStandardState source) { + this.tokenCount = source.tokenCount; + this.stack = source.stack.clone(); + this.heads = source.heads.clone(); + this.relations = source.relations.clone(); + this.assignedDependents = source.assignedDependents.clone(); + this.leftmostDependents = source.leftmostDependents.clone(); + this.rightmostDependents = source.rightmostDependents.clone(); + this.top = source.top; + this.bufferFront = source.bufferFront; + } + + /** + * Creates an independent copy for advancing a search alternative. + * + * @return A copy that can be advanced without affecting this state. Never {@code null}. + */ + ArcStandardState copy() { + return new ArcStandardState(this); + } + + /** + * @return {@code true} if the buffer is empty and only the artificial root remains on + * the stack, so the parse is complete. + */ + public boolean isTerminal() { + return bufferFront == tokenCount && top == 0; + } + + /** + * Checks whether a transition may be applied in the current configuration. + * + * @param transition The transition to check. Must not be {@code null}. + * @return {@code true} if {@link #apply(Transition)} would succeed. + * @throws IllegalArgumentException Thrown if {@code transition} is {@code null}. + */ + public boolean canApply(Transition transition) { + if (transition == null) { + throw new IllegalArgumentException("transition must not be null"); + } + return switch (transition.type()) { + case SHIFT -> bufferFront < tokenCount; + case LEFT_ARC -> top >= 2; + case RIGHT_ARC -> top >= 2 || (top == 1 && bufferFront == tokenCount); + }; + } + + /** + * Applies a transition, updating stack, buffer, and arcs. + * + * @param transition The transition to apply. Must not be {@code null} and must be + * applicable per {@link #canApply(Transition)}. + * @throws IllegalArgumentException Thrown if the transition is {@code null} or not + * applicable in the current configuration. + */ + public void apply(Transition transition) { + if (!canApply(transition)) { + throw new IllegalArgumentException("transition not applicable: " + transition + + " in " + this); + } + switch (transition.type()) { + case SHIFT -> { + top++; + stack[top] = bufferFront++; + } + case LEFT_ARC -> { + final int dependent = stack[top - 1]; + attach(stack[top], dependent, transition.label()); + stack[top - 1] = stack[top]; + top--; + } + case RIGHT_ARC -> { + attach(stack[top - 1], stack[top], transition.label()); + top--; + } + default -> throw new IllegalArgumentException("unsupported type: " + transition.type()); + } + } + + /** + * Records the arc from {@code head} to {@code dependent} and updates the dependent + * bookkeeping of {@code head} when it is a token rather than the artificial root. + * + * @param head The head token index, or {@link #ROOT} for the artificial root. + * @param dependent The zero-based index of the token being attached. + * @param relation The relation label of the arc. + */ + private void attach(int head, int dependent, String relation) { + heads[dependent] = head; + relations[dependent] = relation; + if (head >= 0) { + assignedDependents[head]++; + if (leftmostDependents[head] == NONE || dependent < leftmostDependents[head]) { + leftmostDependents[head] = dependent; + } + if (rightmostDependents[head] == NONE || dependent > rightmostDependents[head]) { + rightmostDependents[head] = dependent; + } + } + } + + /** + * Retrieves a stack element counted from the top. + * + * @param fromTop Zero for the top element, one for the element below it, and so on. + * Must not be negative. + * @return The token index at that position, {@link #ROOT} for the artificial root, or + * {@link #NONE} if the position does not exist. + * @throws IllegalArgumentException Thrown if {@code fromTop} is negative. + */ + public int stack(int fromTop) { + if (fromTop < 0) { + throw new IllegalArgumentException("fromTop must not be negative: " + fromTop); + } + final int position = top - fromTop; + return position < 0 ? NONE : stack[position]; + } + + /** + * Retrieves a buffer element counted from the front. + * + * @param fromFront Zero for the next token to be shifted, one for the token after it, + * and so on. Must not be negative. + * @return The token index at that position, or {@link #NONE} if the position does not + * exist. + * @throws IllegalArgumentException Thrown if {@code fromFront} is negative. + */ + public int buffer(int fromFront) { + if (fromFront < 0) { + throw new IllegalArgumentException("fromFront must not be negative: " + fromFront); + } + if (fromFront >= tokenCount - bufferFront) { + return NONE; + } + return bufferFront + fromFront; + } + + /** {@return the number of tokens in this parse} */ + int tokenCount() { + return tokenCount; + } + + /** + * @return The number of stack elements including the artificial root. + */ + public int stackSize() { + return top + 1; + } + + /** + * @return The number of tokens still in the buffer. + */ + public int bufferSize() { + return tokenCount - bufferFront; + } + + /** + * Retrieves how many dependents have been attached to a token so far. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The number of arcs assigned with the token as head. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int assignedDependents(int index) { + checkTokenIndex(index); + return assignedDependents[index]; + } + + /** + * Rejects a token index outside {@code [0, tokenCount)}. + * + * @param index The zero-based token index to check. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + private void checkTokenIndex(int index) { + if (index < 0 || index >= tokenCount) { + throw new IllegalArgumentException("token index out of range: " + index); + } + } + + /** + * Retrieves the leftmost dependent attached to a token so far. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The dependent's token index, or {@link #NONE} when none is attached. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int leftmostDependent(int index) { + checkTokenIndex(index); + return leftmostDependents[index]; + } + + /** + * Retrieves the rightmost dependent attached to a token so far. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The dependent's token index, or {@link #NONE} when none is attached. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public int rightmostDependent(int index) { + checkTokenIndex(index); + return rightmostDependents[index]; + } + + /** + * Retrieves the relation a token was attached under, when it has been attached. + * + * @param index The zero-based token index. Must be within {@code [0, tokenCount)}. + * @return The relation label, or {@code null} when the token is still unattached. + * @throws IllegalArgumentException Thrown if {@code index} is out of range. + */ + public String assignedRelation(int index) { + checkTokenIndex(index); + return relations[index]; + } + + /** + * Builds the {@link DependencyGraph} of a completed parse. + * + * @return The parsed graph. Never {@code null}. + * @throws IllegalStateException Thrown if the parse is not yet terminal. + */ + public DependencyGraph toGraph() { + if (!isTerminal()) { + throw new IllegalStateException("parse is not terminal: " + this); + } + return DependencyGraph.of(heads, relations); + } + + @Override + public String toString() { + return "stackSize=" + stackSize() + ", bufferSize=" + bufferSize() + + ", tokenCount=" + tokenCount; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java new file mode 100644 index 0000000000..57a2eb07c2 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyContextGenerator.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; + +import opennlp.tools.commons.ThreadSafe; + +/** + * Generates the classification features for one arc-standard configuration: words and + * tags of the topmost stack and frontmost buffer positions, their pairings, the partial + * structure built so far (tags and relations of the leftmost and rightmost dependents, + * valency counts), and a bucketed distance between stack top and buffer front. + * + *

Instances hold no state and are safe to share between threads.

+ * + * @since 3.0.0 + */ +@ThreadSafe +class DependencyContextGenerator { + + /** The feature value standing for the artificial root position. */ + private static final String ROOT_VALUE = "*ROOT*"; + + /** The feature value standing for a position that does not exist. */ + private static final String NONE_VALUE = "*NULL*"; + + /** Separates a word from the tag of the same position within one feature. */ + private static final char WORD_TAG_SEPARATOR = '/'; + + /** Separates the parts of a feature combining several positions. */ + private static final char POSITION_SEPARATOR = '|'; + + /** The number of features {@link #getContext(ArcStandardState, String[], String[])} emits. */ + private static final int FEATURE_COUNT = 37; + + /** Valency counts at or above this bound share one feature value. */ + private static final int MAX_VALENCY = 3; + + /** Distances at or above this bound share the {@link #LONG_DISTANCE} feature value. */ + private static final int MAX_DISTANCE = 4; + + /** The feature value standing for every distance of {@link #MAX_DISTANCE} or more. */ + private static final String LONG_DISTANCE = "4+"; + + /** + * Generates the features of the current configuration. + * + * @param state The configuration to describe. Must not be {@code null}. + * @param tokens The input tokens. Must satisfy the token contract of + * {@link DependencyParser#parse(String[], String[])} and match the state. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null} and must match the state. + * @return The feature strings. Never {@code null}. + * @throws IllegalArgumentException Thrown if a parameter is invalid or the arrays do + * not match the state. + */ + String[] getContext(ArcStandardState state, String[] tokens, String[] tags) { + if (state == null) { + throw new IllegalArgumentException("state must not be null"); + } + ParserInput.check(tokens, tags); + if (tokens.length != state.tokenCount()) { + throw new IllegalArgumentException("tokens and tags must match state token count: " + + tokens.length + " != " + state.tokenCount()); + } + final int s0 = state.stack(0); + final int s1 = state.stack(1); + final int s2 = state.stack(2); + final int b0 = state.buffer(0); + final int b1 = state.buffer(1); + final int b2 = state.buffer(2); + + final String s0w = word(tokens, s0); + final String s0t = tag(tags, s0); + final String s1w = word(tokens, s1); + final String s1t = tag(tags, s1); + final String s2t = tag(tags, s2); + final String b0w = word(tokens, b0); + final String b0t = tag(tags, b0); + final String b1w = word(tokens, b1); + final String b1t = tag(tags, b1); + final String b2t = tag(tags, b2); + + final String s0lct = dependentTag(state, tags, s0, true); + final String s0rct = dependentTag(state, tags, s0, false); + final String s1lct = dependentTag(state, tags, s1, true); + final String s1rct = dependentTag(state, tags, s1, false); + final String s0lcl = dependentRelation(state, s0, true); + final String s0rcl = dependentRelation(state, s0, false); + final String s1rcl = dependentRelation(state, s1, false); + + final List features = new ArrayList<>(FEATURE_COUNT); + features.add("s0w=" + s0w); + features.add("s0t=" + s0t); + features.add("s1w=" + s1w); + features.add("s1t=" + s1t); + features.add("s2t=" + s2t); + features.add("b0w=" + b0w); + features.add("b0t=" + b0t); + features.add("b1w=" + b1w); + features.add("b1t=" + b1t); + features.add("b2t=" + b2t); + features.add("s0wt=" + s0w + WORD_TAG_SEPARATOR + s0t); + features.add("s1wt=" + s1w + WORD_TAG_SEPARATOR + s1t); + features.add("b0wt=" + b0w + WORD_TAG_SEPARATOR + b0t); + features.add("s0w,b0w=" + s0w + POSITION_SEPARATOR + b0w); + features.add("s0t,b0t=" + s0t + POSITION_SEPARATOR + b0t); + features.add("s0w,b0t=" + s0w + POSITION_SEPARATOR + b0t); + features.add("s0t,b0w=" + s0t + POSITION_SEPARATOR + b0w); + features.add("s0wt,b0t=" + s0w + WORD_TAG_SEPARATOR + s0t + POSITION_SEPARATOR + b0t); + features.add("s1t,s0t=" + s1t + POSITION_SEPARATOR + s0t); + features.add("s1t,s0w=" + s1t + POSITION_SEPARATOR + s0w); + features.add("s1w,s0t=" + s1w + POSITION_SEPARATOR + s0t); + features.add("s1t,s0t,b0t=" + s1t + POSITION_SEPARATOR + s0t + POSITION_SEPARATOR + b0t); + features.add("s0t,b0t,b1t=" + s0t + POSITION_SEPARATOR + b0t + POSITION_SEPARATOR + b1t); + features.add("s2t,s1t,s0t=" + s2t + POSITION_SEPARATOR + s1t + POSITION_SEPARATOR + s0t); + features.add("s0lct=" + s0lct); + features.add("s0rct=" + s0rct); + features.add("s1lct=" + s1lct); + features.add("s1rct=" + s1rct); + features.add("s0lcl=" + s0lcl); + features.add("s0rcl=" + s0rcl); + features.add("s1rcl=" + s1rcl); + features.add("s1t,s1rct,s0t=" + s1t + POSITION_SEPARATOR + s1rct + POSITION_SEPARATOR + s0t); + features.add("s0t,s0lct,b0t=" + s0t + POSITION_SEPARATOR + s0lct + POSITION_SEPARATOR + b0t); + features.add("s0deps=" + dependents(state, s0)); + features.add("s1deps=" + dependents(state, s1)); + final String distance = distance(s0, b0); + features.add("dist=" + distance); + features.add("dist,s0t,b0t=" + distance + POSITION_SEPARATOR + s0t + POSITION_SEPARATOR + b0t); + return features.toArray(new String[0]); + } + + /** The word at a position, or the marker value for the root and absent positions. */ + private String word(String[] tokens, int index) { + if (index == ArcStandardState.ROOT) { + return ROOT_VALUE; + } + return index == ArcStandardState.NONE ? NONE_VALUE : tokens[index]; + } + + /** The tag at a position, or the marker value for the root and absent positions. */ + private String tag(String[] tags, int index) { + if (index == ArcStandardState.ROOT) { + return ROOT_VALUE; + } + return index == ArcStandardState.NONE ? NONE_VALUE : tags[index]; + } + + /** The tag of a token's leftmost or rightmost dependent attached so far. */ + private String dependentTag(ArcStandardState state, String[] tags, int index, + boolean leftmost) { + if (index < 0) { + return NONE_VALUE; + } + final int dependent = + leftmost ? state.leftmostDependent(index) : state.rightmostDependent(index); + return tag(tags, dependent); + } + + /** The relation of a token's leftmost or rightmost dependent attached so far. */ + private String dependentRelation(ArcStandardState state, int index, + boolean leftmost) { + if (index < 0) { + return NONE_VALUE; + } + final int dependent = + leftmost ? state.leftmostDependent(index) : state.rightmostDependent(index); + if (dependent < 0) { + return NONE_VALUE; + } + final String relation = state.assignedRelation(dependent); + return relation == null ? NONE_VALUE : relation; + } + + /** A token's dependent count so far, capped at {@link #MAX_VALENCY}. */ + private String dependents(ArcStandardState state, int index) { + return index < 0 ? NONE_VALUE + : Integer.toString(Math.min(state.assignedDependents(index), MAX_VALENCY)); + } + + /** The bucketed distance between stack top and buffer front, capped at {@link #MAX_DISTANCE}. */ + private String distance(int s0, int b0) { + if (s0 < 0 || b0 < 0) { + return NONE_VALUE; + } + final int distance = b0 - s0; + return distance >= MAX_DISTANCE ? LONG_DISTANCE : Integer.toString(distance); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyCrossValidator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyCrossValidator.java new file mode 100644 index 0000000000..8464c4355c --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyCrossValidator.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.function.Predicate; + +import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.eval.CrossValidationPartitioner; +import opennlp.tools.util.eval.Mean; + +/** + * Cross validator for a {@link DependencyParser}. + * + *

The samples are split into {@code k} folds. For each fold a parser is trained on the + * other {@code k - 1} folds through the supplied {@link Trainer} and scored on the + * held-out fold with a {@link DependencyEvaluator}. The unlabeled and labeled attachment + * scores are accumulated over all evaluated tokens, so every token of the input counts + * once, in the fold it was held out from. Both parser implementations of this package can + * be validated, because the trainer is supplied by the caller.

+ * + * @see DependencyEvaluator + * @see CrossValidationPartitioner + * @since 3.0.0 + */ +public class DependencyCrossValidator { + + /** The smallest fold count that leaves training data for every fold. */ + private static final int MIN_FOLDS = 2; + + /** + * Trains a {@link DependencyParser} on the samples of the folds that are not held out. + */ + @FunctionalInterface + public interface Trainer { + + /** + * Trains a parser. + * + * @param samples The training samples of the current fold. Never {@code null}; the + * stream is read once and not closed by the validator. + * @return The trained parser. Must not be {@code null}. + * @throws IOException Thrown if reading the samples or training fails. + */ + DependencyParser train(ObjectStream samples) throws IOException; + } + + private final Trainer trainer; + private final Predicate punctuationTag; + private final Mean uas = new Mean(); + private final Mean las = new Mean(); + private final Mean uasExcludingPunctuation = new Mean(); + private final Mean lasExcludingPunctuation = new Mean(); + + /** + * Initializes a {@link DependencyCrossValidator} that treats tokens tagged + * {@link DependencyEvaluator#UNIVERSAL_PUNCTUATION_TAG} as punctuation. + * + * @param trainer Trains the parser of each fold. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code trainer} is {@code null}. + */ + public DependencyCrossValidator(Trainer trainer) { + this(trainer, DependencyEvaluator.UNIVERSAL_PUNCTUATION_TAG::equals); + } + + /** + * Initializes a {@link DependencyCrossValidator} with a custom notion of punctuation. + * + * @param trainer Trains the parser of each fold. Must not be {@code null}. + * @param punctuationTag Decides from a gold part-of-speech tag whether the token is + * punctuation and therefore left out of the punctuation-free + * scores. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public DependencyCrossValidator(Trainer trainer, Predicate punctuationTag) { + if (trainer == null) { + throw new IllegalArgumentException("trainer must not be null"); + } + if (punctuationTag == null) { + throw new IllegalArgumentException("punctuationTag must not be null"); + } + this.trainer = trainer; + this.punctuationTag = punctuationTag; + } + + /** + * Runs the cross validation and adds the scores of every fold to the totals. + * + * @param samples The samples to train and test with. Must not be {@code null} and must + * support {@link ObjectStream#reset()}, because every fold reads the + * stream from the start. The stream is not closed. + * @param folds The number of folds. Must be at least {@code 2}. + * @throws IOException Thrown if reading the samples or training fails. + * @throws IllegalArgumentException Thrown if {@code samples} is {@code null} or + * {@code folds} is below {@code 2}. + * @throws IllegalStateException Thrown if the trainer returns {@code null}. + */ + public void evaluate(ObjectStream samples, int folds) throws IOException { + if (samples == null) { + throw new IllegalArgumentException("samples must not be null"); + } + if (folds < MIN_FOLDS) { + throw new IllegalArgumentException("folds must be at least " + MIN_FOLDS + ": " + folds); + } + final CrossValidationPartitioner partitioner = + new CrossValidationPartitioner<>(samples, folds); + int fold = 0; + while (partitioner.hasNext()) { + final CrossValidationPartitioner.TrainingSampleStream training = + partitioner.next(); + final DependencyParser parser = trainer.train(training); + if (parser == null) { + throw new IllegalStateException("trainer returned null for fold " + fold); + } + final DependencyEvaluator evaluator = new DependencyEvaluator(parser, punctuationTag); + evaluator.evaluate(training.getTestSampleStream()); + uas.add(evaluator.getUas(), evaluator.getWordCount()); + las.add(evaluator.getLas(), evaluator.getWordCount()); + uasExcludingPunctuation.add(evaluator.getUasExcludingPunctuation(), + evaluator.getWordCountExcludingPunctuation()); + lasExcludingPunctuation.add(evaluator.getLasExcludingPunctuation(), + evaluator.getWordCountExcludingPunctuation()); + fold++; + } + } + + /** + * @return The unlabeled attachment score over all tokens evaluated so far. + */ + public double getUas() { + return uas.mean(); + } + + /** + * @return The labeled attachment score over all tokens evaluated so far. + */ + public double getLas() { + return las.mean(); + } + + /** + * @return The number of tokens evaluated so far; over one complete run this is the + * number of tokens in the samples, because every token is held out once. + */ + public long getWordCount() { + return uas.count(); + } + + /** + * @return The unlabeled attachment score over the evaluated tokens that are not + * punctuation. + */ + public double getUasExcludingPunctuation() { + return uasExcludingPunctuation.mean(); + } + + /** + * @return The labeled attachment score over the evaluated tokens that are not + * punctuation. + */ + public double getLasExcludingPunctuation() { + return lasExcludingPunctuation.mean(); + } + + /** + * @return The number of evaluated tokens that are not punctuation. + */ + public long getWordCountExcludingPunctuation() { + return uasExcludingPunctuation.count(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java new file mode 100644 index 0000000000..7308d4a717 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEvaluator.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.function.Predicate; + +import opennlp.tools.util.eval.Evaluator; +import opennlp.tools.util.eval.Mean; + +/** + * Measures the quality of a {@link DependencyParser} against gold + * {@link DependencySample samples} with the two standard scores: the unlabeled attachment + * score (UAS, the fraction of tokens with the correct head) and the labeled attachment + * score (LAS, the fraction of tokens with the correct head and relation label). + * + *

Both scores are also kept over the tokens that are not punctuation, because + * treebank evaluations customarily leave punctuation out of the attachment scores. A + * token counts as punctuation when its gold part-of-speech tag satisfies the predicate + * given at construction; by default that is the universal tag + * {@link #UNIVERSAL_PUNCTUATION_TAG}.

+ * + * @since 3.0.0 + */ +public class DependencyEvaluator extends Evaluator { + + /** + * The universal part-of-speech tag of punctuation, which the default constructor + * excludes from the punctuation-free scores. + */ + public static final String UNIVERSAL_PUNCTUATION_TAG = "PUNCT"; + + private final DependencyParser parser; + private final Predicate punctuationTag; + private final Mean uas = new Mean(); + private final Mean las = new Mean(); + private final Mean uasExcludingPunctuation = new Mean(); + private final Mean lasExcludingPunctuation = new Mean(); + + /** + * Initializes a {@link DependencyEvaluator} that treats tokens tagged + * {@link #UNIVERSAL_PUNCTUATION_TAG} as punctuation. + * + * @param parser The parser to evaluate. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code parser} is {@code null}. + */ + public DependencyEvaluator(DependencyParser parser) { + this(parser, UNIVERSAL_PUNCTUATION_TAG::equals); + } + + /** + * Initializes a {@link DependencyEvaluator} with a custom notion of punctuation. + * + * @param parser The parser to evaluate. Must not be {@code null}. + * @param punctuationTag Decides from a gold part-of-speech tag whether the token is + * punctuation and therefore left out of the punctuation-free + * scores. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public DependencyEvaluator(DependencyParser parser, Predicate punctuationTag) { + if (parser == null) { + throw new IllegalArgumentException("parser must not be null"); + } + if (punctuationTag == null) { + throw new IllegalArgumentException("punctuationTag must not be null"); + } + this.parser = parser; + this.punctuationTag = punctuationTag; + } + + /** + * {@inheritDoc} + * + *

The returned sample carries the predicted graph over the reference tokens, and + * every token of the reference contributes to both scores.

+ */ + @Override + protected DependencySample processSample(DependencySample reference) { + final DependencyGraph gold = reference.getGraph(); + final String[] tags = reference.getTags(); + final DependencyGraph predicted = parser.parse(reference.getTokens(), tags); + for (int i = 0; i < gold.size(); i++) { + final boolean headMatches = gold.headOf(i) == predicted.headOf(i); + final boolean labelMatches = + headMatches && gold.relationOf(i).equals(predicted.relationOf(i)); + uas.add(headMatches ? 1 : 0); + las.add(labelMatches ? 1 : 0); + if (!punctuationTag.test(tags[i])) { + uasExcludingPunctuation.add(headMatches ? 1 : 0); + lasExcludingPunctuation.add(labelMatches ? 1 : 0); + } + } + return new DependencySample(reference.getTokens(), tags, predicted); + } + + /** + * @return The unlabeled attachment score over all evaluated tokens. + */ + public double getUas() { + return uas.mean(); + } + + /** + * @return The labeled attachment score over all evaluated tokens. + */ + public double getLas() { + return las.mean(); + } + + /** + * @return The number of tokens scored so far. + */ + public long getWordCount() { + return uas.count(); + } + + /** + * @return The unlabeled attachment score over the evaluated tokens that are not + * punctuation. + */ + public double getUasExcludingPunctuation() { + return uasExcludingPunctuation.mean(); + } + + /** + * @return The labeled attachment score over the evaluated tokens that are not + * punctuation. + */ + public double getLasExcludingPunctuation() { + return lasExcludingPunctuation.mean(); + } + + /** + * @return The number of tokens scored so far that are not punctuation. + */ + public long getWordCountExcludingPunctuation() { + return uasExcludingPunctuation.count(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java new file mode 100644 index 0000000000..309b1e34c9 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyEventStream.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.ml.model.Event; +import opennlp.tools.util.ObjectStream; + +/** + * Turns {@link DependencySample samples} into training {@link Event events}: for each + * sample the {@link ArcStandardOracle} derives the gold transitions, and every transition + * becomes one event pairing the configuration features with the encoded transition. + * + *

Samples whose graph has no arc-standard derivation, that is non-projective trees, + * are skipped and counted; the count is logged once the stream is exhausted.

+ */ +class DependencyEventStream implements ObjectStream { + + private static final Logger logger = LoggerFactory.getLogger(DependencyEventStream.class); + + private final ObjectStream samples; + private final DependencyContextGenerator contextGenerator; + private final Deque pending = new ArrayDeque<>(); + + private int skipped; + + /** + * Initializes the stream. + * + * @param samples The samples to convert. Must not be {@code null}. + * @param contextGenerator The feature generator. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if any parameter is {@code null}. + */ + DependencyEventStream(ObjectStream samples, + DependencyContextGenerator contextGenerator) { + if (samples == null) { + throw new IllegalArgumentException("samples must not be null"); + } + if (contextGenerator == null) { + throw new IllegalArgumentException("contextGenerator must not be null"); + } + this.samples = samples; + this.contextGenerator = contextGenerator; + } + + /** + * {@inheritDoc} + * Returns the events of one sample at a time and moves on to the next sample once + * they are exhausted. + * + * @throws IOException Thrown if reading the underlying samples fails. + */ + @Override + public Event read() throws IOException { + while (pending.isEmpty()) { + final DependencySample sample = samples.read(); + if (sample == null) { + if (skipped > 0) { + logger.warn("Skipped {} non-projective sample(s) without an arc-standard derivation.", + skipped); + skipped = 0; + } + return null; + } + if (!ArcStandardOracle.isProjective(sample.getGraph())) { + skipped++; + continue; + } + final List transitions = ArcStandardOracle.transitions(sample.getGraph()); + final ArcStandardState state = new ArcStandardState(sample.getGraph().size()); + final String[] tokens = sample.getTokens(); + final String[] tags = sample.getTags(); + for (final Transition transition : transitions) { + pending.add(new Event(transition.encode(), + contextGenerator.getContext(state, tokens, tags))); + state.apply(transition); + } + } + return pending.poll(); + } + + /** + * {@inheritDoc} + * Also discards buffered events and the skipped sample count. + * + * @throws IOException Thrown if resetting the underlying samples fails. + * @throws UnsupportedOperationException Thrown if the underlying samples cannot be reset. + */ + @Override + public void reset() throws IOException, UnsupportedOperationException { + samples.reset(); + pending.clear(); + skipped = 0; + } + + /** + * {@inheritDoc} + * + * @throws IOException Thrown if closing the underlying samples fails. + */ + @Override + public void close() throws IOException { + samples.close(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java new file mode 100644 index 0000000000..6747875e6f --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyModel.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.Serial; +import java.nio.file.Path; +import java.util.Map; + +import opennlp.tools.ml.model.AbstractModel; +import opennlp.tools.ml.model.MaxentModel; +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.util.model.BaseModel; + +/** + * The persisted form of a trained {@link DependencyParserME}: the transition + * classification model plus the standard model manifest, serialized and loaded through + * the same {@link BaseModel} machinery as every other tool model. + * + * @see DependencyParserME + * @since 3.0.0 + */ +public class DependencyModel extends BaseModel { + + @Serial + private static final long serialVersionUID = -2928968185269611443L; + + /** The component name recorded in the model manifest. */ + private static final String COMPONENT_NAME = "DependencyParserME"; + + /** The artifact map entry holding the transition classification model. */ + static final String PARSER_MODEL_ENTRY_NAME = "depparse.model"; + + /** + * Initializes a {@link DependencyModel} from a trained transition model. + * + * @param languageCode The ISO language code of the training data. Must not be + * {@code null}. + * @param parserModel The transition classification model. Must not be {@code null}. + * @param manifestInfoEntries Additional entries for the manifest, or {@code null}. + * @throws IllegalArgumentException Thrown if {@code languageCode} or + * {@code parserModel} is {@code null}. + */ + public DependencyModel(String languageCode, MaxentModel parserModel, + Map manifestInfoEntries) { + super(COMPONENT_NAME, notNull(languageCode, "languageCode"), manifestInfoEntries); + if (parserModel == null) { + throw new IllegalArgumentException("parserModel must not be null"); + } + artifactMap.put(PARSER_MODEL_ENTRY_NAME, parserModel); + checkArtifactMap(); + } + + /** + * Initializes a {@link DependencyModel} from a serialized model. + * + * @param in The stream to read the model from. Must not be {@code null}. + * @throws IOException Thrown if reading fails or the content is not a valid model. + * @throws IllegalArgumentException Thrown if {@code in} is {@code null}. + */ + public DependencyModel(InputStream in) throws IOException { + super(COMPONENT_NAME, notNull(in, "in")); + } + + /** + * Initializes a {@link DependencyModel} from a serialized model file. + * + * @param modelFile The file to read the model from. Must not be {@code null}. + * @throws IOException Thrown if reading fails or the content is not a valid model. + * @throws IllegalArgumentException Thrown if {@code modelFile} is {@code null}. + */ + public DependencyModel(File modelFile) throws IOException { + super(COMPONENT_NAME, notNull(modelFile, "modelFile")); + } + + /** + * Initializes a {@link DependencyModel} from a serialized model file. + * + * @param modelPath The path to read the model from. Must not be {@code null}. + * @throws IOException Thrown if reading fails or the content is not a valid model. + * @throws IllegalArgumentException Thrown if {@code modelPath} is {@code null}. + */ + public DependencyModel(Path modelPath) throws IOException { + super(COMPONENT_NAME, notNull(modelPath, "modelPath")); + } + + /** + * Rejects a {@code null} constructor argument before it reaches the superclass. + * + * @param value The argument to check. + * @param name The parameter name for the error message. + * @param The argument type. + * @return {@code value}, unchanged. + * @throws IllegalArgumentException Thrown if {@code value} is {@code null}. + */ + private static T notNull(T value, String name) { + if (value == null) { + throw new IllegalArgumentException(name + " must not be null"); + } + return value; + } + + /** + * {@inheritDoc} + * + * @throws InvalidFormatException Thrown if the transition model artifact is missing or + * not a supported model type. + */ + @Override + protected void validateArtifactMap() throws InvalidFormatException { + super.validateArtifactMap(); + if (!(artifactMap.get(PARSER_MODEL_ENTRY_NAME) instanceof AbstractModel)) { + throw new InvalidFormatException("The " + PARSER_MODEL_ENTRY_NAME + + " artifact is missing or not a supported transition model."); + } + } + + /** + * @return The transition classification model. Never {@code null}. + */ + public MaxentModel getParserModel() { + return (MaxentModel) artifactMap.get(PARSER_MODEL_ENTRY_NAME); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java new file mode 100644 index 0000000000..44c1102111 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserME.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.ml.EventTrainer; +import opennlp.tools.ml.TrainerFactory; +import opennlp.tools.ml.TrainerFactory.TrainerType; +import opennlp.tools.ml.model.Event; +import opennlp.tools.ml.model.MaxentModel; +import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.TrainingParameters; + +/** + * A greedy transition-based {@link DependencyParser}: a maximum entropy classifier picks + * the next arc-standard {@link Transition} for each configuration until the parse is + * complete, always taking the highest scoring transition that is applicable. + * + *

The parser holds an immutable model and no per-parse state, so one instance can be + * shared between threads.

+ * + * @see DependencyParser + * @since 3.0.0 + */ +@ThreadSafe +public class DependencyParserME implements DependencyParser { + + private final MaxentModel model; + private final DependencyContextGenerator contextGenerator; + private final Transition[] transitions; + + /** + * Initializes a {@link DependencyParserME} from a {@link DependencyModel}. + * + * @param model The model to parse with. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an + * outcome inventory is invalid or cannot parse a sentence. + */ + public DependencyParserME(DependencyModel model) { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + this.model = model.getParserModel(); + this.contextGenerator = new DependencyContextGenerator(); + this.transitions = decodeOutcomes(this.model); + } + + /** + * Initializes a {@link DependencyParserME} with a raw transition model. + * + * @param model The transition classification model. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an + * outcome inventory is invalid or cannot parse a sentence. + */ + public DependencyParserME(MaxentModel model) { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + this.model = model; + this.contextGenerator = new DependencyContextGenerator(); + this.transitions = decodeOutcomes(model); + } + + /** + * Decodes the outcome inventory once, so that decoding a sentence indexes it instead + * of parsing an outcome string per configuration and outcome. + * + * @param model The transition classification model. + * @return The transitions by outcome index. Never {@code null}. + * @throws IllegalArgumentException Thrown if an outcome does not decode to a + * transition, which means the model is not a dependency parser model. + */ + private static Transition[] decodeOutcomes(MaxentModel model) { + final Transition[] decoded = new Transition[model.getNumOutcomes()]; + final Set seen = new HashSet<>(); + boolean hasShift = false; + boolean hasRightArc = false; + for (int i = 0; i < decoded.length; i++) { + final String outcome = model.getOutcome(i); + try { + decoded[i] = Transition.decode(outcome); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("model outcome is not a transition: " + outcome, e); + } + if (!seen.add(decoded[i])) { + throw new IllegalArgumentException("duplicate model transition: " + outcome); + } + hasShift |= decoded[i].type() == Transition.Type.SHIFT; + hasRightArc |= decoded[i].type() == Transition.Type.RIGHT_ARC; + } + if (!hasShift) { + throw new IllegalArgumentException("model has no SHIFT action"); + } + if (!hasRightArc) { + throw new IllegalArgumentException("model has no RIGHT_ARC action"); + } + return decoded; + } + + /** + * {@inheritDoc} + * + * @throws IllegalStateException Thrown if the model returns scores that do not match + * its outcomes or are not finite. + */ + @Override + public DependencyGraph parse(String[] tokens, String[] tags) { + ParserInput.check(tokens, tags); + final ArcStandardState state = new ArcStandardState(tokens.length); + while (!state.isTerminal()) { + state.apply(bestApplicable(state, tokens, tags)); + } + return state.toGraph(); + } + + /** + * Scores all outcomes for the current configuration and picks the best transition that + * is applicable; inapplicable outcomes are passed over regardless of score. + * + * @param state The current configuration. + * @param tokens The sentence tokens. + * @param tags The part-of-speech tags aligned with {@code tokens}. + * @return The highest scoring applicable transition. Never {@code null}. + * @throws IllegalStateException Thrown if the model returns a score count that does not + * match its outcomes, a non-finite score, or no applicable transition. + */ + private Transition bestApplicable(ArcStandardState state, String[] tokens, String[] tags) { + final double[] probabilities = model.eval(contextGenerator.getContext(state, tokens, tags)); + if (probabilities == null || probabilities.length != transitions.length) { + final int count = probabilities == null ? 0 : probabilities.length; + throw new IllegalStateException("model returned " + count + " scores for " + + transitions.length + " outcomes"); + } + Transition best = null; + double bestProbability = Double.NEGATIVE_INFINITY; + for (int i = 0; i < probabilities.length; i++) { + if (!Double.isFinite(probabilities[i])) { + throw new IllegalStateException("model returned a non-finite score at index " + i); + } + if (probabilities[i] <= bestProbability) { + continue; + } + if (state.canApply(transitions[i])) { + best = transitions[i]; + bestProbability = probabilities[i]; + } + } + if (best == null) { + throw new IllegalStateException( + "no applicable transition among the model outcomes in " + state); + } + return best; + } + + /** + * Trains a greedy arc-standard parser model from dependency samples. + * + *

Non-projective samples have no arc-standard derivation and are skipped during + * event generation.

+ * + * @param languageCode The ISO language code of the training data. Must not be + * {@code null}. + * @param samples The training samples. Must not be {@code null}. + * @param parameters The {@link TrainingParameters}. Must not be {@code null} and must + * select an event model trainer. + * @return A trained {@link DependencyModel}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null} or the + * configured trainer is not an event model trainer. + */ + public static DependencyModel train(String languageCode, + ObjectStream samples, TrainingParameters parameters) + throws IOException { + if (languageCode == null || samples == null || parameters == null) { + throw new IllegalArgumentException( + "languageCode, samples and parameters must not be null"); + } + final TrainerType trainerType = TrainerFactory.getTrainerType(parameters); + if (!TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { + throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); + } + final Map manifestInfoEntries = new HashMap<>(); + final EventTrainer trainer = + TrainerFactory.getEventTrainer(parameters, manifestInfoEntries); + final ObjectStream events = + new DependencyEventStream(samples, new DependencyContextGenerator()); + return new DependencyModel(languageCode, trainer.train(events), manifestInfoEntries); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java new file mode 100644 index 0000000000..d4e3d14773 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * The feature template of the feedforward parser: a fixed set of configuration + * positions whose words, tags, and arc labels are embedded and concatenated into the + * network input, following + * Chen and Manning (2014). + * + *

Positions: the top three stack and buffer items; the leftmost and rightmost + * dependents of the top two stack items; and the leftmost dependent of the leftmost + * dependent and rightmost dependent of the rightmost dependent of the top two stack + * items, capturing second-order structure. Words and tags are read for all positions, + * labels only for the dependent positions, whose relations are already assigned.

+ */ +final class FeedforwardContext { + + /** The number of positions whose word and tag are embedded. */ + static final int POSITIONS = 14; + + /** The number of dependent positions whose arc label is embedded. */ + static final int LABEL_POSITIONS = 8; + + /** The number of symbolic features in one parser configuration. */ + static final int FEATURE_COUNT = 2 * POSITIONS + LABEL_POSITIONS; + + /** The index of the first dependent position; the stack and buffer items precede it. */ + private static final int FIRST_DEPENDENT_POSITION = 6; + + /** Prevents construction of this utility class. */ + private FeedforwardContext() { + } + + /** + * Extracts the symbolic features of a configuration: {@link #POSITIONS} words, then + * {@link #POSITIONS} tags, then {@link #LABEL_POSITIONS} labels; absent positions + * yield {@code null} entries, which the vocabulary maps to its padding symbol. + * + * @param state The configuration to describe. Must not be {@code null}. + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. Must not be + * {@code null}. + * @return The symbolic features, in the order described above. Never {@code null}. + */ + static String[] extract(ArcStandardState state, String[] tokens, String[] tags) { + final int s0 = state.stack(0); + final int s1 = state.stack(1); + final int[] positions = { + s0, s1, state.stack(2), + state.buffer(0), state.buffer(1), state.buffer(2), + leftmost(state, s0), rightmost(state, s0), + leftmost(state, s1), rightmost(state, s1), + leftmost(state, leftmost(state, s0)), rightmost(state, rightmost(state, s0)), + leftmost(state, leftmost(state, s1)), rightmost(state, rightmost(state, s1)) + }; + final String[] features = new String[FEATURE_COUNT]; + for (int i = 0; i < POSITIONS; i++) { + features[i] = symbol(tokens, positions[i]); + features[POSITIONS + i] = symbol(tags, positions[i]); + } + for (int i = 0; i < LABEL_POSITIONS; i++) { + final int position = positions[FIRST_DEPENDENT_POSITION + i]; + features[2 * POSITIONS + i] = + position >= 0 ? state.assignedRelation(position) : null; + } + return features; + } + + /** The leftmost dependent of a position, or the absence marker for absent positions. */ + private static int leftmost(ArcStandardState state, int index) { + return index >= 0 ? state.leftmostDependent(index) : ArcStandardState.NONE; + } + + /** The rightmost dependent of a position, or the absence marker for absent positions. */ + private static int rightmost(ArcStandardState state, int index) { + return index >= 0 ? state.rightmostDependent(index) : ArcStandardState.NONE; + } + + /** The value at a position: the root symbol for the root, {@code null} when absent. */ + private static String symbol(String[] values, int index) { + if (index == ArcStandardState.ROOT) { + return FeedforwardDependencyModel.ROOT_SYMBOL; + } + return index == ArcStandardState.NONE ? null : values[index]; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java new file mode 100644 index 0000000000..4ba91103bf --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java @@ -0,0 +1,951 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReferenceArray; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.util.StringUtil; + +/** + * The weights of the feedforward transition parser: embeddings for words, tags, and arc + * labels, one hidden layer with cube activation, and a transition output layer, stored + * in a plain versioned binary format with no serialization framework involved. The + * architecture follows + * Chen and Manning (2014). + * + *

The network runs with Java arrays and requires no native runtime. The same class + * scores configurations during training and decoding. Unknown words use a learned + * fallback row; words are matched case-insensitively after + * {@link #normalize(String) normalization}.

+ * + *

Instances returned by {@link FeedforwardDependencyTrainer} are immutable and safe + * to share between threads. {@link FeedforwardDependencyTrainer#refine refine} updates + * an independent copy.

+ * + * @see FeedforwardDependencyParser + * @see FeedforwardDependencyTrainer + * @since 3.0.0 + */ +@ThreadSafe +public class FeedforwardDependencyModel { + + /** The format header written before every serialized model; it carries the format version. */ + private static final String MAGIC = "ONLP-FFDP-1"; + + /** The vocabulary label used in load errors for the word map. */ + private static final String WORD_VOCABULARY = "word vocabulary"; + + /** The vocabulary label used in load errors for the tag map. */ + private static final String TAG_VOCABULARY = "tag vocabulary"; + + /** The vocabulary label used in load errors for the label map. */ + private static final String LABEL_VOCABULARY = "label vocabulary"; + + /** Maximum combined entries across the word, tag, and label maps. */ + private static final int MAX_VOCABULARY_ENTRIES = 2_000_000; + + /** Maximum transitions accepted from a serialized model. */ + private static final int MAX_TRANSITIONS = 100_000; + + /** Maximum embedding width accepted from a serialized model. */ + static final int MAX_EMBEDDING_SIZE = 4_096; + + /** Maximum hidden-layer width accepted from a serialized model. */ + static final int MAX_HIDDEN_SIZE = 65_536; + + /** Maximum float values allocated while loading a serialized model. */ + static final long MAX_MODEL_FLOAT_VALUES = 100_000_000L; + + /** U+03A3, GREEK CAPITAL LETTER SIGMA, the one code point with a contextual lowering. */ + private static final int GREEK_CAPITAL_SIGMA = 0x03A3; + + /** U+03C2, GREEK SMALL LETTER FINAL SIGMA, the word-final lowering of the capital. */ + private static final char GREEK_SMALL_FINAL_SIGMA = '\u03C2'; + + /** The vocabulary key of every word, tag, or label the model has no embedding row for. */ + static final String UNKNOWN = "*UNK*"; + + /** The vocabulary key of a template position that does not exist in a configuration. */ + static final String ABSENT = "*NULL*"; + + /** The vocabulary key of the artificial root node. */ + static final String ROOT_SYMBOL = "*ROOT*"; + + /** The lazy scoring cache; {@code null} until {@link #enableScoringCache()}. */ + private volatile ContributionCache cache; + + private final Map wordIds; + private final Map tagIds; + private final Map labelIds; + private final String[] transitions; + + private final int embeddingSize; + private final float[][] embeddings; + private final float[][] hiddenWeights; + private final float[] hiddenBias; + private final float[][] outputWeights; + private final float[] outputBias; + + /** + * Assembles a model from its vocabularies and weights, copying the maps so later + * changes by the caller do not reach the model. + * + * @param wordIds The word vocabulary, mapping each normalized word to its row. + * @param tagIds The tag vocabulary. + * @param labelIds The dependency label vocabulary. + * @param transitions The transition inventory, indexed by output row. + * @param embeddingSize The embedding dimensionality. + * @param embeddings The embedding rows for words, tags, and labels. + * @param hiddenWeights The hidden layer weights. + * @param hiddenBias The hidden layer bias. + * @param outputWeights The output layer weights. + * @param outputBias The output layer bias. + */ + FeedforwardDependencyModel(Map wordIds, Map tagIds, + Map labelIds, String[] transitions, int embeddingSize, + float[][] embeddings, float[][] hiddenWeights, float[] hiddenBias, + float[][] outputWeights, float[] outputBias) { + this.wordIds = Map.copyOf(wordIds); + this.tagIds = Map.copyOf(tagIds); + this.labelIds = Map.copyOf(labelIds); + this.transitions = transitions; + this.embeddingSize = embeddingSize; + this.embeddings = embeddings; + this.hiddenWeights = hiddenWeights; + this.hiddenBias = hiddenBias; + this.outputWeights = outputWeights; + this.outputBias = outputBias; + } + + /** + * Scores every transition for a configuration described by embedding row indices. + * + * @param features The embedding rows of the configuration, as produced by + * {@link #featureIds(String[])}. Must not be {@code null}. + * @return One unnormalized score per transition, indexed like + * {@link #transitions()}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code features} is {@code null}, does not + * have the required length, or contains an invalid embedding index. + * @throws IllegalStateException If the model produces a non-finite transition score. + */ + double[] score(int[] features) { + if (features == null) { + throw new IllegalArgumentException("features must not be null"); + } + if (features.length != FeedforwardContext.FEATURE_COUNT) { + throw new IllegalArgumentException("features must contain " + + FeedforwardContext.FEATURE_COUNT + " embedding indices"); + } + for (int i = 0; i < features.length; i++) { + if (features[i] < 0 || features[i] >= embeddings.length) { + throw new IllegalArgumentException( + "feature embedding out of range at index " + i + ": " + features[i]); + } + } + final int hidden = hiddenBias.length; + final double[] h = new double[hidden]; + for (int j = 0; j < hidden; j++) { + h[j] = hiddenBias[j]; + } + final ContributionCache cache = this.cache; + for (int f = 0; f < features.length; f++) { + final int row = features[f]; + final double[] contribution = cache == null ? null : cache.contribution(this, f, row); + if (contribution != null) { + for (int j = 0; j < hidden; j++) { + h[j] += contribution[j]; + } + } else { + final float[] embedding = embeddings[row]; + final int offset = f * embeddingSize; + for (int j = 0; j < hidden; j++) { + final float[] weights = hiddenWeights[j]; + double sum = 0.0; + for (int d = 0; d < embeddingSize; d++) { + sum += (double) weights[offset + d] * embedding[d]; + } + h[j] += sum; + } + } + } + for (int j = 0; j < hidden; j++) { + h[j] = h[j] * h[j] * h[j]; + } + final double[] scores = new double[transitions.length]; + for (int o = 0; o < scores.length; o++) { + final float[] row = outputWeights[o]; + double sum = outputBias[o]; + for (int j = 0; j < hidden; j++) { + sum += row[j] * h[j]; + } + if (!Double.isFinite(sum)) { + throw new IllegalStateException("the model produced a non-finite transition score"); + } + scores[o] = sum; + } + return scores; + } + + /** + * Caches hidden-layer contributions by template position and embedding row. + * + *

Contributions retain double precision so cached and direct scoring use the + * same values. Training and refinement use uncached copies; {@link #copy()} + * does not copy the cache.

+ */ + synchronized void enableScoringCache() { + if (cache == null) { + cache = new ContributionCache(FeedforwardContext.FEATURE_COUNT, embeddings.length); + } + } + + /** + * Stores contributions on first use up to a shared entry limit. Reference tables + * are allocated for every template position and embedding row. Pairs beyond the + * entry limit use direct scoring. + */ + private static final class ContributionCache { + + /** The maximum number of cached (position, row) pairs. */ + private static final int MAX_PAIRS = 32768; + + /** The cached contribution vectors, indexed by template position and embedding row. */ + private final AtomicReferenceArray[] byPosition; + + /** The number of pairs the cache may still add before the budget is spent. */ + private final AtomicInteger remaining = new AtomicInteger(MAX_PAIRS); + + /** + * Creates an empty cache. + * + * @param positions The number of feature positions. + * @param rows The number of embedding rows a position can hold. + */ + @SuppressWarnings("unchecked") + private ContributionCache(int positions, int rows) { + byPosition = new AtomicReferenceArray[positions]; + for (int f = 0; f < positions; f++) { + byPosition[f] = new AtomicReferenceArray<>(rows); + } + } + + /** + * Returns the cached hidden-layer contribution of one pair, computing and + * publishing it on first sight while the budget lasts. + * + * @param model The immutable model the contributions derive from. + * @param position The template position. + * @param row The embedding row at that position. + * @return The contribution vector, or {@code null} when the budget is spent and + * the pair is not cached. + */ + private double[] contribution(FeedforwardDependencyModel model, int position, int row) { + final AtomicReferenceArray slots = byPosition[position]; + double[] contribution = slots.get(row); + if (contribution != null) { + return contribution; + } + if (!reserve()) { + return null; + } + boolean published = false; + try { + final int hidden = model.hiddenBias.length; + final float[] embedding = model.embeddings[row]; + final int offset = position * model.embeddingSize; + contribution = new double[hidden]; + for (int j = 0; j < hidden; j++) { + final float[] weights = model.hiddenWeights[j]; + double sum = 0.0; + for (int d = 0; d < model.embeddingSize; d++) { + sum += (double) weights[offset + d] * embedding[d]; + } + contribution[j] = sum; + } + published = slots.compareAndSet(row, null, contribution); + return published ? contribution : slots.get(row); + } finally { + if (!published) { + remaining.incrementAndGet(); + } + } + } + + /** + * Reserves capacity before allocating a contribution array. + * + * @return Whether one entry was reserved. + */ + private boolean reserve() { + int available = remaining.get(); + while (available > 0) { + if (remaining.compareAndSet(available, available - 1)) { + return true; + } + available = remaining.get(); + } + return false; + } + } + + /** + * Maps the symbolic features of {@link FeedforwardContext} onto embedding rows. + * + * @param symbols The symbolic features. Must not be {@code null}. + * @return The embedding row per feature. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code symbols} is {@code null} or does + * not have the required length. + */ + int[] featureIds(String[] symbols) { + if (symbols == null) { + throw new IllegalArgumentException("symbols must not be null"); + } + if (symbols.length != FeedforwardContext.FEATURE_COUNT) { + throw new IllegalArgumentException("symbols must contain " + + FeedforwardContext.FEATURE_COUNT + " features"); + } + final int[] ids = new int[symbols.length]; + for (int i = 0; i < FeedforwardContext.POSITIONS; i++) { + ids[i] = lookup(wordIds, normalize(symbols[i])); + } + for (int i = FeedforwardContext.POSITIONS; i < 2 * FeedforwardContext.POSITIONS; i++) { + ids[i] = lookup(tagIds, symbols[i]); + } + for (int i = 2 * FeedforwardContext.POSITIONS; i < symbols.length; i++) { + ids[i] = lookup(labelIds, symbols[i]); + } + return ids; + } + + /** + * @return The transition outcome strings by output index. Never {@code null}. + */ + public String[] transitions() { + return transitions.clone(); + } + + /** + * Lowercases a word symbol; special symbols and absences pass through. + * + *

Case is mapped per code point like + * {@link StringUtil#toLowerCase(CharSequence)}. Greek capital sigma also applies the + * cased-letter and case-ignorable context defined by the Unicode + * SpecialCasing + * data within one token.

+ * + * @param word The word to normalize. May be {@code null}. + * @return The vocabulary key of {@code word}, or {@code null} if {@code word} is + * {@code null}. + */ + static String normalize(String word) { + if (word == null) { + return null; + } + if (isSpecialSymbol(word)) { + return word; + } + final String simple = StringUtil.toLowerCase(word); + if (simple.equals(word)) { + return word; + } + StringBuilder contextual = null; + int sourceIndex = 0; + int loweredIndex = 0; + while (sourceIndex < word.length()) { + final int cp = word.codePointAt(sourceIndex); + final int width = Character.charCount(cp); + final int loweredCp = simple.codePointAt(loweredIndex); + final int loweredWidth = Character.charCount(loweredCp); + if (cp == GREEK_CAPITAL_SIGMA && hasCasedLetterBefore(word, sourceIndex) + && !hasCasedLetterAfter(word, sourceIndex + width)) { + if (contextual == null) { + contextual = new StringBuilder(simple.length()); + contextual.append(simple, 0, loweredIndex); + } + contextual.append(GREEK_SMALL_FINAL_SIGMA); + } else if (contextual != null) { + contextual.appendCodePoint(loweredCp); + } + sourceIndex += width; + loweredIndex += loweredWidth; + } + return contextual == null ? simple : contextual.toString(); + } + + /** + * Checks the final-sigma prefix context, skipping case-ignorable code points. + * + * @param word The word being normalized. + * @param index The index of the capital sigma in {@code word}. + * @return {@code true} if a cased letter precedes {@code index}. + */ + private static boolean hasCasedLetterBefore(String word, int index) { + int current = index; + while (current > 0) { + final int cp = word.codePointBefore(current); + current -= Character.charCount(cp); + if (!isCaseIgnorable(cp)) { + return isCased(cp); + } + } + return false; + } + + /** + * Checks the final-sigma suffix context, skipping case-ignorable code points. + * + * @param word The word being normalized. + * @param index The index just after the capital sigma in {@code word}. + * @return {@code true} if a cased letter follows at or after {@code index}. + */ + private static boolean hasCasedLetterAfter(String word, int index) { + int current = index; + while (current < word.length()) { + final int cp = word.codePointAt(current); + if (!isCaseIgnorable(cp)) { + return isCased(cp); + } + current += Character.charCount(cp); + } + return false; + } + + /** + * @param cp The code point to test. + * @return {@code true} if {@code cp} is uppercase, lowercase, or titlecase. + */ + private static boolean isCased(int cp) { + return Character.isUpperCase(cp) || Character.isLowerCase(cp) + || Character.isTitleCase(cp); + } + + /** + * @param cp The code point to test. + * @return {@code true} if {@code cp} is skipped when looking for cased neighbors. + */ + private static boolean isCaseIgnorable(int cp) { + final boolean categoryMatches = switch (Character.getType(cp)) { + case Character.NON_SPACING_MARK, Character.ENCLOSING_MARK, + Character.FORMAT, Character.MODIFIER_LETTER, Character.MODIFIER_SYMBOL -> true; + default -> false; + }; + if (categoryMatches) { + return true; + } + return switch (cp) { + case '\'', '.', ':', '\u00B7', '\u0387', '\u055F', '\u05F4', + '\u2018', '\u2019', '\u2024', '\u2027', '\uFE13', '\uFE52', + '\uFE55', '\uFF07', '\uFF0E', '\uFF1A' -> true; + default -> false; + }; + } + + /** + * @param symbol The symbol to test. May be {@code null}. + * @return {@code true} if {@code symbol} is one of the reserved feature values. + */ + static boolean isSpecialSymbol(String symbol) { + return UNKNOWN.equals(symbol) || ABSENT.equals(symbol) || ROOT_SYMBOL.equals(symbol); + } + + /** + * Resolves a symbol to its embedding row: absences map to {@link #ABSENT}, symbols + * without a row of their own fall back to {@link #UNKNOWN}. + * + * @param ids The vocabulary to resolve against. + * @param symbol The symbol to resolve, or {@code null} for an absent position. + * @return The embedding row of the symbol or of its fallback. + * @throws IllegalStateException Thrown if {@code ids} has no {@link #UNKNOWN} row. + */ + private static int lookup(Map ids, String symbol) { + Integer id = ids.get(symbol == null ? ABSENT : symbol); + if (id == null) { + id = ids.get(UNKNOWN); + } + if (id == null) { + throw new IllegalStateException("vocabulary has no " + UNKNOWN + " row to fall back on"); + } + return id; + } + + /** + * Writes the model in the versioned binary format. + * + * @param out The stream to write to. Must not be {@code null}. Not closed. + * @throws IOException Thrown if writing fails. + */ + public void serialize(OutputStream out) throws IOException { + if (out == null) { + throw new IllegalArgumentException("out must not be null"); + } + final DataOutputStream data = new DataOutputStream(new BufferedOutputStream(out)); + data.writeUTF(MAGIC); + writeVocabulary(data, wordIds); + writeVocabulary(data, tagIds); + writeVocabulary(data, labelIds); + data.writeInt(transitions.length); + for (final String transition : transitions) { + data.writeUTF(transition); + } + data.writeInt(embeddingSize); + writeMatrix(data, embeddings); + writeMatrix(data, hiddenWeights); + writeVector(data, hiddenBias); + writeMatrix(data, outputWeights); + writeVector(data, outputBias); + data.flush(); + } + + /** + * Loads a model from the versioned binary format. + * + * @param in The stream to read from. Must not be {@code null}. Not closed. + * @return The loaded model. Never {@code null}. + * @throws IOException Thrown if reading fails. + * @throws InvalidFormatException Thrown if the content is not a valid model. + */ + public static FeedforwardDependencyModel load(InputStream in) throws IOException { + if (in == null) { + throw new IllegalArgumentException("in must not be null"); + } + final DataInputStream data = new DataInputStream(new BufferedInputStream(in)); + final String magic = data.readUTF(); + if (!MAGIC.equals(magic)) { + throw new InvalidFormatException("not a feedforward dependency model: " + magic); + } + final Map wordIds = readVocabulary(data, WORD_VOCABULARY); + final Map tagIds = readVocabulary(data, TAG_VOCABULARY); + final Map labelIds = readVocabulary(data, LABEL_VOCABULARY); + final int embeddingRows = validateVocabularies(wordIds, tagIds, labelIds); + final String[] transitions = new String[ + readCount(data, "transition count", MAX_TRANSITIONS, false)]; + final Set transitionSet = new HashSet<>(); + boolean hasShift = false; + boolean hasRightArc = false; + for (int i = 0; i < transitions.length; i++) { + transitions[i] = data.readUTF(); + if (!transitionSet.add(transitions[i])) { + throw new InvalidFormatException("duplicate transition: " + transitions[i]); + } + try { + final Transition transition = Transition.decode(transitions[i]); + hasShift |= transition.type() == Transition.Type.SHIFT; + hasRightArc |= transition.type() == Transition.Type.RIGHT_ARC; + } catch (IllegalArgumentException e) { + throw new InvalidFormatException("invalid transition: " + transitions[i], e); + } + } + if (!hasShift) { + throw new InvalidFormatException("transition inventory has no SHIFT action"); + } + if (!hasRightArc) { + throw new InvalidFormatException("transition inventory has no RIGHT_ARC action"); + } + final int embeddingSize = readCount(data, "embedding size", MAX_EMBEDDING_SIZE, false); + final long[] remainingFloats = {MAX_MODEL_FLOAT_VALUES}; + final float[][] embeddings = readMatrix(data, embeddingRows, embeddingSize, + MAX_VOCABULARY_ENTRIES, remainingFloats, "embedding matrix"); + final int inputSize = FeedforwardContext.FEATURE_COUNT * embeddingSize; + final float[][] hiddenWeights = readMatrix(data, -1, inputSize, + MAX_HIDDEN_SIZE, remainingFloats, "hidden matrix"); + final float[] hiddenBias = readVector(data, hiddenWeights.length, + remainingFloats, "hidden bias"); + final float[][] outputWeights = readMatrix(data, transitions.length, + hiddenWeights.length, MAX_TRANSITIONS, remainingFloats, "output matrix"); + final float[] outputBias = readVector(data, transitions.length, + remainingFloats, "output bias"); + if (data.read() != -1) { + throw new InvalidFormatException("trailing data after feedforward dependency model"); + } + return new FeedforwardDependencyModel(wordIds, tagIds, labelIds, transitions, + embeddingSize, embeddings, hiddenWeights, hiddenBias, outputWeights, outputBias); + } + + /** + * Loads a model from a file. + * + * @param path The file to read. Must not be {@code null}. + * @return The loaded model. Never {@code null}. + * @throws IOException Thrown if reading fails. + * @throws InvalidFormatException Thrown if the content is not a valid model. + */ + public static FeedforwardDependencyModel load(Path path) throws IOException { + if (path == null) { + throw new IllegalArgumentException("path must not be null"); + } + try (InputStream in = Files.newInputStream(path)) { + return load(in); + } + } + + /** + * Writes one vocabulary as its size followed by (symbol, id) pairs in ascending id + * order, so that serializing the same model yields the same bytes on every JVM. + * + * @param data The output to write to. + * @param ids The vocabulary to write. + * @throws IOException Thrown if writing fails. + */ + private static void writeVocabulary(DataOutputStream data, Map ids) + throws IOException { + data.writeInt(ids.size()); + final List> entries = new ArrayList<>(ids.entrySet()); + entries.sort(Map.Entry.comparingByValue()); + for (final Map.Entry entry : entries) { + data.writeUTF(entry.getKey()); + data.writeInt(entry.getValue()); + } + } + + /** + * Reads one vocabulary written by {@link #writeVocabulary}. + * + * @param data The input to read from. + * @param label The vocabulary name used in error messages. + * @return The symbol to id map. Never {@code null}. + * @throws IOException Thrown if reading fails, the size is out of range, or a symbol + * repeats. + */ + private static Map readVocabulary(DataInputStream data, String label) + throws IOException { + final int size = readCount(data, label + " size", MAX_VOCABULARY_ENTRIES, true); + final Map ids = new HashMap<>(size * 2); + for (int i = 0; i < size; i++) { + final String symbol = data.readUTF(); + final int id = data.readInt(); + if (ids.put(symbol, id) != null) { + throw new InvalidFormatException("duplicate symbol in " + label + ": " + symbol); + } + } + return ids; + } + + /** + * Checks that all vocabulary ids form one consecutive embedding index range and that + * each map has its required special symbols. + * + * @param wordIds The word vocabulary. + * @param tagIds The tag vocabulary. + * @param labelIds The label vocabulary. + * @return The number of embedding rows the vocabularies address. + * @throws IOException Thrown if the combined size exceeds the limit, a required symbol + * is missing, or the ids are not a consecutive range starting at zero. + */ + private static int validateVocabularies(Map wordIds, + Map tagIds, Map labelIds) throws IOException { + final long total = (long) wordIds.size() + tagIds.size() + labelIds.size(); + if (total > MAX_VOCABULARY_ENTRIES) { + throw new InvalidFormatException("combined vocabulary size exceeds " + MAX_VOCABULARY_ENTRIES); + } + final boolean[] present = new boolean[(int) total]; + validateVocabulary(wordIds, present, WORD_VOCABULARY, UNKNOWN, ABSENT, ROOT_SYMBOL); + validateVocabulary(tagIds, present, TAG_VOCABULARY, UNKNOWN, ABSENT, ROOT_SYMBOL); + validateVocabulary(labelIds, present, LABEL_VOCABULARY, UNKNOWN, ABSENT); + for (int i = 0; i < present.length; i++) { + if (!present[i]) { + throw new InvalidFormatException("missing embedding id: " + i); + } + } + return present.length; + } + + /** + * Checks one vocabulary's required symbols and embedding ids. + * + * @param ids The vocabulary to check. + * @param present The ids seen so far across all vocabularies; updated in place. + * @param label The vocabulary name used in error messages. + * @param requiredSymbols The symbols the vocabulary must contain. + * @throws IOException Thrown if a required symbol is missing or an id is out of range + * or already taken. + */ + private static void validateVocabulary(Map ids, boolean[] present, + String label, String... requiredSymbols) throws IOException { + for (final String required : requiredSymbols) { + if (!ids.containsKey(required)) { + throw new InvalidFormatException(label + " has no " + required + " symbol"); + } + } + for (final Map.Entry entry : ids.entrySet()) { + final int id = entry.getValue(); + if (id < 0 || id >= present.length) { + throw new InvalidFormatException(label + " id out of range for " + entry.getKey() + ": " + id); + } + if (present[id]) { + throw new InvalidFormatException("duplicate embedding id: " + id); + } + present[id] = true; + } + } + + /** + * Writes a rectangular matrix as its dimensions followed by its values in row order. + * + * @param data The output to write to. + * @param matrix The matrix to write. + * @throws IOException Thrown if writing fails. + */ + private static void writeMatrix(DataOutputStream data, float[][] matrix) + throws IOException { + data.writeInt(matrix.length); + data.writeInt(matrix.length == 0 ? 0 : matrix[0].length); + for (final float[] row : matrix) { + for (final float value : row) { + data.writeFloat(value); + } + } + } + + /** + * Reads a matrix written by {@link #writeMatrix}. + * + * @param data The input to read from. + * @param expectedRows The required row count, or a negative value to accept any count + * up to {@code maxRows}. + * @param expectedColumns The required column count. + * @param maxRows The largest row count accepted. + * @param remainingFloats The one-element allocation budget, decremented in place. + * @param label The matrix name used in error messages. + * @return The matrix. Never {@code null}. + * @throws IOException Thrown if reading fails, a dimension is out of range or does not + * match, the budget is exceeded, or a value is not finite. + */ + private static float[][] readMatrix(DataInputStream data, int expectedRows, + int expectedColumns, int maxRows, long[] remainingFloats, String label) + throws IOException { + final int rows = readCount(data, label + " rows", maxRows, false); + final int columns = readCount(data, label + " columns", Integer.MAX_VALUE, false); + if (expectedRows >= 0 && rows != expectedRows) { + throw new InvalidFormatException(label + " row count is " + rows + ", expected " + expectedRows); + } + if (columns != expectedColumns) { + throw new InvalidFormatException(label + " column count is " + columns + + ", expected " + expectedColumns); + } + reserveFloats(remainingFloats, (long) rows * columns, label); + final float[][] matrix = new float[rows][columns]; + for (int r = 0; r < rows; r++) { + for (int c = 0; c < columns; c++) { + matrix[r][c] = readFiniteFloat(data, label); + } + } + return matrix; + } + + /** + * Writes a vector as its length followed by its values. + * + * @param data The output to write to. + * @param vector The vector to write. + * @throws IOException Thrown if writing fails. + */ + private static void writeVector(DataOutputStream data, float[] vector) throws IOException { + data.writeInt(vector.length); + for (final float value : vector) { + data.writeFloat(value); + } + } + + /** + * Reads a vector written by {@link #writeVector}. + * + * @param data The input to read from. + * @param expectedLength The required length. + * @param remainingFloats The one-element allocation budget, decremented in place. + * @param label The vector name used in error messages. + * @return The vector. Never {@code null}. + * @throws IOException Thrown if reading fails, the length does not match, the budget + * is exceeded, or a value is not finite. + */ + private static float[] readVector(DataInputStream data, int expectedLength, + long[] remainingFloats, String label) throws IOException { + final int length = readCount(data, label + " length", Integer.MAX_VALUE, false); + if (length != expectedLength) { + throw new InvalidFormatException(label + " length is " + length + ", expected " + expectedLength); + } + reserveFloats(remainingFloats, length, label); + final float[] vector = new float[length]; + for (int i = 0; i < vector.length; i++) { + vector[i] = readFiniteFloat(data, label); + } + return vector; + } + + /** + * Reads a bounded count from the model. + * + * @param data The input to read from. + * @param label The count name used in error messages. + * @param maximum The largest value accepted. + * @param allowZero Whether zero is accepted. + * @return The count. + * @throws IOException Thrown if reading fails or the value is out of range. + */ + private static int readCount(DataInputStream data, String label, int maximum, + boolean allowZero) throws IOException { + final int value = data.readInt(); + if (value < 0 || !allowZero && value == 0 || value > maximum) { + throw new InvalidFormatException(label + " out of range: " + value); + } + return value; + } + + /** + * Reserves float entries before allocating a matrix or vector. + * + * @param remaining The one-element allocation budget, decremented in place. + * @param count The number of values about to be allocated. + * @param label The structure name used in error messages. + * @throws IOException Thrown if {@code count} exceeds the remaining budget. + */ + private static void reserveFloats(long[] remaining, long count, String label) + throws IOException { + if (count > remaining[0]) { + throw new InvalidFormatException(label + " exceeds the model allocation limit"); + } + remaining[0] -= count; + } + + /** + * Reads one model weight. + * + * @param data The input to read from. + * @param label The structure name used in error messages. + * @return The weight. + * @throws IOException Thrown if reading fails or the value is NaN or infinite. + */ + private static float readFiniteFloat(DataInputStream data, String label) + throws IOException { + final float value = data.readFloat(); + if (!Float.isFinite(value)) { + throw new InvalidFormatException(label + " contains a non-finite value"); + } + return value; + } + + /** + * Creates an independent copy of this model: the weights and the transition + * inventory array are deep-copied, and the vocabularies are shared because their + * maps are immutable. Training passes update the copy, never a model a caller holds. + * + * @return A copy of this model sharing no mutable state with it. Never {@code null}. + */ + FeedforwardDependencyModel copy() { + return new FeedforwardDependencyModel(wordIds, tagIds, labelIds, transitions.clone(), + embeddingSize, copyOf(embeddings), copyOf(hiddenWeights), hiddenBias.clone(), + copyOf(outputWeights), outputBias.clone()); + } + + /** + * Deep-copies a matrix, row by row. + * + * @param matrix The matrix to copy. + * @return A copy sharing no rows with {@code matrix}. Never {@code null}. + */ + private static float[][] copyOf(float[][] matrix) { + final float[][] copy = new float[matrix.length][]; + for (int r = 0; r < matrix.length; r++) { + copy[r] = matrix[r].clone(); + } + return copy; + } + + /** + * @return The immutable map from a normalized word to its embedding row. Never {@code null}. + */ + Map wordIds() { + return wordIds; + } + + /** + * @return The immutable map from a tag to its embedding row. Never {@code null}. + */ + Map tagIds() { + return tagIds; + } + + /** + * @return The immutable map from an arc label to its embedding row. Never {@code null}. + */ + Map labelIds() { + return labelIds; + } + + /** + * @return The width of one embedding row. + */ + int embeddingSize() { + return embeddingSize; + } + + /** + * @return The live embedding matrix, one row per vocabulary entry, not a copy: the + * trainer writes its updates into it. Never {@code null}. + */ + float[][] embeddings() { + return embeddings; + } + + /** + * @return The live hidden layer weights, not a copy. Never {@code null}. + */ + float[][] hiddenWeights() { + return hiddenWeights; + } + + /** + * @return The live hidden layer bias, not a copy. Never {@code null}. + */ + float[] hiddenBias() { + return hiddenBias; + } + + /** + * @return The live output layer weights, one row per transition, not a copy. Never + * {@code null}. + */ + float[][] outputWeights() { + return outputWeights; + } + + /** + * @return The live output layer bias, one entry per transition, not a copy. Never + * {@code null}. + */ + float[] outputBias() { + return outputBias; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java new file mode 100644 index 0000000000..c0684a22f8 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import opennlp.tools.commons.ThreadSafe; + +/** + * The pure-Java neural {@link DependencyParser}: an arc-standard decoder over the + * {@link FeedforwardDependencyModel}, greedy by default and beamed when constructed + * with a beam size above one. + * + *

Beam decoding retains the highest scoring transition sequences by summed + * log-probability. Complete arc-standard derivations for a sentence have equal length, + * so their scores need no length normalization.

+ * + *

Inference is ordinary array arithmetic with no native runtime involved, so this + * parser has the same input and output API as the classical parser while scoring with + * learned dense representations instead of sparse feature conjunctions.

+ * + *

The parser holds an immutable model and no per-parse state, so one instance can be + * shared between threads.

+ * + * @see FeedforwardDependencyTrainer + * @since 3.0.0 + */ +@ThreadSafe +public class FeedforwardDependencyParser implements DependencyParser { + + /** The beam size at which decoding is greedy. */ + private static final int GREEDY_BEAM_SIZE = 1; + + private final FeedforwardDependencyModel model; + private final Transition[] transitions; + private final int beamSize; + + /** + * Initializes a greedy {@link FeedforwardDependencyParser}. + * + * @param model The model to parse with. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null} or an + * outcome of the model does not decode to a transition. + */ + public FeedforwardDependencyParser(FeedforwardDependencyModel model) { + this(model, GREEDY_BEAM_SIZE); + } + + /** + * Initializes a {@link FeedforwardDependencyParser} with a beam. + * + * @param model The model to parse with. Must not be {@code null}. + * @param beamSize The number of transition sequences to retain. Must be + * greater than zero; {@code 1} decodes greedily. + * @throws IllegalArgumentException Thrown if {@code model} is {@code null}, + * {@code beamSize} is not positive, or an outcome of the model does not + * decode to a transition. + */ + public FeedforwardDependencyParser(FeedforwardDependencyModel model, int beamSize) { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + if (beamSize < GREEDY_BEAM_SIZE) { + throw new IllegalArgumentException("beamSize must be positive: " + beamSize); + } + this.model = model; + this.beamSize = beamSize; + // Parser instances read immutable models. Training and refinement use uncached copies. + model.enableScoringCache(); + final String[] outcomes = model.transitions(); + this.transitions = new Transition[outcomes.length]; + for (int i = 0; i < outcomes.length; i++) { + transitions[i] = Transition.decode(outcomes[i]); + } + } + + /** + * {@inheritDoc} + * + * @throws IllegalStateException If no model outcome is applicable in a configuration + * or the model produces a non-finite transition score. + */ + @Override + public DependencyGraph parse(String[] tokens, String[] tags) { + ParserInput.check(tokens, tags); + if (beamSize == GREEDY_BEAM_SIZE) { + return greedyParse(tokens, tags); + } + return beamParse(tokens, tags); + } + + /** + * Decodes greedily: the highest scoring applicable transition wins each step. + * + * @param tokens The sentence tokens. + * @param tags The POS tags, aligned with {@code tokens}. + * @return The parse. Never {@code null}. + * @throws IllegalStateException If no model outcome is applicable in a configuration + * or the model produces a non-finite transition score. + */ + private DependencyGraph greedyParse(String[] tokens, String[] tags) { + final ArcStandardState state = new ArcStandardState(tokens.length); + while (!state.isTerminal()) { + final double[] scores = model.score( + model.featureIds(FeedforwardContext.extract(state, tokens, tags))); + Transition best = null; + double bestScore = Double.NEGATIVE_INFINITY; + for (int i = 0; i < scores.length; i++) { + if (scores[i] > bestScore && state.canApply(transitions[i])) { + best = transitions[i]; + bestScore = scores[i]; + } + } + if (best == null) { + throw new IllegalStateException( + "no applicable transition among the model outcomes in " + state); + } + state.apply(best); + } + return state.toGraph(); + } + + /** + * One search alternative of the beam. + * + * @param state The configuration reached so far. + * @param score The summed log-probability of the transitions taken to reach it. + * @param next The transition that would advance {@code state}, or {@code null} once + * it has been applied or the state is complete. + */ + private record Alternative(ArcStandardState state, double score, Transition next) { + } + + /** + * Decodes with a beam and returns its highest scoring complete sequence. + * + * @param tokens The sentence tokens. + * @param tags The POS tags, aligned with {@code tokens}. + * @return The parse. Never {@code null}. + * @throws IllegalStateException If no beam alternative can be advanced by a model + * outcome or the model produces a non-finite transition score. + */ + private DependencyGraph beamParse(String[] tokens, String[] tags) { + List beam = + List.of(new Alternative(new ArcStandardState(tokens.length), 0.0, null)); + while (true) { + boolean advanced = false; + final List expansions = new ArrayList<>(); + for (final Alternative alternative : beam) { + if (alternative.state().isTerminal()) { + expansions.add(alternative); + continue; + } + advanced = true; + final double[] logProbabilities = logSoftmax(model.score( + model.featureIds(FeedforwardContext.extract(alternative.state(), tokens, tags)))); + for (int i = 0; i < logProbabilities.length; i++) { + if (alternative.state().canApply(transitions[i])) { + expansions.add(new Alternative(alternative.state(), + alternative.score() + logProbabilities[i], transitions[i])); + } + } + } + if (!advanced) { + break; + } + expansions.sort(Comparator.comparingDouble(Alternative::score).reversed()); + final List survivors = + new ArrayList<>(Math.min(beamSize, expansions.size())); + for (int i = 0; i < expansions.size() && survivors.size() < beamSize; i++) { + final Alternative expansion = expansions.get(i); + if (expansion.next() == null) { + survivors.add(expansion); + } else { + final ArcStandardState state = expansion.state().copy(); + state.apply(expansion.next()); + survivors.add(new Alternative(state, expansion.score(), null)); + } + } + if (survivors.isEmpty()) { + throw new IllegalStateException( + "no applicable transition among the model outcomes in the beam"); + } + beam = survivors; + } + return beam.get(0).state().toGraph(); + } + + /** + * Normalizes raw transition scores to log-probabilities. + * + * @param scores The finite, non-empty raw output scores. + * @return The log-softmax of {@code scores}. Never {@code null}. + */ + private double[] logSoftmax(double[] scores) { + double max = Double.NEGATIVE_INFINITY; + for (final double score : scores) { + max = Math.max(max, score); + } + double sum = 0.0; + for (final double score : scores) { + sum += Math.exp(score - max); + } + final double logSum = Math.log(sum); + final double[] logProbabilities = new double[scores.length]; + for (int i = 0; i < scores.length; i++) { + logProbabilities[i] = (scores[i] - max) - logSum; + } + return logProbabilities; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java new file mode 100644 index 0000000000..d52d7fb7eb --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java @@ -0,0 +1,1175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.Function; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.util.ObjectStream; + +/** + * Trains the {@link FeedforwardDependencyModel} entirely in Java: oracle-derived + * transition examples, minibatch AdaGrad over a softmax cross-entropy loss, cube + * activation, and inverted dropout on the hidden layer, the training recipe of + * Chen and Manning (2014). Training and + * inference use Java arrays and require no external training framework. + * + *

Words below the frequency cutoff share a learned unknown embedding; absent + * template positions share a learned padding embedding. Non-projective samples have no + * arc-standard derivation and are skipped. Training is deterministic for a fixed + * {@link Settings#seed()}.

+ * + * @since 3.0.0 + */ +public final class FeedforwardDependencyTrainer { + + private static final Logger logger = + LoggerFactory.getLogger(FeedforwardDependencyTrainer.class); + + /** The AdaGrad denominator offset that keeps the first steps finite. */ + private static final double ADAGRAD_EPSILON = 1e-6; + + /** The derivative factor of the cube activation, {@code d/dx x^3 = 3x^2}. */ + private static final double CUBE_DERIVATIVE_FACTOR = 3.0; + + /** The numerator of the Glorot uniform bound, {@code sqrt(6 / (fanIn + fanOut))}. */ + private static final double GLOROT_NUMERATOR = 6.0; + + /** The half-width of the uniform range embeddings are drawn from. */ + private static final double EMBEDDING_INIT_SCALE = 0.01; + + /** The smallest probability the reported training loss takes the log of. */ + private static final double LOSS_PROBABILITY_FLOOR = 1e-12; + + /** The sentence loss reported when refinement made no update for a sentence. */ + private static final double NO_UPDATE = -1.0; + + /** Special-symbol rows present before any training vocabulary is added. */ + private static final int MIN_VOCABULARY_ROWS = 8; + + /** SHIFT and at least one RIGHT_ARC are required to parse a sentence. */ + private static final int MIN_TRANSITIONS = 2; + + /** Prevents construction of this utility class. */ + private FeedforwardDependencyTrainer() { + } + + /** + * The training hyperparameters. + * + * @param embeddingSize The embedding dimensionality. Must be positive. + * @param hiddenSize The hidden layer width. Must be positive. + * @param epochs The number of passes over the examples. Must be positive. + * @param batchSize The minibatch size. Must be positive. + * @param learningRate The AdaGrad step size. Must be positive. + * @param l2 The L2 penalty applied to the dense weights. Must not be negative. + * @param dropout The hidden dropout probability. Must be in {@code [0, 1)}. + * @param wordCutoff The minimum frequency for a word to get its own embedding. Must + * not be negative. + * @param seed The random seed making a run reproducible. + */ + public record Settings(int embeddingSize, int hiddenSize, int epochs, int batchSize, + double learningRate, double l2, double dropout, int wordCutoff, long seed) { + + /** The default embedding dimensionality. */ + private static final int DEFAULT_EMBEDDING_SIZE = 50; + + /** The default hidden layer width. */ + private static final int DEFAULT_HIDDEN_SIZE = 200; + + /** The default number of epochs. */ + private static final int DEFAULT_EPOCHS = 10; + + /** The default minibatch size. */ + private static final int DEFAULT_BATCH_SIZE = 256; + + /** The default AdaGrad step size. */ + private static final double DEFAULT_LEARNING_RATE = 0.02; + + /** The default L2 penalty. */ + private static final double DEFAULT_L2 = 1e-8; + + /** The default hidden dropout probability. */ + private static final double DEFAULT_DROPOUT = 0.5; + + /** The default minimum word frequency for an embedding row. */ + private static final int DEFAULT_WORD_CUTOFF = 2; + + /** The default random seed. */ + private static final long DEFAULT_SEED = 17L; + + /** + * Validates the hyperparameters. + * + * @throws IllegalArgumentException Thrown if a value is outside its documented + * range. + */ + public Settings { + if (embeddingSize <= 0 || hiddenSize <= 0 || epochs <= 0 || batchSize <= 0) { + throw new IllegalArgumentException("sizes, epochs and batch must be positive"); + } + if (embeddingSize > FeedforwardDependencyModel.MAX_EMBEDDING_SIZE) { + throw new IllegalArgumentException("embeddingSize exceeds the model format limit: " + + embeddingSize); + } + if (hiddenSize > FeedforwardDependencyModel.MAX_HIDDEN_SIZE) { + throw new IllegalArgumentException("hiddenSize exceeds the model format limit: " + + hiddenSize); + } + final long minimumModelValues = modelFloatValues(MIN_VOCABULARY_ROWS, + MIN_TRANSITIONS, embeddingSize, hiddenSize); + if (minimumModelValues > FeedforwardDependencyModel.MAX_MODEL_FLOAT_VALUES) { + throw new IllegalArgumentException("embeddingSize and hiddenSize exceed the model " + + "format allocation limit"); + } + if (!Double.isFinite(learningRate) || learningRate <= 0.0 + || !Double.isFinite(l2) || l2 < 0.0) { + throw new IllegalArgumentException( + "learningRate must be finite and positive, l2 finite and not negative"); + } + if (!(dropout >= 0.0 && dropout < 1.0)) { + throw new IllegalArgumentException("dropout must be in [0, 1): " + dropout); + } + if (wordCutoff < 0) { + throw new IllegalArgumentException("wordCutoff must not be negative"); + } + } + + /** + * @return The default hyperparameters. Never {@code null}. + */ + public static Settings defaults() { + return new Settings(DEFAULT_EMBEDDING_SIZE, DEFAULT_HIDDEN_SIZE, DEFAULT_EPOCHS, + DEFAULT_BATCH_SIZE, DEFAULT_LEARNING_RATE, DEFAULT_L2, DEFAULT_DROPOUT, + DEFAULT_WORD_CUTOFF, DEFAULT_SEED); + } + } + + /** + * Trains a model from dependency samples. + * + * @param samples The training samples. Must not be {@code null}. + * @param settings The hyperparameters. Must not be {@code null}. + * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null} or no + * trainable example can be derived from the samples. + */ + public static FeedforwardDependencyModel train(ObjectStream samples, + Settings settings) throws IOException { + return train(samples, settings, null); + } + + /** + * Trains a model from dependency samples, seeding word embeddings from a pretrained + * source. + * + *

The provider is consulted once per vocabulary word during initialization; words + * it returns {@code null} for keep their random initialization, and all embeddings + * remain trainable afterwards. The pretrained source is a training-time ingredient + * only: the learned embeddings ship inside the model, so parsing carries no + * dependency on the source.

+ * + * @param samples The training samples. Must not be {@code null}. + * @param settings The hyperparameters. Must not be {@code null}. + * @param pretrained Maps a normalized word to its pretrained vector of exactly + * {@link Settings#embeddingSize()} dimensions, or {@code null} for + * unknown words. May be {@code null} to disable seeding. + * @return A trained {@link FeedforwardDependencyModel}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if {@code samples} or {@code settings} is + * {@code null}, no trainable example can be derived, or a pretrained vector + * has the wrong dimensionality. + */ + public static FeedforwardDependencyModel train(ObjectStream samples, + Settings settings, Function pretrained) + throws IOException { + if (samples == null) { + throw new IllegalArgumentException("samples must not be null"); + } + if (settings == null) { + throw new IllegalArgumentException("settings must not be null"); + } + final List corpus = readAll(samples); + final FeedforwardDependencyModel model = initialize(corpus, settings); + if (pretrained != null) { + seed(model, pretrained, settings); + } + final List featureList = new ArrayList<>(); + final List goldList = new ArrayList<>(); + collectExamples(corpus, model, featureList, goldList); + if (featureList.isEmpty()) { + throw new IllegalArgumentException("no trainable examples in the samples"); + } + optimize(model, featureList, goldList, settings); + return model; + } + + /** + * Refines a locally trained model with beam search and the early-update method of + * Collins and Roark (2004). The loss + * is conditional likelihood over candidate paths scored by summed log-probabilities. + * + *

Refinement updates a copy with per-sentence AdaGrad steps and no dropout. + * {@link Settings#epochs()} sets the number of refinement passes. Use the same beam + * size for parsing the result.

+ * + *

The transition inventory comes from {@code model} and is not extended, because + * its size is the width of the trained output layer. A refinement corpus using a + * relation label the original training set lacked is therefore rejected rather than + * silently ignored.

+ * + * @param model The locally trained model to refine. Left untouched. Must not be + * {@code null}. + * @param samples The training samples. Must not be {@code null}. + * @param settings The hyperparameters; {@code epochs}, {@code learningRate}, + * {@code l2}, and {@code seed} apply. Must not be {@code null}. + * @param beamSize The beam width to track the gold derivation in. Must be at least 2. + * @return A new refined model, distinct from {@code model}. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}, + * {@code beamSize} is below 2, no trainable sample can be derived, or a sample + * requires a transition {@code model} does not know. + */ + public static FeedforwardDependencyModel refine(FeedforwardDependencyModel model, + ObjectStream samples, Settings settings, int beamSize) + throws IOException { + if (model == null) { + throw new IllegalArgumentException("model must not be null"); + } + if (samples == null) { + throw new IllegalArgumentException("samples must not be null"); + } + if (settings == null) { + throw new IllegalArgumentException("settings must not be null"); + } + if (beamSize < 2) { + throw new IllegalArgumentException("beamSize must be at least 2: " + beamSize); + } + final List corpus = readAll(samples); + final String[] outcomes = model.transitions(); + final Map transitionIds = new HashMap<>(); + final Transition[] transitions = new Transition[outcomes.length]; + for (int i = 0; i < transitions.length; i++) { + transitionIds.put(outcomes[i], i); + transitions[i] = Transition.decode(outcomes[i]); + } + + final List trainable = new ArrayList<>(); + final List oracles = new ArrayList<>(); + for (final DependencySample s : corpus) { + if (!ArcStandardOracle.isProjective(s.getGraph())) { + continue; + } + final List oracle = ArcStandardOracle.transitions(s.getGraph()); + final int[] encoded = new int[oracle.size()]; + for (int i = 0; i < encoded.length; i++) { + final String outcome = oracle.get(i).encode(); + final Integer id = transitionIds.get(outcome); + if (id == null) { + // The outcome space was fixed by the original training set, so a relation + // label it never saw has no output unit to push probability onto. + throw new IllegalArgumentException( + "unknown transition in the refinement samples: " + outcome); + } + encoded[i] = id; + } + trainable.add(s); + oracles.add(encoded); + } + if (trainable.isEmpty()) { + throw new IllegalArgumentException("no trainable samples for refinement"); + } + + final FeedforwardDependencyModel refined = model.copy(); + final GlobalOptimizer optimizer = new GlobalOptimizer(refined, settings); + final Random random = new Random(settings.seed()); + final int[] order = new int[trainable.size()]; + for (int i = 0; i < order.length; i++) { + order[i] = i; + } + for (int epoch = 1; epoch <= settings.epochs(); epoch++) { + final long epochStart = System.currentTimeMillis(); + shuffle(order, random); + double loss = 0.0; + int updates = 0; + for (final int index : order) { + final double sentenceLoss = optimizer.refineSentence(trainable.get(index), + oracles.get(index), transitions, beamSize); + if (sentenceLoss >= 0.0) { + loss += sentenceLoss; + updates++; + } + } + checkFinite(refined); + logger.info("refine epoch {}: loss {} over {} updates in {} ms", epoch, + loss / Math.max(updates, 1), updates, System.currentTimeMillis() - epochStart); + } + return refined; + } + + /** One candidate path in the refinement beam: the parent link forms the history. */ + private static final class BeamNode { + private final BeamNode parent; + private final int[] features; + private final int transition; + private final double score; + private final boolean gold; + private ArcStandardState state; + + /** + * Extends {@code parent} by one transition; the start node passes {@code null}. + * + * @param parent The node this one extends, or {@code null} for the start node. + * @param features The embedding rows the transition was scored on. + * @param transition The transition index taken from {@code parent}. + * @param score The summed log-probability of the path. + * @param gold Whether the path is the gold derivation so far. + */ + private BeamNode(BeamNode parent, int[] features, int transition, double score, + boolean gold) { + this.parent = parent; + this.features = features; + this.transition = transition; + this.score = score; + this.gold = gold; + } + } + + /** The forward, backward, and AdaGrad state for global refinement. */ + private static final class GlobalOptimizer { + private final FeedforwardDependencyModel model; + private final Settings settings; + private final int embeddingSize; + private final int hiddenSize; + private final int outputSize; + private final int inputSize; + + private final double[][] embeddingAccumulator; + private final double[][] hiddenAccumulator; + private final double[] hiddenBiasAccumulator; + private final double[][] outputAccumulator; + private final double[] outputBiasAccumulator; + + private final double[][] hiddenGradient; + private final double[] hiddenBiasGradient; + private final double[][] outputGradient; + private final double[] outputBiasGradient; + private final Map embeddingGradients = new HashMap<>(); + + private final double[] x; + private final double[] pre; + private final double[] hidden; + private final double[] probabilities; + private final double[] hiddenDelta; + private final double[] inputDelta; + + /** + * Sizes the accumulators and scratch buffers for one refinement run. + * + * @param model The model copy the run updates in place. + * @param settings The hyperparameters; {@code learningRate} and {@code l2} apply. + */ + private GlobalOptimizer(FeedforwardDependencyModel model, Settings settings) { + this.model = model; + this.settings = settings; + this.embeddingSize = model.embeddings()[0].length; + this.hiddenSize = model.hiddenBias().length; + this.outputSize = model.outputBias().length; + this.inputSize = FeedforwardContext.FEATURE_COUNT * embeddingSize; + this.embeddingAccumulator = + new double[model.embeddings().length][embeddingSize]; + this.hiddenAccumulator = new double[hiddenSize][inputSize]; + this.hiddenBiasAccumulator = new double[hiddenSize]; + this.outputAccumulator = new double[outputSize][hiddenSize]; + this.outputBiasAccumulator = new double[outputSize]; + this.hiddenGradient = new double[hiddenSize][inputSize]; + this.hiddenBiasGradient = new double[hiddenSize]; + this.outputGradient = new double[outputSize][hiddenSize]; + this.outputBiasGradient = new double[outputSize]; + this.x = new double[inputSize]; + this.pre = new double[hiddenSize]; + this.hidden = new double[hiddenSize]; + this.probabilities = new double[outputSize]; + this.hiddenDelta = new double[hiddenSize]; + this.inputDelta = new double[inputSize]; + } + + /** + * Decodes one sentence with the beam, updating on the early-update point or the + * final beam. + * + * @param sample The sentence. + * @param oracle The gold transition indexes. + * @param transitions The decoded transition inventory. + * @param beamSize The beam width. + * @return The sentence loss, or {@link FeedforwardDependencyTrainer#NO_UPDATE} when the + * sentence produced no update. + */ + private double refineSentence(DependencySample sample, int[] oracle, + Transition[] transitions, int beamSize) { + final String[] tokens = sample.getTokens(); + final String[] tags = sample.getTags(); + final BeamNode root = new BeamNode(null, null, -1, 0.0, true); + root.state = new ArcStandardState(tokens.length); + List beam = List.of(root); + + for (int step = 0; step < oracle.length; step++) { + final List expansions = new ArrayList<>(); + BeamNode goldChild = null; + for (final BeamNode node : beam) { + final int[] features = + model.featureIds(FeedforwardContext.extract(node.state, tokens, tags)); + forward(features); + logSoftmaxInPlace(probabilities); + for (int i = 0; i < outputSize; i++) { + if (node.state.canApply(transitions[i])) { + final boolean goldNext = node.gold && i == oracle[step]; + final BeamNode child = new BeamNode(node, features, i, + node.score + probabilities[i], goldNext); + expansions.add(child); + if (goldNext) { + goldChild = child; + } + } + } + } + expansions.sort((a, b) -> Double.compare(b.score, a.score)); + final List survivors = + new ArrayList<>(expansions.subList(0, Math.min(beamSize, expansions.size()))); + boolean goldRetained = false; + for (final BeamNode survivor : survivors) { + if (survivor.gold) { + goldRetained = true; + break; + } + } + if (!goldRetained) { + if (goldChild == null) { + // The gold transition was not applicable in the gold configuration, so + // this sentence yields no update. + return NO_UPDATE; + } + survivors.add(goldChild); + return updateFromCandidates(survivors); + } + if (step == oracle.length - 1) { + return updateFromCandidates(survivors); + } + for (final BeamNode survivor : survivors) { + survivor.state = survivor.parent.state.copy(); + survivor.state.apply(transitions[survivor.transition]); + } + beam = survivors; + } + return NO_UPDATE; + } + + /** + * Applies the conditional-likelihood update over the candidate paths. + * + * @param candidates The surviving paths plus, on early update, the gold path. + * @return The negative log-probability of the gold path under the candidates. + */ + private double updateFromCandidates(List candidates) { + double max = Double.NEGATIVE_INFINITY; + double goldScore = Double.NEGATIVE_INFINITY; + for (final BeamNode candidate : candidates) { + max = Math.max(max, candidate.score); + if (candidate.gold) { + goldScore = candidate.score; + } + } + double normalizer = 0.0; + for (final BeamNode candidate : candidates) { + normalizer += Math.exp(candidate.score - max); + } + final double logNormalizer = max + Math.log(normalizer); + + zero(hiddenGradient); + Arrays.fill(hiddenBiasGradient, 0.0); + zero(outputGradient); + Arrays.fill(outputBiasGradient, 0.0); + embeddingGradients.clear(); + for (final BeamNode candidate : candidates) { + final double weight = Math.exp(candidate.score - logNormalizer) + - (candidate.gold ? 1.0 : 0.0); + if (weight == 0.0) { + continue; + } + for (BeamNode node = candidate; node.parent != null; node = node.parent) { + backward(node.features, node.transition, weight); + } + } + update(model.hiddenWeights(), hiddenGradient, hiddenAccumulator, 1, settings); + updateVector(model.hiddenBias(), hiddenBiasGradient, hiddenBiasAccumulator, 1, + settings); + update(model.outputWeights(), outputGradient, outputAccumulator, 1, settings); + updateVector(model.outputBias(), outputBiasGradient, outputBiasAccumulator, 1, + settings); + for (final Map.Entry entry : embeddingGradients.entrySet()) { + final float[] embeddingRow = model.embeddings()[entry.getKey()]; + final double[] accumulatorRow = embeddingAccumulator[entry.getKey()]; + final double[] gradientRow = entry.getValue(); + for (int d = 0; d < embeddingSize; d++) { + final double gradient = gradientRow[d]; + accumulatorRow[d] += gradient * gradient; + embeddingRow[d] -= settings.learningRate() * gradient + / (Math.sqrt(accumulatorRow[d]) + ADAGRAD_EPSILON); + } + } + return logNormalizer - goldScore; + } + + /** + * Computes hidden activations and raw output scores for one feature vector into + * the scratch buffers. + * + * @param features The embedding rows of the configuration. + */ + private void forward(int[] features) { + final float[][] embeddings = model.embeddings(); + for (int f = 0; f < features.length; f++) { + final float[] embedding = embeddings[features[f]]; + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + x[offset + d] = embedding[d]; + } + } + final float[][] hiddenWeights = model.hiddenWeights(); + final float[] hiddenBias = model.hiddenBias(); + for (int j = 0; j < hiddenSize; j++) { + final float[] weightRow = hiddenWeights[j]; + double sum = hiddenBias[j]; + for (int k = 0; k < inputSize; k++) { + sum += weightRow[k] * x[k]; + } + pre[j] = sum; + hidden[j] = sum * sum * sum; + } + final float[][] outputWeights = model.outputWeights(); + final float[] outputBias = model.outputBias(); + for (int o = 0; o < outputSize; o++) { + final float[] weightRow = outputWeights[o]; + double sum = outputBias[o]; + for (int j = 0; j < hiddenSize; j++) { + sum += weightRow[j] * hidden[j]; + } + probabilities[o] = sum; + } + } + + /** + * Accumulates gradients for one decoded step: the weighted difference between the + * step's softmax and its chosen transition. + * + * @param features The step's input features. + * @param chosen The transition the path took at this step. + * @param weight The path's weight in the candidate distribution. + */ + private void backward(int[] features, int chosen, double weight) { + forward(features); + double max = Double.NEGATIVE_INFINITY; + for (int o = 0; o < outputSize; o++) { + max = Math.max(max, probabilities[o]); + } + double normalizer = 0.0; + for (int o = 0; o < outputSize; o++) { + probabilities[o] = Math.exp(probabilities[o] - max); + normalizer += probabilities[o]; + } + Arrays.fill(hiddenDelta, 0.0); + Arrays.fill(inputDelta, 0.0); + final float[][] outputWeights = model.outputWeights(); + for (int o = 0; o < outputSize; o++) { + // dL/dlogit for a path's step under the conditional likelihood: the path weight + // times how the step's log-probability responds to this logit + final double delta = + weight * ((o == chosen ? 1.0 : 0.0) - probabilities[o] / normalizer); + outputBiasGradient[o] += delta; + final double[] gradientRow = outputGradient[o]; + final float[] weightRow = outputWeights[o]; + for (int j = 0; j < hiddenSize; j++) { + gradientRow[j] += delta * hidden[j]; + hiddenDelta[j] += delta * weightRow[j]; + } + } + final float[][] hiddenWeights = model.hiddenWeights(); + for (int j = 0; j < hiddenSize; j++) { + final double preDelta = hiddenDelta[j] * CUBE_DERIVATIVE_FACTOR * pre[j] * pre[j]; + hiddenBiasGradient[j] += preDelta; + final double[] gradientRow = hiddenGradient[j]; + final float[] weightRow = hiddenWeights[j]; + for (int k = 0; k < inputSize; k++) { + gradientRow[k] += preDelta * x[k]; + inputDelta[k] += preDelta * weightRow[k]; + } + } + for (int f = 0; f < features.length; f++) { + final double[] embeddingGradient = embeddingGradients + .computeIfAbsent(features[f], key -> new double[embeddingSize]); + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + embeddingGradient[d] += inputDelta[offset + d]; + } + } + } + + /** + * Turns raw scores into log-probabilities in place. + * + * @param scores The raw scores, overwritten with their log-softmax. + */ + private void logSoftmaxInPlace(double[] scores) { + double max = Double.NEGATIVE_INFINITY; + for (final double score : scores) { + max = Math.max(max, score); + } + double sum = 0.0; + for (final double score : scores) { + sum += Math.exp(score - max); + } + final double logSum = Math.log(sum); + for (int i = 0; i < scores.length; i++) { + scores[i] = (scores[i] - max) - logSum; + } + } + } + + /** + * Reads a sample stream into memory; both trainers pass over the corpus repeatedly. + * + * @param samples The stream to drain. + * @return All samples in stream order. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + */ + private static List readAll(ObjectStream samples) + throws IOException { + final List corpus = new ArrayList<>(); + DependencySample sample; + while ((sample = samples.read()) != null) { + corpus.add(sample); + } + return corpus; + } + + /** + * Overwrites the random word rows with pretrained vectors where available. + * + * @param model The freshly initialized model whose word rows are seeded. + * @param pretrained The pretrained vector source, returning {@code null} for a word it + * does not cover. + * @param settings The hyperparameters, which fix the expected vector width. + * @throws IllegalArgumentException Thrown if a pretrained vector has a different width + * than the embedding size or contains a non-finite value. + */ + private static void seed(FeedforwardDependencyModel model, + Function pretrained, Settings settings) { + int seeded = 0; + for (final Map.Entry entry : model.wordIds().entrySet()) { + if (FeedforwardDependencyModel.isSpecialSymbol(entry.getKey())) { + // The special unknown, padding, and root symbols have no pretrained + // counterpart, so they keep their random initialization. + continue; + } + final float[] vector = pretrained.apply(entry.getKey()); + if (vector == null) { + continue; + } + if (vector.length != settings.embeddingSize()) { + throw new IllegalArgumentException("pretrained vector for '" + entry.getKey() + + "' has " + vector.length + " dimensions, expected " + + settings.embeddingSize()); + } + for (final float value : vector) { + if (!Float.isFinite(value)) { + throw new IllegalArgumentException( + "pretrained vector for '" + entry.getKey() + "' contains a non-finite value"); + } + } + System.arraycopy(vector, 0, model.embeddings()[entry.getValue()], 0, vector.length); + seeded++; + } + logger.info("seeded {} of {} word embeddings from the pretrained source", seeded, + model.wordIds().size()); + } + + /** + * Builds the vocabularies and randomly initialized weights. A word, tag, or label in + * the training data that spells a reserved symbol shares the reserved row, as an + * unknown symbol does, so the reserved rows are never displaced. + * + * @param corpus The training samples. + * @param settings The hyperparameters. + * @return The untrained model. Never {@code null}. + * @throws IllegalArgumentException Thrown if the vocabularies and settings would + * produce more model values than the model format can store. + */ + private static FeedforwardDependencyModel initialize(List corpus, + Settings settings) { + final Map wordCounts = new HashMap<>(); + final Map tagIds = new HashMap<>(); + final Map labelIds = new HashMap<>(); + final Map transitionIds = new HashMap<>(); + for (final DependencySample s : corpus) { + if (!ArcStandardOracle.isProjective(s.getGraph())) { + continue; + } + final List oracle = ArcStandardOracle.transitions(s.getGraph()); + for (final String token : s.getTokens()) { + wordCounts.merge(FeedforwardDependencyModel.normalize(token), 1, Integer::sum); + } + for (final String tag : s.getTags()) { + tagIds.putIfAbsent(tag, 0); + } + final DependencyGraph graph = s.getGraph(); + for (int i = 0; i < graph.size(); i++) { + labelIds.putIfAbsent(graph.relationOf(i), 0); + } + for (final Transition transition : oracle) { + transitionIds.putIfAbsent(transition.encode(), 0); + } + } + + int row = 0; + final Map wordIds = new HashMap<>(); + row = addSpecialSymbols(wordIds, row, FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, FeedforwardDependencyModel.ROOT_SYMBOL); + for (final Map.Entry entry : wordCounts.entrySet()) { + if (entry.getValue() >= settings.wordCutoff() && !wordIds.containsKey(entry.getKey())) { + wordIds.put(entry.getKey(), row++); + } + } + final Map tags = new HashMap<>(); + row = addSpecialSymbols(tags, row, FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, FeedforwardDependencyModel.ROOT_SYMBOL); + for (final String tag : tagIds.keySet()) { + if (!tags.containsKey(tag)) { + tags.put(tag, row++); + } + } + final Map labels = new HashMap<>(); + row = addSpecialSymbols(labels, row, FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT); + for (final String label : labelIds.keySet()) { + if (!labels.containsKey(label)) { + labels.put(label, row++); + } + } + + final String[] transitions = transitionIds.keySet().toArray(String[]::new); + Arrays.sort(transitions); + + final long modelValues = modelFloatValues(row, transitions.length, + settings.embeddingSize(), settings.hiddenSize()); + if (modelValues > FeedforwardDependencyModel.MAX_MODEL_FLOAT_VALUES) { + throw new IllegalArgumentException("training vocabulary and settings require " + + modelValues + " model values, limit is " + + FeedforwardDependencyModel.MAX_MODEL_FLOAT_VALUES); + } + + final Random random = new Random(settings.seed()); + final int inputSize = FeedforwardContext.FEATURE_COUNT * settings.embeddingSize(); + final float[][] embeddings = uniform(random, row, settings.embeddingSize(), + EMBEDDING_INIT_SCALE); + final float[][] hiddenWeights = uniform(random, settings.hiddenSize(), inputSize, + Math.sqrt(GLOROT_NUMERATOR / (inputSize + settings.hiddenSize()))); + final float[][] outputWeights = uniform(random, transitions.length, + settings.hiddenSize(), + Math.sqrt(GLOROT_NUMERATOR / (settings.hiddenSize() + transitions.length))); + return new FeedforwardDependencyModel(wordIds, tags, labels, transitions, + settings.embeddingSize(), embeddings, hiddenWeights, + new float[settings.hiddenSize()], outputWeights, new float[transitions.length]); + } + + /** + * Assigns the next embedding rows to the special symbols of one vocabulary. + * + * @param ids The vocabulary to fill. + * @param row The next free embedding row. + * @param symbols The special symbols, in the order they take their rows. + * @return The next free embedding row after the symbols. + */ + private static int addSpecialSymbols(Map ids, int row, String... symbols) { + int next = row; + for (final String symbol : symbols) { + ids.put(symbol, next); + next++; + } + return next; + } + + /** + * @param vocabularyRows The combined word, tag, and label row count. + * @param transitionCount The output layer width. + * @param embeddingSize The embedding dimensionality. + * @param hiddenSize The hidden layer width. + * @return The number of float values a model of that shape stores. + */ + private static long modelFloatValues(int vocabularyRows, int transitionCount, + int embeddingSize, int hiddenSize) { + final long inputSize = (long) FeedforwardContext.FEATURE_COUNT * embeddingSize; + return (long) vocabularyRows * embeddingSize + + (long) hiddenSize * inputSize + + hiddenSize + + (long) transitionCount * hiddenSize + + transitionCount; + } + + /** + * Replays the oracle over every projective sample, emitting one example per step. + * + * @param corpus The training samples. + * @param model The initialized model whose vocabularies map the features. + * @param featureList Receives the embedding rows of each configuration. + * @param goldList Receives the gold transition index of each configuration. + */ + private static void collectExamples(List corpus, + FeedforwardDependencyModel model, List featureList, List goldList) { + final Map transitionIds = new HashMap<>(); + final String[] transitions = model.transitions(); + for (int i = 0; i < transitions.length; i++) { + transitionIds.put(transitions[i], i); + } + int skipped = 0; + for (final DependencySample sample : corpus) { + if (!ArcStandardOracle.isProjective(sample.getGraph())) { + skipped++; + continue; + } + final List oracle = ArcStandardOracle.transitions(sample.getGraph()); + final ArcStandardState state = new ArcStandardState(sample.getGraph().size()); + final String[] tokens = sample.getTokens(); + final String[] tags = sample.getTags(); + for (final Transition transition : oracle) { + featureList.add(model.featureIds(FeedforwardContext.extract(state, tokens, tags))); + goldList.add(transitionIds.get(transition.encode())); + state.apply(transition); + } + } + if (skipped > 0) { + logger.warn("Skipped {} non-projective sample(s) without an arc-standard derivation.", + skipped); + } + } + + /** + * Minibatch AdaGrad over softmax cross-entropy with cube activation and dropout, + * updating {@code model} in place. + * + * @param model The initialized model. + * @param featureList The embedding rows of every example. + * @param goldList The gold transition index of every example. + * @param settings The hyperparameters. + * @throws IllegalStateException Thrown if an epoch produces a non-finite parameter. + */ + private static void optimize(FeedforwardDependencyModel model, List featureList, + List goldList, Settings settings) { + final int exampleCount = featureList.size(); + final int[][] features = featureList.toArray(new int[0][]); + final int[] gold = new int[exampleCount]; + for (int i = 0; i < exampleCount; i++) { + gold[i] = goldList.get(i); + } + + final float[][] embeddings = model.embeddings(); + final float[][] hiddenWeights = model.hiddenWeights(); + final float[] hiddenBias = model.hiddenBias(); + final float[][] outputWeights = model.outputWeights(); + final float[] outputBias = model.outputBias(); + final int embeddingSize = settings.embeddingSize(); + final int hiddenSize = settings.hiddenSize(); + final int outputSize = outputBias.length; + final int inputSize = features[0].length * embeddingSize; + + final double[][] embeddingAccumulator = + new double[embeddings.length][embeddingSize]; + final double[][] hiddenAccumulator = new double[hiddenSize][inputSize]; + final double[] hiddenBiasAccumulator = new double[hiddenSize]; + final double[][] outputAccumulator = new double[outputSize][hiddenSize]; + final double[] outputBiasAccumulator = new double[outputSize]; + + final double[][] hiddenGradient = new double[hiddenSize][inputSize]; + final double[] hiddenBiasGradient = new double[hiddenSize]; + final double[][] outputGradient = new double[outputSize][hiddenSize]; + final double[] outputBiasGradient = new double[outputSize]; + final Map embeddingGradients = new HashMap<>(); + + final Random random = new Random(settings.seed()); + final int[] order = new int[exampleCount]; + for (int i = 0; i < exampleCount; i++) { + order[i] = i; + } + + final double keep = 1.0 - settings.dropout(); + final double[] x = new double[inputSize]; + final double[] pre = new double[hiddenSize]; + final double[] hidden = new double[hiddenSize]; + final boolean[] mask = new boolean[hiddenSize]; + final double[] probabilities = new double[outputSize]; + final double[] hiddenDelta = new double[hiddenSize]; + final double[] inputDelta = new double[inputSize]; + + for (int epoch = 1; epoch <= settings.epochs(); epoch++) { + final long epochStart = System.currentTimeMillis(); + shuffle(order, random); + double loss = 0.0; + for (int batchStart = 0; batchStart < exampleCount; + batchStart += settings.batchSize()) { + final int batchEnd = Math.min(batchStart + settings.batchSize(), exampleCount); + final int batch = batchEnd - batchStart; + zero(hiddenGradient); + Arrays.fill(hiddenBiasGradient, 0.0); + zero(outputGradient); + Arrays.fill(outputBiasGradient, 0.0); + embeddingGradients.clear(); + + for (int b = batchStart; b < batchEnd; b++) { + final int[] feats = features[order[b]]; + final int goldTransition = gold[order[b]]; + for (int f = 0; f < feats.length; f++) { + final float[] embedding = embeddings[feats[f]]; + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + x[offset + d] = embedding[d]; + } + } + for (int j = 0; j < hiddenSize; j++) { + mask[j] = random.nextDouble() < keep; + if (!mask[j]) { + pre[j] = 0.0; + hidden[j] = 0.0; + continue; + } + final float[] weightRow = hiddenWeights[j]; + double sum = hiddenBias[j]; + for (int k = 0; k < inputSize; k++) { + sum += weightRow[k] * x[k]; + } + pre[j] = sum; + hidden[j] = sum * sum * sum / keep; + } + double max = Double.NEGATIVE_INFINITY; + for (int o = 0; o < outputSize; o++) { + final float[] weightRow = outputWeights[o]; + double sum = outputBias[o]; + for (int j = 0; j < hiddenSize; j++) { + sum += weightRow[j] * hidden[j]; + } + probabilities[o] = sum; + max = Math.max(max, sum); + } + double normalizer = 0.0; + for (int o = 0; o < outputSize; o++) { + probabilities[o] = Math.exp(probabilities[o] - max); + normalizer += probabilities[o]; + } + for (int o = 0; o < outputSize; o++) { + probabilities[o] /= normalizer; + } + loss -= Math.log(Math.max(probabilities[goldTransition], LOSS_PROBABILITY_FLOOR)); + + Arrays.fill(hiddenDelta, 0.0); + Arrays.fill(inputDelta, 0.0); + for (int o = 0; o < outputSize; o++) { + final double delta = probabilities[o] - (o == goldTransition ? 1.0 : 0.0); + outputBiasGradient[o] += delta; + final double[] gradientRow = outputGradient[o]; + final float[] weightRow = outputWeights[o]; + for (int j = 0; j < hiddenSize; j++) { + gradientRow[j] += delta * hidden[j]; + hiddenDelta[j] += delta * weightRow[j]; + } + } + for (int j = 0; j < hiddenSize; j++) { + if (!mask[j]) { + continue; + } + final double preDelta = + hiddenDelta[j] * CUBE_DERIVATIVE_FACTOR * pre[j] * pre[j] / keep; + hiddenBiasGradient[j] += preDelta; + final double[] gradientRow = hiddenGradient[j]; + final float[] weightRow = hiddenWeights[j]; + for (int k = 0; k < inputSize; k++) { + gradientRow[k] += preDelta * x[k]; + inputDelta[k] += preDelta * weightRow[k]; + } + } + for (int f = 0; f < feats.length; f++) { + final double[] embeddingGradient = embeddingGradients + .computeIfAbsent(feats[f], key -> new double[embeddingSize]); + final int offset = f * embeddingSize; + for (int d = 0; d < embeddingSize; d++) { + embeddingGradient[d] += inputDelta[offset + d]; + } + } + } + + update(hiddenWeights, hiddenGradient, hiddenAccumulator, batch, settings); + updateVector(hiddenBias, hiddenBiasGradient, hiddenBiasAccumulator, batch, settings); + update(outputWeights, outputGradient, outputAccumulator, batch, settings); + updateVector(outputBias, outputBiasGradient, outputBiasAccumulator, batch, settings); + for (final Map.Entry entry : embeddingGradients.entrySet()) { + final float[] embeddingRow = embeddings[entry.getKey()]; + final double[] accumulatorRow = embeddingAccumulator[entry.getKey()]; + final double[] gradientRow = entry.getValue(); + for (int d = 0; d < embeddingSize; d++) { + final double gradient = gradientRow[d] / batch; + accumulatorRow[d] += gradient * gradient; + embeddingRow[d] -= settings.learningRate() * gradient + / (Math.sqrt(accumulatorRow[d]) + ADAGRAD_EPSILON); + } + } + } + checkFinite(model); + logger.info("epoch {}: loss {} in {} ms", epoch, loss / exampleCount, + System.currentTimeMillis() - epochStart); + } + } + + /** + * One AdaGrad step on a weight matrix, with the L2 penalty folded into the gradient. + * + * @param weights The weights, updated in place. + * @param gradients The summed gradients of the batch. + * @param accumulators The AdaGrad squared-gradient sums, updated in place. + * @param batch The number of examples the gradients sum over. + * @param settings The hyperparameters; {@code learningRate} and {@code l2} apply. + */ + private static void update(float[][] weights, double[][] gradients, + double[][] accumulators, int batch, Settings settings) { + for (int r = 0; r < weights.length; r++) { + final float[] weightRow = weights[r]; + final double[] gradientRow = gradients[r]; + final double[] accumulatorRow = accumulators[r]; + for (int c = 0; c < weightRow.length; c++) { + final double gradient = gradientRow[c] / batch + settings.l2() * weightRow[c]; + accumulatorRow[c] += gradient * gradient; + weightRow[c] -= settings.learningRate() * gradient + / (Math.sqrt(accumulatorRow[c]) + ADAGRAD_EPSILON); + } + } + } + + /** + * One AdaGrad step on a bias vector; biases carry no L2 penalty. + * + * @param weights The biases, updated in place. + * @param gradients The summed gradients of the batch. + * @param accumulators The AdaGrad squared-gradient sums, updated in place. + * @param batch The number of examples the gradients sum over. + * @param settings The hyperparameters; {@code learningRate} applies. + */ + private static void updateVector(float[] weights, double[] gradients, + double[] accumulators, int batch, Settings settings) { + for (int i = 0; i < weights.length; i++) { + final double gradient = gradients[i] / batch; + accumulators[i] += gradient * gradient; + weights[i] -= settings.learningRate() * gradient + / (Math.sqrt(accumulators[i]) + ADAGRAD_EPSILON); + } + } + + /** + * Rejects numerical overflow before returning or continuing to train a model. + * + * @param model The model to check. + * @throws IllegalStateException Thrown if any parameter is NaN or infinite. + */ + private static void checkFinite(FeedforwardDependencyModel model) { + checkFinite(model.embeddings()); + checkFinite(model.hiddenWeights()); + checkFinite(model.hiddenBias()); + checkFinite(model.outputWeights()); + checkFinite(model.outputBias()); + } + + /** + * @param matrix The matrix to check. + * @throws IllegalStateException Thrown if any value is NaN or infinite. + */ + private static void checkFinite(float[][] matrix) { + for (final float[] row : matrix) { + checkFinite(row); + } + } + + /** + * @param vector The vector to check. + * @throws IllegalStateException Thrown if any value is NaN or infinite. + */ + private static void checkFinite(float[] vector) { + for (final float value : vector) { + if (!Float.isFinite(value)) { + throw new IllegalStateException("training produced a non-finite model parameter"); + } + } + } + + /** + * @param random The source of randomness. + * @param rows The row count. + * @param columns The column count. + * @param scale The half-width of the uniform range. + * @return A matrix drawn uniformly from {@code [-scale, scale]}. Never {@code null}. + */ + private static float[][] uniform(Random random, int rows, int columns, double scale) { + final float[][] matrix = new float[rows][columns]; + for (int r = 0; r < rows; r++) { + for (int c = 0; c < columns; c++) { + matrix[r][c] = (float) ((random.nextDouble() * 2.0 - 1.0) * scale); + } + } + return matrix; + } + + /** + * Fills a matrix with zeros. + * + * @param matrix The matrix to clear in place. + */ + private static void zero(double[][] matrix) { + for (final double[] row : matrix) { + Arrays.fill(row, 0.0); + } + } + + /** + * Fisher-Yates shuffle of the visit order. + * + * @param order The permutation to shuffle in place. + * @param random The source of randomness. + */ + private static void shuffle(int[] order, Random random) { + for (int i = order.length - 1; i > 0; i--) { + final int j = random.nextInt(i + 1); + final int swap = order[i]; + order[i] = order[j]; + order[j] = swap; + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ParserInput.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ParserInput.java new file mode 100644 index 0000000000..810e3cfe73 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ParserInput.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +/** + * Validates the token and tag arrays a parser receives. The checks match the ones a + * {@link DependencySample} applies to its arrays; keeping a copy here avoids a call + * into a package-private member of another module. + */ +final class ParserInput { + + /** Prevents construction of this utility class. */ + private ParserInput() { + } + + /** + * Checks that tokens and tags are present, non-empty, aligned, and free of nulls. + * + * @param tokens The tokens of one input. + * @param tags The part-of-speech tag of each token. + * @throws IllegalArgumentException Thrown if an array is {@code null} or empty, the + * lengths do not match, or an element is {@code null}. + */ + static void check(String[] tokens, String[] tags) { + if (tokens == null || tags == null) { + throw new IllegalArgumentException("tokens and tags must not be null"); + } + if (tokens.length == 0) { + throw new IllegalArgumentException("tokens must not be empty"); + } + if (tokens.length != tags.length) { + throw new IllegalArgumentException("tokens and tags must have the same length: " + + tokens.length + " != " + tags.length); + } + for (int i = 0; i < tokens.length; i++) { + if (tokens[i] == null) { + throw new IllegalArgumentException("token must not be null at index " + i); + } + if (tags[i] == null) { + throw new IllegalArgumentException("tag must not be null at index " + i); + } + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java new file mode 100644 index 0000000000..f2896ec4b4 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/Transition.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import opennlp.tools.util.StringUtil; + +/** + * One action of the arc-standard transition system: shift the next buffer token onto the + * stack, or attach one of the two topmost stack tokens to the other under a relation label. + * + *

{@link #encode()} renders a transition as a classification outcome and + * {@link #decode(String)} restores it.

+ * + * @param type The kind of action. Must not be {@code null}. + * @param label The relation label for arc actions, {@code null} for {@link Type#SHIFT}. + * + * @since 3.0.0 + */ +public record Transition(Type type, String label) { + + /** + * The kinds of arc-standard action. + */ + public enum Type { + /** Pushes the front of the buffer onto the stack. */ + SHIFT, + /** Attaches the second stack token to the top one and removes the second. */ + LEFT_ARC, + /** Attaches the top stack token to the second one and removes the top. */ + RIGHT_ARC + } + + /** The single shift transition; shifts carry no label. */ + public static final Transition SHIFT = new Transition(Type.SHIFT, null); + + /** Separates the type name from the label in an encoded arc transition. */ + private static final char SEPARATOR = ':'; + + /** + * Validates the pairing of type and label. + * + * @throws IllegalArgumentException Thrown if {@code type} is {@code null}, a shift + * carries a label, or an arc action has a {@code null} or blank label. + */ + public Transition { + if (type == null) { + throw new IllegalArgumentException("type must not be null"); + } + if (type == Type.SHIFT) { + if (label != null) { + throw new IllegalArgumentException("a shift must not carry a label: " + label); + } + } else if (label == null || StringUtil.isBlank(label)) { + throw new IllegalArgumentException("an arc transition needs a relation label"); + } + } + + /** + * Creates a left-arc transition. + * + * @param label The relation label. Must not be {@code null} or blank. + * @return A {@link Transition} of {@link Type#LEFT_ARC}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code label} is {@code null} or blank. + */ + public static Transition leftArc(String label) { + return new Transition(Type.LEFT_ARC, label); + } + + /** + * Creates a right-arc transition. + * + * @param label The relation label. Must not be {@code null} or blank. + * @return A {@link Transition} of {@link Type#RIGHT_ARC}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code label} is {@code null} or blank. + */ + public static Transition rightArc(String label) { + return new Transition(Type.RIGHT_ARC, label); + } + + /** + * Renders the transition as a model outcome string, for example {@code SHIFT} or + * {@code LEFT_ARC:nsubj}. + * + * @return The outcome string. Never {@code null}. + */ + public String encode() { + return type == Type.SHIFT ? type.name() : type.name() + SEPARATOR + label; + } + + /** + * Restores a transition from a model outcome string produced by {@link #encode()}. + * + * @param outcome The outcome string. Must not be {@code null}. + * @return The decoded {@link Transition}. Never {@code null}. + * @throws IllegalArgumentException Thrown if {@code outcome} is {@code null} or does not + * name a valid transition. + */ + public static Transition decode(String outcome) { + if (outcome == null) { + throw new IllegalArgumentException("outcome must not be null"); + } + if (Type.SHIFT.name().equals(outcome)) { + return SHIFT; + } + final int separator = outcome.indexOf(SEPARATOR); + if (separator < 0) { + throw new IllegalArgumentException("not a transition outcome: " + outcome); + } + final Type type; + try { + type = Type.valueOf(outcome.substring(0, separator)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("not a transition outcome: " + outcome, e); + } + return new Transition(type, outcome.substring(separator + 1)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java new file mode 100644 index 0000000000..40e1271e70 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardOracleTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests that {@link ArcStandardOracle} derivations replay to the gold graph through + * {@link ArcStandardState}, and that non-projective input is rejected. + */ +public class ArcStandardOracleTest { + + /** + * Derives the oracle transitions for {@code gold} and replays them on a fresh state. + * + * @param gold The gold graph to derive from. + * @return The graph the replayed derivation builds. Never {@code null}. + */ + private static DependencyGraph replay(DependencyGraph gold) { + final List transitions = ArcStandardOracle.transitions(gold); + // every token is shifted once and attached once + assertEquals(2 * gold.size(), transitions.size()); + final ArcStandardState state = new ArcStandardState(gold.size()); + for (final Transition transition : transitions) { + assertTrue(state.canApply(transition)); + state.apply(transition); + } + return state.toGraph(); + } + + @Test + void testRoundTripSimpleSentence() { + final DependencyGraph gold = DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testRoundTripSingleToken() { + final DependencyGraph gold = DependencyGraph.of(new int[] {-1}, new String[] {"root"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testRoundTripRightBranching() { + // "eat fresh fish now": root with a right dependent that has its own left dependent + final DependencyGraph gold = DependencyGraph.of(new int[] {-1, 2, 0, 0}, + new String[] {"root", "amod", "obj", "advmod"}); + assertEquals(gold, replay(gold)); + } + + @Test + void testRoundTripDeepChain() { + final DependencyGraph gold = DependencyGraph.of(new int[] {1, 2, 3, -1}, + new String[] {"a", "b", "c", "root"}); + assertEquals(gold, replay(gold)); + } + + /** Checks the projectivity test on a graph with crossed arcs, a chain, and a null. */ + @Test + void testIsProjective() { + assertFalse(ArcStandardOracle.isProjective(DependencyGraph.of(new int[] {2, 3, -1, 2}, + new String[] {"a", "b", "root", "c"}))); + assertTrue(ArcStandardOracle.isProjective(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}))); + assertThrows(IllegalArgumentException.class, () -> ArcStandardOracle.isProjective(null)); + } + + @Test + void testNonProjectiveThrows() { + // arcs (2,0) and (3,1) cross, so there is no arc-standard derivation + final DependencyGraph nonProjective = DependencyGraph.of(new int[] {2, 3, -1, 2}, + new String[] {"a", "b", "root", "c"}); + assertThrows(IllegalArgumentException.class, + () -> ArcStandardOracle.transitions(nonProjective)); + } + + @ParameterizedTest(name = "all trees with {0} token(s)") + @ValueSource(ints = {1, 2, 3, 4, 5}) + void testAllTreesOfSize(int size) { + checkHeadAssignments(new int[size], 0); + } + + /** + * Enumerates every head assignment and checks each valid tree against the + * arc-crossing definition of projectivity. + * + * @param heads The head assignment under construction; positions before {@code index} + * are fixed. + * @param index The next position to assign a head to. + */ + private static void checkHeadAssignments(int[] heads, int index) { + if (index < heads.length) { + for (int head = DependencyArc.ROOT_HEAD; head < heads.length; head++) { + heads[index] = head; + checkHeadAssignments(heads, index + 1); + } + return; + } + + final String[] relations = new String[heads.length]; + for (int i = 0; i < relations.length; i++) { + relations[i] = heads[i] == DependencyArc.ROOT_HEAD ? "root" : "dep"; + } + final DependencyGraph graph; + try { + graph = DependencyGraph.of(heads, relations); + } catch (IllegalArgumentException e) { + return; + } + + assertEquals(isProjective(graph), ArcStandardOracle.isProjective(graph), graph.toString()); + if (isProjective(graph)) { + assertEquals(graph, replay(graph), graph.toString()); + } else { + assertThrows(IllegalArgumentException.class, + () -> ArcStandardOracle.transitions(graph), graph.toString()); + } + } + + /** + * Decides projectivity by the arc-crossing definition. + * + * @param graph The graph to inspect. + * @return {@code true} if no pair of arcs crosses in token order. + */ + private static boolean isProjective(DependencyGraph graph) { + for (int first = 0; first < graph.size(); first++) { + final int firstLow = Math.min(first, graph.headOf(first)); + final int firstHigh = Math.max(first, graph.headOf(first)); + for (int second = first + 1; second < graph.size(); second++) { + final int secondLow = Math.min(second, graph.headOf(second)); + final int secondHigh = Math.max(second, graph.headOf(second)); + if (firstLow < secondLow && secondLow < firstHigh && firstHigh < secondHigh + || secondLow < firstLow && firstLow < secondHigh && secondHigh < firstHigh) { + return false; + } + } + } + return true; + } + + @Test + void testNullGraphThrows() { + assertThrows(IllegalArgumentException.class, () -> ArcStandardOracle.transitions(null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java new file mode 100644 index 0000000000..eb4c71a9a4 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/ArcStandardStateTest.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the configuration mechanics of {@link ArcStandardState}: the start + * configuration, transition applicability at the boundaries, the bookkeeping of attached + * dependents, copy independence, and accessor validation. + */ +public class ArcStandardStateTest { + + @Test + void testInitialConfiguration() { + final ArcStandardState state = new ArcStandardState(3); + assertEquals(ArcStandardState.ROOT, state.stack(0)); + assertEquals(ArcStandardState.NONE, state.stack(1)); + assertEquals(0, state.buffer(0)); + assertEquals(1, state.buffer(1)); + assertEquals(2, state.buffer(2)); + assertEquals(ArcStandardState.NONE, state.buffer(3)); + assertEquals(1, state.stackSize()); + assertEquals(3, state.bufferSize()); + assertFalse(state.isTerminal()); + + state.apply(Transition.SHIFT); + assertEquals(ArcStandardState.NONE, state.buffer(Integer.MAX_VALUE)); + } + + @Test + void testSingleTokenDerivationIsForced() { + // With one token the system permits exactly one derivation: shift the token, then + // attach it to the artificial root with a right arc. + final ArcStandardState state = new ArcStandardState(1); + assertTrue(state.canApply(Transition.SHIFT)); + assertFalse(state.canApply(Transition.leftArc("det"))); + assertFalse(state.canApply(Transition.rightArc("root"))); + + state.apply(Transition.SHIFT); + assertFalse(state.canApply(Transition.SHIFT)); + assertFalse(state.canApply(Transition.leftArc("det"))); + assertTrue(state.canApply(Transition.rightArc("root"))); + + state.apply(Transition.rightArc("root")); + assertTrue(state.isTerminal()); + assertEquals(DependencyGraph.of(new int[] {-1}, new String[] {"root"}), + state.toGraph()); + } + + @Test + void testDependentBookkeepingDuringADerivation() { + // Derives "the dog barks" (the<-dog via det, dog<-barks via nsubj, barks<-root) and + // checks the partial-structure accessors after every attachment. + final ArcStandardState state = new ArcStandardState(3); + state.apply(Transition.SHIFT); + state.apply(Transition.SHIFT); + assertEquals(0, state.assignedDependents(1)); + assertNull(state.assignedRelation(0)); + + state.apply(Transition.leftArc("det")); + assertEquals(1, state.assignedDependents(1)); + assertEquals(0, state.leftmostDependent(1)); + assertEquals(0, state.rightmostDependent(1)); + assertEquals("det", state.assignedRelation(0)); + + state.apply(Transition.SHIFT); + state.apply(Transition.leftArc("nsubj")); + assertEquals(1, state.leftmostDependent(2)); + assertEquals("nsubj", state.assignedRelation(1)); + + state.apply(Transition.rightArc("root")); + assertTrue(state.isTerminal()); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), state.toGraph()); + } + + @Test + void testInapplicableTransitionIsRejected() { + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalArgumentException.class, + () -> state.apply(Transition.leftArc("det"))); + assertThrows(IllegalArgumentException.class, + () -> state.apply(Transition.rightArc("root"))); + assertThrows(IllegalArgumentException.class, () -> state.apply(null)); + assertThrows(IllegalArgumentException.class, () -> state.canApply(null)); + } + + @Test + void testToGraphBeforeTerminalIsRejected() { + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalStateException.class, state::toGraph); + state.apply(Transition.SHIFT); + assertThrows(IllegalStateException.class, state::toGraph); + } + + @Test + void testCopyIsIndependentOfTheOriginal() { + final ArcStandardState original = new ArcStandardState(2); + original.apply(Transition.SHIFT); + final ArcStandardState copy = original.copy(); + + copy.apply(Transition.SHIFT); + copy.apply(Transition.leftArc("nsubj")); + // The copy advanced by two transitions while the original still has one token + // buffered and one on the stack. + assertEquals(2, original.stackSize()); + assertEquals(1, original.bufferSize()); + assertEquals(0, original.assignedDependents(1)); + assertEquals(1, copy.assignedDependents(1)); + } + + @Test + void testAccessorValidation() { + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalArgumentException.class, () -> state.stack(-1)); + assertThrows(IllegalArgumentException.class, () -> state.buffer(-1)); + assertThrows(IllegalArgumentException.class, () -> state.assignedDependents(-1)); + assertThrows(IllegalArgumentException.class, () -> state.assignedDependents(2)); + assertThrows(IllegalArgumentException.class, () -> state.leftmostDependent(2)); + assertThrows(IllegalArgumentException.class, () -> state.rightmostDependent(-1)); + assertThrows(IllegalArgumentException.class, () -> state.assignedRelation(2)); + } + + @Test + void testTokenCountMustBePositive() { + assertThrows(IllegalArgumentException.class, () -> new ArcStandardState(0)); + assertThrows(IllegalArgumentException.class, () -> new ArcStandardState(-1)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyCrossValidatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyCrossValidatorTest.java new file mode 100644 index 0000000000..f50291caf7 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyCrossValidatorTest.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import opennlp.tools.util.ObjectStream; +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static opennlp.tools.depparse.DependencyTestSamples.corpus; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests {@link DependencyCrossValidator}: the folds it trains, the tokens it counts, its + * argument checks, and its agreement with a single {@link DependencyEvaluator} on a corpus + * every fold can memorize. + */ +public class DependencyCrossValidatorTest { + + /** The tokens in {@link DependencyTestSamples#corpus()}: 40 repetitions of 3 + 2 + 3. */ + private static final int CORPUS_WORDS = 320; + + /** The sentences in {@link DependencyTestSamples#corpus()}. */ + private static final int CORPUS_SENTENCES = 120; + + /** + * Trains the maximum-entropy transition parser with a zero cutoff so the small test + * corpus keeps every feature. + * + * @param samples The training samples. + * @return The trained parser. Never {@code null}. + * @throws IOException Thrown if reading the samples fails. + */ + private static DependencyParser trainTransitionParser(ObjectStream samples) + throws IOException { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + return new DependencyParserME(DependencyParserME.train("eng", samples, parameters)); + } + + @ParameterizedTest(name = "folds = {0}") + @ValueSource(ints = {2, 3, 5, 8}) + void testTrainsOneParserPerFoldOnTheOtherFolds(int folds) throws IOException { + final List trainingSizes = new ArrayList<>(); + final DependencyCrossValidator validator = new DependencyCrossValidator(samples -> { + trainingSizes.add(count(samples)); + return trainTransitionParser(ObjectStreamUtils.createObjectStream(corpus())); + }); + validator.evaluate(ObjectStreamUtils.createObjectStream(corpus()), folds); + assertEquals(folds, trainingSizes.size()); + int heldOut = 0; + for (int size : trainingSizes) { + heldOut += CORPUS_SENTENCES - size; + } + assertEquals(CORPUS_SENTENCES, heldOut, "every sentence is held out exactly once"); + } + + @Test + void testCountsEveryTokenOnce() throws IOException { + final DependencyCrossValidator validator = + new DependencyCrossValidator(DependencyCrossValidatorTest::trainTransitionParser); + validator.evaluate(ObjectStreamUtils.createObjectStream(corpus()), 4); + assertEquals(CORPUS_WORDS, validator.getWordCount()); + assertEquals(CORPUS_WORDS, validator.getWordCountExcludingPunctuation(), + "the corpus has no punctuation, so no token is left out"); + } + + @Test + void testAgreesWithSingleEvaluatorOnMemorizableCorpus() throws IOException { + final DependencyCrossValidator validator = + new DependencyCrossValidator(DependencyCrossValidatorTest::trainTransitionParser); + validator.evaluate(ObjectStreamUtils.createObjectStream(corpus()), 4); + + final DependencyEvaluator evaluator = new DependencyEvaluator( + trainTransitionParser(ObjectStreamUtils.createObjectStream(corpus()))); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + + assertEquals(evaluator.getWordCount(), validator.getWordCount()); + assertEquals(evaluator.getUas(), validator.getUas()); + assertEquals(evaluator.getLas(), validator.getLas()); + assertEquals(evaluator.getUasExcludingPunctuation(), + validator.getUasExcludingPunctuation()); + assertEquals(evaluator.getLasExcludingPunctuation(), + validator.getLasExcludingPunctuation()); + assertEquals(1.0d, validator.getUas()); + assertEquals(1.0d, validator.getLas()); + } + + @Test + void testAccumulatesAcrossRuns() throws IOException { + final DependencyCrossValidator validator = + new DependencyCrossValidator(DependencyCrossValidatorTest::trainTransitionParser); + validator.evaluate(ObjectStreamUtils.createObjectStream(corpus()), 2); + validator.evaluate(ObjectStreamUtils.createObjectStream(corpus()), 2); + assertEquals(2 * CORPUS_WORDS, validator.getWordCount()); + } + + @Test + void testScoresPunctuationSeparately() throws IOException { + final DependencySample punctuated = DependencyTestSamples.sample( + new String[] {"dogs", "bark", "."}, new String[] {"NOUN", "VERB", "PUNCT"}, + new int[] {1, -1, 1}, new String[] {"nsubj", "root", "punct"}); + final DependencyGraph misattachedPunctuation = DependencyGraph.of( + new int[] {1, -1, 0}, new String[] {"nsubj", "root", "punct"}); + final DependencyCrossValidator validator = + new DependencyCrossValidator(samples -> (tokens, tags) -> misattachedPunctuation); + validator.evaluate(ObjectStreamUtils.createObjectStream( + List.of(punctuated, punctuated)), 2); + assertEquals(6, validator.getWordCount()); + assertEquals(4, validator.getWordCountExcludingPunctuation()); + assertEquals(2.0d / 3.0d, validator.getUas(), 1e-12); + assertEquals(2.0d / 3.0d, validator.getLas(), 1e-12); + assertEquals(1.0d, validator.getUasExcludingPunctuation()); + assertEquals(1.0d, validator.getLasExcludingPunctuation()); + } + + @Test + void testEmptyValidatorScoresZero() { + final DependencyCrossValidator validator = + new DependencyCrossValidator(DependencyCrossValidatorTest::trainTransitionParser); + assertEquals(0, validator.getWordCount()); + assertEquals(0.0d, validator.getUas()); + assertEquals(0.0d, validator.getLas()); + } + + @ParameterizedTest(name = "folds = {0}") + @ValueSource(ints = {Integer.MIN_VALUE, -1, 0, 1}) + void testRejectsFoldCountBelowTwo(int folds) { + final AtomicInteger trainings = new AtomicInteger(); + final DependencyCrossValidator validator = new DependencyCrossValidator(samples -> { + trainings.incrementAndGet(); + return trainTransitionParser(samples); + }); + assertThrows(IllegalArgumentException.class, + () -> validator.evaluate(ObjectStreamUtils.createObjectStream(corpus()), folds)); + assertEquals(0, trainings.get(), "nothing is trained when the fold count is invalid"); + } + + @Test + void testRejectsNullArguments() { + assertThrows(IllegalArgumentException.class, () -> new DependencyCrossValidator(null)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyCrossValidator(null, DependencyEvaluator.UNIVERSAL_PUNCTUATION_TAG::equals)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyCrossValidator(DependencyCrossValidatorTest::trainTransitionParser, null)); + final DependencyCrossValidator validator = + new DependencyCrossValidator(DependencyCrossValidatorTest::trainTransitionParser); + assertThrows(IllegalArgumentException.class, () -> validator.evaluate(null, 2)); + } + + @Test + void testRejectsTrainerReturningNull() { + final DependencyCrossValidator validator = new DependencyCrossValidator(samples -> null); + assertThrows(IllegalStateException.class, + () -> validator.evaluate(ObjectStreamUtils.createObjectStream(corpus()), 2)); + } + + /** + * Drains a stream and counts its samples. + * + * @param samples The stream to drain. + * @return The number of samples read. + * @throws IOException Thrown if reading fails. + */ + private static int count(ObjectStream samples) throws IOException { + int count = 0; + while (samples.read() != null) { + count++; + } + return count; + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyEvaluatorTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyEvaluatorTest.java new file mode 100644 index 0000000000..bf664620c9 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyEvaluatorTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.ObjectStreamUtils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the scores of {@link DependencyEvaluator} against a parser with a known error + * pattern, in particular the split between all tokens and the tokens that are not + * punctuation. + */ +public class DependencyEvaluatorTest { + + private static final String[] TOKENS = {"dogs", "bark", "loudly", "."}; + private static final String[] UNIVERSAL_TAGS = {"NOUN", "VERB", "ADV", "PUNCT"}; + private static final String[] PENN_TAGS = {"NNS", "VBP", "RB", "."}; + + private static final DependencyGraph GOLD = DependencyGraph.of( + new int[] {1, -1, 1, 1}, new String[] {"nsubj", "root", "advmod", "punct"}); + + /** Right heads throughout, but a wrong label on the adverb and a wrong head on the period. */ + private static final DependencyGraph PREDICTED = DependencyGraph.of( + new int[] {1, -1, 1, 2}, new String[] {"nsubj", "root", "obl", "punct"}); + + /** + * Evaluates the fixed prediction against the gold tree with the given tags. + * + * @param evaluator The evaluator under test. + * @param tags The gold tags of the fixture sentence. + * @throws IOException Thrown if reading the in-memory sample fails. + */ + private static void evaluate(DependencyEvaluator evaluator, String[] tags) throws IOException { + evaluator.evaluate(ObjectStreamUtils.createObjectStream( + List.of(new DependencySample(TOKENS, tags, GOLD)))); + } + + @Test + void testScoresAllTokensAndNonPunctuationSeparately() throws IOException { + final DependencyEvaluator evaluator = new DependencyEvaluator((tokens, tags) -> PREDICTED); + evaluate(evaluator, UNIVERSAL_TAGS); + assertEquals(4, evaluator.getWordCount()); + assertEquals(0.75d, evaluator.getUas()); + assertEquals(0.5d, evaluator.getLas()); + assertEquals(3, evaluator.getWordCountExcludingPunctuation()); + assertEquals(1.0d, evaluator.getUasExcludingPunctuation()); + assertEquals(2.0d / 3.0d, evaluator.getLasExcludingPunctuation(), 1e-12); + } + + @Test + void testDefaultPunctuationIsTheUniversalTag() throws IOException { + final DependencyEvaluator evaluator = new DependencyEvaluator((tokens, tags) -> PREDICTED); + evaluate(evaluator, PENN_TAGS); + assertEquals(4, evaluator.getWordCountExcludingPunctuation(), + "a Penn tag set has no PUNCT tag, so nothing is excluded"); + assertEquals(evaluator.getUas(), evaluator.getUasExcludingPunctuation()); + assertEquals(evaluator.getLas(), evaluator.getLasExcludingPunctuation()); + } + + @Test + void testCustomPunctuationPredicate() throws IOException { + final DependencyEvaluator evaluator = + new DependencyEvaluator((tokens, tags) -> PREDICTED, "."::equals); + evaluate(evaluator, PENN_TAGS); + assertEquals(3, evaluator.getWordCountExcludingPunctuation()); + assertEquals(1.0d, evaluator.getUasExcludingPunctuation()); + } + + @Test + void testEmptyEvaluatorScoresZero() { + final DependencyEvaluator evaluator = new DependencyEvaluator((tokens, tags) -> PREDICTED); + assertEquals(0, evaluator.getWordCount()); + assertEquals(0, evaluator.getWordCountExcludingPunctuation()); + assertEquals(0.0d, evaluator.getUas()); + assertEquals(0.0d, evaluator.getUasExcludingPunctuation()); + } + + @Test + void testRejectsNullArguments() { + assertThrows(IllegalArgumentException.class, () -> new DependencyEvaluator(null)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyEvaluator(null, DependencyEvaluator.UNIVERSAL_PUNCTUATION_TAG::equals)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyEvaluator((tokens, tags) -> PREDICTED, null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java new file mode 100644 index 0000000000..096331c1f1 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserEdgeCaseTest.java @@ -0,0 +1,321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static opennlp.tools.depparse.DependencyTestSamples.corpus; +import static opennlp.tools.depparse.DependencyTestSamples.sample; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the boundary behavior of both dependency parsers: empty and single-token input, + * non-projective sentences during training and parsing, and the file round trip of both + * model formats, which must reproduce the exact parses of the original models. + */ +public class DependencyParserEdgeCaseTest { + + /** The language code of the test corpus. */ + private static final String LANGUAGE = "eng"; + + /** The random seed making the feedforward training runs reproducible. */ + private static final long SEED = 17L; + + /** + * Single-epoch feedforward settings for tests that only inspect the trained inventory. + */ + private static final FeedforwardDependencyTrainer.Settings SINGLE_EPOCH_SETTINGS = + new FeedforwardDependencyTrainer.Settings(8, 8, 1, 32, 0.05, 0.0, 0.0, 1, SEED); + + /** The tokens of the first corpus sentence. */ + private static final String[] THE_DOG_BARKS_TOKENS = {"the", "dog", "barks"}; + + /** The tags of the first corpus sentence. */ + private static final String[] THE_DOG_BARKS_TAGS = {"DT", "NN", "VBZ"}; + + /** The gold graph of the first corpus sentence. */ + private static final DependencyGraph THE_DOG_BARKS_GRAPH = + DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); + + /** The tokens of the third corpus sentence. */ + private static final String[] SHE_EATS_FISH_TOKENS = {"she", "eats", "fish"}; + + /** The tags of the third corpus sentence. */ + private static final String[] SHE_EATS_FISH_TAGS = {"PRP", "VBZ", "NN"}; + + /** The gold graph of the third corpus sentence. */ + private static final DependencyGraph SHE_EATS_FISH_GRAPH = + DependencyGraph.of(new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"}); + + /** The number of threads parsing concurrently in the sharing test. */ + private static final int THREADS = 8; + + /** The number of parses each thread performs in the sharing test. */ + private static final int ITERATIONS_PER_THREAD = 50; + + private static DependencyModel maxentModel; + private static DependencyParserME maxentParser; + private static FeedforwardDependencyModel feedforwardModel; + private static FeedforwardDependencyParser feedforwardParser; + + /** + * Builds a four-token sample whose gold arcs (2,0) and (3,1) cross, so the tree is + * non-projective and has no arc-standard derivation. + * + * @return The non-projective sample. Never {@code null}. + */ + private static DependencySample nonProjectiveSample() { + return sample(new String[] {"the", "dog", "barks", "today"}, + new String[] {"DT", "NN", "VBZ", "RB"}, + new int[] {2, 3, -1, 2}, new String[] {"det", "nsubj", "root", "advmod"}); + } + + /** + * Trains one classical and one neural model on the shared corpus. The feedforward + * settings disable dropout and fix the seed, so the test network memorizes the corpus + * deterministically. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ + @BeforeAll + static void trainParsers() throws IOException { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + maxentModel = DependencyParserME.train(LANGUAGE, + ObjectStreamUtils.createObjectStream(corpus()), parameters); + maxentParser = new DependencyParserME(maxentModel); + + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 60, 32, 0.05, 0.0, 0.0, 1, SEED); + feedforwardModel = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + feedforwardParser = new FeedforwardDependencyParser(feedforwardModel); + } + + @Test + void testEmptySentenceIsRejectedByBothParsers() { + assertThrows(IllegalArgumentException.class, + () -> maxentParser.parse(new String[0], new String[0])); + assertThrows(IllegalArgumentException.class, + () -> feedforwardParser.parse(new String[0], new String[0])); + // The transition system itself has no configuration for zero tokens either. + assertThrows(IllegalArgumentException.class, () -> new ArcStandardState(0)); + } + + @Test + void testNullTokenOrTagIsRejectedByBothParsers() { + assertThrows(IllegalArgumentException.class, + () -> maxentParser.parse(new String[] {null}, new String[] {"NN"})); + assertThrows(IllegalArgumentException.class, + () -> maxentParser.parse(new String[] {"word"}, new String[] {null})); + assertThrows(IllegalArgumentException.class, + () -> feedforwardParser.parse(new String[] {null}, new String[] {"NN"})); + assertThrows(IllegalArgumentException.class, + () -> feedforwardParser.parse(new String[] {"word"}, new String[] {null})); + } + + @Test + void testContextGeneratorRejectsMisalignedInput() { + final DependencyContextGenerator generator = new DependencyContextGenerator(); + final ArcStandardState state = new ArcStandardState(2); + assertThrows(IllegalArgumentException.class, + () -> generator.getContext(state, new String[] {"one"}, new String[] {"NN"})); + } + + @Test + void testSingleTokenSentenceAttachesToTheRoot() { + // A single token permits only the derivation shift then right-arc, so the head is + // forced to the artificial root and the model only chooses the relation label. + final DependencyGraph maxentParse = + maxentParser.parse(new String[] {"Run"}, new String[] {"VB"}); + assertEquals(DependencyGraph.of(new int[] {-1}, new String[] {"root"}), maxentParse); + + final DependencyGraph feedforwardParse = + feedforwardParser.parse(new String[] {"Run"}, new String[] {"VB"}); + assertEquals(DependencyGraph.of(new int[] {-1}, new String[] {"root"}), + feedforwardParse); + } + + @Test + void testNonProjectiveSamplesAreSkippedDuringTraining() throws IOException { + // One non-projective sample joins the corpus; it cannot yield events, so training + // proceeds on the remaining samples and still memorizes the projective sentences. + final List mixed = new ArrayList<>(corpus()); + mixed.add(nonProjectiveSample()); + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + final DependencyModel model = DependencyParserME.train(LANGUAGE, + ObjectStreamUtils.createObjectStream(mixed), parameters); + final DependencyParserME parser = new DependencyParserME(model); + assertEquals(THE_DOG_BARKS_GRAPH, parser.parse(THE_DOG_BARKS_TOKENS, THE_DOG_BARKS_TAGS)); + assertEquals(SHE_EATS_FISH_GRAPH, parser.parse(SHE_EATS_FISH_TOKENS, SHE_EATS_FISH_TAGS)); + } + + @Test + void testFeedforwardTrainingOmitsNonProjectiveLabels() throws IOException { + final List mixed = new ArrayList<>(corpus()); + mixed.add(sample(new String[] {"a", "b", "c", "d"}, + new String[] {"DT", "NN", "VBZ", "RB"}, new int[] {2, 3, -1, 2}, + new String[] {"det", "dislocated", "root", "obj"})); + + final FeedforwardDependencyModel trained = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(mixed), SINGLE_EPOCH_SETTINGS); + + assertEquals(0, List.of(trained.transitions()).stream() + .filter(transition -> transition.contains("dislocated")).count()); + } + + @Test + void testNonProjectiveGoldDecodesToAProjectiveTree() { + // The parser can only emit arc-standard derivations, so for a sentence whose gold + // tree is non-projective the prediction is necessarily a different, projective tree. + final DependencySample gold = nonProjectiveSample(); + final DependencyGraph parsed = maxentParser.parse(gold.getTokens(), gold.getTags()); + assertNotEquals(gold.getGraph(), parsed); + assertEquals(0, crossingArcCount(parsed)); + // The expected projective result is deterministic for the test model. + assertEquals(DependencyGraph.of(new int[] {1, 2, 3, -1}, + new String[] {"det", "nsubj", "nsubj", "root"}), parsed); + } + + @Test + void testFeedforwardTrainingRejectsNoProjectiveSamples() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(List.of(nonProjectiveSample())), + SINGLE_EPOCH_SETTINGS)); + assertEquals("no trainable examples in the samples", e.getMessage()); + } + + @Test + void testRefinementRejectsNoProjectiveSamples() { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 1, 32, 0.01, 0.0, 0.0, 1, SEED); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.refine(feedforwardModel, + ObjectStreamUtils.createObjectStream(List.of(nonProjectiveSample())), + settings, 2)); + assertEquals("no trainable samples for refinement", e.getMessage()); + } + + @Test + void testMaxentModelFileRoundTripParsesIdentically(@TempDir Path dir) + throws IOException { + final Path file = dir.resolve("depparse.bin"); + maxentModel.serialize(file); + final DependencyParserME reloaded = new DependencyParserME(new DependencyModel(file)); + for (final DependencySample sample : corpus()) { + assertEquals(maxentParser.parse(sample.getTokens(), sample.getTags()), + reloaded.parse(sample.getTokens(), sample.getTags()), + Arrays.toString(sample.getTokens())); + } + assertEquals(THE_DOG_BARKS_GRAPH, reloaded.parse(THE_DOG_BARKS_TOKENS, THE_DOG_BARKS_TAGS)); + } + + @Test + void testFeedforwardModelFileRoundTripParsesIdentically(@TempDir Path dir) + throws IOException { + final Path file = dir.resolve("depparse-ff.bin"); + try (OutputStream out = Files.newOutputStream(file)) { + feedforwardModel.serialize(out); + } + final FeedforwardDependencyParser reloaded = + new FeedforwardDependencyParser(FeedforwardDependencyModel.load(file)); + for (final DependencySample sample : corpus()) { + assertEquals(feedforwardParser.parse(sample.getTokens(), sample.getTags()), + reloaded.parse(sample.getTokens(), sample.getTags()), + Arrays.toString(sample.getTokens())); + } + assertEquals(SHE_EATS_FISH_GRAPH, reloaded.parse(SHE_EATS_FISH_TOKENS, SHE_EATS_FISH_TAGS)); + } + + @Test + void testParserInstancesCanBeSharedBetweenThreads() throws Exception { + final List> tasks = new ArrayList<>(); + for (int task = 0; task < THREADS; task++) { + tasks.add(() -> { + for (int iteration = 0; iteration < ITERATIONS_PER_THREAD; iteration++) { + assertEquals(THE_DOG_BARKS_GRAPH, + maxentParser.parse(THE_DOG_BARKS_TOKENS, THE_DOG_BARKS_TAGS)); + assertEquals(THE_DOG_BARKS_GRAPH, + feedforwardParser.parse(THE_DOG_BARKS_TOKENS, THE_DOG_BARKS_TAGS)); + } + return null; + }); + } + + final ExecutorService executor = Executors.newFixedThreadPool(THREADS); + try { + final List> results = executor.invokeAll(tasks); + for (final Future result : results) { + result.get(); + } + } finally { + executor.shutdownNow(); + } + } + + /** + * Counts the pairs of crossing arcs in a graph, treating the root arc as spanning + * from a virtual position left of the sentence to its dependent. A projective tree + * has zero crossing pairs. + * + * @param graph The graph to inspect. Must not be {@code null}. + * @return The number of crossing arc pairs. + * @throws IllegalArgumentException Thrown if {@code graph} is {@code null}. + */ + private static int crossingArcCount(DependencyGraph graph) { + if (graph == null) { + throw new IllegalArgumentException("graph must not be null"); + } + int crossings = 0; + for (int i = 0; i < graph.size(); i++) { + for (int j = i + 1; j < graph.size(); j++) { + final int iLow = Math.min(i, graph.headOf(i)); + final int iHigh = Math.max(i, graph.headOf(i)); + final int jLow = Math.min(j, graph.headOf(j)); + final int jHigh = Math.max(j, graph.headOf(j)); + if ((iLow < jLow && jLow < iHigh && iHigh < jHigh) + || (jLow < iLow && iLow < jHigh && jHigh < iHigh)) { + crossings++; + } + } + } + return crossings; + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java new file mode 100644 index 0000000000..126a9f87d6 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserMETest.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.ml.model.MaxentModel; +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +import static opennlp.tools.depparse.DependencyTestSamples.corpus; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests {@link DependencyParserME} end to end: training on a small corpus must let the + * greedy parser reproduce the training sentences, which proves the oracle, event stream, + * feature generation, and decode loop agree with each other. + */ +public class DependencyParserMETest { + + /** The language code of the test corpus. */ + private static final String LANGUAGE = "eng"; + + /** The tokens of the first corpus sentence. */ + private static final String[] THE_DOG_BARKS_TOKENS = {"the", "dog", "barks"}; + + /** The tags of the first corpus sentence. */ + private static final String[] THE_DOG_BARKS_TAGS = {"DT", "NN", "VBZ"}; + + /** The gold graph of the first corpus sentence. */ + private static final DependencyGraph THE_DOG_BARKS_GRAPH = + DependencyGraph.of(new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}); + + /** The total token count of {@link DependencyTestSamples#corpus()}. */ + private static final int CORPUS_WORD_COUNT = 320; + + /** The encoded shift outcome. */ + private static final String SHIFT_OUTCOME = "SHIFT"; + + /** The encoded right arc outcome attaching a token to the root. */ + private static final String ROOT_ARC_OUTCOME = "RIGHT_ARC:root"; + + private static DependencyModel model; + private static DependencyParserME parser; + + /** + * Trains the shared model once for all tests; the zero cutoff keeps every feature of + * the test corpus. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ + @BeforeAll + static void trainParser() throws IOException { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, 0); + model = DependencyParserME.train(LANGUAGE, + ObjectStreamUtils.createObjectStream(corpus()), parameters); + parser = new DependencyParserME(model); + } + + @Test + void testMemorizesTrainingSentences() { + final DependencyGraph parsed = parser.parse(THE_DOG_BARKS_TOKENS, THE_DOG_BARKS_TAGS); + assertEquals(THE_DOG_BARKS_GRAPH, parsed); + } + + @Test + void testParseAlwaysYieldsASingleRootedTree() { + // an unseen sentence must still decode to a valid graph, whatever its quality + final DependencyGraph parsed = parser.parse(new String[] {"cats", "sleep"}, + new String[] {"NNS", "VBP"}); + assertEquals(2, parsed.size()); + parsed.root(); + } + + @Test + void testEvaluatorScoresPerfectlyOnTrainingData() throws IOException { + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + assertEquals(CORPUS_WORD_COUNT, evaluator.getWordCount()); + } + + @Test + void testParseValidatesArguments() { + assertThrows(IllegalArgumentException.class, + () -> parser.parse(null, new String[] {"DT"})); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[] {"the"}, null)); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[0], new String[0])); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[] {"the"}, new String[] {"DT", "NN"})); + } + + @Test + void testConstructorRejectsNullModel() { + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME((DependencyModel) null)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME((MaxentModel) null)); + } + + @Test + void testModelConstructorsRejectNullArguments() { + assertThrows(IllegalArgumentException.class, + () -> new DependencyModel(null, model.getParserModel(), null)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyModel(LANGUAGE, null, null)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyModel((InputStream) null)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyModel((File) null)); + assertThrows(IllegalArgumentException.class, + () -> new DependencyModel((Path) null)); + } + + @Test + void testTrainValidatesArguments() { + assertThrows(IllegalArgumentException.class, + () -> DependencyParserME.train(LANGUAGE, null, TrainingParameters.defaultParams())); + assertThrows(IllegalArgumentException.class, + () -> DependencyParserME.train(LANGUAGE, + ObjectStreamUtils.createObjectStream(corpus()), null)); + assertThrows(IllegalArgumentException.class, + () -> DependencyParserME.train(null, + ObjectStreamUtils.createObjectStream(corpus()), + TrainingParameters.defaultParams())); + } + + @Test + void testModelRoundTripThroughSerialization() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.serialize(out); + final DependencyModel reloaded = new DependencyModel( + new ByteArrayInputStream(out.toByteArray())); + final DependencyGraph parsed = new DependencyParserME(reloaded) + .parse(THE_DOG_BARKS_TOKENS, THE_DOG_BARKS_TAGS); + assertEquals(THE_DOG_BARKS_GRAPH, parsed); + } + + @Test + void testModelRejectsNullParserModel() { + assertThrows(IllegalArgumentException.class, + () -> new DependencyModel(LANGUAGE, null, null)); + } + + @Test + void testModelWithForeignOutcomesIsRejectedAtConstruction() { + // The outcome inventory is decoded once up front, so a model trained for another + // task is rejected when the parser is built rather than mid-sentence. + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME(new OutcomeOnlyModel("NN", "VB"))); + } + + @Test + void testIncompleteActionInventoriesAreRejectedAtConstruction() { + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME(new OutcomeOnlyModel(SHIFT_OUTCOME))); + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME(new OutcomeOnlyModel(ROOT_ARC_OUTCOME))); + } + + @Test + void testDuplicateActionsAreRejectedAtConstruction() { + assertThrows(IllegalArgumentException.class, + () -> new DependencyParserME( + new OutcomeOnlyModel(SHIFT_OUTCOME, SHIFT_OUTCOME, ROOT_ARC_OUTCOME))); + } + + @Test + void testModelScoreCountIsValidated() { + final DependencyParserME invalid = new DependencyParserME( + new OutcomeOnlyModel(new double[] {1.0}, SHIFT_OUTCOME, ROOT_ARC_OUTCOME)); + final IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> invalid.parse(new String[] {"word"}, new String[] {"NN"})); + assertEquals("model returned 1 scores for 2 outcomes", exception.getMessage()); + } + + @Test + void testNonFiniteModelScoreIsRejected() { + final DependencyParserME invalid = new DependencyParserME( + new OutcomeOnlyModel(new double[] {Double.NaN, 1.0}, + SHIFT_OUTCOME, ROOT_ARC_OUTCOME)); + assertThrows(IllegalStateException.class, + () -> invalid.parse(new String[] {"word"}, new String[] {"NN"})); + } + + /** + * A {@link MaxentModel} that only knows its outcome inventory, enough to build a + * parser from; any other use fails. + */ + private static final class OutcomeOnlyModel implements MaxentModel { + + private final String[] outcomes; + private final double[] scores; + + /** + * Initializes a model that fails on every evaluation. + * + * @param outcomes The outcome inventory, in index order. + */ + private OutcomeOnlyModel(String... outcomes) { + this(null, outcomes); + } + + /** + * Initializes a model returning fixed scores. + * + * @param scores The scores every evaluation returns, or {@code null} to fail instead. + * @param outcomes The outcome inventory, in index order. + */ + private OutcomeOnlyModel(double[] scores, String... outcomes) { + this.scores = scores; + this.outcomes = outcomes; + } + + @Override + public String getOutcome(int i) { + return outcomes[i]; + } + + @Override + public int getNumOutcomes() { + return outcomes.length; + } + + @Override + public double[] eval(String[] context) { + if (scores == null) { + throw new UnsupportedOperationException(); + } + return scores.clone(); + } + + @Override + public double[] eval(String[] context, double[] probs) { + throw new UnsupportedOperationException(); + } + + @Override + public double[] eval(String[] context, float[] values) { + throw new UnsupportedOperationException(); + } + + @Override + public String getBestOutcome(double[] outcomeScores) { + throw new UnsupportedOperationException(); + } + + @Override + public String getAllOutcomes(double[] outcomeScores) { + throw new UnsupportedOperationException(); + } + + @Override + public int getIndex(String outcome) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java new file mode 100644 index 0000000000..bb238c3865 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyTestSamples.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.util.ArrayList; +import java.util.List; + +/** + * The gold samples shared by the dependency parser tests. + */ +final class DependencyTestSamples { + + /** How often the distinct sentences are repeated in {@link #corpus()}. */ + private static final int REPETITIONS = 40; + + /** Prevents construction of this utility class. */ + private DependencyTestSamples() { + } + + /** + * Builds one gold sample from its parallel arrays. + * + * @param tokens The sentence tokens. Must not be {@code null}. + * @param tags The part-of-speech tags aligned with {@code tokens}. + * @param heads The zero-based head per token, {@code -1} for the root. + * @param relations The relation label per token. + * @return The assembled sample. Never {@code null}. + */ + static DependencySample sample(String[] tokens, String[] tags, int[] heads, + String[] relations) { + return new DependencySample(tokens, tags, DependencyGraph.of(heads, relations)); + } + + /** + * Builds the three distinct projective sentences of the test corpus. + * + * @return One sample per sentence. Never {@code null}. + */ + static List sentences() { + return List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"}), + sample(new String[] {"dogs", "bark"}, new String[] {"NNS", "VBP"}, + new int[] {1, -1}, new String[] {"nsubj", "root"}), + sample(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}, + new int[] {1, -1, 1}, new String[] {"nsubj", "root", "obj"})); + } + + /** + * Builds the training corpus: {@link #sentences()} repeated {@code REPETITIONS} + * times, which makes the small parsers memorize them deterministically. + * + * @return The training samples. Never {@code null}. + */ + static List corpus() { + final List distinct = sentences(); + final List corpus = new ArrayList<>(REPETITIONS * distinct.size()); + for (int i = 0; i < REPETITIONS; i++) { + corpus.addAll(distinct); + } + return corpus; + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java new file mode 100644 index 0000000000..c089981388 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java @@ -0,0 +1,746 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; + +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.util.ObjectStreamUtils; +import opennlp.tools.util.StringUtil; + +import static opennlp.tools.depparse.DependencyTestSamples.corpus; +import static opennlp.tools.depparse.DependencyTestSamples.sample; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests feedforward training, parsing, refinement, and model serialization. + */ +public class FeedforwardDependencyParserTest { + + /** The random seed shared by every training run in this class. */ + private static final long SEED = 17L; + + /** The header every serialized model starts with, pinned independently of the model. */ + private static final String MAGIC = "ONLP-FFDP-1"; + + /** The embedding width of the test networks. */ + private static final int EMBEDDING_SIZE = 16; + + /** The hidden width of the test networks. */ + private static final int HIDDEN_SIZE = 32; + + /** The minibatch size of the test networks. */ + private static final int BATCH_SIZE = 32; + + /** The word cutoff that gives every corpus word its own embedding row. */ + private static final int WORD_CUTOFF = 1; + + /** The number of vocabularies a serialized model starts with. */ + private static final int VOCABULARY_COUNT = 3; + + /** The first tag row in the hand-written minimal model, after three word rows. */ + private static final int FIRST_TAG_ID = 3; + + /** The first label row in the hand-written minimal model, after three tag rows. */ + private static final int FIRST_LABEL_ID = 6; + + /** The total embedding rows of the hand-written minimal model. */ + private static final int MINIMAL_ROWS = 8; + + private static FeedforwardDependencyModel model; + private static FeedforwardDependencyParser parser; + + /** + * Builds the shared test hyperparameters: no dropout, no L2, and a fixed seed. + * + * @param epochs The number of passes over the corpus. + * @param learningRate The AdaGrad step size. + * @return The settings. Never {@code null}. + */ + private static FeedforwardDependencyTrainer.Settings settings(int epochs, + double learningRate) { + return new FeedforwardDependencyTrainer.Settings(EMBEDDING_SIZE, HIDDEN_SIZE, epochs, + BATCH_SIZE, learningRate, 0.0, 0.0, WORD_CUTOFF, SEED); + } + + /** + * @return The distinct training sentences, one per parameterized invocation. Never + * {@code null}. + */ + private static List trainingSamples() { + return DependencyTestSamples.sentences(); + } + + /** + * Trains the shared model once for all tests, with dropout off and a fixed seed so + * the test network memorizes the corpus deterministically. + * + * @throws IOException Thrown if reading the in-memory samples fails. + */ + @BeforeAll + static void trainParser() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = settings(120, 0.05); + model = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + parser = new FeedforwardDependencyParser(model); + } + + /** Checks that the greedy parser reproduces a training sentence. */ + @Test + void testMemorizesTrainingSentences() { + final DependencyGraph parsed = parser.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"}); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), parsed); + } + + /** Checks UAS and LAS of 1.0 on the training corpus. */ + @Test + void testEvaluatorScoresPerfectlyOnTrainingData() throws IOException { + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + } + + /** Checks that unseen words still parse to a single-rooted tree. */ + @Test + void testUnknownWordsStillYieldASingleRootedTree() { + final DependencyGraph parsed = parser.parse(new String[] {"unseen", "words"}, + new String[] {"JJ", "NNS"}); + assertEquals(2, parsed.size()); + parsed.root(); + } + + /** Checks that the transition inventory holds only actions the oracle produced. */ + @Test + void testTransitionInventoryContainsOnlyObservedActions() { + assertFalse(List.of(model.transitions()).contains("LEFT_ARC:root")); + } + + /** + * Checks that a beam of one yields the greedy parse for every training sample. + * + * @param sample The training sample to parse both ways. + */ + @ParameterizedTest(name = "beam of one matches greedy for {0}") + @MethodSource("trainingSamples") + void testBeamOfOneMatchesGreedy(DependencySample sample) { + final FeedforwardDependencyParser beamed = new FeedforwardDependencyParser(model, 1); + assertEquals(parser.parse(sample.getTokens(), sample.getTags()), + beamed.parse(sample.getTokens(), sample.getTags())); + } + + /** Checks that beam search reproduces a training sentence. */ + @Test + void testBeamedParserReproducesTrainingSentences() { + final FeedforwardDependencyParser beamed = new FeedforwardDependencyParser(model, 4); + assertEquals(DependencyGraph.of(new int[] {1, 2, -1}, + new String[] {"det", "nsubj", "root"}), + beamed.parse(new String[] {"the", "dog", "barks"}, + new String[] {"DT", "NN", "VBZ"})); + } + + /** Checks that beam search is deterministic and single-rooted on unseen words. */ + @Test + void testBeamedParseIsDeterministicAndSingleRooted() { + final FeedforwardDependencyParser beamed = new FeedforwardDependencyParser(model, 8); + final String[] tokens = {"unseen", "words", "everywhere"}; + final String[] tags = {"JJ", "NNS", "RB"}; + final DependencyGraph first = beamed.parse(tokens, tags); + assertEquals(first, beamed.parse(tokens, tags)); + assertEquals(3, first.size()); + first.root(); + } + + /** Checks that refinement keeps perfect scores on the training corpus. */ + @Test + void testRefinementKeepsToyPerformance() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = settings(60, 0.05); + final FeedforwardDependencyModel local = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + final FeedforwardDependencyTrainer.Settings refineSettings = settings(2, 0.01); + final FeedforwardDependencyModel refined = FeedforwardDependencyTrainer.refine( + local, ObjectStreamUtils.createObjectStream(corpus()), refineSettings, 2); + + final DependencyEvaluator evaluator = + new DependencyEvaluator(new FeedforwardDependencyParser(refined, 2)); + evaluator.evaluate(ObjectStreamUtils.createObjectStream(corpus())); + assertEquals(1.0d, evaluator.getUas()); + assertEquals(1.0d, evaluator.getLas()); + } + + /** Checks that refinement fails loudly on a relation label the model never saw. */ + @Test + void testRefineRejectsAnUnknownRelation() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = settings(1, 0.01); + final List unseenRelation = List.of( + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "dislocated", "root"})); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.refine(model, + ObjectStreamUtils.createObjectStream(unseenRelation), settings, 2)); + assertEquals("unknown transition in the refinement samples: LEFT_ARC:dislocated", + e.getMessage()); + } + + /** Checks that refinement returns a distinct model and leaves the input scores unchanged. */ + @Test + void testRefineReturnsANewModelAndLeavesTheOriginalUntouched() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = settings(60, 0.05); + final FeedforwardDependencyModel local = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings); + final String[] tokens = {"the", "dog", "barks"}; + final String[] tags = {"DT", "NN", "VBZ"}; + final int[] features = local.featureIds( + FeedforwardContext.extract(new ArcStandardState(tokens.length), tokens, tags)); + final double[] before = local.score(features); + + final FeedforwardDependencyTrainer.Settings refineSettings = settings(2, 0.01); + final FeedforwardDependencyModel refined = FeedforwardDependencyTrainer.refine( + local, ObjectStreamUtils.createObjectStream(corpus()), refineSettings, 2); + + assertNotSame(local, refined); + assertArrayEquals(before, local.score(features)); + assertFalse(Arrays.equals(before, refined.score(features))); + } + + /** + * Checks the final-sigma rule applied after code-point case mapping. + * + * @param input The capitalized word. + * @param expected The normalized word. + */ + @ParameterizedTest(name = "normalize({0}) is {1}") + @CsvSource({ + // ODOS, road, all caps: the trailing sigma position lowers to U+03C2 + "\u039F\u0394\u039F\u03A3, \u03BF\u03B4\u03BF\u03C2", + // SOFIA: the word-initial sigma is not final and lowers to the medial U+03C3 + "\u03A3\u039F\u03A6\u0399\u0391, \u03C3\u03BF\u03C6\u03B9\u03B1", + // a lone capital sigma has no preceding letter, so the rule does not fire + "\u03A3, \u03C3", + // case-ignorable combining marks and apostrophes do not change the cased-letter test + "\u039F\u0301\u03A3, \u03BF\u0301\u03C2", + "\u039F\u03A3\u0301\u0391, \u03BF\u03C3\u0301\u03B1", + "\u039F\u2019\u03A3, \u03BF\u2019\u03C2", + "\u039F\u03A3\u2019\u0391, \u03BF\u03C3\u2019\u03B1" + }) + void testNormalizeAppliesTheFinalSigmaRule(String input, String expected) { + assertEquals(expected, FeedforwardDependencyModel.normalize(input)); + } + + /** Checks that an uncased predecessor does not trigger the final-sigma rule. */ + @Test + void testFinalSigmaRequiresCasedLetterContext() { + assertEquals("\u4E2D\u03C3", FeedforwardDependencyModel.normalize("\u4E2D\u03A3")); + } + + /** + * Checks that normalization reuses a string when no case mapping is needed. + */ + @Test + void testNormalizeReturnsTheSameInstanceForLowercaseWords() { + final String plain = "barks"; + assertSame(plain, FeedforwardDependencyModel.normalize(plain)); + // lowercase Greek with its native final sigma is already normalized + final String greek = "\u03BF\u03B4\u03BF\u03C2"; + assertSame(greek, FeedforwardDependencyModel.normalize(greek)); + } + + /** + * Checks that non-projective samples are skipped before transition validation. + */ + @Test + void testNonProjectiveSampleWithUnknownRelationIsSkippedNotFatal() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = settings(1, 0.01); + // heads {2, 3, -1, 2}: the arcs from 2 to 0 and from 3 to 1 cross, so the graph + // is non-projective, and "dislocated" is a relation the model was never trained on + final List mixed = List.of( + sample(new String[] {"a", "b", "c", "d"}, new String[] {"DT", "NN", "VBZ", "NN"}, + new int[] {2, 3, -1, 2}, new String[] {"det", "dislocated", "root", "obj"}), + sample(new String[] {"the", "dog", "barks"}, new String[] {"DT", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"det", "nsubj", "root"})); + + final FeedforwardDependencyModel refined = FeedforwardDependencyTrainer.refine( + model, ObjectStreamUtils.createObjectStream(mixed), settings, 2); + assertNotSame(model, refined); + } + + /** + * Checks that serialized vocabulary entries follow their numeric ids. + */ + @Test + void testSerializedVocabulariesAreWrittenInAscendingIdOrder() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.serialize(out); + try (DataInputStream data = new DataInputStream( + new ByteArrayInputStream(out.toByteArray()))) { + data.readUTF(); + for (int vocab = 0; vocab < VOCABULARY_COUNT; vocab++) { + final int size = data.readInt(); + int previous = Integer.MIN_VALUE; + for (int entry = 0; entry < size; entry++) { + data.readUTF(); + final int id = data.readInt(); + assertTrue(id > previous, + "vocabulary " + vocab + " must be written in ascending id order"); + previous = id; + } + } + } + } + + /** + * Compares cached and direct scoring, including repeated cache reads. + */ + @Test + void testScoringCacheMatchesTheDirectPath() { + final FeedforwardDependencyModel uncached = model.copy(); + final String[] tokens = {"the", "dog", "barks"}; + final String[] tags = {"DT", "NN", "VBZ"}; + final int[] features = model.featureIds( + FeedforwardContext.extract(new ArcStandardState(tokens.length), tokens, tags)); + + for (int round = 0; round < 3; round++) { + final double[] cached = model.score(features); + final double[] direct = uncached.score(features); + assertArrayEquals(direct, cached); + } + } + + /** Checks per-code-point lowering and reserved-symbol pass-through. */ + @Test + void testNormalizeUsesTheUnicodeDataCaseMapping() { + // StringUtil maps per code point via UnicodeData, so no character expands; the JDK's + // String.toLowerCase would render this word as "i" + COMBINING DOT ABOVE instead. + assertEquals(StringUtil.toLowerCase("\u0130STANBUL"), + FeedforwardDependencyModel.normalize("\u0130STANBUL")); + assertEquals("istanbul", FeedforwardDependencyModel.normalize("\u0130STANBUL")); + assertEquals("*hello", FeedforwardDependencyModel.normalize("*HELLO")); + // reserved symbols still pass through untouched + assertEquals(FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.normalize(FeedforwardDependencyModel.UNKNOWN)); + assertNull(FeedforwardDependencyModel.normalize(null)); + } + + /** Checks the null and beam-size guards of refinement. */ + @Test + void testRefineValidation() { + final FeedforwardDependencyTrainer.Settings settings = + FeedforwardDependencyTrainer.Settings.defaults(); + assertThrows(IllegalArgumentException.class, () -> FeedforwardDependencyTrainer + .refine(null, ObjectStreamUtils.createObjectStream(corpus()), settings, 4)); + assertThrows(IllegalArgumentException.class, () -> FeedforwardDependencyTrainer + .refine(model, ObjectStreamUtils.createObjectStream(corpus()), settings, 1)); + } + + /** Checks the parser constructor guards. */ + @Test + void testBeamSizeValidation() { + assertThrows(IllegalArgumentException.class, + () -> new FeedforwardDependencyParser(model, 0)); + assertThrows(IllegalArgumentException.class, + () -> new FeedforwardDependencyParser(null, 4)); + } + + /** Checks that a serialized and reloaded model parses like the original. */ + @Test + void testModelRoundTripThroughSerialization() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.serialize(out); + final FeedforwardDependencyModel reloaded = + FeedforwardDependencyModel.load(new ByteArrayInputStream(out.toByteArray())); + final DependencyGraph parsed = new FeedforwardDependencyParser(reloaded) + .parse(new String[] {"she", "eats", "fish"}, new String[] {"PRP", "VBZ", "NN"}); + assertEquals(DependencyGraph.of(new int[] {1, -1, 1}, + new String[] {"nsubj", "root", "obj"}), parsed); + } + + /** Checks that arbitrary bytes are rejected with an IOException. */ + @Test + void testCorruptModelIsRejected() { + assertThrows(IOException.class, () -> FeedforwardDependencyModel.load( + new ByteArrayInputStream("not a model".getBytes(StandardCharsets.UTF_8)))); + } + + /** Checks that a negative vocabulary size is rejected with an InvalidFormatException. */ + @Test + void testNegativeVocabularyCountFailsWithInvalidFormat() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (DataOutputStream data = new DataOutputStream(out)) { + data.writeUTF(MAGIC); + data.writeInt(-1); + } + assertThrows(InvalidFormatException.class, () -> FeedforwardDependencyModel.load( + new ByteArrayInputStream(out.toByteArray()))); + } + + /** Checks that a hidden matrix of the wrong width is rejected. */ + @Test + void testInconsistentModelDimensionsFailDuringLoading() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (DataOutputStream data = new DataOutputStream(out)) { + writeMinimalModel(data, FIRST_TAG_ID, 1, 0.0f); + } + assertThrows(InvalidFormatException.class, () -> FeedforwardDependencyModel.load( + new ByteArrayInputStream(out.toByteArray()))); + } + + /** Checks that overlapping vocabulary ids are rejected. */ + @Test + void testDuplicateVocabularyIdsFailDuringLoading() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (DataOutputStream data = new DataOutputStream(out)) { + writeMinimalModel(data, 0, FeedforwardContext.FEATURE_COUNT, 0.0f); + } + assertThrows(InvalidFormatException.class, () -> FeedforwardDependencyModel.load( + new ByteArrayInputStream(out.toByteArray()))); + } + + /** Checks that a NaN weight is rejected. */ + @Test + void testNonFiniteWeightFailsDuringLoading() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (DataOutputStream data = new DataOutputStream(out)) { + writeMinimalModel(data, FIRST_TAG_ID, FeedforwardContext.FEATURE_COUNT, Float.NaN); + } + assertThrows(InvalidFormatException.class, () -> FeedforwardDependencyModel.load( + new ByteArrayInputStream(out.toByteArray()))); + } + + /** Checks that bytes after the model are rejected. */ + @Test + void testTrailingModelDataFailsDuringLoading() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.serialize(out); + out.write(1); + assertThrows(InvalidFormatException.class, () -> FeedforwardDependencyModel.load( + new ByteArrayInputStream(out.toByteArray()))); + } + + /** Checks that an inventory without SHIFT is rejected. */ + @Test + void testModelWithoutShiftFailsDuringLoading() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (DataOutputStream data = new DataOutputStream(out)) { + writeModelWithTransitions(data, Transition.rightArc("root").encode()); + } + assertThrows(InvalidFormatException.class, () -> FeedforwardDependencyModel.load( + new ByteArrayInputStream(out.toByteArray()))); + } + + /** Checks that an inventory without RIGHT_ARC is rejected. */ + @Test + void testModelWithoutRootArcFailsDuringLoading() throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (DataOutputStream data = new DataOutputStream(out)) { + writeModelWithTransitions(data, Transition.SHIFT.encode()); + } + assertThrows(InvalidFormatException.class, () -> FeedforwardDependencyModel.load( + new ByteArrayInputStream(out.toByteArray()))); + } + + /** + * Writes a test vocabulary with consecutive embedding indices. + * + * @param data The output to write to. + * @param first The embedding row of the first symbol. + * @param symbols The symbols, in row order. + * @throws IOException Thrown if writing fails. + */ + private static void writeVocabulary(DataOutputStream data, int first, String... symbols) + throws IOException { + data.writeInt(symbols.length); + for (int i = 0; i < symbols.length; i++) { + data.writeUTF(symbols[i]); + data.writeInt(first + i); + } + } + + /** + * Writes the smallest complete model, with selected fields exposed for corruption. + * + * @param data The output to write to. + * @param firstTagId The embedding row of the first tag symbol. + * @param hiddenColumns The hidden matrix width. + * @param outputBias The single output bias value. + * @throws IOException Thrown if writing fails. + */ + private static void writeMinimalModel(DataOutputStream data, int firstTagId, + int hiddenColumns, float outputBias) throws IOException { + data.writeUTF(MAGIC); + writeVocabulary(data, 0, + FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, + FeedforwardDependencyModel.ROOT_SYMBOL); + writeVocabulary(data, firstTagId, + FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, + FeedforwardDependencyModel.ROOT_SYMBOL); + writeVocabulary(data, FIRST_LABEL_ID, + FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT); + data.writeInt(1); + data.writeUTF(Transition.SHIFT.encode()); + data.writeInt(1); + writeMatrix(data, MINIMAL_ROWS, 1); + writeMatrix(data, 1, hiddenColumns); + writeVector(data, 1); + writeMatrix(data, 1, 1); + data.writeInt(1); + data.writeFloat(outputBias); + } + + /** + * Writes a structurally complete model with the selected transition inventory. + * + * @param data The output to write to. + * @param transitions The encoded transitions, in output order. + * @throws IOException Thrown if writing fails. + */ + private static void writeModelWithTransitions(DataOutputStream data, + String... transitions) throws IOException { + data.writeUTF(MAGIC); + writeVocabulary(data, 0, + FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, + FeedforwardDependencyModel.ROOT_SYMBOL); + writeVocabulary(data, FIRST_TAG_ID, + FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT, + FeedforwardDependencyModel.ROOT_SYMBOL); + writeVocabulary(data, FIRST_LABEL_ID, + FeedforwardDependencyModel.UNKNOWN, + FeedforwardDependencyModel.ABSENT); + data.writeInt(transitions.length); + for (final String transition : transitions) { + data.writeUTF(transition); + } + data.writeInt(1); + writeMatrix(data, MINIMAL_ROWS, 1); + writeMatrix(data, 1, FeedforwardContext.FEATURE_COUNT); + writeVector(data, 1); + writeMatrix(data, transitions.length, 1); + writeVector(data, transitions.length); + } + + /** + * Writes a zero-filled matrix in the model format. + * + * @param data The output to write to. + * @param rows The row count. + * @param columns The column count. + * @throws IOException Thrown if writing fails. + */ + private static void writeMatrix(DataOutputStream data, int rows, int columns) + throws IOException { + data.writeInt(rows); + data.writeInt(columns); + for (int i = 0; i < rows * columns; i++) { + data.writeFloat(0.0f); + } + } + + /** + * Writes a zero-filled vector in the model format. + * + * @param data The output to write to. + * @param length The vector length. + * @throws IOException Thrown if writing fails. + */ + private static void writeVector(DataOutputStream data, int length) throws IOException { + data.writeInt(length); + for (int i = 0; i < length; i++) { + data.writeFloat(0.0f); + } + } + + /** + * Checks that each out-of-range hyperparameter is rejected. + * + * @param embeddingSize The embedding width. + * @param hiddenSize The hidden width. + * @param learningRate The step size. + * @param l2 The L2 penalty. + * @param dropout The dropout probability. + */ + @ParameterizedTest(name = "embedding {0}, hidden {1}, rate {2}, l2 {3}, dropout {4}") + @CsvSource({ + "0, 32, 0.05, 0.0, 0.0", + "16, 32, -1.0, 0.0, 0.0", + "16, 32, 0.05, 0.0, 1.0", + "16, 32, NaN, 0.0, 0.0", + "16, 32, 0.05, Infinity, 0.0", + "4097, 32, 0.05, 0.0, 0.0", + "16, 65537, 0.05, 0.0, 0.0", + "1000, 3000, 0.05, 0.0, 0.0" + }) + void testSettingsValidation(int embeddingSize, int hiddenSize, double learningRate, + double l2, double dropout) { + assertThrows(IllegalArgumentException.class, () -> new FeedforwardDependencyTrainer + .Settings(embeddingSize, hiddenSize, 10, BATCH_SIZE, learningRate, l2, dropout, + WORD_CUTOFF, SEED)); + } + + /** Checks that a diverging run fails with an IllegalStateException. */ + @Test + void testTrainingRejectsNonFiniteWeights() { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings( + 4, 4, 2, 1, Double.MAX_VALUE, 0.0, 0.0, WORD_CUTOFF, SEED); + assertThrows(IllegalStateException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings)); + } + + /** Checks the feature length guards of the model. */ + @Test + void testModelRejectsInvalidFeatureArrays() { + assertThrows(IllegalArgumentException.class, () -> model.featureIds(new String[0])); + assertThrows(IllegalArgumentException.class, () -> model.score(new int[0])); + } + + /** Checks that pretrained vectors seed rows and malformed vectors are rejected. */ + @Test + void testPretrainedSeedingAppliesAndValidates() throws IOException { + // near-zero learning keeps the seeded row observable after one epoch + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(4, 8, 1, BATCH_SIZE, 1e-9, 0.0, 0.0, + WORD_CUTOFF, SEED); + final float[] vector = {0.25f, -0.5f, 0.75f, -1.0f}; + final FeedforwardDependencyModel seeded = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings, + word -> "dog".equals(word) ? vector.clone() : null); + final int row = seeded.wordIds().get("dog"); + for (int d = 0; d < vector.length; d++) { + assertEquals(vector[d], seeded.embeddings()[row][d], 1e-4); + } + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings, + word -> new float[] {1.0f})); + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), settings, + word -> new float[] {Float.NaN, 0.0f, 0.0f, 0.0f})); + } + + /** Checks the null and length guards of the parser and trainer. */ + @Test + void testArgumentValidation() { + assertThrows(IllegalArgumentException.class, + () -> new FeedforwardDependencyParser(null)); + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train(null, + FeedforwardDependencyTrainer.Settings.defaults())); + assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(corpus()), null)); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(null, new String[] {"DT"})); + assertThrows(IllegalArgumentException.class, + () -> parser.parse(new String[] {"the"}, new String[] {"DT", "NN"})); + } + + /** + * Verifies that training data using the model's reserved symbols as ordinary tags, + * labels, or tokens does not displace the reserved rows: the model reloads and the + * colliding symbols share the reserved rows the way unknown symbols do. + * + * @throws IOException Thrown if serialization or loading fails. + */ + @Test + void testReservedSymbolsInTrainingDataSurviveReload() throws IOException { + final FeedforwardDependencyTrainer.Settings settings = settings(1, 0.01); + final List colliding = List.of( + sample(new String[] {"*ROOT*", "*UNK*", "barks"}, new String[] {"*UNK*", "NN", "VBZ"}, + new int[] {1, 2, -1}, new String[] {"*NULL*", "nsubj", "root"}), + sample(new String[] {"dogs", "bark"}, new String[] {"*NULL*", "*ROOT*"}, + new int[] {1, -1}, new String[] {"*UNK*", "root"})); + final FeedforwardDependencyModel trained = FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(colliding), settings); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + trained.serialize(out); + final FeedforwardDependencyModel reloaded = + FeedforwardDependencyModel.load(new ByteArrayInputStream(out.toByteArray())); + final String[] tokens = {"*ROOT*", "*UNK*", "barks"}; + final String[] tags = {"*UNK*", "NN", "VBZ"}; + assertEquals(new FeedforwardDependencyParser(trained).parse(tokens, tags), + new FeedforwardDependencyParser(reloaded).parse(tokens, tags)); + } + /** + * Checks that training on samples without an arc-standard derivation fails with the + * documented message instead of producing an empty model. + */ + @Test + void testTrainingWithOnlyNonProjectiveSamplesFailsLoud() { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 1, 32, 0.01, 0.0, 0.0, 1, 17L); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.train( + ObjectStreamUtils.createObjectStream(nonProjectiveCorpus()), settings)); + assertEquals("no trainable examples in the samples", e.getMessage()); + } + + /** + * Checks that refining on samples without an arc-standard derivation fails with the + * documented message instead of returning the model unchanged. + */ + @Test + void testRefiningWithOnlyNonProjectiveSamplesFailsLoud() { + final FeedforwardDependencyTrainer.Settings settings = + new FeedforwardDependencyTrainer.Settings(16, 32, 1, 32, 0.01, 0.0, 0.0, 1, 17L); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FeedforwardDependencyTrainer.refine(model, + ObjectStreamUtils.createObjectStream(nonProjectiveCorpus()), settings, 2)); + assertEquals("no trainable samples for refinement", e.getMessage()); + } + + /** + * Builds samples whose arcs cross, so that none has an arc-standard derivation. + * + * @return Two non-projective samples. Never {@code null}. + */ + private static List nonProjectiveCorpus() { + return List.of( + sample(new String[] {"a", "b", "c", "d"}, new String[] {"DT", "NN", "VBZ", "NN"}, + new int[] {2, 3, -1, 2}, new String[] {"det", "dislocated", "root", "obj"}), + sample(new String[] {"the", "dog", "barks", "loud"}, new String[] {"DT", "NN", "VBZ", "RB"}, + new int[] {2, 3, -1, 2}, new String[] {"det", "nsubj", "root", "advmod"})); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyScoringTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyScoringTest.java new file mode 100644 index 0000000000..50336c3e09 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyScoringTest.java @@ -0,0 +1,456 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReferenceArray; +import java.util.stream.DoubleStream; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** Checks numerical precision and concurrent cache use during dependency parsing. */ +class FeedforwardDependencyScoringTest { + + private static final int CACHE_ENTRY_LIMIT = 32768; + private static final int FEATURES = FeedforwardContext.FEATURE_COUNT; + private static final int VOCABULARY_ROWS = 8; + private static final String[] TRANSITIONS = {"SHIFT", "LEFT_ARC:dep", "RIGHT_ARC:root"}; + private static final String[] TOKENS = {"birds", "sing"}; + private static final String[] TAGS = {"NOUN", "VERB"}; + + /** + * A hidden-layer calculation with a known result. + * + * @param name The arithmetic condition. + * @param inputs The embedding values. + * @param weights The hidden-layer weights. + * @param bias The hidden bias. + * @param outputScale The output weight magnitude. + * @param hidden The expected hidden sum before activation. + */ + private record Calculation(String name, float[] inputs, float[] weights, + float bias, float outputScale, double hidden) { + } + + /** {@return finite products and sums covering overflow, underflow and cancellation} */ + private static Stream calculations() { + final float[] inputs = new float[256]; + final float[] weights = new float[256]; + Arrays.fill(inputs, Math.scalb(1f, 80)); + Arrays.fill(weights, Math.scalb(1f, 40)); + return Stream.of( + new Calculation("product overflow", new float[] {Math.scalb(1f, 80)}, + new float[] {Math.scalb(1f, 80)}, 0, Math.scalb(1f, -120), Math.scalb(1d, 160)), + new Calculation("product underflow", new float[] {Math.scalb(1f, -80)}, + new float[] {Math.scalb(1f, -80)}, 0, Math.scalb(1f, 120), Math.scalb(1d, -160)), + new Calculation("product cancellation", new float[] {Math.nextUp(1f), 1f}, + new float[] {Math.nextUp(1f), -(1f + Math.scalb(1f, -22))}, 0, + Math.scalb(1f, 120), Math.scalb(1d, -46)), + new Calculation("cache range", inputs, weights, 0, Math.scalb(1f, -120), + Math.scalb(1d, 128)), + new Calculation("cache cancellation", new float[] {1f, Math.scalb(1f, -24)}, + new float[] {1f, 1f}, -1f, Math.scalb(1f, 80), Math.scalb(1d, -24))) + .flatMap(calculation -> Stream.of(false, true).map(negative -> + Arguments.of(calculation.name(), calculation, negative))); + } + + /** + * Direct, cached, copied and reloaded models retain the calculated values. + * + * @param name The arithmetic condition. + * @param calculation The input values and expected result. + * @param negative Whether to negate the hidden sum. + * @throws IOException If model serialization or loading fails. + */ + @ParameterizedTest(name = "{0}, negative={2}") + @MethodSource("calculations") + void testKnownScores(String name, Calculation calculation, boolean negative) throws IOException { + final FeedforwardDependencyModel model = calculationModel(calculation, negative); + final double hidden = negative ? -calculation.hidden() : calculation.hidden(); + final double score = hidden * hidden * hidden * calculation.outputScale(); + final double[] expected = {0, -score, score}; + final int[] features = new int[FEATURES]; + assertArrayEquals(expected, model.score(features), name); + final byte[] serialized = serialize(model); + model.enableScoringCache(); + assertArrayEquals(expected, model.score(features)); + assertArrayEquals(expected, model.score(features)); + assertArrayEquals(expected, model.copy().score(features)); + assertArrayEquals(serialized, serialize(model)); + final FeedforwardDependencyModel loaded = FeedforwardDependencyModel.load( + new ByteArrayInputStream(serialized)); + assertArrayEquals(expected, loaded.score(features)); + final DependencyGraph expectedGraph = negative + ? DependencyGraph.of(new int[] {1, -1}, new String[] {"dep", "root"}) + : DependencyGraph.of(new int[] {-1, 0}, new String[] {"root", "root"}); + assertEquals(expectedGraph, new FeedforwardDependencyParser(loaded).parse(TOKENS, TAGS)); + } + + /** + * Builds one active input block with opposite arc output weights. + * + * @param calculation The input values. + * @param negative Whether to negate weights and bias. + * @return The model. + */ + private FeedforwardDependencyModel calculationModel(Calculation calculation, boolean negative) { + final int width = calculation.inputs().length; + final float[][] embeddings = new float[VOCABULARY_ROWS][width]; + for (int row = 0; row < embeddings.length; row++) { + embeddings[row] = calculation.inputs().clone(); + } + final float[][] weights = new float[1][FEATURES * width]; + for (int i = 0; i < width; i++) { + weights[0][i] = negative ? -calculation.weights()[i] : calculation.weights()[i]; + } + return model(embeddings, weights, + new float[] {negative ? -calculation.bias() : calculation.bias()}, + new float[][] {{0}, {-calculation.outputScale()}, {calculation.outputScale()}}); + } + + /** + * Builds a model with disjoint reserved vocabularies and valid transitions. + * + * @param embeddings The embedding matrix. + * @param weights The hidden weights. + * @param bias The hidden bias. + * @param outputs The output weights. + * @return The model. + */ + private FeedforwardDependencyModel model(float[][] embeddings, float[][] weights, + float[] bias, float[][] outputs) { + return new FeedforwardDependencyModel( + Map.of(FeedforwardDependencyModel.UNKNOWN, 0, FeedforwardDependencyModel.ABSENT, 1, + FeedforwardDependencyModel.ROOT_SYMBOL, 2), + Map.of(FeedforwardDependencyModel.UNKNOWN, 3, FeedforwardDependencyModel.ABSENT, 4, + FeedforwardDependencyModel.ROOT_SYMBOL, 5), + Map.of(FeedforwardDependencyModel.UNKNOWN, 6, FeedforwardDependencyModel.ABSENT, 7), + TRANSITIONS.clone(), embeddings[0].length, embeddings, weights, bias, outputs, + new float[TRANSITIONS.length]); + } + + /** {@return invalid values in each output for greedy and beam parsing} */ + private static Stream invalidScores() { + return Stream.of(Float.NaN, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY) + .flatMap(value -> IntStream.range(0, TRANSITIONS.length).boxed().flatMap(index -> + Stream.of(1, 4).map(beam -> Arguments.of(value, index, beam)))); + } + + /** + * Corrupt in-memory scores fail at either decoding entry point. + * + * @param value The injected output bias. + * @param index The affected transition. + * @param beam The decoder beam size. + */ + @ParameterizedTest + @MethodSource("invalidScores") + void testInvalidScores(float value, int index, int beam) { + final FeedforwardDependencyModel model = model(new float[VOCABULARY_ROWS][1], + new float[1][FEATURES], new float[1], new float[TRANSITIONS.length][1]); + model.outputBias()[index] = value; + final FeedforwardDependencyParser parser = new FeedforwardDependencyParser(model, beam); + final IllegalStateException error = assertThrows(IllegalStateException.class, + () -> parser.parse(TOKENS, TAGS)); + assertTrue(error.getMessage().contains("non-finite"), error.getMessage()); + } + + /** {@return common score offsets for parsing and refinement normalization} */ + private static Stream normalizationCases() { + return DoubleStream.of(0, 1, -1, 1e20, -1e20, Float.MAX_VALUE, -Float.MAX_VALUE, 1e100) + .boxed().flatMap(offset -> Stream.of(false, true) + .map(refinement -> Arguments.of(offset, refinement))); + } + + /** + * A common finite score offset leaves equal transition probabilities unchanged. + * + * @param offset The score shared by all transitions. + * @param refinement Whether to use the refinement optimizer's normalization. + * @throws ReflectiveOperationException If the normalization method cannot be called. + */ + @ParameterizedTest + @MethodSource("normalizationCases") + void testEqualLogProbabilities(double offset, boolean refinement) + throws ReflectiveOperationException { + final double[] scores = new double[TRANSITIONS.length]; + Arrays.fill(scores, offset); + final double[] expected = new double[TRANSITIONS.length]; + Arrays.fill(expected, -Math.log(TRANSITIONS.length)); + assertArrayEquals(expected, normalizedScores(scores, refinement), 1e-15); + } + + /** {@return offsets for three unequal, representable scores in both normalizers} */ + private static Stream unequalNormalizationCases() { + return DoubleStream.of(0, 1, -1, 1e12, -1e12, Math.scalb(1d, 48)).boxed() + .flatMap(offset -> Stream.of(false, true).map(refinement -> Arguments.of(offset, refinement))); + } + + /** + * Unequal probabilities retain their values and sum to one after a common shift. + * + * @param offset The common score shift. + * @param refinement Whether to use refinement normalization. + * @throws ReflectiveOperationException If the normalization method cannot be called. + */ + @ParameterizedTest + @MethodSource("unequalNormalizationCases") + void testUnequalLogProbabilities(double offset, boolean refinement) + throws ReflectiveOperationException { + final double logSum = Math.log(Math.exp(-0.5) + Math.exp(-0.25) + 1); + final double[] expected = {-0.5 - logSum, -0.25 - logSum, -logSum}; + final double[] actual = normalizedScores(new double[] {offset, offset + 0.25, offset + 0.5}, + refinement); + assertArrayEquals(expected, actual, 1e-15); + assertEquals(1.0, Arrays.stream(actual).map(Math::exp).sum(), 1e-15); + } + + /** + * Calls a private normalizer without exposing it through production API. + * + * @param scores The finite raw scores; refinement overwrites them. + * @param refinement Whether to use refinement normalization. + * @return The log probabilities. + * @throws ReflectiveOperationException If the normalizer cannot be called. + */ + private double[] normalizedScores(double[] scores, boolean refinement) + throws ReflectiveOperationException { + final FeedforwardDependencyModel model = model( + new float[VOCABULARY_ROWS][1], new float[1][FEATURES], new float[1], + new float[TRANSITIONS.length][1]); + if (refinement) { + final Class type = Class.forName( + FeedforwardDependencyTrainer.class.getName() + "$GlobalOptimizer"); + final var constructor = type.getDeclaredConstructor(FeedforwardDependencyModel.class, + FeedforwardDependencyTrainer.Settings.class); + constructor.setAccessible(true); + final Object optimizer = constructor.newInstance(model, + FeedforwardDependencyTrainer.Settings.defaults()); + final var method = type.getDeclaredMethod("logSoftmaxInPlace", double[].class); + method.setAccessible(true); + method.invoke(optimizer, (Object) scores); + return scores; + } else { + final var method = FeedforwardDependencyParser.class.getDeclaredMethod( + "logSoftmax", double[].class); + method.setAccessible(true); + return (double[]) method.invoke(new FeedforwardDependencyParser(model, 4), (Object) scores); + } + } + + /** {@return independent shared-key and distinct-key contention runs} */ + private static Stream contentionCases() { + return IntStream.range(0, 8).boxed().flatMap(round -> + Stream.of(false, true).map(shared -> Arguments.of(round, shared))); + } + + /** + * Concurrent cache misses respect the capacity and refund duplicate reservations. + * + * @param round The independent run. + * @param shared Whether requests use the same embedding row. + * @throws Exception If a worker or cache inspection fails. + */ + @ParameterizedTest + @MethodSource("contentionCases") + void testConcurrentBudget(int round, boolean shared) throws Exception { + final float[][] embeddings = new float[VOCABULARY_ROWS][128]; + for (int row = 0; row < embeddings.length; row++) { + Arrays.fill(embeddings[row], (row + 1) * 0.125f); + } + final float[][] weights = new float[256][FEATURES * 128]; + for (float[] row : weights) { + Arrays.fill(row, 0.125f); + } + final float[][] outputs = new float[TRANSITIONS.length][256]; + outputs[0][0] = 1; + final FeedforwardDependencyModel model = model(embeddings, weights, new float[256], outputs); + model.enableScoringCache(); + final Object cache = field(model, "cache"); + final AtomicInteger remaining = (AtomicInteger) field(cache, "remaining"); + final int budget = shared ? 8 : 1; + remaining.set(budget); + final CountDownLatch ready = new CountDownLatch(8); + final CountDownLatch start = new CountDownLatch(1); + try (var workers = Executors.newFixedThreadPool(8)) { + final var futures = IntStream.range(0, 8).mapToObj(index -> workers.submit(() -> { + final int[] features = new int[FEATURES]; + Arrays.fill(features, shared ? 0 : index); + ready.countDown(); + assertTrue(start.await(10, TimeUnit.SECONDS), "workers did not start"); + return model.score(features); + })).toList(); + try { + assertTrue(ready.await(10, TimeUnit.SECONDS), "workers were not ready"); + } finally { + start.countDown(); + } + for (int i = 0; i < futures.size(); i++) { + final double hidden = FEATURES * 2.0 * (shared ? 1 : i + 1); + assertArrayEquals(new double[] {hidden * hidden * hidden, 0, 0}, + futures.get(i).get(10, TimeUnit.SECONDS)); + } + } + final int entries = cachedEntries(cache); + assertTrue(entries <= budget, + "round " + round + " stored " + entries + " entries with budget " + budget); + assertEquals(budget - entries, remaining.get()); + model.score(new int[FEATURES]); + assertEquals(budget, cachedEntries(cache)); + assertEquals(0, remaining.get()); + } + + /** + * Concurrent parser constructors initialize one cache on a shared model. + * + * @param round The independent run. + * @throws Exception If a worker or cache inspection fails. + */ + @ParameterizedTest + @ValueSource(ints = {0, 1, 2, 3}) + void testConcurrentInitialization(int round) throws Exception { + final FeedforwardDependencyModel model = model(new float[100000][1], + new float[1][FEATURES], new float[1], new float[TRANSITIONS.length][1]); + final CountDownLatch ready = new CountDownLatch(8); + final CountDownLatch start = new CountDownLatch(1); + final Callable initialize = () -> { + ready.countDown(); + assertTrue(start.await(10, TimeUnit.SECONDS), "workers did not start"); + new FeedforwardDependencyParser(model); + return field(model, "cache"); + }; + final Set caches = Collections.newSetFromMap(new IdentityHashMap<>()); + try (var workers = Executors.newFixedThreadPool(8)) { + final var futures = IntStream.range(0, 8).mapToObj(index -> workers.submit(initialize)).toList(); + try { + assertTrue(ready.await(10, TimeUnit.SECONDS), "workers were not ready"); + } finally { + start.countDown(); + } + for (var future : futures) { + caches.add(future.get(10, TimeUnit.SECONDS)); + } + } + assertEquals(1, caches.size(), "cache identities in round " + round); + } + + /** + * Full caches retain their capacity and agree with direct calculations on misses. + * + * @param budget The cache capacity to exercise. + * @throws ReflectiveOperationException If cache inspection fails. + */ + @ParameterizedTest + @ValueSource(ints = {0, 1, 3, CACHE_ENTRY_LIMIT}) + void testFullCache(int budget) throws ReflectiveOperationException { + final int rows = Math.max(VOCABULARY_ROWS, (budget + FEATURES - 1) / FEATURES + 2); + final float[][] embeddings = new float[rows][1]; + for (int row = 0; row < rows; row++) { + embeddings[row][0] = (row + 1) * 0.125f; + } + final float[][] weights = new float[1][FEATURES]; + Arrays.fill(weights[0], 0.5f); + final FeedforwardDependencyModel model = model(embeddings, weights, new float[1], + new float[][] {{0}, {-1}, {1}}); + model.enableScoringCache(); + final Object cache = field(model, "cache"); + final AtomicInteger remaining = (AtomicInteger) field(cache, "remaining"); + assertEquals(CACHE_ENTRY_LIMIT, remaining.get()); + remaining.set(budget); + for (int row = 0; row < rows; row++) { + final int[] features = new int[FEATURES]; + Arrays.fill(features, row); + final double hidden = FEATURES * (row + 1) / 16.0; + final double score = hidden * hidden * hidden; + final double[] expected = {0, -score, score}; + assertArrayEquals(expected, model.score(features)); + assertArrayEquals(expected, model.score(features)); + } + assertEquals(budget, cachedEntries(cache)); + assertEquals(0, remaining.get()); + } + + /** + * Counts published contribution arrays. + * + * @param cache The model cache. + * @return The retained entry count. + * @throws ReflectiveOperationException If cache inspection fails. + */ + private int cachedEntries(Object cache) throws ReflectiveOperationException { + int entries = 0; + for (AtomicReferenceArray position : (AtomicReferenceArray[]) field(cache, "byPosition")) { + for (int row = 0; row < position.length(); row++) { + if (position.get(row) != null) { + entries++; + } + } + } + return entries; + } + + /** + * Reads private cache state without adding inspection methods to the model. + * + * @param owner The field owner. + * @param name The field name. + * @return The field value. + * @throws ReflectiveOperationException If field access fails. + */ + private Object field(Object owner, String name) throws ReflectiveOperationException { + final Field field = owner.getClass().getDeclaredField(name); + field.setAccessible(true); + return field.get(owner); + } + + /** + * Serializes the model for comparisons before and after scoring. + * + * @param model The model. + * @return Its binary representation. + * @throws IOException If writing fails. + */ + private byte[] serialize(FeedforwardDependencyModel model) throws IOException { + final ByteArrayOutputStream output = new ByteArrayOutputStream(); + model.serialize(output); + return output.toByteArray(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java new file mode 100644 index 0000000000..c35489401e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/TransitionTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.depparse; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests valid transition encodings and rejection of malformed outcomes. + */ +public class TransitionTest { + + @Test + void testEncodeRendersTheOutcomeStrings() { + assertEquals("SHIFT", Transition.SHIFT.encode()); + assertEquals("LEFT_ARC:nsubj", Transition.leftArc("nsubj").encode()); + assertEquals("RIGHT_ARC:obj", Transition.rightArc("obj").encode()); + } + + @Test + void testDecodeRestoresEncodedTransitions() { + assertSame(Transition.SHIFT, Transition.decode("SHIFT")); + assertEquals(Transition.leftArc("nsubj"), Transition.decode("LEFT_ARC:nsubj")); + assertEquals(Transition.rightArc("obj"), Transition.decode("RIGHT_ARC:obj")); + } + + @Test + void testLabelContainingTheSeparatorRoundTrips() { + // Only the first separator divides the type from the label. + final Transition transition = Transition.leftArc("nmod:poss"); + assertEquals("LEFT_ARC:nmod:poss", transition.encode()); + assertEquals(transition, Transition.decode(transition.encode())); + } + + @Test + void testDecodeRejectsMalformedOutcomes() { + assertThrows(IllegalArgumentException.class, () -> Transition.decode(null)); + assertThrows(IllegalArgumentException.class, () -> Transition.decode("UNKNOWN")); + assertThrows(IllegalArgumentException.class, () -> Transition.decode("UNKNOWN:det")); + // A labeled shift is contradictory and must be rejected by the record invariant. + assertThrows(IllegalArgumentException.class, () -> Transition.decode("SHIFT:det")); + } + + @Test + void testConstructorValidation() { + assertThrows(IllegalArgumentException.class, () -> new Transition(null, "det")); + assertThrows(IllegalArgumentException.class, + () -> new Transition(Transition.Type.SHIFT, "det")); + assertThrows(IllegalArgumentException.class, () -> Transition.leftArc(null)); + assertThrows(IllegalArgumentException.class, () -> Transition.rightArc(" ")); + } +} diff --git a/opennlp-docs/src/docbkx/dependency.xml b/opennlp-docs/src/docbkx/dependency.xml new file mode 100644 index 0000000000..3357f9a9ab --- /dev/null +++ b/opennlp-docs/src/docbkx/dependency.xml @@ -0,0 +1,138 @@ + + + + + + + Dependency Parsing + +
+ Introduction + + Dependency parsing assigns each token a syntactic head and a relation label. + Both parser implementations accept tokens with aligned part-of-speech tags and + return a DependencyGraph. DependencyParserME uses a + maximum entropy model. FeedforwardDependencyParser uses a neural + model implemented in Java. + +
+ +
+ Training data + + ConlluDependencySampleStream reads the basic HEAD and + DEPREL columns of a CoNLL-U treebank. Select + ConlluTagset.U for universal tags or ConlluTagset.X + for language-specific tags. The reader retains syntactic words from multiword + tokens and skips sentences without a complete dependency tree. + +
+ +
+ Maximum entropy parser + + Train and save the standard OpenNLP model: + + Files.newInputStream(Path.of("train.conllu")); + +TrainingParameters parameters = TrainingParameters.defaultParams(); +parameters.put(Parameters.CUTOFF_PARAM, 0); +DependencyModel model; +try (ObjectStream samples = + new ConlluDependencySampleStream(trainingData, ConlluTagset.U)) { + model = DependencyParserME.train("eng", samples, parameters); +} +model.serialize(Path.of("en-depparse.bin")); + +DependencyParser parser = + new DependencyParserME(new DependencyModel(Path.of("en-depparse.bin")));]]> + + +
+ +
+ Feedforward parser + + The feedforward trainer has its own versioned model format. This example trains, + saves, reloads, and creates a parser with a beam size of four: + + samples = + new ConlluDependencySampleStream(trainingData, ConlluTagset.U)) { + model = FeedforwardDependencyTrainer.train( + samples, FeedforwardDependencyTrainer.Settings.defaults()); +} +try (OutputStream out = Files.newOutputStream(Path.of("en-depparse-ff.bin"))) { + model.serialize(out); +} + +DependencyParser parser = new FeedforwardDependencyParser( + FeedforwardDependencyModel.load(Path.of("en-depparse-ff.bin")), 4);]]> + + + + Scoring uses double precision for input products and cached contributions. + Cached and direct scoring produce the same values. The shared cache retains + at most 32,768 template-position and embedding-row pairs; other pairs use + direct calculation. Greedy and beam parsing throw IllegalStateException + if the model produces a non-finite transition score. + +
+ +
+ Parsing and evaluation + + Both implementations use the same parsing API. Graph indices refer to the input + token array: + + " + head + " (" + arc.relation() + ")"); +}]]> + + DependencyEvaluator reports the unlabeled attachment score (the fraction + of tokens with the correct head) and the labeled attachment score (the fraction with + the correct head and relation label), both over all tokens and, through the + ExcludingPunctuation getters, over the tokens whose gold tag is not + PUNCT. DependencyCrossValidator splits a sample stream into + folds, trains a parser on the other folds through a trainer you supply, and scores + the held-out fold, so both parser implementations can be cross validated: + + new DependencyParserME( + DependencyParserME.train("eng", samples, parameters))); +validator.evaluate(trainingSamples, 4); +double uas = validator.getUas(); +double lasExcludingPunctuation = validator.getLasExcludingPunctuation();]]> + + The provided treebank evaluation, UniversalDependencyParserEval in + opennlp-eval-tests, trains each parser from scratch on a Universal + Dependencies 2.0 training split and pins all four scores on data the parser has not + seen: the transition parser on the English, German, Spanish AnCora, and French + development splits and in a five-fold cross validation of the English training split, + the feedforward parser on the English and Spanish AnCora development splits. It uses + gold sentence boundaries, tokens, and tags, so it measures the dependency parser + rather than a complete text pipeline. The arc-standard trainers skip non-projective + training samples. The treebank download helper and the pinned scores are documented + in opennlp-formats/dev. + +
+ diff --git a/opennlp-docs/src/docbkx/opennlp.xml b/opennlp-docs/src/docbkx/opennlp.xml index 0761fc95ff..4493b16d7d 100644 --- a/opennlp-docs/src/docbkx/opennlp.xml +++ b/opennlp-docs/src/docbkx/opennlp.xml @@ -113,6 +113,7 @@ under the License. + diff --git a/opennlp-eval-tests/src/test/java/opennlp/tools/eval/UniversalDependencyParserEval.java b/opennlp-eval-tests/src/test/java/opennlp/tools/eval/UniversalDependencyParserEval.java new file mode 100644 index 0000000000..f94dd3673e --- /dev/null +++ b/opennlp-eval-tests/src/test/java/opennlp/tools/eval/UniversalDependencyParserEval.java @@ -0,0 +1,352 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.eval; + +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.depparse.DependencyCrossValidator; +import opennlp.tools.depparse.DependencyEvaluator; +import opennlp.tools.depparse.DependencyModel; +import opennlp.tools.depparse.DependencyParser; +import opennlp.tools.depparse.DependencyParserME; +import opennlp.tools.depparse.FeedforwardDependencyModel; +import opennlp.tools.depparse.FeedforwardDependencyParser; +import opennlp.tools.depparse.FeedforwardDependencyTrainer; +import opennlp.tools.formats.conllu.ConlluDependencySampleStream; +import opennlp.tools.formats.conllu.ConlluTagset; +import opennlp.tools.util.MarkableFileInputStreamFactory; +import opennlp.tools.util.Parameters; +import opennlp.tools.util.TrainingParameters; + +/** + * Measures the accuracy of the two dependency parsers on Universal Dependencies 2.0 + * treebanks and pins the exact scores. + * + *

Every test trains from scratch on a treebank's training split and scores the parser + * on sentences it has not seen: the treebank's development split, or, for the English + * cross validation, the held-out fold. The treebank's sentence segmentation, tokens, and + * universal part-of-speech tags are the parser input, so the scores measure parsing by + * itself and not the errors of an upstream pipeline. Four numbers are pinned per run: + * the unlabeled attachment score (UAS, the fraction of tokens with the correct head), the + * labeled attachment score (LAS, the fraction with the correct head and relation label), + * and both again over the tokens that are not tagged {@code PUNCT}, the customary + * reporting convention for Universal Dependencies.

+ * + *

The transition parser is the maximum-entropy arc-standard parser trained with a + * feature cutoff of {@value #TRANSITION_CUTOFF}; the feedforward parser is trained with + * {@link FeedforwardDependencyTrainer.Settings#defaults()} and parsed greedily. The + * transition parser is evaluated on English, German, Spanish, and French, the feedforward + * parser on English and Spanish. The English cross validation uses + * {@value #ENGLISH_FOLDS} folds of the training split, the same fold count as the + * constituency parser evaluation.

+ */ +public class UniversalDependencyParserEval extends AbstractEvalTest { + + /** The feature cutoff of the transition parser. */ + private static final int TRANSITION_CUTOFF = 5; + + /** The fold count of the English cross validation. */ + private static final int ENGLISH_FOLDS = 5; + + /** + * One treebank: its language code and its training and development splits. + * + * @param language The ISO 639-3 code passed to the transition trainer. + * @param train The training split. + * @param dev The development split. + */ + private record Treebank(String language, File train, File dev) { + + /** + * Locates a treebank under {@code ud20} in the data directory. + * + * @param language The ISO 639-3 code. + * @param directory The treebank directory name, for example {@code UD_English}. + * @param prefix The split file prefix, for example {@code en}. + * @return The treebank. Never {@code null}. + * @throws IOException Thrown if the data directory is not set or missing. + */ + static Treebank of(String language, String directory, String prefix) throws IOException { + final File base = new File(getOpennlpDataDir(), "ud20/" + directory); + return new Treebank(language, new File(base, prefix + "-ud-train.conllu"), + new File(base, prefix + "-ud-dev.conllu")); + } + } + + /** + * The scores of one run. + * + * @param words The number of scored tokens. + * @param wordsExcludingPunctuation The number of scored tokens not tagged {@code PUNCT}. + * @param uas The unlabeled attachment score over all tokens. + * @param las The labeled attachment score over all tokens. + * @param uasExcludingPunctuation The unlabeled attachment score without punctuation. + * @param lasExcludingPunctuation The labeled attachment score without punctuation. + */ + private record Scores(long words, long wordsExcludingPunctuation, double uas, double las, + double uasExcludingPunctuation, double lasExcludingPunctuation) { + + /** + * Reads the scores of a finished evaluation. + * + * @param evaluator The evaluator after {@link DependencyEvaluator#evaluate}. + * @return The scores. Never {@code null}. + */ + static Scores of(DependencyEvaluator evaluator) { + return new Scores(evaluator.getWordCount(), evaluator.getWordCountExcludingPunctuation(), + evaluator.getUas(), evaluator.getLas(), evaluator.getUasExcludingPunctuation(), + evaluator.getLasExcludingPunctuation()); + } + + /** + * Reads the scores of a finished cross validation. + * + * @param validator The validator after {@link DependencyCrossValidator#evaluate}. + * @return The scores. Never {@code null}. + */ + static Scores of(DependencyCrossValidator validator) { + return new Scores(validator.getWordCount(), validator.getWordCountExcludingPunctuation(), + validator.getUas(), validator.getLas(), validator.getUasExcludingPunctuation(), + validator.getLasExcludingPunctuation()); + } + } + + private static Treebank english; + private static Treebank german; + private static Treebank spanish; + private static Treebank french; + + /** + * Locates the treebank splits and verifies their digests. + * + * @throws Exception Thrown if the data directory or a split is missing or changed. + */ + @BeforeAll + static void verifyTrainingData() throws Exception { + english = Treebank.of("eng", "UD_English", "en"); + german = Treebank.of("deu", "UD_German", "de"); + spanish = Treebank.of("spa", "UD_Spanish-AnCora", "es_ancora"); + french = Treebank.of("fra", "UD_French", "fr"); + verifyFileChecksum(english.train().toPath(), + new BigInteger("240180699945832517506080812986784120023")); + verifyFileChecksum(english.dev().toPath(), + new BigInteger("77584448353380940340541536944256636015")); + verifyFileChecksum(german.train().toPath(), + new BigInteger("290029973628740099051462227636742189201")); + verifyFileChecksum(german.dev().toPath(), + new BigInteger("133330826309481994758690698938095410310")); + verifyFileChecksum(spanish.train().toPath(), + new BigInteger("224942804200733453179524127037951530195")); + verifyFileChecksum(spanish.dev().toPath(), + new BigInteger("280996187464384493180190898172297941708")); + verifyFileChecksum(french.train().toPath(), + new BigInteger("147594341855033303213328942622831640563")); + verifyFileChecksum(french.dev().toPath(), + new BigInteger("146783792803312408008054489850921785757")); + } + + /** + * Cross validates the transition parser on the English training split: five parsers, + * each trained on four fifths of the split and scored on the remaining fifth, so every + * training sentence is scored once by a parser that did not see it. + * + * @throws IOException Thrown if reading or training fails. + */ + @Test + void crossValidateTransitionParserEnglish() throws IOException { + final DependencyCrossValidator validator = new DependencyCrossValidator( + samples -> new DependencyParserME( + DependencyParserME.train(english.language(), samples, transitionParameters()))); + try (ConlluDependencySampleStream train = samples(english.train())) { + validator.evaluate(train, ENGLISH_FOLDS); + } + assertScores(new Scores(204585, 180906, 0.8205440281545567d, 0.7885182198108366d, + 0.8421666500834688d, 0.8065404132532917d), Scores.of(validator)); + } + + /** + * Trains the transition parser on the English training split and scores it on the + * development split. + * + * @throws IOException Thrown if reading or training fails. + */ + @Test + void trainAndEvalTransitionParserEnglish() throws IOException { + assertScores(new Scores(25148, 22065, 0.8181565134404326d, 0.7861460155877207d, + 0.8372082483571267d, 0.8014502605937004d), + evaluate(transitionParser(english), english.dev())); + } + + /** + * Trains the transition parser on the German training split and scores it on the + * development split. + * + * @throws IOException Thrown if reading or training fails. + */ + @Test + void trainAndEvalTransitionParserGerman() throws IOException { + assertScores(new Scores(12348, 10730, 0.7843375445416262d, 0.7305636540330418d, + 0.802982292637465d, 0.7413793103448276d), + evaluate(transitionParser(german), german.dev())); + } + + /** + * Trains the transition parser on the Spanish AnCora training split and scores it on + * the development split. + * + * @throws IOException Thrown if reading or training fails. + */ + @Test + void trainAndEvalTransitionParserSpanishAncora() throws IOException { + assertScores(new Scores(52336, 46058, 0.8355243044940385d, 0.7914437480892693d, + 0.8598506231273612d, 0.8098918754613748d), + evaluate(transitionParser(spanish), spanish.dev())); + } + + /** + * Trains the transition parser on the French training split and scores it on the + * development split. + * + * @throws IOException Thrown if reading or training fails. + */ + @Test + void trainAndEvalTransitionParserFrench() throws IOException { + assertScores(new Scores(35766, 31947, 0.8501370016216518d, 0.8191578594195604d, + 0.8794879018374182d, 0.8450245719472878d), + evaluate(transitionParser(french), french.dev())); + } + + /** + * Trains the feedforward parser with its default settings on the English training + * split and scores it on the development split. + * + * @throws IOException Thrown if reading or training fails. + */ + @Test + void trainAndEvalFeedforwardParserEnglish() throws IOException { + assertScores(new Scores(25148, 22065, 0.8412597423254334d, 0.8174009861619215d, + 0.8539768864717879d, 0.8272830274189894d), + evaluate(feedforwardParser(english), english.dev())); + } + + /** + * Trains the feedforward parser with its default settings on the Spanish AnCora + * training split and scores it on the development split. + * + * @throws IOException Thrown if reading or training fails. + */ + @Test + void trainAndEvalFeedforwardParserSpanishAncora() throws IOException { + assertScores(new Scores(52336, 46058, 0.8617013145826964d, 0.8279195964536838d, + 0.8780016500933605d, 0.839615267705936d), + evaluate(feedforwardParser(spanish), spanish.dev())); + } + + /** + * @return The training parameters of the transition parser. Never {@code null}. + */ + private static TrainingParameters transitionParameters() { + final TrainingParameters parameters = TrainingParameters.defaultParams(); + parameters.put(Parameters.CUTOFF_PARAM, TRANSITION_CUTOFF); + return parameters; + } + + /** + * Trains the maximum-entropy transition parser on a treebank's training split. + * + * @param treebank The treebank. + * @return The trained parser. Never {@code null}. + * @throws IOException Thrown if reading or training fails. + */ + private static DependencyParser transitionParser(Treebank treebank) throws IOException { + final DependencyModel model; + try (ConlluDependencySampleStream train = samples(treebank.train())) { + model = DependencyParserME.train(treebank.language(), train, transitionParameters()); + } + return new DependencyParserME(model); + } + + /** + * Trains the feedforward parser with its default settings on a treebank's training + * split. + * + * @param treebank The treebank. + * @return The trained parser, decoding greedily. Never {@code null}. + * @throws IOException Thrown if reading or training fails. + */ + private static DependencyParser feedforwardParser(Treebank treebank) throws IOException { + final FeedforwardDependencyModel model; + try (ConlluDependencySampleStream train = samples(treebank.train())) { + model = FeedforwardDependencyTrainer.train(train, + FeedforwardDependencyTrainer.Settings.defaults()); + } + return new FeedforwardDependencyParser(model); + } + + /** + * Scores a parser on one split. + * + * @param parser The parser to score. + * @param split The CoNLL-U file with the gold trees. + * @return The scores. Never {@code null}. + * @throws IOException Thrown if reading the split fails. + */ + private static Scores evaluate(DependencyParser parser, File split) throws IOException { + final DependencyEvaluator evaluator = new DependencyEvaluator(parser); + try (ConlluDependencySampleStream dev = samples(split)) { + evaluator.evaluate(dev); + } + return Scores.of(evaluator); + } + + /** + * Asserts that the measured scores match the pinned ones. + * + * @param expected The pinned scores. + * @param actual The measured scores. + */ + private static void assertScores(Scores expected, Scores actual) { + Assertions.assertEquals(expected.words(), actual.words(), "words"); + Assertions.assertEquals(expected.wordsExcludingPunctuation(), + actual.wordsExcludingPunctuation(), "words excluding punctuation"); + Assertions.assertEquals(expected.uas(), actual.uas(), ACCURACY_DELTA, "UAS"); + Assertions.assertEquals(expected.las(), actual.las(), ACCURACY_DELTA, "LAS"); + Assertions.assertEquals(expected.uasExcludingPunctuation(), + actual.uasExcludingPunctuation(), ACCURACY_DELTA, "UAS excluding punctuation"); + Assertions.assertEquals(expected.lasExcludingPunctuation(), + actual.lasExcludingPunctuation(), ACCURACY_DELTA, "LAS excluding punctuation"); + } + + /** + * Opens one split with the universal tagset. + * + * @param split The CoNLL-U file. + * @return The sample stream. + * @throws IOException Thrown if the file cannot be opened. + */ + private static ConlluDependencySampleStream samples(File split) throws IOException { + return new ConlluDependencySampleStream(new MarkableFileInputStreamFactory(split), + ConlluTagset.U); + } +}