> getAvailableModels() {
if (availableModels == null) {
try {
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/ResourceInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/ResourceInstaller.java
new file mode 100644
index 0000000000..54b215b6e3
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/ResourceInstaller.java
@@ -0,0 +1,1558 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util;
+
+import java.io.BufferedInputStream;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystemNotFoundException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.ProviderNotFoundException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.Duration;
+import java.util.Comparator;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Stream;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipException;
+import java.util.zip.ZipFile;
+import java.util.zip.ZipInputStream;
+
+import opennlp.tools.util.archive.TarStream;
+
+/**
+ * Fetches a third-party resource, such as a training corpus, a dictionary archive, or a
+ * lexicon, into a local directory. The caller supplies the location and thereby accepts
+ * that resource's license; no locations are built in and no data is bundled. Only
+ * {@code http}, {@code https}, and {@code file} locations are accepted.
+ *
+ * A checksum is required for http and https sources and optional for file sources.
+ * It is verified against the downloaded bytes before anything is unpacked: a
+ * 64-character hex digest selects SHA-256, a 128-character one SHA-512.
+ * The content format is detected from the bytes, not from the name: gzip-compressed
+ * tar archives and zip archives are unpacked with their relative structure. Invalid,
+ * escaping, and duplicate file paths are rejected. Plain gzip files are decompressed,
+ * and other content is stored as a file under the source name. One name rule overrides
+ * byte detection: a {@code *.bin} source is always stored packed because an OpenNLP
+ * model file is itself a zip archive that its consumers load packed.
+ *
+ * Each installation is bounded by {@link Limits}: http and https fetches use
+ * connection and read timeouts, follow at most a fixed number of redirects, reject
+ * redirects that leave the http and https schemes or downgrade https to http, and
+ * abort once the download or the expanded content crosses its size limit or the
+ * archive crosses its entry limit. Compressed content, gzip and zip alike, may
+ * expand to at most {@link Limits#maxExpansionRatio()} times its compressed size,
+ * with a floor of {@value #MIN_EXPANSION_BYTES} bytes for small sources. The defaults in
+ * {@link Limits#DEFAULT} apply when no limits are given, and {@link Limits#builder()}
+ * starts from them.
+ *
+ * Installation is staged: content is unpacked into a hidden staging directory on
+ * the same filesystem and moved into the target only after the download was verified
+ * and every entry unpacked cleanly. A fetch, verification, or unpacking failure
+ * promotes no files into the target directory.
+ * Promotion does not replace a file that already exists in the target and detects
+ * the collision before moving anything, so refreshing a resource means removing its
+ * old files first. Work files left in the target by an installation that was killed
+ * are removed at the start of the next installation into that target, so concurrent
+ * installations into one target directory are not supported.
+ *
+ * @see DownloadUtil
+ * @since 3.0.0
+ */
+public final class ResourceInstaller {
+
+ private static final String SHA_256 = "SHA-256";
+ private static final String SHA_512 = "SHA-512";
+ private static final int SHA_256_HEX_LENGTH = 64;
+ private static final int SHA_512_HEX_LENGTH = 128;
+ private static final String GZIP_SUFFIX = ".gz";
+
+ /** OpenNLP model files are packed zip archives; install them packed. */
+ private static final String MODEL_SUFFIX = ".bin";
+ private static final String DEFAULT_RESOURCE_NAME = "resource";
+ private static final String STAGING_PREFIX = ".opennlp-staging";
+ private static final String DOWNLOAD_PREFIX = ".opennlp-download";
+ private static final String DOWNLOAD_SUFFIX = ".part";
+ private static final int BUFFER_SIZE = 8192;
+ private static final int MAGIC_LENGTH = 4;
+ private static final int GZIP_MAGIC_FIRST = 0x1F;
+ private static final int GZIP_MAGIC_SECOND = 0x8B;
+ private static final int ZIP_MAGIC_FIRST = 'P';
+ private static final int ZIP_MAGIC_SECOND = 'K';
+ private static final int ZIP_LOCAL_HEADER_THIRD = 3;
+ private static final int ZIP_LOCAL_HEADER_FOURTH = 4;
+ private static final int ZIP_END_HEADER_THIRD = 5;
+ private static final int ZIP_END_HEADER_FOURTH = 6;
+ private static final int ZIP_END_HEADER_LENGTH = 22;
+ private static final int ZIP_DISK_OFFSET = 4;
+ private static final int ZIP_CENTRAL_DISK_OFFSET = 6;
+ private static final int ZIP_DISK_ENTRIES_OFFSET = 8;
+ private static final int ZIP_TOTAL_ENTRIES_OFFSET = 10;
+ private static final int ZIP_CENTRAL_SIZE_OFFSET = 12;
+ private static final int ZIP_CENTRAL_OFFSET_OFFSET = 16;
+ private static final int ZIP_COMMENT_LENGTH_OFFSET = 20;
+ private static final int TAR_END_BLOCKS_LENGTH = 1024;
+
+ /** The expanded size every compressed source may reach regardless of the ratio. */
+ private static final long MIN_EXPANSION_BYTES = 1L << 20;
+ private static final int HTTP_TEMPORARY_REDIRECT = 307;
+ private static final int HTTP_PERMANENT_REDIRECT = 308;
+ private static final String SCHEME_HTTP = "http";
+ private static final String SCHEME_HTTPS = "https";
+ private static final String SCHEME_FILE = "file";
+ private static final String MALFORMED_ZIP_ERROR = "malformed zip archive";
+ private static final String ZIP_MISMATCH_ERROR =
+ "zip local headers and central directory list different files";
+
+ /**
+ * Safety limits and network behavior for one installation.
+ *
+ * @param connectTimeout How long to wait for a connection to be established. Must
+ * be positive.
+ * @param readTimeout How long to wait for data on an established connection. Must
+ * be positive.
+ * @param maxRedirects How many http redirects to follow before failing. Must not be
+ * negative; zero rejects all redirects.
+ * @param maxDownloadBytes The largest download accepted, in bytes. Must be positive.
+ * @param maxExpandedBytes The largest expanded byte count accepted. For gzip content,
+ * this counts the entire decompressed stream; otherwise, it
+ * counts installed file content. Must be positive.
+ * @param maxEntries The largest number of archive entries accepted, counting every
+ * entry including directories, so an archive of many tiny files
+ * cannot exhaust directory entries while staying under the byte
+ * limits. Must be positive.
+ * @param maxExpansionRatio The largest expanded size accepted per compressed byte of
+ * a source, so a small archive cannot expand to the whole
+ * expansion limit. Applies to gzip and zip content; a source
+ * may always expand to {@value #MIN_EXPANSION_BYTES} bytes
+ * regardless of it. Must be positive.
+ */
+ public record Limits(Duration connectTimeout, Duration readTimeout, int maxRedirects,
+ long maxDownloadBytes, long maxExpandedBytes, long maxEntries,
+ long maxExpansionRatio) {
+
+ /** The system property overriding the default download limit in bytes. */
+ public static final String MAX_DOWNLOAD_BYTES_PROPERTY = "opennlp.download.max.bytes";
+
+ /** The system property overriding the default expansion limit in bytes. */
+ public static final String MAX_EXPANDED_BYTES_PROPERTY =
+ "opennlp.install.max.total.bytes";
+
+ /** The system property overriding the default archive entry limit. */
+ public static final String MAX_ENTRIES_PROPERTY = "opennlp.install.max.entries";
+
+ /** The system property overriding the default expansion ratio. */
+ public static final String MAX_EXPANSION_RATIO_PROPERTY =
+ "opennlp.install.max.expansion.ratio";
+
+ /**
+ * The limits applied when none are given: 20 second connect timeout, 60 second
+ * read timeout, at most 5 redirects, a 1 GiB download limit, a 4 GiB expansion
+ * limit, 100000 archive entries, and an expansion ratio of 100. Each limit can be
+ * raised or lowered at startup through its system property
+ * ({@link #MAX_DOWNLOAD_BYTES_PROPERTY}, {@link #MAX_EXPANDED_BYTES_PROPERTY},
+ * {@link #MAX_ENTRIES_PROPERTY}, {@link #MAX_EXPANSION_RATIO_PROPERTY}), read once
+ * at class load; a value that is absent, not a number, or not positive falls back
+ * to the built-in default.
+ */
+ public static final Limits DEFAULT = new Limits(Duration.ofSeconds(20),
+ Duration.ofSeconds(60), 5,
+ longProperty(MAX_DOWNLOAD_BYTES_PROPERTY, 1L << 30),
+ longProperty(MAX_EXPANDED_BYTES_PROPERTY, 4L << 30),
+ longProperty(MAX_ENTRIES_PROPERTY, 100_000L),
+ longProperty(MAX_EXPANSION_RATIO_PROPERTY, 100L));
+
+ /**
+ * Reads a limit override from a system property, trimmed before parsing.
+ *
+ * @param name The property name.
+ * @param fallback The built-in default.
+ * @return The property's value, or {@code fallback} when the property is absent,
+ * not a number, or not positive.
+ */
+ static long longProperty(String name, long fallback) {
+ final String value = System.getProperty(name);
+ if (value == null) {
+ return fallback;
+ }
+ final long parsed;
+ try {
+ parsed = Long.parseLong(value.trim());
+ } catch (NumberFormatException e) {
+ return fallback;
+ }
+ return parsed > 0 ? parsed : fallback;
+ }
+
+ /**
+ * Validates the limit values before constructing an instance.
+ *
+ * @param connectTimeout How long to wait for a connection to be established.
+ * @param readTimeout How long to wait for data on an established connection.
+ * @param maxRedirects How many http redirects to follow before failing.
+ * @param maxDownloadBytes The largest download accepted, in bytes.
+ * @param maxExpandedBytes The largest expanded byte count accepted.
+ * @param maxEntries The largest number of archive entries accepted.
+ * @param maxExpansionRatio The largest expanded size accepted per compressed byte.
+ * @throws IllegalArgumentException Thrown if either timeout is {@code null}, zero,
+ * or negative, a limit is not positive, or the redirect limit is
+ * negative.
+ */
+ public Limits(Duration connectTimeout, Duration readTimeout, int maxRedirects,
+ long maxDownloadBytes, long maxExpandedBytes, long maxEntries,
+ long maxExpansionRatio) {
+ if (connectTimeout == null) {
+ throw new IllegalArgumentException("connectTimeout must not be null");
+ }
+ if (connectTimeout.isZero() || connectTimeout.isNegative()) {
+ throw new IllegalArgumentException("connectTimeout must be positive");
+ }
+ if (readTimeout == null) {
+ throw new IllegalArgumentException("readTimeout must not be null");
+ }
+ if (readTimeout.isZero() || readTimeout.isNegative()) {
+ throw new IllegalArgumentException("readTimeout must be positive");
+ }
+ if (maxRedirects < 0) {
+ throw new IllegalArgumentException("maxRedirects must not be negative");
+ }
+ if (maxDownloadBytes <= 0) {
+ throw new IllegalArgumentException("maxDownloadBytes must be positive");
+ }
+ if (maxExpandedBytes <= 0) {
+ throw new IllegalArgumentException("maxExpandedBytes must be positive");
+ }
+ if (maxEntries <= 0) {
+ throw new IllegalArgumentException("maxEntries must be positive");
+ }
+ if (maxExpansionRatio <= 0) {
+ throw new IllegalArgumentException("maxExpansionRatio must be positive");
+ }
+ this.connectTimeout = connectTimeout;
+ this.readTimeout = readTimeout;
+ this.maxRedirects = maxRedirects;
+ this.maxDownloadBytes = maxDownloadBytes;
+ this.maxExpandedBytes = maxExpandedBytes;
+ this.maxEntries = maxEntries;
+ this.maxExpansionRatio = maxExpansionRatio;
+ }
+
+ /**
+ * Starts from {@link #DEFAULT} so a caller can state only the limits that differ
+ * from it, instead of repeating all seven in the canonical constructor.
+ *
+ * @return A builder holding the default limits. Not {@code null}.
+ */
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ /**
+ * Collects limit values and validates them on {@link #build()}. Each setter returns
+ * this builder. Not thread safe; the {@link Limits} it builds is immutable.
+ */
+ public static final class Builder {
+
+ private Duration connectTimeout = DEFAULT.connectTimeout();
+ private Duration readTimeout = DEFAULT.readTimeout();
+ private int maxRedirects = DEFAULT.maxRedirects();
+ private long maxDownloadBytes = DEFAULT.maxDownloadBytes();
+ private long maxExpandedBytes = DEFAULT.maxExpandedBytes();
+ private long maxEntries = DEFAULT.maxEntries();
+ private long maxExpansionRatio = DEFAULT.maxExpansionRatio();
+
+ /** Initializes a builder with {@link Limits#DEFAULT}. */
+ private Builder() {
+ }
+
+ /**
+ * Sets how long to wait for a connection to be established.
+ *
+ * @param connectTimeout How long to wait for a connection to be established.
+ * Must be positive.
+ * @return This builder. Not {@code null}.
+ */
+ public Builder connectTimeout(Duration connectTimeout) {
+ this.connectTimeout = connectTimeout;
+ return this;
+ }
+
+ /**
+ * Sets how long to wait for data on an established connection.
+ *
+ * @param readTimeout How long to wait for data on an established connection.
+ * Must be positive.
+ * @return This builder. Not {@code null}.
+ */
+ public Builder readTimeout(Duration readTimeout) {
+ this.readTimeout = readTimeout;
+ return this;
+ }
+
+ /**
+ * Sets how many http redirects to follow before failing.
+ *
+ * @param maxRedirects How many http redirects to follow before failing. Must not
+ * be negative; zero rejects all redirects.
+ * @return This builder. Not {@code null}.
+ */
+ public Builder maxRedirects(int maxRedirects) {
+ this.maxRedirects = maxRedirects;
+ return this;
+ }
+
+ /**
+ * Sets the largest download accepted.
+ *
+ * @param maxDownloadBytes The largest download accepted, in bytes. Must be
+ * positive.
+ * @return This builder. Not {@code null}.
+ */
+ public Builder maxDownloadBytes(long maxDownloadBytes) {
+ this.maxDownloadBytes = maxDownloadBytes;
+ return this;
+ }
+
+ /**
+ * Sets the largest expanded byte count accepted.
+ *
+ * @param maxExpandedBytes The largest expanded byte count accepted. For gzip
+ * content, this counts the entire decompressed stream;
+ * otherwise, it counts installed file content. Must be
+ * positive.
+ * @return This builder. Not {@code null}.
+ */
+ public Builder maxExpandedBytes(long maxExpandedBytes) {
+ this.maxExpandedBytes = maxExpandedBytes;
+ return this;
+ }
+
+ /**
+ * Sets the largest number of archive entries accepted.
+ *
+ * @param maxEntries The largest number of archive entries accepted, counting
+ * every entry including directories. Must be positive.
+ * @return This builder. Not {@code null}.
+ */
+ public Builder maxEntries(long maxEntries) {
+ this.maxEntries = maxEntries;
+ return this;
+ }
+
+ /**
+ * Sets the largest expanded size accepted per compressed byte of a source.
+ *
+ * @param maxExpansionRatio The largest expanded size accepted per compressed
+ * byte of a source. Must be positive.
+ * @return This builder. Not {@code null}.
+ */
+ public Builder maxExpansionRatio(long maxExpansionRatio) {
+ this.maxExpansionRatio = maxExpansionRatio;
+ return this;
+ }
+
+ /**
+ * Builds the limits.
+ *
+ * @return The limits collected so far. Not {@code null}.
+ * @throws IllegalArgumentException Thrown if any value is outside its documented
+ * range.
+ */
+ public Limits build() {
+ return new Limits(connectTimeout, readTimeout, maxRedirects, maxDownloadBytes,
+ maxExpandedBytes, maxEntries, maxExpansionRatio);
+ }
+ }
+ }
+
+ /** Prevents construction of this utility class. */
+ private ResourceInstaller() {
+ }
+
+ /**
+ * Unpacks a resource without checksum verification, under {@link Limits#DEFAULT}.
+ * This overload treats the source as trusted caller input and performs no
+ * cryptographic integrity verification, so it accepts only {@code file} sources; an
+ * http or https source must go through an overload that takes its checksum.
+ *
+ * @param source The resource location, a {@code file} URI. Not {@code null}.
+ * @param targetDirectory The directory to install into; created when absent. Must
+ * not be {@code null}.
+ * @return The target directory. Not {@code null}.
+ * @throws IOException Thrown if fetching or unpacking fails.
+ * @throws IllegalArgumentException Thrown if {@code source} or
+ * {@code targetDirectory} is {@code null}, {@code source} contains a scheme
+ * other than {@code file}, or its last path segment is not a valid local file
+ * name.
+ */
+ public static Path install(URI source, Path targetDirectory) throws IOException {
+ return install(source, targetDirectory, null);
+ }
+
+ /**
+ * Fetches, verifies, and unpacks a resource under {@link Limits#DEFAULT}.
+ *
+ * @param source The resource location, an {@code http}, {@code https}, or
+ * {@code file} URI. Not {@code null}.
+ * @param targetDirectory The directory to install into; created when absent. Must
+ * not be {@code null}.
+ * @param checksum The expected digest of the downloaded bytes as a hex string,
+ * compared case-insensitively and ignoring leading and trailing
+ * whitespace: 64 characters select SHA-256, 128 characters SHA-512.
+ * Required for an http or https source; pass {@code null} to skip
+ * verification for a {@code file} source.
+ * @return The target directory. Not {@code null}.
+ * @throws IOException Thrown if fetching fails, the checksum does not match, or
+ * unpacking fails.
+ * @throws IllegalArgumentException Thrown if {@code source} or
+ * {@code targetDirectory} is {@code null}, {@code source} contains a scheme
+ * other than {@code http}, {@code https}, or {@code file}, {@code checksum}
+ * is not a 64-character or 128-character hex string, an http or https source
+ * contains no checksum, or the source does not provide a valid local file
+ * name.
+ */
+ public static Path install(URI source, Path targetDirectory, String checksum)
+ throws IOException {
+ return install(source, targetDirectory, checksum, Limits.DEFAULT);
+ }
+
+ /**
+ * Fetches, verifies, and unpacks a resource under the given {@link Limits}.
+ *
+ * @param source The resource location, an {@code http}, {@code https}, or
+ * {@code file} URI. Not {@code null}.
+ * @param targetDirectory The directory to install into; created when absent. Must
+ * not be {@code null}.
+ * @param checksum The expected digest of the downloaded bytes as a hex string,
+ * compared case-insensitively and ignoring leading and trailing
+ * whitespace: 64 characters select SHA-256, 128 characters SHA-512.
+ * Required for an http or https source; pass {@code null} to skip
+ * verification for a {@code file} source.
+ * @param limits The timeouts, redirect allowance, and size and entry limits to
+ * enforce. Not {@code null}.
+ * @return The target directory. Not {@code null}.
+ * @throws IOException Thrown if fetching fails, a limit is exceeded, the checksum
+ * does not match, or unpacking fails.
+ * @throws IllegalArgumentException Thrown if {@code source}, {@code targetDirectory},
+ * or {@code limits} is {@code null}, {@code source} contains a scheme other
+ * than {@code http}, {@code https}, or {@code file}, {@code checksum} is
+ * not a 64-character or 128-character hex string, an http or https source
+ * contains no checksum, or the source does not provide a valid local file
+ * name.
+ */
+ public static Path install(URI source, Path targetDirectory, String checksum,
+ Limits limits) throws IOException {
+ return install(source, targetDirectory, checksum, null, limits);
+ }
+
+ /**
+ * Installs a catalog entry under its preferred file name when it is not an archive.
+ *
+ * @param source The resource location.
+ * @param targetDirectory The directory to install into.
+ * @param checksum The expected digest, or {@code null} for a file source.
+ * @param name The preferred name for non-archive content.
+ * @return The target directory.
+ * @throws IOException Thrown if fetching, verification, or unpacking fails.
+ * @throws IllegalArgumentException Thrown if an argument is invalid.
+ */
+ static Path installNamed(URI source, Path targetDirectory, String checksum,
+ String name) throws IOException {
+ if (name == null) {
+ throw new IllegalArgumentException("name must not be null");
+ }
+ return install(source, targetDirectory, checksum, name, Limits.DEFAULT);
+ }
+
+ /**
+ * Validates the request, downloads and verifies the resource, and installs it from a
+ * staging directory.
+ *
+ * @param source The resource location.
+ * @param targetDirectory The directory to install into.
+ * @param checksum The expected digest, or {@code null} for a file source.
+ * @param name The preferred name for non-archive content, or {@code null} to use the
+ * source name.
+ * @param limits The limits to enforce.
+ * @return The target directory. Not {@code null}.
+ * @throws IOException Thrown if fetching, verification, or installation fails.
+ * @throws IllegalArgumentException Thrown if an argument is invalid.
+ */
+ private static Path install(URI source, Path targetDirectory, String checksum,
+ String name, Limits limits) throws IOException {
+ if (source == null) {
+ throw new IllegalArgumentException("source must not be null");
+ }
+ if (targetDirectory == null) {
+ throw new IllegalArgumentException("targetDirectory must not be null");
+ }
+ if (limits == null) {
+ throw new IllegalArgumentException("limits must not be null");
+ }
+ validateSource(source);
+ final String expected = validateChecksum(checksum);
+ if (expected == null && isHttp(source.getScheme())) {
+ throw new IllegalArgumentException(
+ "checksum must be given for an http or https source: " + source);
+ }
+ final String resourceName = validateSourceName(
+ name == null ? sourceName(source) : name);
+ final boolean createdTarget = Files.notExists(targetDirectory);
+ Files.createDirectories(targetDirectory);
+ removeStaleWorkFiles(targetDirectory);
+ final Path downloaded = createDownloadFile(targetDirectory);
+ try {
+ download(source, downloaded, limits);
+ if (expected != null) {
+ verify(downloaded, expected);
+ }
+ installStaged(downloaded, resourceName, targetDirectory, limits);
+ return targetDirectory;
+ } catch (IOException e) {
+ Files.deleteIfExists(downloaded);
+ if (createdTarget) {
+ removeIfEmpty(targetDirectory, e);
+ }
+ throw e;
+ } finally {
+ Files.deleteIfExists(downloaded);
+ }
+ }
+
+ /**
+ * Removes download files and staging directories that an earlier installation left in
+ * the target because its process ended before cleanup. Only entries with the hidden
+ * work-file prefixes are touched.
+ *
+ * @param targetDirectory The directory to install into.
+ * @throws IOException Thrown if listing or deleting fails.
+ */
+ private static void removeStaleWorkFiles(Path targetDirectory) throws IOException {
+ final List stale;
+ try (Stream entries = Files.list(targetDirectory)) {
+ stale = entries.filter(ResourceInstaller::isWorkFile).toList();
+ }
+ for (final Path entry : stale) {
+ if (Files.isDirectory(entry, LinkOption.NOFOLLOW_LINKS)) {
+ deleteRecursively(entry);
+ } else {
+ Files.deleteIfExists(entry);
+ }
+ }
+ }
+
+ /**
+ * Classifies a target directory entry as a work file of this class.
+ *
+ * @param entry The entry to inspect.
+ * @return {@code true} if the entry name carries a work-file prefix.
+ */
+ private static boolean isWorkFile(Path entry) {
+ final String fileName = entry.getFileName().toString();
+ return fileName.startsWith(STAGING_PREFIX) || fileName.startsWith(DOWNLOAD_PREFIX);
+ }
+
+ /**
+ * Removes a target directory this installation created when the failed installation
+ * left nothing in it, so a failed first attempt leaves the filesystem as it was.
+ *
+ * @param targetDirectory The directory this installation created.
+ * @param failure The failure being reported; a cleanup error is added to it.
+ */
+ private static void removeIfEmpty(Path targetDirectory, IOException failure) {
+ try (Stream entries = Files.list(targetDirectory)) {
+ if (entries.findAny().isEmpty()) {
+ Files.deleteIfExists(targetDirectory);
+ }
+ } catch (IOException cleanup) {
+ failure.addSuppressed(cleanup);
+ }
+ }
+
+ /**
+ * Creates the file the download is written to. It is placed on the target's
+ * filesystem, not in the system temporary directory, so a large download cannot
+ * exhaust the system temporary directory while the target has room. Its hidden prefix
+ * distinguishes it from installed content.
+ *
+ * @param targetDirectory The directory to install into. Must already exist.
+ * @return The newly created, empty download file. Not {@code null}.
+ * @throws IOException Thrown if the file cannot be created.
+ */
+ static Path createDownloadFile(Path targetDirectory) throws IOException {
+ return Files.createTempFile(targetDirectory, DOWNLOAD_PREFIX, DOWNLOAD_SUFFIX);
+ }
+
+ /**
+ * Validates the checksum argument and normalizes it for comparison.
+ *
+ * @param checksum The digest as given by the caller, or {@code null} to skip.
+ * @return The stripped digest, or {@code null} when verification is skipped.
+ * @throws IllegalArgumentException Thrown if the digest is not a 64-character or
+ * 128-character hex string.
+ */
+ private static String validateChecksum(String checksum) {
+ if (checksum == null) {
+ return null;
+ }
+ final String trimmed = checksum.strip();
+ if ((trimmed.length() == SHA_256_HEX_LENGTH || trimmed.length() == SHA_512_HEX_LENGTH)
+ && isHex(trimmed)) {
+ return trimmed;
+ }
+ throw new IllegalArgumentException(
+ "checksum must be 64 (SHA-256) or 128 (SHA-512) hex characters; pass null to skip");
+ }
+
+ /**
+ * Checks whether a string is made up entirely of hexadecimal digits.
+ *
+ * @param value The string to inspect.
+ * @return {@code true} if every character is a hexadecimal digit.
+ */
+ private static boolean isHex(String value) {
+ for (int i = 0; i < value.length(); i++) {
+ final char c = value.charAt(i);
+ final boolean digit = c >= '0' && c <= '9';
+ final boolean lower = c >= 'a' && c <= 'f';
+ final boolean upper = c >= 'A' && c <= 'F';
+ if (!digit && !lower && !upper) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Fetches the source into the given file, bounded by the download limit. Http and
+ * https locations are fetched with timeouts and the redirect policy; a {@code file}
+ * location is read directly. The scheme was accepted by {@link #validateSource(URI)}
+ * at the public boundary.
+ *
+ * @param source The resource location.
+ * @param file The file receiving the downloaded bytes.
+ * @param limits The limits to enforce.
+ * @throws IOException Thrown if fetching fails or a limit is exceeded.
+ */
+ private static void download(URI source, Path file, Limits limits) throws IOException {
+ final Budget budget = new Budget(limits.maxDownloadBytes(),
+ "download exceeds the limit of " + limits.maxDownloadBytes() + " bytes");
+ if (isHttp(source.getScheme())) {
+ downloadHttp(source, file, limits, budget);
+ } else {
+ try (InputStream in = Files.newInputStream(localFile(source))) {
+ copyBounded(in, file, budget);
+ }
+ }
+ }
+
+ /**
+ * Rejects a source the installer will not fetch. Only {@code http}, {@code https}, and
+ * {@code file} are accepted: any other scheme would be passed to any URL handler
+ * the runtime happens to have installed, outside the connection timeout, read timeout,
+ * and redirect policy this class enforces.
+ *
+ * @param source The resource location as given by the caller.
+ * @throws IllegalArgumentException Thrown if the scheme is absent or unsupported.
+ */
+ private static void validateSource(URI source) {
+ final String scheme = source.getScheme();
+ if (!isHttp(scheme) && !SCHEME_FILE.equalsIgnoreCase(scheme)) {
+ throw new IllegalArgumentException(
+ "source scheme must be http, https, or file, but was: " + source);
+ }
+ }
+
+ /**
+ * Classifies a scheme as one the http fetch path handles.
+ *
+ * @param scheme The URI scheme, or {@code null} when the location has none.
+ * @return {@code true} for {@code http} and {@code https}, ignoring case.
+ */
+ private static boolean isHttp(String scheme) {
+ return SCHEME_HTTP.equalsIgnoreCase(scheme) || SCHEME_HTTPS.equalsIgnoreCase(scheme);
+ }
+
+ /**
+ * Resolves a {@code file} location to a path on the default filesystem.
+ *
+ * @param source The {@code file} location, already validated as such.
+ * @return The local path. Not {@code null}.
+ * @throws IOException Thrown if the location does not name a file this runtime can
+ * open, such as a {@code file} URI naming a remote host.
+ */
+ private static Path localFile(URI source) throws IOException {
+ try {
+ return Path.of(source);
+ } catch (IllegalArgumentException | FileSystemNotFoundException e) {
+ throw new IOException("not a readable local file location: " + source, e);
+ }
+ }
+
+ /**
+ * Fetches an http or https source with connection and read timeouts, following at
+ * most the allowed number of redirects under the redirect policy, checking any
+ * declared content length against the download limit before reading the body, and
+ * bounding the transferred bytes against the same limit.
+ *
+ * @param source The resource location as requested by the caller.
+ * @param file The file receiving the downloaded bytes.
+ * @param limits The limits to enforce.
+ * @param budget The download budget shared with the caller.
+ * @throws IOException Thrown if fetching fails, the server answers with a status
+ * other than 200, the redirect policy is violated, or a limit is exceeded.
+ */
+ private static void downloadHttp(URI source, Path file, Limits limits, Budget budget)
+ throws IOException {
+ URI current = source;
+ int redirects = 0;
+ while (true) {
+ final HttpURLConnection connection =
+ (HttpURLConnection) current.toURL().openConnection();
+ connection.setInstanceFollowRedirects(false);
+ connection.setConnectTimeout(timeoutMillis(limits.connectTimeout()));
+ connection.setReadTimeout(timeoutMillis(limits.readTimeout()));
+ try {
+ final int status = connection.getResponseCode();
+ if (isRedirect(status)) {
+ if (redirects >= limits.maxRedirects()) {
+ throw new IOException(
+ "more than " + limits.maxRedirects() + " redirects: " + source);
+ }
+ current = resolveRedirect(current, connection.getHeaderField("Location"));
+ redirects++;
+ continue;
+ }
+ if (status != HttpURLConnection.HTTP_OK) {
+ throw new IOException(
+ "download failed with HTTP status " + status + ": " + current);
+ }
+ final long declared = connection.getContentLengthLong();
+ if (declared > limits.maxDownloadBytes()) {
+ throw new IOException("declared content length " + declared
+ + " exceeds the download limit of " + limits.maxDownloadBytes()
+ + " bytes");
+ }
+ try (InputStream in = connection.getInputStream()) {
+ copyBounded(in, file, budget);
+ }
+ return;
+ } finally {
+ connection.disconnect();
+ }
+ }
+ }
+
+ /**
+ * Classifies a response status as a redirect the installer follows.
+ *
+ * @param status The HTTP response status.
+ * @return {@code true} if the status is one of the redirect statuses 301, 302, 303,
+ * 307, or 308.
+ */
+ private static boolean isRedirect(int status) {
+ return status == HttpURLConnection.HTTP_MOVED_PERM
+ || status == HttpURLConnection.HTTP_MOVED_TEMP
+ || status == HttpURLConnection.HTTP_SEE_OTHER
+ || status == HTTP_TEMPORARY_REDIRECT
+ || status == HTTP_PERMANENT_REDIRECT;
+ }
+
+ /**
+ * Resolves a redirect location against the redirected request and enforces the
+ * redirect policy: the target must be an http or https location, and an https
+ * request must not be redirected to plain http.
+ *
+ * @param from The location that returned the redirect.
+ * @param location The Location header value, absolute or relative, or {@code null}
+ * when the header is absent.
+ * @return The resolved redirect target. Not {@code null}.
+ * @throws IOException Thrown if the location is absent or malformed, leaves the
+ * http and https schemes, or downgrades https to http.
+ */
+ static URI resolveRedirect(URI from, String location) throws IOException {
+ if (location == null || location.isEmpty()) {
+ throw new IOException("redirect from " + from + " contains no Location header");
+ }
+ final URI target;
+ try {
+ target = from.resolve(location);
+ } catch (IllegalArgumentException e) {
+ throw new IOException(
+ "redirect from " + from + " contains a malformed Location: " + location, e);
+ }
+ final String scheme = target.getScheme();
+ final boolean https = SCHEME_HTTPS.equalsIgnoreCase(scheme);
+ if (!https && !SCHEME_HTTP.equalsIgnoreCase(scheme)) {
+ throw new IOException(
+ "redirect target is not an http or https location: " + target);
+ }
+ if (SCHEME_HTTPS.equalsIgnoreCase(from.getScheme()) && !https) {
+ throw new IOException("redirect downgrades https to http: " + target);
+ }
+ return target;
+ }
+
+ /**
+ * Converts a timeout to the millisecond form the connection setters take. A positive
+ * timeout shorter than a millisecond becomes one millisecond because
+ * {@link HttpURLConnection#setReadTimeout(int) zero disables the timeout}. A timeout
+ * too large for the int range is capped.
+ *
+ * @param timeout The timeout as a duration. Must be positive.
+ * @return The timeout in milliseconds, at least {@code 1} and at most
+ * {@link Integer#MAX_VALUE}.
+ */
+ private static int timeoutMillis(Duration timeout) {
+ final long millis;
+ try {
+ millis = timeout.toMillis();
+ } catch (ArithmeticException e) {
+ return Integer.MAX_VALUE;
+ }
+ return Math.clamp(millis, 1, Integer.MAX_VALUE);
+ }
+
+ /**
+ * Computes the file's digest and compares it with the expected hex digest, ignoring
+ * hex letter case. The digest length selects the algorithm: 64 characters SHA-256,
+ * 128 characters SHA-512.
+ *
+ * @param file The file to digest.
+ * @param expected The expected hex digest, already trimmed.
+ * @throws IOException Thrown if the file cannot be read or the digests differ.
+ */
+ private static void verify(Path file, String expected) throws IOException {
+ final String algorithm =
+ expected.length() == SHA_512_HEX_LENGTH ? SHA_512 : SHA_256;
+ final MessageDigest digest;
+ try {
+ digest = MessageDigest.getInstance(algorithm);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException(algorithm + " is unavailable in this runtime", e);
+ }
+ try (InputStream in = Files.newInputStream(file)) {
+ final byte[] buffer = new byte[BUFFER_SIZE];
+ int read;
+ while ((read = in.read(buffer)) >= 0) {
+ digest.update(buffer, 0, read);
+ }
+ }
+ final String actual = HexFormat.of().formatHex(digest.digest());
+ if (!actual.equalsIgnoreCase(expected)) {
+ throw new IOException(
+ "checksum mismatch: expected " + expected + " but downloaded " + actual);
+ }
+ }
+
+ /**
+ * Unpacks the downloaded content into a hidden staging directory beneath the target
+ * and promotes it into the target only after every entry unpacked cleanly. The
+ * staging directory lives on the target's filesystem so promotion is a sequence of
+ * renames, and it is removed whether the installation succeeds or fails.
+ *
+ * @param downloaded The fetched and verified file.
+ * @param name The file name derived from the source location.
+ * @param target The directory to install into.
+ * @param limits The limits to enforce while unpacking.
+ * @throws IOException Thrown if unpacking fails, a limit is exceeded, or promotion
+ * or staging cleanup fails.
+ */
+ private static void installStaged(Path downloaded, String name, Path target,
+ Limits limits) throws IOException {
+ final Path staging = Files.createTempDirectory(target, STAGING_PREFIX);
+ try {
+ unpack(downloaded, name, staging, limits);
+ promote(staging, target);
+ } catch (IOException e) {
+ try {
+ deleteRecursively(staging);
+ } catch (IOException cleanup) {
+ e.addSuppressed(cleanup);
+ }
+ throw e;
+ }
+ deleteRecursively(staging);
+ }
+
+ /**
+ * Moves all staged regular files to their relative locations beneath the target
+ * without replacing anything that already exists there. All destinations are
+ * checked before the first move, so a collision leaves the target without a mix of
+ * old and new files, and the move itself refuses an existing destination as well.
+ *
+ * @param staging The staging directory holding the fully unpacked content.
+ * @param target The directory to install into.
+ * @throws IOException Thrown if a destination already exists, a move fails, or a
+ * directory on the way to a destination is an existing symbolic link.
+ */
+ private static void promote(Path staging, Path target) throws IOException {
+ final List files;
+ try (Stream walk = Files.walk(staging)) {
+ files = walk.filter(Files::isRegularFile).toList();
+ }
+ for (final Path file : files) {
+ ensureVacant(target, staging.relativize(file));
+ }
+ for (final Path file : files) {
+ moveIntoPlace(file, destination(target, staging.relativize(file)));
+ }
+ }
+
+ /**
+ * Moves one staged file to its destination without replacing an existing file. The
+ * move is not requested atomically: on POSIX filesystems an atomic move renames over
+ * an existing destination, which would void the vacancy check.
+ *
+ * @param file The staged file.
+ * @param destination The destination beneath the target.
+ * @throws IOException Thrown if the destination exists or the move fails.
+ */
+ static void moveIntoPlace(Path file, Path destination) throws IOException {
+ Files.move(file, destination);
+ }
+
+ /**
+ * Checks that one staged file's destination is free to receive it, without creating
+ * anything. A missing directory on the way proves the destination vacant.
+ *
+ * @param target The directory to install into.
+ * @param relative The staged file's path relative to the staging directory.
+ * @throws IOException Thrown if the destination already exists, or a directory on
+ * the way is a symbolic link or exists as something other than a directory.
+ */
+ private static void ensureVacant(Path target, Path relative) throws IOException {
+ final Path destination = destination(target, relative, false);
+ if (destination != null && Files.exists(destination, LinkOption.NOFOLLOW_LINKS)) {
+ throw new IOException("target already contains: " + destination);
+ }
+ }
+
+ /**
+ * Resolves one staged file's destination beneath the target, creating the directories
+ * leading to it one at a time.
+ *
+ * @param target The directory to install into.
+ * @param relative The staged file's path relative to the staging directory.
+ * @return The destination path beneath the target. Not {@code null}.
+ * @throws IOException Thrown if a directory on the way is a symbolic link or exists as
+ * something other than a directory, or if a directory cannot be created.
+ */
+ private static Path destination(Path target, Path relative) throws IOException {
+ return destination(target, relative, true);
+ }
+
+ /**
+ * Walks the directories leading to one staged file's destination without descending
+ * through a symbolic link that is already there. An entry name that stays inside the
+ * staging directory can still land outside the target if a directory below the target
+ * is a link to somewhere else.
+ *
+ * This covers links present when the installation runs. It is not a defense against
+ * a link created concurrently, between the check here and the move that follows.
+ *
+ * @param target The directory to install into.
+ * @param relative The staged file's path relative to the staging directory.
+ * @param create Whether to create a missing directory on the way; when {@code false},
+ * a missing directory ends the walk.
+ * @return The destination path beneath the target, or {@code null} when a directory
+ * on the way is missing and {@code create} is {@code false}.
+ * @throws IOException Thrown if a directory on the way is a symbolic link or exists as
+ * something other than a directory, or if a directory cannot be created.
+ */
+ private static Path destination(Path target, Path relative, boolean create)
+ throws IOException {
+ Path directory = target;
+ for (int i = 0; i < relative.getNameCount() - 1; i++) {
+ directory = directory.resolve(relative.getName(i));
+ if (Files.isSymbolicLink(directory)) {
+ throw new IOException(
+ "installation path crosses a symbolic link: " + directory);
+ }
+ if (!Files.exists(directory)) {
+ if (!create) {
+ return null;
+ }
+ Files.createDirectory(directory);
+ } else if (!Files.isDirectory(directory)) {
+ throw new IOException(
+ "installation path crosses an existing file: " + directory);
+ }
+ }
+ return directory.resolve(relative.getFileName());
+ }
+
+ /**
+ * Removes the given directory tree, deepest entries first.
+ *
+ * @param root The directory to remove.
+ * @throws IOException Thrown if a deletion fails.
+ */
+ private static void deleteRecursively(Path root) throws IOException {
+ final List paths;
+ try (Stream walk = Files.walk(root)) {
+ paths = walk.sorted(Comparator.reverseOrder()).toList();
+ }
+ for (final Path path : paths) {
+ Files.deleteIfExists(path);
+ }
+ }
+
+ /**
+ * Detects the content format from its leading bytes and unpacks accordingly,
+ * bounding the total expanded bytes against the expansion limit. One exception: a
+ * source named {@code *.bin}, in any letter case, is stored verbatim even when its
+ * bytes are a zip archive. OpenNLP model consumers load the packed zip artifact. Unpacking it would
+ * place the internal entries ({@code manifest.properties}, {@code *.model}) in the
+ * target instead of the model.
+ *
+ * @param downloaded The fetched file.
+ * @param name The file name derived from the source location.
+ * @param staging The staging directory to unpack into.
+ * @param limits The limits to enforce.
+ * @throws IOException Thrown if reading or unpacking fails or the expansion limit
+ * is exceeded.
+ */
+ private static void unpack(Path downloaded, String name, Path staging, Limits limits)
+ throws IOException {
+ final Budget budget = new Budget(limits.maxExpandedBytes(),
+ "expanded content exceeds the limit of " + limits.maxExpandedBytes()
+ + " bytes");
+ final Budget entryBudget = new Budget(limits.maxEntries(),
+ "archive entry count exceeds the limit of " + limits.maxEntries()
+ + " entries");
+ try (InputStream raw = new BufferedInputStream(Files.newInputStream(downloaded))) {
+ raw.mark(MAGIC_LENGTH);
+ final byte[] magic = raw.readNBytes(MAGIC_LENGTH);
+ raw.reset();
+ if (endsWithIgnoreCase(name, MODEL_SUFFIX)) {
+ copyBounded(raw, safeChild(staging, name), budget);
+ } else if (hasMagic(magic, GZIP_MAGIC_FIRST, GZIP_MAGIC_SECOND)) {
+ unpackGzip(raw, name, staging,
+ expansionBudget(Files.size(downloaded), limits, budget), entryBudget);
+ } else if (hasMagic(magic, ZIP_MAGIC_FIRST, ZIP_MAGIC_SECOND,
+ ZIP_LOCAL_HEADER_THIRD, ZIP_LOCAL_HEADER_FOURTH)) {
+ final Set unpacked = unpackZip(raw, staging,
+ expansionBudget(Files.size(downloaded), limits, budget), entryBudget);
+ validateZip(downloaded, unpacked);
+ } else if (hasMagic(magic, ZIP_MAGIC_FIRST, ZIP_MAGIC_SECOND,
+ ZIP_END_HEADER_THIRD, ZIP_END_HEADER_FOURTH)) {
+ validateEmptyZip(raw);
+ } else {
+ copyBounded(raw, safeChild(staging, name), budget);
+ }
+ }
+ }
+
+ /**
+ * Bounds expansion by the ratio as well as the absolute limit, so a small source
+ * cannot expand to the whole absolute limit. Deflate reaches roughly 1000 to 1, so
+ * the absolute limit alone lets a few megabytes fill the target filesystem.
+ *
+ * @param compressedSize The size of the compressed source in bytes.
+ * @param limits The limits holding the accepted expansion ratio.
+ * @param budget The expansion budget under the absolute limit.
+ * @return The tighter of the two budgets. Not {@code null}.
+ */
+ private static Budget expansionBudget(long compressedSize, Limits limits,
+ Budget budget) {
+ final long ratio = limits.maxExpansionRatio();
+ final long ratioCeiling = compressedSize > Long.MAX_VALUE / ratio
+ ? Long.MAX_VALUE
+ : Math.max(MIN_EXPANSION_BYTES, compressedSize * ratio);
+ if (ratioCeiling >= budget.limit()) {
+ return budget;
+ }
+ return new Budget(ratioCeiling, "content expands beyond "
+ + ratio + " times its compressed size");
+ }
+
+ /**
+ * Compares a name's ending with a suffix, ignoring letter case.
+ *
+ * @param name The file name.
+ * @param suffix The suffix to look for.
+ * @return {@code true} if the name ends with the suffix in any letter case.
+ */
+ private static boolean endsWithIgnoreCase(String name, String suffix) {
+ return name.length() >= suffix.length() && name.regionMatches(true,
+ name.length() - suffix.length(), suffix, 0, suffix.length());
+ }
+
+ /**
+ * Checks whether the bytes at the start of a resource match the given signature.
+ *
+ * @param actual The bytes read from the resource.
+ * @param expected The unsigned byte values in the signature.
+ * @return {@code true} when the resource begins with the expected values.
+ */
+ private static boolean hasMagic(byte[] actual, int... expected) {
+ if (actual.length < expected.length) {
+ return false;
+ }
+ for (int i = 0; i < expected.length; i++) {
+ if ((actual[i] & 0xFF) != expected[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Checks that a zip archive contains a valid central directory listing the same files
+ * the local headers delivered, before staged content is promoted. The two listings
+ * can disagree in a crafted archive, and the local headers are what was unpacked.
+ *
+ * @param archive The downloaded archive.
+ * @param unpacked The names of the file entries read from the local headers.
+ * @throws IOException Thrown if the archive is malformed, the listings differ, or the
+ * archive cannot be read.
+ */
+ private static void validateZip(Path archive, Set unpacked) throws IOException {
+ final Set listed = new HashSet<>();
+ try (ZipFile zip = new ZipFile(archive.toFile())) {
+ final Enumeration entries = zip.entries();
+ while (entries.hasMoreElements()) {
+ final ZipEntry entry = entries.nextElement();
+ if (!entry.isDirectory()) {
+ listed.add(entry.getName());
+ }
+ }
+ } catch (UnsupportedOperationException e) {
+ listed.addAll(listZipOnNonDefaultFileSystem(archive));
+ } catch (ZipException e) {
+ throw new IOException(MALFORMED_ZIP_ERROR, e);
+ }
+ if (!listed.equals(unpacked)) {
+ throw new IOException(ZIP_MISMATCH_ERROR);
+ }
+ }
+
+ /**
+ * Lists the file entries of an archive stored by a file-system provider that cannot
+ * supply a {@link java.io.File} to {@link ZipFile}.
+ *
+ * @param archive The downloaded archive.
+ * @return The file entry names from the central directory. Not {@code null}.
+ * @throws IOException Thrown if the archive is malformed or cannot be read.
+ */
+ private static Set listZipOnNonDefaultFileSystem(Path archive)
+ throws IOException {
+ final Set listed = new HashSet<>();
+ try (FileSystem zip = FileSystems.newFileSystem(archive)) {
+ for (final Path root : zip.getRootDirectories()) {
+ try (Stream walk = Files.walk(root)) {
+ walk.filter(Files::isRegularFile)
+ .map(path -> root.relativize(path).toString())
+ .forEach(listed::add);
+ }
+ }
+ } catch (ZipException | ProviderNotFoundException e) {
+ throw new IOException(MALFORMED_ZIP_ERROR, e);
+ }
+ return listed;
+ }
+
+ /**
+ * Validates an empty zip archive from its end-of-central-directory record. An empty
+ * archive has no local entry headers for {@link ZipInputStream} to validate.
+ *
+ * @param raw The zip content, positioned at its first byte.
+ * @throws IOException Thrown if the record is truncated, declares entries or central
+ * directory data, or has bytes beyond its declared comment.
+ */
+ private static void validateEmptyZip(InputStream raw) throws IOException {
+ final byte[] header = raw.readNBytes(ZIP_END_HEADER_LENGTH);
+ if (header.length != ZIP_END_HEADER_LENGTH
+ || littleEndianShort(header, ZIP_DISK_OFFSET) != 0
+ || littleEndianShort(header, ZIP_CENTRAL_DISK_OFFSET) != 0
+ || littleEndianShort(header, ZIP_DISK_ENTRIES_OFFSET) != 0
+ || littleEndianShort(header, ZIP_TOTAL_ENTRIES_OFFSET) != 0
+ || littleEndianInt(header, ZIP_CENTRAL_SIZE_OFFSET) != 0
+ || littleEndianInt(header, ZIP_CENTRAL_OFFSET_OFFSET) != 0) {
+ throw new IOException(MALFORMED_ZIP_ERROR);
+ }
+ final int commentLength = littleEndianShort(header, ZIP_COMMENT_LENGTH_OFFSET);
+ if (raw.readNBytes(commentLength).length != commentLength || raw.read() >= 0) {
+ throw new IOException(MALFORMED_ZIP_ERROR);
+ }
+ }
+
+ /**
+ * Reads an unsigned 16-bit little-endian value.
+ *
+ * @param bytes The source bytes.
+ * @param offset The first byte to read.
+ * @return The decoded value.
+ */
+ private static int littleEndianShort(byte[] bytes, int offset) {
+ return bytes[offset] & 0xFF | (bytes[offset + 1] & 0xFF) << 8;
+ }
+
+ /**
+ * Reads an unsigned 32-bit little-endian value.
+ *
+ * @param bytes The source bytes.
+ * @param offset The first byte to read.
+ * @return The decoded value.
+ */
+ private static long littleEndianInt(byte[] bytes, int offset) {
+ return littleEndianShort(bytes, offset)
+ | (long) littleEndianShort(bytes, offset + 2) << 16;
+ }
+
+ /**
+ * Unpacks gzip content: a tar archive inside when present, a plain file otherwise. A
+ * plain file omits the {@code .gz} suffix of its source name, in any letter case. If the source name is
+ * only that suffix, the installed file is named {@value #DEFAULT_RESOURCE_NAME}.
+ *
+ * @param raw The gzip-compressed content.
+ * @param name The file name derived from the source location.
+ * @param staging The staging directory to unpack into.
+ * @param budget The expansion budget.
+ * @param entryBudget The entry-count budget.
+ * @throws IOException Thrown if decompressing or unpacking fails or a limit is
+ * exceeded.
+ */
+ private static void unpackGzip(InputStream raw, String name, Path staging,
+ Budget budget, Budget entryBudget) throws IOException {
+ final InputStream decompressed = new BufferedInputStream(
+ new BudgetInputStream(new GZIPInputStream(raw), budget), BUFFER_SIZE);
+ if (TarStream.startsWithHeader(decompressed) || startsWithEmptyTar(decompressed)) {
+ unpackTar(decompressed, staging, entryBudget);
+ decompressed.transferTo(OutputStream.nullOutputStream());
+ } else {
+ final String strippedName = endsWithIgnoreCase(name, GZIP_SUFFIX)
+ ? name.substring(0, name.length() - GZIP_SUFFIX.length()) : name;
+ final String plainName = strippedName.isEmpty()
+ ? DEFAULT_RESOURCE_NAME : strippedName;
+ copy(decompressed, safeChild(staging, plainName));
+ }
+ }
+
+ /**
+ * Checks for the two zero blocks that make up an empty tar archive. There is no entry
+ * header for {@link TarStream#startsWithHeader(InputStream)} to recognize in this case.
+ *
+ * @param in The decompressed content. Must support mark and reset.
+ * @return {@code true} when the content starts with two zero tar blocks.
+ * @throws IOException Thrown if reading or resetting the stream fails.
+ */
+ private static boolean startsWithEmptyTar(InputStream in) throws IOException {
+ in.mark(TAR_END_BLOCKS_LENGTH);
+ try {
+ final byte[] blocks = in.readNBytes(TAR_END_BLOCKS_LENGTH);
+ if (blocks.length != TAR_END_BLOCKS_LENGTH) {
+ return false;
+ }
+ for (final byte b : blocks) {
+ if (b != 0) {
+ return false;
+ }
+ }
+ return true;
+ } finally {
+ in.reset();
+ }
+ }
+
+ /**
+ * Unpacks every regular tar entry to its relative location beneath the staging
+ * directory.
+ *
+ * @param decompressed The uncompressed tar content.
+ * @param staging The staging directory to unpack into.
+ * @param entryBudget The archive-header limit.
+ * @throws IOException Thrown if the archive is malformed, an entry escapes the
+ * staging directory, or the entry limit is exceeded.
+ */
+ private static void unpackTar(InputStream decompressed, Path staging,
+ Budget entryBudget) throws IOException {
+ final TarStream entries = new TarStream(decompressed, entryBudget.limit());
+ while (entries.next()) {
+ if (!entries.isFile()) {
+ safeChild(staging, entries.name());
+ continue;
+ }
+ final Path file = newArchiveFile(staging, entries.name());
+ copy(entries.entryStream(), file);
+ }
+ }
+
+ /**
+ * Unpacks every regular zip entry to its relative location beneath the staging
+ * directory.
+ *
+ * @param raw The zip content.
+ * @param staging The staging directory to unpack into.
+ * @param budget The expansion budget.
+ * @param entryBudget The entry-count budget, charged for every entry including
+ * directories.
+ * @return The names of the file entries unpacked. Not {@code null}.
+ * @throws IOException Thrown if the archive is malformed, an entry escapes the
+ * staging directory, or a limit is exceeded.
+ */
+ private static Set unpackZip(InputStream raw, Path staging, Budget budget,
+ Budget entryBudget) throws IOException {
+ final ZipInputStream zip = new ZipInputStream(raw);
+ final Set unpacked = new HashSet<>();
+ boolean foundEntry = false;
+ ZipEntry entry;
+ while ((entry = zip.getNextEntry()) != null) {
+ foundEntry = true;
+ entryBudget.spend(1);
+ if (entry.isDirectory()) {
+ safeChild(staging, entry.getName());
+ consumeBounded(zip, budget);
+ continue;
+ }
+ final Path file = newArchiveFile(staging, entry.getName());
+ copyBounded(zip, file, budget);
+ unpacked.add(entry.getName());
+ }
+ if (!foundEntry) {
+ throw new IOException(MALFORMED_ZIP_ERROR);
+ }
+ return unpacked;
+ }
+
+ /**
+ * Resolves a file entry beneath the staging directory and creates its parent
+ * directories. A second entry that normalizes to the same path is rejected.
+ *
+ * @param staging The staging directory.
+ * @param entryName The path stored in the archive.
+ * @return The new file path. Not {@code null}.
+ * @throws IOException Thrown if the path escapes the staging directory, duplicates
+ * another file entry, or its parent directories cannot be created.
+ */
+ private static Path newArchiveFile(Path staging, String entryName) throws IOException {
+ final Path file = safeChild(staging, entryName);
+ if (Files.exists(file, LinkOption.NOFOLLOW_LINKS)) {
+ throw new IOException("archive contains duplicate file entry: " + entryName);
+ }
+ Files.createDirectories(file.getParent());
+ return file;
+ }
+
+ /**
+ * Copies the stream into the file, charging every byte against the budget before it
+ * is written, so an oversized transfer aborts within one buffer of its limit.
+ *
+ * @param in The content to copy.
+ * @param file The file to write.
+ * @param budget The byte budget to charge.
+ * @throws IOException Thrown if reading or writing fails or the budget is exceeded.
+ */
+ private static void copyBounded(InputStream in, Path file, Budget budget)
+ throws IOException {
+ try (OutputStream out = Files.newOutputStream(file)) {
+ new BudgetInputStream(in, budget).transferTo(out);
+ }
+ }
+
+ /**
+ * Reads and discards an entry while charging each byte against the expansion limit.
+ *
+ * @param in The entry content.
+ * @param budget The byte budget to charge.
+ * @throws IOException Thrown if reading fails or the budget is exceeded.
+ */
+ private static void consumeBounded(InputStream in, Budget budget) throws IOException {
+ new BudgetInputStream(in, budget).transferTo(OutputStream.nullOutputStream());
+ }
+
+ /**
+ * Copies the stream into the file. The input stream must already enforce any byte
+ * limit.
+ *
+ * @param in The content to copy.
+ * @param file The file to write.
+ * @throws IOException Thrown if reading or writing fails.
+ */
+ private static void copy(InputStream in, Path file) throws IOException {
+ try (OutputStream out = Files.newOutputStream(file)) {
+ in.transferTo(out);
+ }
+ }
+
+ /**
+ * Resolves an archive entry inside the staging directory, rejecting escaping paths.
+ *
+ * @param staging The staging directory to unpack into.
+ * @param entryName The entry name as stored in the archive.
+ * @return The resolved path beneath the staging directory. Not {@code null}.
+ * @throws IOException Thrown if the entry resolves outside the staging directory.
+ */
+ private static Path safeChild(Path staging, String entryName) throws IOException {
+ final Path resolved;
+ try {
+ resolved = staging.resolve(entryName).normalize();
+ } catch (InvalidPathException e) {
+ throw new IOException("archive entry has an invalid path: " + entryName, e);
+ }
+ if (!resolved.startsWith(staging.normalize())) {
+ throw new IOException("archive entry escapes the target directory: " + entryName);
+ }
+ return resolved;
+ }
+
+ /**
+ * Derives a file name from the source URI for non-archive content.
+ *
+ * @param source The resource location.
+ * @return The last path segment, or {@code resource} if the location has none.
+ */
+ private static String sourceName(URI source) {
+ final String path = source.getPath();
+ if (path == null || path.isEmpty()) {
+ return DEFAULT_RESOURCE_NAME;
+ }
+ final int slash = path.lastIndexOf('/');
+ final String name = slash < 0 ? path : path.substring(slash + 1);
+ return name.isEmpty() ? DEFAULT_RESOURCE_NAME : name;
+ }
+
+ /**
+ * Rejects names that are empty, path-like, or contain a NUL character.
+ *
+ * @param name The candidate local file name.
+ * @return The validated name.
+ * @throws IllegalArgumentException Thrown if {@code name} is not a file name.
+ */
+ static String validateSourceName(String name) {
+ if (name.isEmpty() || ".".equals(name) || "..".equals(name)) {
+ throw new IllegalArgumentException("name must be a file name");
+ }
+ for (int i = 0; i < name.length(); i++) {
+ final char c = name.charAt(i);
+ if (c == '/' || c == '\\' || c == 0) {
+ throw new IllegalArgumentException("name must be a file name");
+ }
+ }
+ return name;
+ }
+
+ /**
+ * A unit budget, counting bytes or archive entries: {@link #spend(long)} accumulates
+ * spent units and fails once the limit is crossed.
+ */
+ private static final class Budget {
+
+ private final long limit;
+ private final String message;
+ private long used;
+
+ /**
+ * Creates a budget that has spent zero units.
+ *
+ * @param limit The largest total number of units accepted.
+ * @param message The failure message raised when the limit is crossed.
+ */
+ Budget(long limit, String message) {
+ this.limit = limit;
+ this.message = message;
+ }
+
+ /** {@return the maximum number of units accepted} */
+ long limit() {
+ return limit;
+ }
+
+ /**
+ * Charges the given number of units against the budget.
+ *
+ * @param units The number of units to charge.
+ * @throws IOException Thrown if the total charged units exceed the limit.
+ */
+ void spend(long units) throws IOException {
+ used += units;
+ if (used > limit) {
+ throw new IOException(message);
+ }
+ }
+ }
+
+ /**
+ * Charges every byte read or skipped from an expanded stream against a shared budget.
+ */
+ private static final class BudgetInputStream extends FilterInputStream {
+
+ private final Budget budget;
+
+ /**
+ * Initializes a budgeted stream.
+ *
+ * @param in The expanded stream to read.
+ * @param budget The budget to charge.
+ */
+ BudgetInputStream(InputStream in, Budget budget) {
+ super(in);
+ this.budget = budget;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public int read() throws IOException {
+ final int value = super.read();
+ if (value >= 0) {
+ budget.spend(1);
+ }
+ return value;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public int read(byte[] buffer, int offset, int length) throws IOException {
+ final int read = super.read(buffer, offset, length);
+ if (read > 0) {
+ budget.spend(read);
+ }
+ return read;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public long skip(long bytes) throws IOException {
+ final long skipped = super.skip(bytes);
+ budget.spend(skipped);
+ return skipped;
+ }
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/archive/TarStream.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/archive/TarStream.java
new file mode 100644
index 0000000000..c016a566fa
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/archive/TarStream.java
@@ -0,0 +1,659 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util.archive;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+import opennlp.tools.commons.Internal;
+
+/**
+ * A forward-only reader for classic v7, POSIX ustar, GNU, and pax tar streams.
+ * {@link #next()} advances to the following entry and {@link #entryStream()} exposes
+ * only the current entry's bytes.
+ *
+ * The reader validates header checksums, supports ustar name prefixes, GNU long
+ * names, pax {@code path} and {@code size} records, and GNU base-256 sizes. Sparse
+ * entries and global pax records that change paths or sizes are rejected because this
+ * reader cannot reproduce their content or global semantics.
+ *
+ * @since 3.0.0
+ */
+@Internal
+public final class TarStream {
+
+ private static final int BLOCK = 512;
+ private static final int NAME_LENGTH = 100;
+ private static final int CHECKSUM_OFFSET = 148;
+ private static final int CHECKSUM_LENGTH = 8;
+ private static final int SIZE_OFFSET = 124;
+ private static final int SIZE_LENGTH = 12;
+ private static final int TYPE_OFFSET = 156;
+ private static final int MAGIC_OFFSET = 257;
+ private static final int PREFIX_OFFSET = 345;
+ private static final int PREFIX_LENGTH = 155;
+ private static final String USTAR_MAGIC = "ustar";
+ private static final char TYPE_REGULAR_FILE = '0';
+ private static final char TYPE_REGULAR_FILE_CLASSIC = '\0';
+ private static final char TYPE_GNU_LONG_NAME = 'L';
+ private static final char TYPE_GNU_LONG_LINK = 'K';
+ private static final char TYPE_GNU_SPARSE = 'S';
+ private static final char TYPE_PAX_EXTENDED = 'x';
+ private static final char TYPE_PAX_GLOBAL = 'g';
+ private static final int MAX_EXTENSION_BYTES = 1 << 20;
+ private static final String MALFORMED_RECORD = "malformed pax extended header record";
+ private static final String KEYWORD_PATH = "path";
+ private static final String KEYWORD_SIZE = "size";
+ private static final String SPARSE_PREFIX = "GNU.sparse.";
+ private static final int BASE_256_MARKER = 0x80;
+ private static final int BASE_256_NEGATIVE = 0x40;
+ private static final int BASE_256_FIRST_BYTE_BITS = 0x7F;
+
+ private final InputStream in;
+ private final long maxEntries;
+ private final byte[] header = new byte[BLOCK];
+
+ private String name;
+ private long size;
+ private char type;
+ private long remaining;
+ private long entries;
+ private boolean ended;
+
+ /** The name an extension header supplied for the entry that follows it, else null. */
+ private String pendingPath;
+
+ /** The size a pax header supplied for the entry that follows it, else {@code -1}. */
+ private long pendingSize = -1;
+
+ /**
+ * Initializes the reader.
+ *
+ * @param in The tar content. Not {@code null}. Not closed by this class.
+ * @throws IllegalArgumentException Thrown if {@code in} is {@code null}.
+ */
+ public TarStream(InputStream in) {
+ this(in, Long.MAX_VALUE);
+ }
+
+ /**
+ * Initializes a reader with an archive-entry limit. Extension headers count toward the
+ * limit.
+ *
+ * @param in The tar content. Not {@code null}. Not closed by this class.
+ * @param maxEntries The maximum number of archive headers to read. Must be positive.
+ * @throws IllegalArgumentException Thrown if {@code in} is {@code null} or
+ * {@code maxEntries} is not positive.
+ */
+ public TarStream(InputStream in, long maxEntries) {
+ if (in == null) {
+ throw new IllegalArgumentException("in must not be null");
+ }
+ if (maxEntries <= 0) {
+ throw new IllegalArgumentException("maxEntries must be positive");
+ }
+ this.in = in;
+ this.maxEntries = maxEntries;
+ }
+
+ /**
+ * Checks whether the given stream is positioned at a tar entry header, leaving its
+ * position unchanged.
+ *
+ * @param in The stream to inspect. Not {@code null} and must support
+ * {@link InputStream#mark(int) mark} and {@link InputStream#reset() reset}.
+ * @return {@code true} if the next 512 bytes read as a tar header, {@code false} if
+ * they do not or if fewer than 512 bytes are available.
+ * @throws IOException Thrown if reading from or repositioning the stream fails.
+ * @throws IllegalArgumentException Thrown if {@code in} is {@code null} or does not
+ * support mark and reset.
+ */
+ public static boolean startsWithHeader(InputStream in) throws IOException {
+ if (in == null) {
+ throw new IllegalArgumentException("in must not be null");
+ }
+ if (!in.markSupported()) {
+ throw new IllegalArgumentException("in must support mark and reset");
+ }
+ in.mark(BLOCK);
+ try {
+ final byte[] block = new byte[BLOCK];
+ return in.readNBytes(block, 0, BLOCK) == BLOCK && isHeader(block);
+ } finally {
+ in.reset();
+ }
+ }
+
+ /**
+ * Advances to the next entry.
+ *
+ * @return {@code true} if an entry is available, {@code false} at the end of the
+ * archive.
+ * @throws IOException Thrown if the archive is truncated or a header is malformed.
+ */
+ public boolean next() throws IOException {
+ if (ended) {
+ return false;
+ }
+ skip(remaining);
+ skip(padding(size));
+ while (true) {
+ if (!readBlock() || isEndBlock()) {
+ ended = true;
+ return false;
+ }
+ entries++;
+ if (entries > maxEntries) {
+ throw new IOException("archive entry count exceeds the limit of "
+ + maxEntries + " entries");
+ }
+ if (!hasValidChecksum(header)) {
+ throw new IOException("malformed tar header checksum");
+ }
+ type = (char) header[TYPE_OFFSET];
+ size = parseSize();
+ remaining = size;
+ if (type == TYPE_GNU_SPARSE) {
+ throw new IOException("sparse tar entries are not supported: "
+ + "the archived bytes describe file holes, not contiguous content");
+ }
+ if (type == TYPE_PAX_EXTENDED || type == TYPE_PAX_GLOBAL) {
+ readRecords(readExtensionPayload(), type == TYPE_PAX_GLOBAL);
+ continue;
+ }
+ if (type == TYPE_GNU_LONG_NAME) {
+ final byte[] payload = readExtensionPayload();
+ pendingPath = trimNul(decodeUtf8(payload, 0, payload.length, "GNU long name"));
+ continue;
+ }
+ if (type == TYPE_GNU_LONG_LINK) {
+ // The link target of an entry this reader does not expose.
+ readExtensionPayload();
+ continue;
+ }
+ name = pendingPath != null ? pendingPath : readName();
+ pendingPath = null;
+ if (pendingSize >= 0) {
+ size = pendingSize;
+ remaining = size;
+ pendingSize = -1;
+ }
+ if (name.isEmpty()) {
+ throw new IOException("tar entry header contains an empty name");
+ }
+ return true;
+ }
+ }
+
+ /**
+ * Reads the payload of the extension header just read, leaving the stream positioned on
+ * the header that follows it.
+ *
+ * @return The payload bytes. Not {@code null}.
+ * @throws IOException Thrown if the payload is larger than
+ * {@link #MAX_EXTENSION_BYTES} or the archive ends inside it.
+ */
+ private byte[] readExtensionPayload() throws IOException {
+ if (size > MAX_EXTENSION_BYTES) {
+ throw new IOException("tar extension header of " + size + " bytes exceeds the "
+ + MAX_EXTENSION_BYTES + " byte limit");
+ }
+ final byte[] payload = new byte[(int) size];
+ if (in.readNBytes(payload, 0, payload.length) < payload.length) {
+ throw new IOException("truncated tar archive");
+ }
+ skip(padding(size));
+ size = 0;
+ remaining = 0;
+ return payload;
+ }
+
+ /**
+ * Reads a pax extended header payload, which is a sequence of
+ * {@code " =\n"} records. Each length counts the complete
+ * record, including the length digits, blank, and newline. Records are parsed from raw
+ * bytes because the length is a byte count and a multibyte value would shift later records.
+ *
+ * @param payload The raw extended header payload.
+ * @param global Whether this is a global header, which applies to all following entries.
+ * @throws IOException Thrown if a record is malformed, if a global header contains a
+ * keyword that would change the entries after it, or if the entry is sparse.
+ */
+ private void readRecords(byte[] payload, boolean global) throws IOException {
+ int offset = 0;
+ while (offset < payload.length) {
+ int blank = offset;
+ while (blank < payload.length && payload[blank] != ' ') {
+ blank++;
+ }
+ if (blank == payload.length || blank == offset) {
+ throw new IOException(MALFORMED_RECORD);
+ }
+ int length = 0;
+ for (int i = offset; i < blank; i++) {
+ final byte b = payload[i];
+ if (b < '0' || b > '9') {
+ throw new IOException(MALFORMED_RECORD);
+ }
+ length = length * 10 + (b - '0');
+ if (length > payload.length) {
+ throw new IOException(MALFORMED_RECORD);
+ }
+ }
+ final int end = offset + length;
+ if (length <= blank - offset || end > payload.length || payload[end - 1] != '\n') {
+ throw new IOException(MALFORMED_RECORD);
+ }
+ int equals = blank + 1;
+ while (equals < end - 1 && payload[equals] != '=') {
+ equals++;
+ }
+ if (equals >= end - 1 || equals == blank + 1) {
+ throw new IOException(MALFORMED_RECORD);
+ }
+ apply(decodeUtf8(payload, blank + 1, equals - blank - 1, "pax record"),
+ decodeUtf8(payload, equals + 1, end - equals - 2, "pax record"), global);
+ offset = end;
+ }
+ }
+
+ /**
+ * Decodes archive text without replacing malformed input, which would silently
+ * change an archive path.
+ *
+ * @param bytes The bytes containing the text.
+ * @param offset The first byte to decode.
+ * @param length The number of bytes to decode.
+ * @param subject The field name to use in an error message.
+ * @return The decoded text. Not {@code null}.
+ * @throws IOException Thrown if the bytes are not valid UTF-8.
+ */
+ private static String decodeUtf8(byte[] bytes, int offset, int length, String subject)
+ throws IOException {
+ try {
+ return StandardCharsets.UTF_8.newDecoder()
+ .decode(ByteBuffer.wrap(bytes, offset, length)).toString();
+ } catch (CharacterCodingException e) {
+ throw new IOException(subject + " is not valid UTF-8", e);
+ }
+ }
+
+ /**
+ * Applies one pax record. Only {@code path} and {@code size} change what this reader
+ * reports, so other keywords are ignored, except sparse entries, which cannot be
+ * unpacked.
+ *
+ * @param keyword The record's keyword.
+ * @param value The record's value.
+ * @param global Whether the record came from a global header.
+ * @throws IOException Thrown if the entry is sparse, if a global header contains a
+ * keyword that would change the entries after it, or if {@code size} is not a
+ * number.
+ */
+ private void apply(String keyword, String value, boolean global) throws IOException {
+ if (keyword.startsWith(SPARSE_PREFIX)) {
+ throw new IOException("sparse tar entries are not supported: "
+ + "the archived bytes describe file holes, not contiguous content");
+ }
+ if (!KEYWORD_PATH.equals(keyword) && !KEYWORD_SIZE.equals(keyword)) {
+ return;
+ }
+ if (global) {
+ throw new IOException("pax global header contains " + keyword
+ + ", which would change every entry after it");
+ }
+ if (KEYWORD_PATH.equals(keyword)) {
+ pendingPath = value;
+ return;
+ }
+ try {
+ pendingSize = Long.parseLong(value);
+ } catch (NumberFormatException e) {
+ throw new IOException("pax size record is not a number: " + value, e);
+ }
+ if (pendingSize < 0) {
+ throw new IOException("pax size record is negative: " + value);
+ }
+ }
+
+ /**
+ * Drops everything from the first NUL onward, which is how a GNU long-name header
+ * terminates the name it contains.
+ *
+ * @param value The decoded payload.
+ * @return The name without its terminator. Not {@code null}.
+ */
+ private static String trimNul(String value) {
+ final int nul = value.indexOf('\0');
+ return nul < 0 ? value : value.substring(0, nul);
+ }
+
+ /**
+ * Reads the current header's entry name. On a POSIX ustar header a non-empty name
+ * prefix is joined to the name field with {@code /}, which is how a name longer than
+ * the 100-byte name field is stored when no extension header contains it.
+ *
+ * @return The entry name. Not {@code null}.
+ * @throws IOException Thrown if the stored name or prefix is not valid UTF-8.
+ */
+ private String readName() throws IOException {
+ final String stored = field(0, NAME_LENGTH, "tar entry name");
+ if (!hasPosixUstarMagic(header)) {
+ return stored;
+ }
+ final String prefix = field(PREFIX_OFFSET, PREFIX_LENGTH, "tar entry name prefix");
+ return prefix.isEmpty() ? stored : prefix + "/" + stored;
+ }
+
+ /**
+ * Reads a NUL-terminated text field of the current header.
+ *
+ * @param offset The field's offset in the header block.
+ * @param length The field's length in bytes.
+ * @param subject The field name to use in an error message.
+ * @return The field content up to its first NUL, decoded as UTF-8. Not {@code null}.
+ * @throws IOException Thrown if the field is not valid UTF-8.
+ */
+ private String field(int offset, int length, String subject) throws IOException {
+ int end = 0;
+ while (end < length && header[offset + end] != 0) {
+ end++;
+ }
+ return decodeUtf8(header, offset, end, subject);
+ }
+
+ /**
+ * @return The current entry's name as stored in the archive. Not {@code null}
+ * after a successful {@link #next()}.
+ */
+ public String name() {
+ return name;
+ }
+
+ /**
+ * @return The current entry's size in bytes.
+ */
+ public long size() {
+ return size;
+ }
+
+ /**
+ * @return {@code true} if the current entry is a regular file.
+ */
+ public boolean isFile() {
+ return type == TYPE_REGULAR_FILE || type == TYPE_REGULAR_FILE_CLASSIC;
+ }
+
+ /**
+ * Opens the current entry's content.
+ *
+ * @return A stream over exactly this entry's bytes; reading past the end returns end
+ * of stream, and a zero-length read returns {@code 0} as
+ * {@link InputStream#read(byte[], int, int)} requires. Not {@code null}.
+ * Closing it is not required.
+ */
+ public InputStream entryStream() {
+ return new InputStream() {
+ @Override
+ public int read() throws IOException {
+ if (remaining <= 0) {
+ return -1;
+ }
+ final int b = in.read();
+ if (b < 0) {
+ throw new IOException("truncated tar entry: " + name);
+ }
+ remaining--;
+ return b;
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * The range is checked before the entry state, so invalid arguments are reported
+ * when the entry is exhausted or the requested length is zero. This override uses
+ * the exceptions specified by {@link InputStream}.
+ *
+ * @throws NullPointerException Thrown if {@code buffer} is {@code null}.
+ * @throws IndexOutOfBoundsException Thrown if {@code offset} or {@code length} is
+ * negative, or {@code length} is greater than
+ * {@code buffer.length - offset}.
+ */
+ @Override
+ public int read(byte[] buffer, int offset, int length) throws IOException {
+ Objects.checkFromIndexSize(offset, length, buffer.length);
+ if (length == 0) {
+ return 0;
+ }
+ if (remaining <= 0) {
+ return -1;
+ }
+ final int read = in.read(buffer, offset, (int) Math.min(length, remaining));
+ if (read < 0) {
+ throw new IOException("truncated tar entry: " + name);
+ }
+ remaining -= read;
+ return read;
+ }
+ };
+ }
+
+ /**
+ * Checks whether a full 512-byte block reads as a tar entry header, which it does when
+ * it starts with a name and its stored checksum matches the block. Both classic and
+ * ustar headers contain that checksum, so it identifies both without relying on the
+ * ustar magic, which arbitrary content can also contain.
+ *
+ * @param block The block to inspect. Must be 512 bytes long.
+ * @return {@code true} if the block reads as a tar header.
+ */
+ private static boolean isHeader(byte[] block) {
+ return block[0] != 0 && hasValidChecksum(block);
+ }
+
+ /**
+ * Verifies a header block against the checksum stored in it. The checksum is the sum
+ * of every header byte with the checksum field itself read as eight blanks. Historical
+ * writers used signed-byte sums, so both totals are accepted.
+ *
+ * @param block The header block to verify. Must be 512 bytes long.
+ * @return {@code true} if the stored checksum is well formed and matches the block.
+ */
+ private static boolean hasValidChecksum(byte[] block) {
+ long stored = 0;
+ boolean digits = false;
+ boolean trailingPadding = false;
+ for (int i = CHECKSUM_OFFSET; i < CHECKSUM_OFFSET + CHECKSUM_LENGTH; i++) {
+ final byte b = block[i];
+ if (b == 0 || b == ' ') {
+ if (digits) {
+ trailingPadding = true;
+ }
+ continue;
+ }
+ if (!isOctalDigit(b) || trailingPadding) {
+ return false;
+ }
+ stored = stored * 8 + (b - '0');
+ digits = true;
+ }
+ if (!digits) {
+ return false;
+ }
+ int unsigned = 0;
+ int signed = 0;
+ for (int i = 0; i < BLOCK; i++) {
+ final byte b = i >= CHECKSUM_OFFSET && i < CHECKSUM_OFFSET + CHECKSUM_LENGTH
+ ? (byte) ' ' : block[i];
+ unsigned += b & 0xFF;
+ signed += b;
+ }
+ return stored == unsigned || stored == signed;
+ }
+
+ /**
+ * Checks for the POSIX ustar magic specifically, which is {@code "ustar"} followed by a
+ * NUL. GNU writes {@code "ustar"} followed by two blanks and a NUL in the same field,
+ * and its headers must not be read as ustar: GNU stores {@code atime} at the offset
+ * ustar gives to the name prefix, so a GNU incremental archive would otherwise deliver
+ * every entry under a directory named after an octal timestamp.
+ *
+ * @param block The block to inspect. Must be 512 bytes long.
+ * @return {@code true} if the block contains the POSIX ustar magic.
+ */
+ private static boolean hasPosixUstarMagic(byte[] block) {
+ for (int i = 0; i < USTAR_MAGIC.length(); i++) {
+ if (block[MAGIC_OFFSET + i] != USTAR_MAGIC.charAt(i)) {
+ return false;
+ }
+ }
+ return block[MAGIC_OFFSET + USTAR_MAGIC.length()] == 0;
+ }
+
+ /**
+ * @param b The byte to classify.
+ * @return {@code true} if the byte is one of the digits {@code 0} to {@code 7}.
+ */
+ private static boolean isOctalDigit(byte b) {
+ return b >= '0' && b <= '7';
+ }
+
+ /**
+ * Fills the header buffer with the next 512-byte block.
+ *
+ * @return {@code true} when a full block was read, {@code false} at a clean end of
+ * the stream before any byte of the block.
+ * @throws IOException Thrown if the stream ends inside the block.
+ */
+ private boolean readBlock() throws IOException {
+ final int filled = in.readNBytes(header, 0, header.length);
+ if (filled == 0) {
+ return false;
+ }
+ if (filled < header.length) {
+ throw new IOException("truncated tar header");
+ }
+ return true;
+ }
+
+ /**
+ * @return {@code true} if the current header buffer is one of the all-zero blocks
+ * that terminate a tar archive.
+ */
+ private boolean isEndBlock() {
+ for (final byte b : header) {
+ if (b != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Parses the size field of the current header, tolerating NUL and blank padding around
+ * the octal digits. A field with its leading bit set is in the base-256 encoding
+ * instead, and is read by {@link #parseBase256Size()}.
+ *
+ * @return The entry size in bytes.
+ * @throws IOException Thrown if the field contains a character that is not an octal
+ * digit, a blank, or NUL padding, or if the base-256 form is negative or too
+ * large for a {@code long}.
+ */
+ private long parseSize() throws IOException {
+ if ((header[SIZE_OFFSET] & BASE_256_MARKER) != 0) {
+ return parseBase256Size();
+ }
+ long value = 0;
+ boolean digitSeen = false;
+ boolean trailingPadding = false;
+ for (int i = SIZE_OFFSET; i < SIZE_OFFSET + SIZE_LENGTH; i++) {
+ final byte b = header[i];
+ if (b == 0 || b == ' ') {
+ if (digitSeen) {
+ trailingPadding = true;
+ }
+ continue;
+ }
+ if (!isOctalDigit(b) || trailingPadding) {
+ throw new IOException("malformed tar size field in entry header");
+ }
+ digitSeen = true;
+ value = value * 8 + (b - '0');
+ }
+ return value;
+ }
+
+ /**
+ * Reads a size field in the base-256 encoding, which GNU writes when a value does not
+ * fit the 11 octal digits the field otherwise contains, so entries of 8 GiB and above
+ * can state their length.
+ *
+ * The leading bit marks the encoding, the next bit is the sign, and the remaining
+ * bits of that byte followed by all later bytes form a big-endian two's complement
+ * number. Negative sizes are rejected.
+ *
+ * @return The entry size in bytes.
+ * @throws IOException Thrown if the encoded value is negative or does not fit a
+ * {@code long}.
+ */
+ private long parseBase256Size() throws IOException {
+ if ((header[SIZE_OFFSET] & BASE_256_NEGATIVE) != 0) {
+ throw new IOException("tar size field is negative");
+ }
+ long value = 0;
+ for (int i = SIZE_OFFSET; i < SIZE_OFFSET + SIZE_LENGTH; i++) {
+ final int b = i == SIZE_OFFSET
+ ? header[i] & BASE_256_FIRST_BYTE_BITS : header[i] & 0xFF;
+ if (value >>> (Long.SIZE - Byte.SIZE - 1) != 0) {
+ throw new IOException(
+ "tar size field exceeds the largest length this reader can represent");
+ }
+ value = value << Byte.SIZE | b;
+ }
+ return value;
+ }
+
+ /**
+ * @param entrySize The size of an entry's content in bytes.
+ * @return The number of padding bytes that align the entry to the next 512-byte
+ * block boundary.
+ */
+ private long padding(long entrySize) {
+ final long remainder = entrySize % BLOCK;
+ return remainder == 0 ? 0 : BLOCK - remainder;
+ }
+
+ /**
+ * Consumes and discards the given number of bytes from the underlying stream.
+ *
+ * @param bytes The number of bytes to discard.
+ * @throws IOException Thrown if the stream ends before all bytes were consumed.
+ */
+ private void skip(long bytes) throws IOException {
+ try {
+ in.skipNBytes(bytes);
+ } catch (EOFException e) {
+ throw new IOException("truncated tar archive", e);
+ }
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownloadTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownloadTest.java
index 67d85af698..7c219fd4f3 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownloadTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownloadTest.java
@@ -25,11 +25,13 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.function.Executable;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import opennlp.tools.util.DictionaryCatalog;
import opennlp.tools.util.DigestTestUtil;
-import opennlp.tools.util.DownloadUtil;
/**
* Pins the Hunspell catalog download gate; network fetches are not exercised here.
@@ -42,16 +44,18 @@ public class HunspellDictionaryDownloadTest {
*
* @param target A scratch directory managed by the test framework.
* @throws IOException Thrown if the local catalog cannot be prepared.
- */
+ */
@Test
void testDownloadRequiresRemoteProperty(@TempDir Path target) throws IOException {
final DictionaryCatalog catalog = localCatalog(target);
- final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY);
- System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY);
+ final String previous =
+ System.getProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
+ System.clearProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
try {
final IOException e = Assertions.assertThrows(IOException.class,
() -> HunspellDictionaryDownload.downloadFromCatalog(catalog, "demo", target));
- Assertions.assertTrue(e.getMessage().contains(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY));
+ Assertions.assertTrue(
+ e.getMessage().contains(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY));
} finally {
restore(previous);
}
@@ -68,8 +72,9 @@ void testDownloadRequiresRemoteProperty(@TempDir Path target) throws IOException
void testDownloadsFromApplicationCatalog(@TempDir Path target) throws IOException {
final DictionaryCatalog catalog = localCatalog(target);
final Path output = target.resolve("output");
- final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY);
- System.setProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY, "true");
+ final String previous =
+ System.getProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
+ System.setProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY, "true");
try {
HunspellDictionaryDownload.downloadFromCatalog(catalog, "demo", output);
Assertions.assertEquals("SET UTF-8\n",
@@ -85,18 +90,28 @@ void testDownloadsFromApplicationCatalog(@TempDir Path target) throws IOExceptio
/**
* Verifies that each required parameter is checked before a download begins.
*
+ * @param argument The invalid method parameter.
* @param target A scratch directory managed by the test framework.
* @throws IOException Thrown if the local catalog cannot be prepared.
*/
- @Test
- void testRejectsNullParameters(@TempDir Path target) throws IOException {
+ @ParameterizedTest(name = "{0}")
+ @ValueSource(strings = {"catalog", "dictionaryId", "targetDirectory"})
+ void testRejectsNullParameters(String argument, @TempDir Path target)
+ throws IOException {
final DictionaryCatalog catalog = localCatalog(target);
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> HunspellDictionaryDownload.downloadFromCatalog(null, "demo", target));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> HunspellDictionaryDownload.downloadFromCatalog(catalog, null, target));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> HunspellDictionaryDownload.downloadFromCatalog(catalog, "demo", null));
+ final Executable download = switch (argument) {
+ case "catalog" -> () ->
+ HunspellDictionaryDownload.downloadFromCatalog(null, "demo", target);
+ case "dictionaryId" -> () ->
+ HunspellDictionaryDownload.downloadFromCatalog(catalog, null, target);
+ case "targetDirectory" -> () ->
+ HunspellDictionaryDownload.downloadFromCatalog(catalog, "demo", null);
+ default -> throw new IllegalArgumentException("unknown argument: " + argument);
+ };
+
+ final IllegalArgumentException thrown =
+ Assertions.assertThrows(IllegalArgumentException.class, download);
+ Assertions.assertEquals(argument + " must not be null", thrown.getMessage());
}
/**
@@ -140,9 +155,9 @@ private static String entry(String id, Path source, byte[] content, String filen
/** Restores the remote-download property to its previous value. */
private static void restore(String previous) {
if (previous == null) {
- System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY);
+ System.clearProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
} else {
- System.setProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY, previous);
+ System.setProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY, previous);
}
}
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java
index fc2d59a6b7..7e3f3e5a88 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java
@@ -29,6 +29,7 @@
import org.junit.jupiter.api.io.TempDir;
import opennlp.tools.util.Span;
+import opennlp.tools.util.archive.TarArchives;
/**
* Demonstrates the intended end-to-end usage of this package with miniature,
@@ -60,7 +61,7 @@ void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work)
throws IOException {
// A minimal but complete dictionary: one lexicon file plus the three definition
// files every MeCab-format distribution contains, wrapped like a release archive.
- final byte[] archive = TarGzArchives.gzippedTar(new String[][] {
+ final byte[] archive = TarArchives.gzippedTar(new String[][] {
{"mini-dict-0.1/lexicon.csv", String.join("\n",
"\u6771\u4EAC,0,0,3000,noun,proper",
"\u4EAC\u90FD,0,0,3000,noun,proper",
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java
index 3fe57e1b87..cbbfe4915b 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java
@@ -18,26 +18,41 @@
package opennlp.tools.tokenize.lattice;
import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.function.Executable;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.util.DictionaryCatalog;
+import opennlp.tools.util.DigestTestUtil;
+import opennlp.tools.util.archive.TarArchives;
/**
* Tests the installer against project-authored, in-memory archives; no external
- * dictionary data and no network access are involved.
+ * dictionary data and no network access are involved. Fetch, verification, and
+ * unpacking limits are exercised in {@code opennlp.tools.util.ResourceInstallerTest},
+ * which tests the shared installation path this installer delegates to.
*/
public class MecabDictionaryInstallerTest {
@Test
- void testExtractsDictionaryFilesAndFlattensPaths(@TempDir Path target)
- throws IOException {
- final byte[] archive = TarGzArchives.gzippedTar(new String[][] {
+ void testInstallsDictionaryFilesAndFlattensPaths(@TempDir Path source,
+ @TempDir Path target) throws IOException {
+ final Path archiveFile = archive(source, new String[][] {
{"dict-1.0/lexicon.csv", "cat,0,0,100,noun\n"},
{"dict-1.0/matrix.def", "1 1\n0 0 0\n"},
{"dict-1.0/char.def", "DEFAULT 0 1 0\n"},
@@ -45,10 +60,10 @@ void testExtractsDictionaryFilesAndFlattensPaths(@TempDir Path target)
{"dict-1.0/README", "not a dictionary file"},
{"dict-1.0/dicrc", "config"}});
- final int extracted = MecabDictionaryInstaller.extract(
- new ByteArrayInputStream(archive), target);
+ final int installed =
+ MecabDictionaryInstaller.install(archiveFile.toUri(), target);
- Assertions.assertEquals(5, extracted);
+ Assertions.assertEquals(5, installed);
Assertions.assertTrue(Files.exists(target.resolve("lexicon.csv")));
Assertions.assertTrue(Files.exists(target.resolve("matrix.def")));
Assertions.assertTrue(Files.exists(target.resolve("char.def")));
@@ -62,180 +77,257 @@ void testExtractsDictionaryFilesAndFlattensPaths(@TempDir Path target)
/**
* Verifies that only files at the archive root count as dictionary payload.
* mecab-ko-dic ships template user dictionaries under {@code user-dic/} whose
- * numeric fields are empty, input for {@code mecab-dict-index} rather than loadable
+ * numeric fields are empty. They are input for {@code mecab-dict-index}, not loadable
* lexicon data. Flattening them next to the real lexicon fails the subsequent load,
* and on a case-insensitive file system a template can silently overwrite a real
* lexicon file of the same base name.
*/
@Test
- void testNestedTemplateFilesAreNotExtracted(@TempDir Path target) throws IOException {
- final byte[] archive = TarGzArchives.gzippedTar(new String[][] {
+ void testNestedTemplateFilesAreNotInstalled(@TempDir Path source,
+ @TempDir Path target) throws IOException {
+ final Path archiveFile = archive(source, new String[][] {
{"dict-1.0/NNP.csv", "cat,1786,3546,2953,noun\n"},
{"dict-1.0/matrix.def", "1 1\n0 0 0\n"},
{"dict-1.0/user-dic/person.csv", "template,,,,noun\n"}});
- final int extracted = MecabDictionaryInstaller.extract(
- new ByteArrayInputStream(archive), target);
+ final int installed =
+ MecabDictionaryInstaller.install(archiveFile.toUri(), target);
- Assertions.assertEquals(2, extracted);
+ Assertions.assertEquals(2, installed);
Assertions.assertTrue(Files.exists(target.resolve("NNP.csv")));
Assertions.assertTrue(Files.notExists(target.resolve("person.csv")));
}
+ /**
+ * Verifies that two payload entries that flatten to the same base name are rejected,
+ * because keeping either one silently would install an ambiguous dictionary.
+ */
+ @Test
+ void testEntriesFlatteningToTheSameNameAreRejected(@TempDir Path source,
+ @TempDir Path target) throws IOException {
+ final Path archiveFile = archive(source, new String[][] {
+ {"words.csv", "cat,0,0,100,noun\n"},
+ {"d/words.csv", "dog,0,0,100,noun\n"}});
+
+ final IOException e = Assertions.assertThrows(IOException.class,
+ () -> MecabDictionaryInstaller.install(archiveFile.toUri(), target));
+ Assertions.assertEquals(
+ "the archive flattens two entries to the same name: words.csv",
+ e.getMessage());
+ Assertions.assertTrue(Files.notExists(target.resolve("words.csv")));
+ }
+
@Test
void testInstallReadsAFileUri(@TempDir Path source, @TempDir Path target)
throws IOException {
- final Path archiveFile = source.resolve("dict.tar.gz");
- Files.write(archiveFile, TarGzArchives.gzippedTar(new String[][] {
+ final Path archiveFile = archive(source, new String[][] {
{"d/words.csv", "cat,0,0,100,noun\n"},
- {"d/matrix.def", "1 1\n0 0 0\n"}}));
+ {"d/matrix.def", "1 1\n0 0 0\n"}});
- final int extracted =
+ final int installed =
MecabDictionaryInstaller.install(archiveFile.toUri(), target);
- Assertions.assertEquals(2, extracted);
+ Assertions.assertEquals(2, installed);
Assertions.assertTrue(Files.exists(target.resolve("words.csv")));
}
- @Test
- void testNonFileInstallIsRejected(@TempDir Path target) {
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> MecabDictionaryInstaller.install(
- URI.create("https://example.invalid/dict.tar.gz"), target));
- }
-
- @Test
- void testArchivesWithoutDictionaryFilesFailLoud(@TempDir Path target)
- throws IOException {
- final byte[] archive =
- TarGzArchives.gzippedTar(new String[][] {{"readme.txt", "nothing here"}});
- Assertions.assertThrows(IOException.class, () -> MecabDictionaryInstaller.extract(
- new ByteArrayInputStream(archive), target));
- }
-
/**
- * Verifies that a tar entry with a declared size that is above the per-entry limit is
- * rejected before any payload is written. The fixture stores only the oversized
- * header so the test does not allocate the declared size.
+ * Checks installation when the target uses a different filesystem provider.
+ *
+ * @param source The directory containing the fixture archive.
+ * @param scratch The directory containing the ZIP filesystem.
+ * @throws IOException Thrown if creating or installing the fixture fails.
*/
@Test
- void testOversizedEntryFailsLoud(@TempDir Path target) throws IOException {
- final long limit = 64;
- final byte[] archive = TarGzArchives.gzippedTar(
- TarGzArchives.Entry.withDeclaredSize("huge.csv", new byte[0], limit + 1));
- final IOException e = Assertions.assertThrows(IOException.class,
- () -> MecabDictionaryInstaller.extract(new ByteArrayInputStream(archive),
- target, limit, 1024, 16, 100));
- Assertions.assertEquals("tar entry size exceeds safe limit of " + limit,
- e.getMessage());
+ void testInstallIntoANonDefaultFileSystem(@TempDir Path source, @TempDir Path scratch)
+ throws IOException {
+ final Path archiveFile = archive(source, new String[][] {
+ {"d/words.csv", "cat,0,0,100,noun\n"},
+ {"d/matrix.def", "1 1\n0 0 0\n"}});
+ final URI zip = URI.create("jar:" + scratch.resolve("target.zip").toUri());
+
+ try (FileSystem targetFileSystem =
+ FileSystems.newFileSystem(zip, Map.of("create", "true"))) {
+ final Path target = targetFileSystem.getPath("/dictionary");
+
+ Assertions.assertEquals(2,
+ MecabDictionaryInstaller.install(archiveFile.toUri(), target));
+ Assertions.assertEquals("cat,0,0,100,noun\n",
+ Files.readString(target.resolve("words.csv")));
+ }
}
/**
- * Verifies that extracting dictionary files with sizes that sum above the total-bytes
- * limit fails with {@link IOException}.
+ * Verifies that a pax long-name entry installs under its real name. The common
+ * distributions ship plain ustar today, but {@code bsdtar} and
+ * {@code tar --format=posix} write pax archives, whose over-100-byte names live in an
+ * extension header instead of the header name field.
*/
@Test
- void testTotalExtractedBytesBudgetFailsLoud(@TempDir Path target) throws IOException {
- final long limit = 30;
- final byte[] archive = TarGzArchives.gzippedTar(new String[][] {
- {"a.csv", "01234567890123456789"},
- {"b.csv", "01234567890123456789"}});
- final IOException e = Assertions.assertThrows(IOException.class,
- () -> MecabDictionaryInstaller.extract(new ByteArrayInputStream(archive),
- target, 1024, limit, 16, 100));
- Assertions.assertEquals("extracted archive size exceeds safe limit of " + limit,
- e.getMessage());
+ void testPaxLongNamedEntryInstallsUnderItsRealName(@TempDir Path source,
+ @TempDir Path target) throws IOException {
+ final String longBaseName = "a".repeat(110) + ".csv";
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ TarArchives.entry(tar, "PaxHeaders.0/lexicon",
+ TarArchives.paxRecord("path", "dict-1.0/" + longBaseName), 'x');
+ TarArchives.entry(tar, "dict-1.0/" + "a".repeat(88),
+ "cat,0,0,100,noun\n".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TarArchives.TERMINATOR_SIZE]);
+ final Path archiveFile = source.resolve("dict.tar.gz");
+ Files.write(archiveFile, TarArchives.gzip(tar.toByteArray()));
+
+ final int installed =
+ MecabDictionaryInstaller.install(archiveFile.toUri(), target);
+
+ Assertions.assertEquals(1, installed);
+ Assertions.assertEquals("cat,0,0,100,noun\n",
+ Files.readString(target.resolve(longBaseName)));
}
/**
- * Verifies that an archive with more dictionary files than the entry-count limit
- * fails on the entry that would exceed it.
+ * Verifies that installing into a target that already holds a dictionary file of the
+ * same name is rejected, leaving the first installation in place. Refreshing a
+ * dictionary means removing its old files first.
*/
@Test
- void testExtractedEntryCountBudgetFailsLoud(@TempDir Path target) throws IOException {
- final int limit = 2;
- final byte[] archive = TarGzArchives.gzippedTar(new String[][] {
- {"a.csv", "a\n"},
- {"b.def", "b\n"},
- {"c.csv", "c\n"}});
+ void testReinstallOverAnExistingDictionaryIsRejected(@TempDir Path source,
+ @TempDir Path target) throws IOException {
+ final Path archiveFile = archive(source, new String[][] {
+ {"d/words.csv", "cat,0,0,100,noun\n"}});
+ Assertions.assertEquals(1,
+ MecabDictionaryInstaller.install(archiveFile.toUri(), target));
+
final IOException e = Assertions.assertThrows(IOException.class,
- () -> MecabDictionaryInstaller.extract(new ByteArrayInputStream(archive),
- target, 1024, 1024, limit, 100));
- Assertions.assertEquals("extracted entry count exceeds safe limit of " + limit,
- e.getMessage());
+ () -> MecabDictionaryInstaller.install(archiveFile.toUri(), target));
+ Assertions.assertTrue(e.getMessage().contains("target already contains: "));
+ Assertions.assertEquals("cat,0,0,100,noun\n",
+ Files.readString(target.resolve("words.csv")));
}
- /**
- * Verifies that a highly compressible payload whose expansion exceeds the gzip
- * ratio limit fails before the inflated content is kept.
- */
@Test
- void testGzipExpansionRatioBudgetFailsLoud(@TempDir Path target) throws IOException {
- final int ratio = 2;
- final byte[] zeros = new byte[64 * 1024];
- Arrays.fill(zeros, (byte) 0);
- final byte[] archive = TarGzArchives.gzippedTar(
- TarGzArchives.Entry.of("zeros.csv", zeros));
- final IOException e = Assertions.assertThrows(IOException.class,
- () -> MecabDictionaryInstaller.extract(new ByteArrayInputStream(archive),
- target, zeros.length, zeros.length, 16, ratio));
- Assertions.assertEquals("gzip expansion ratio exceeds safe limit of " + ratio,
- e.getMessage());
+ void testRemoteInstallWithoutDigestIsRejected(@TempDir Path target) {
+ final IllegalArgumentException e =
+ Assertions.assertThrows(IllegalArgumentException.class,
+ () -> MecabDictionaryInstaller.install(
+ URI.create("https://example.invalid/dict.tar.gz"), target));
+ Assertions.assertTrue(
+ e.getMessage().contains("checksum must be given for an http or https source"));
}
@Test
- void testInvalidTarChecksumIsRejected(@TempDir Path target) throws IOException {
- final byte[] archive = TarGzArchives.gzippedTarWithInvalidHeaderChecksum(
- TarGzArchives.Entry.of("words.csv", "cat,0,0,100,noun\n"));
+ void testInstallVerifiesDigest(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final Path archiveFile = archive(source, new String[][] {
+ {"d/words.csv", "cat,0,0,100,noun\n"},
+ {"d/matrix.def", "1 1\n0 0 0\n"}});
+ final byte[] archive = Files.readAllBytes(archiveFile);
+
+ final int installed = MecabDictionaryInstaller.install(
+ archiveFile.toUri(), target, DigestTestUtil.sha512(archive));
+ Assertions.assertEquals(2, installed);
+
final IOException e = Assertions.assertThrows(IOException.class,
- () -> MecabDictionaryInstaller.extract(new ByteArrayInputStream(archive), target));
- Assertions.assertEquals("tar header checksum does not match", e.getMessage());
- Assertions.assertTrue(Files.notExists(target.resolve("words.csv")));
+ () -> MecabDictionaryInstaller.install(archiveFile.toUri(),
+ target.resolve("other"), DigestTestUtil.sha512(new byte[] {1})));
+ Assertions.assertTrue(e.getMessage().contains("checksum mismatch: expected"));
}
@Test
- void testFailedExtractionDoesNotPublishEarlierEntries(@TempDir Path target)
+ void testInstallFromCatalogRequiresRemoteProperty(@TempDir Path target)
throws IOException {
- final long limit = 64;
- final byte[] archive = TarGzArchives.gzippedTar(
- TarGzArchives.Entry.of("words.csv", "cat,0,0,100,noun\n"),
- TarGzArchives.Entry.withDeclaredSize("matrix.def", new byte[0], limit + 1));
- Assertions.assertThrows(IOException.class,
- () -> MecabDictionaryInstaller.extract(new ByteArrayInputStream(archive),
- target, limit, 1024, 16, 100));
- Assertions.assertTrue(Files.notExists(target.resolve("words.csv")));
+ final DictionaryCatalog catalog = emptyCatalog();
+ final String previous =
+ System.getProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
+ System.clearProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
+ try {
+ final IOException e = Assertions.assertThrows(IOException.class,
+ () -> MecabDictionaryInstaller.installFromCatalog(
+ catalog, "mecab.ipadic", target));
+ Assertions.assertTrue(
+ e.getMessage().contains(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY));
+ } finally {
+ if (previous == null) {
+ System.clearProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
+ } else {
+ System.setProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY, previous);
+ }
+ }
}
@Test
- void testExistingFileIsNotReplaced(@TempDir Path target) throws IOException {
- final Path existing = target.resolve("words.csv");
- Files.writeString(existing, "existing\n");
- final byte[] archive = TarGzArchives.gzippedTar(
- TarGzArchives.Entry.of("words.csv", "replacement\n"));
-
+ void testArchivesWithoutDictionaryFilesAreRejected(@TempDir Path source,
+ @TempDir Path target) throws IOException {
+ final Path archiveFile =
+ archive(source, new String[][] {{"readme.txt", "nothing here"}});
final IOException e = Assertions.assertThrows(IOException.class,
- () -> MecabDictionaryInstaller.extract(new ByteArrayInputStream(archive), target));
- Assertions.assertEquals("dictionary file already exists: " + existing, e.getMessage());
- Assertions.assertEquals("existing\n", Files.readString(existing));
+ () -> MecabDictionaryInstaller.install(archiveFile.toUri(), target));
+ Assertions.assertEquals("the archive contains no dictionary file", e.getMessage());
}
- @Test
- void testInvalidArguments(@TempDir Path target) {
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> MecabDictionaryInstaller.install(null, target));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> MecabDictionaryInstaller.install(target.toUri(), null));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> MecabDictionaryInstaller.extract(null, target));
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> MecabDictionaryInstaller.extract(
- new ByteArrayInputStream(new byte[0]), null));
+ /**
+ * Checks each public installer parameter independently.
+ *
+ * @param argument The invalid parameter.
+ * @param target A scratch directory managed by the test framework.
+ * @throws IOException Thrown if the empty catalog cannot be loaded.
+ */
+ @ParameterizedTest(name = "{0}")
+ @ValueSource(strings = {"archive", "targetDirectory", "catalog", "dictionaryId",
+ "catalog targetDirectory"})
+ void testInvalidArguments(String argument, @TempDir Path target) throws IOException {
+ final DictionaryCatalog catalog = emptyCatalog();
+ final Executable install = switch (argument) {
+ case "archive" -> () -> MecabDictionaryInstaller.install(null, target);
+ case "targetDirectory" -> () ->
+ MecabDictionaryInstaller.install(target.toUri(), null);
+ case "catalog" -> () ->
+ MecabDictionaryInstaller.installFromCatalog(null, "mecab.ipadic", target);
+ case "dictionaryId" -> () ->
+ MecabDictionaryInstaller.installFromCatalog(catalog, null, target);
+ case "catalog targetDirectory" -> () ->
+ MecabDictionaryInstaller.installFromCatalog(catalog, "mecab.ipadic", null);
+ default -> throw new IllegalArgumentException("unknown argument: " + argument);
+ };
+
+ final IllegalArgumentException thrown =
+ Assertions.assertThrows(IllegalArgumentException.class, install);
+ final String parameter = argument.startsWith("catalog ")
+ ? argument.substring("catalog ".length()) : argument;
+ Assertions.assertEquals(parameter + " must not be null", thrown.getMessage());
+ }
+
+ private static DictionaryCatalog emptyCatalog() throws IOException {
+ return DictionaryCatalog.load(new ByteArrayInputStream(new byte[0]));
+ }
+
+ /**
+ * Writes a gzip-compressed tar archive of the given entries to a file.
+ *
+ * @param directory The directory to write the archive into.
+ * @param entries The entries as {@code {name, content}} pairs.
+ * @return The archive file. Never {@code null}.
+ * @throws IOException Thrown if writing fails.
+ */
+ private static Path archive(Path directory, String[][] entries) throws IOException {
+ final Path archiveFile = directory.resolve("dict.tar.gz");
+ Files.write(archiveFile, TarArchives.gzippedTar(entries));
+ return archiveFile;
}
@Test
- void testDefaultBudgetsWithoutOverrides() {
- Assertions.assertEquals(512L * 1024 * 1024, MecabDictionaryInstaller.MAX_ENTRY_BYTES);
- Assertions.assertEquals(2L * 1024 * 1024 * 1024,
- MecabDictionaryInstaller.MAX_TOTAL_EXTRACTED_BYTES);
+ void testStaleScratchOfAKilledInstallIsRemoved(@TempDir Path source,
+ @TempDir Path target) throws IOException {
+ final Path stale = Files.createDirectories(target.resolve(".mecab-dict-OLD"));
+ Files.writeString(stale.resolve("words.csv"), "half");
+ final Path archiveFile = archive(source, new String[][] {
+ {"d/words.csv", "cat,0,0,100,noun\n"}});
+
+ Assertions.assertEquals(1,
+ MecabDictionaryInstaller.install(archiveFile.toUri(), target));
+
+ try (Stream entries = Files.list(target)) {
+ Assertions.assertEquals(List.of("words.csv"),
+ entries.map(path -> path.getFileName().toString()).sorted().toList());
+ }
}
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java
deleted file mode 100644
index bb883773bc..0000000000
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.tools.tokenize.lattice;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.zip.GZIPOutputStream;
-
-/**
- * Builds miniature, project-authored gzip-compressed ustar archives in memory for the
- * tests of this package; no external archive data is involved.
- */
-final class TarGzArchives {
-
- /** The tar block size; headers, content, and padding are all complete blocks. */
- private static final int BLOCK = 512;
-
- /** The header field offsets and lengths this builder writes, in bytes. */
- private static final int NAME_LENGTH = 100;
- private static final int MODE_OFFSET = 100;
- private static final int SIZE_OFFSET = 124;
- private static final int SIZE_LENGTH = 12;
- private static final int CHECKSUM_OFFSET = 148;
- private static final int CHECKSUM_LENGTH = 8;
- private static final int TYPE_OFFSET = 156;
-
- /** The type flag of a regular file entry. */
- private static final char REGULAR_FILE = '0';
-
- private TarGzArchives() {
- }
-
- /**
- * One archive entry: a path name, the bytes stored after the header, and the size
- * field written into the header (which may differ from the stored content length so
- * budget checks can be exercised without allocating the declared payload).
- *
- * @param name The entry name including any directory prefix.
- * @param content The bytes written after the header; may be shorter than
- * {@code declaredSize}.
- * @param declaredSize The octal size field stored in the header.
- */
- record Entry(String name, byte[] content, long declaredSize) {
-
- /**
- * Builds an entry with a declared size that matches its UTF-8 content length.
- *
- * @param name The entry name.
- * @param content The entry text.
- * @return The entry. Not {@code null}.
- */
- static Entry of(String name, String content) {
- final byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
- return new Entry(name, bytes, bytes.length);
- }
-
- /**
- * Builds an entry with a declared size that matches its content length.
- *
- * @param name The entry name.
- * @param content The entry bytes.
- * @return The entry. Not {@code null}.
- */
- static Entry of(String name, byte[] content) {
- return new Entry(name, content, content.length);
- }
-
- /**
- * Builds an entry with a header size field that is set independently of the stored
- * content, for oversized-entry budget tests.
- *
- * @param name The entry name.
- * @param content The bytes stored after the header; typically empty for header-only
- * oversized cases.
- * @param declaredSize The size field written into the header.
- * @return The entry. Not {@code null}.
- */
- static Entry withDeclaredSize(String name, byte[] content, long declaredSize) {
- return new Entry(name, content, declaredSize);
- }
- }
-
- /**
- * Builds a gzip-compressed tar archive from name and content pairs, the layout a
- * dictionary distribution ships in.
- *
- * @param entries The entries as {@code {name, content}} pairs. Must not be
- * {@code null}.
- * @return The compressed archive bytes. Not {@code null}.
- * @throws IOException Thrown if writing to the in-memory streams fails.
- */
- static byte[] gzippedTar(String[][] entries) throws IOException {
- final Entry[] typed = new Entry[entries.length];
- for (int i = 0; i < entries.length; i++) {
- typed[i] = Entry.of(entries[i][0], entries[i][1]);
- }
- return gzippedTar(typed);
- }
-
- /**
- * Builds a gzip-compressed tar archive from typed entries.
- *
- * @param entries The entries to store. Must not be {@code null}.
- * @return The compressed archive bytes. Not {@code null}.
- * @throws IOException Thrown if writing to the in-memory streams fails.
- */
- static byte[] gzippedTar(Entry... entries) throws IOException {
- return gzip(tar(entries));
- }
-
- /**
- * Builds an archive with an incorrect checksum in its first header.
- *
- * @param entries The entries to store. Must not be {@code null} or empty.
- * @return The compressed archive bytes. Not {@code null}.
- * @throws IOException Thrown if writing to the in-memory streams fails.
- */
- static byte[] gzippedTarWithInvalidHeaderChecksum(Entry... entries) throws IOException {
- final byte[] tar = tar(entries);
- tar[CHECKSUM_OFFSET] = tar[CHECKSUM_OFFSET] == '0' ? (byte) '1' : (byte) '0';
- return gzip(tar);
- }
-
- /**
- * Builds the uncompressed tar image.
- *
- * @param entries The entries to store. Must not be {@code null}.
- * @return The tar bytes. Not {@code null}.
- * @throws IOException Thrown if writing to the in-memory stream fails.
- */
- private static byte[] tar(Entry... entries) throws IOException {
- final ByteArrayOutputStream tar = new ByteArrayOutputStream();
- for (final Entry entry : entries) {
- tarEntry(tar, entry);
- }
- // Two zero blocks end a tar archive.
- tar.write(new byte[2 * BLOCK]);
- return tar.toByteArray();
- }
-
- /**
- * Compresses a tar image with gzip.
- *
- * @param tar The tar bytes. Must not be {@code null}.
- * @return The compressed bytes. Not {@code null}.
- * @throws IOException Thrown if compression fails.
- */
- private static byte[] gzip(byte[] tar) throws IOException {
- final ByteArrayOutputStream compressed = new ByteArrayOutputStream();
- try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) {
- gzip.write(tar);
- }
- return compressed.toByteArray();
- }
-
- /**
- * Appends one ustar file entry to a growing tar image: a 512-byte header block
- * followed by the stored content padded to a block boundary of the declared size
- * when content is present, or the header alone when the test supplies no payload.
- *
- * @param tar The tar image under construction. Must not be {@code null}.
- * @param entry The entry to append. Must not be {@code null}.
- * @throws IOException Thrown if writing to the in-memory stream fails.
- * @throws IllegalArgumentException Thrown if {@code name} does not fit the header or
- * {@code declaredSize} is negative.
- */
- private static void tarEntry(ByteArrayOutputStream tar, Entry entry) throws IOException {
- final byte[] nameBytes = entry.name().getBytes(StandardCharsets.UTF_8);
- if (nameBytes.length == 0 || nameBytes.length > NAME_LENGTH) {
- throw new IllegalArgumentException(
- "entry name must be 1.." + NAME_LENGTH + " bytes, got " + nameBytes.length);
- }
- if (entry.declaredSize() < 0) {
- throw new IllegalArgumentException("declaredSize must not be negative");
- }
- final byte[] header = new byte[BLOCK];
- System.arraycopy(nameBytes, 0, header, 0, nameBytes.length);
- final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII);
- System.arraycopy(mode, 0, header, MODE_OFFSET, mode.length);
- // Both numeric fields hold octal digits followed by one terminator byte.
- final byte[] size = String.format("%0" + (SIZE_LENGTH - 1) + "o", entry.declaredSize())
- .getBytes(StandardCharsets.US_ASCII);
- System.arraycopy(size, 0, header, SIZE_OFFSET, size.length);
- header[TYPE_OFFSET] = REGULAR_FILE;
- // The checksum is computed with its own field read as spaces.
- for (int i = CHECKSUM_OFFSET; i < CHECKSUM_OFFSET + CHECKSUM_LENGTH; i++) {
- header[i] = ' ';
- }
- int checksum = 0;
- for (final byte b : header) {
- checksum += b & 0xFF;
- }
- final byte[] checksumText = String.format("%0" + (CHECKSUM_LENGTH - 2) + "o", checksum)
- .getBytes(StandardCharsets.US_ASCII);
- System.arraycopy(checksumText, 0, header, CHECKSUM_OFFSET, checksumText.length);
- header[CHECKSUM_OFFSET + CHECKSUM_LENGTH - 2] = 0;
- header[CHECKSUM_OFFSET + CHECKSUM_LENGTH - 1] = ' ';
- tar.write(header);
- if (entry.declaredSize() == 0) {
- return;
- }
- if (entry.content().length == 0) {
- // Header-only oversized fixtures: the extractor rejects on the size field before
- // reading a payload, so the declared bytes are not materialised here.
- return;
- }
- tar.write(entry.content());
- final long missing = Math.max(0, entry.declaredSize() - entry.content().length);
- if (missing > 0) {
- writeZeros(tar, missing);
- }
- final int padding = (BLOCK - (int) (entry.declaredSize() % BLOCK)) % BLOCK;
- tar.write(new byte[padding]);
- }
-
- /**
- * Writes {@code count} zero bytes to the stream.
- *
- * @param out The stream to write to.
- * @param count The number of zero bytes.
- * @throws IOException Thrown if writing fails.
- */
- private static void writeZeros(ByteArrayOutputStream out, long count) throws IOException {
- final byte[] zeros = new byte[8192];
- long remaining = count;
- while (remaining > 0) {
- final int chunk = (int) Math.min(zeros.length, remaining);
- out.write(zeros, 0, chunk);
- remaining -= chunk;
- }
- }
-}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java
index 9233b297af..cbb65dfb3c 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java
@@ -20,13 +20,17 @@
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.function.Executable;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import opennlp.tools.stemmer.hunspell.HunspellDictionary;
@@ -36,47 +40,152 @@
*/
public class DictionaryCatalogTest {
+ private static final String SHA_512 = "0".repeat(128);
+
+ /** Checks a catalog entry with all required fields. */
+ @Test
+ void testEntryAcceptsItsDocumentedFields() {
+ final URI absolute = URI.create("https://example.invalid/dictionary.tar.gz");
+
+ Assertions.assertDoesNotThrow(() ->
+ new DictionaryCatalog.Entry("demo", absolute, "A".repeat(128), null));
+ }
+
+ /**
+ * Checks each invalid field of the public catalog entry record.
+ *
+ * @param field The invalid record field.
+ */
+ @ParameterizedTest(name = "{0}")
+ @ValueSource(strings = {"id", "uri", "relative uri", "sha512", "invalid sha512",
+ "empty filename", "path filename"})
+ void testEntryRejectsInvalidFields(String field) {
+ final URI absolute = URI.create("https://example.invalid/dictionary.tar.gz");
+ final Executable construction = switch (field) {
+ case "id" -> () -> new DictionaryCatalog.Entry(null, absolute, SHA_512, null);
+ case "uri" -> () -> new DictionaryCatalog.Entry("demo", null, SHA_512, null);
+ case "relative uri" -> () -> new DictionaryCatalog.Entry(
+ "demo", URI.create("dictionary.tar.gz"), SHA_512, null);
+ case "sha512" -> () -> new DictionaryCatalog.Entry("demo", absolute, null, null);
+ case "invalid sha512" ->
+ () -> new DictionaryCatalog.Entry("demo", absolute, "not-a-digest", null);
+ case "empty filename" ->
+ () -> new DictionaryCatalog.Entry("demo", absolute, SHA_512, "");
+ case "path filename" ->
+ () -> new DictionaryCatalog.Entry("demo", absolute, SHA_512, "../dict.bin");
+ default -> throw new IllegalArgumentException("unknown field: " + field);
+ };
+
+ final IllegalArgumentException thrown =
+ Assertions.assertThrows(IllegalArgumentException.class, construction);
+ final String expectedMessage = switch (field) {
+ case "id" -> "id must not be null";
+ case "uri" -> "uri must not be null";
+ case "relative uri" -> "uri must be absolute";
+ case "sha512" -> "sha512 must not be null";
+ case "invalid sha512" -> "sha512 must be 128 hex digits";
+ case "empty filename", "path filename" -> "filename must be a file name";
+ default -> throw new IllegalArgumentException("unknown field: " + field);
+ };
+ Assertions.assertEquals(expectedMessage, thrown.getMessage());
+ }
+
+ /**
+ * Checks that invalid property values are reported as catalog I/O errors.
+ *
+ * @throws IOException Thrown if a fixture catalog cannot be loaded.
+ */
+ @Test
+ void testGetRejectsInvalidCatalogValues() throws IOException {
+ final DictionaryCatalog relativeUri = DictionaryCatalog.load(new ByteArrayInputStream(
+ ("demo.url=dictionary.tar.gz\ndemo.sha512=" + SHA_512 + "\n")
+ .getBytes(StandardCharsets.UTF_8)));
+ final DictionaryCatalog invalidDigest = DictionaryCatalog.load(new ByteArrayInputStream(
+ "demo.url=https://example.invalid/dictionary.tar.gz\ndemo.sha512=invalid\n"
+ .getBytes(StandardCharsets.UTF_8)));
+
+ Assertions.assertAll(
+ () -> Assertions.assertThrows(IOException.class, () -> relativeUri.get("demo")),
+ () -> Assertions.assertThrows(IOException.class, () -> invalidDigest.get("demo")));
+ }
+
+ /**
+ * Checks argument validation before the opt-in remote setting is read.
+ *
+ * @param argument The invalid method parameter.
+ * @param dir A scratch directory managed by the test framework.
+ * @throws IOException Thrown if the fixture catalog cannot be loaded.
+ */
+ @ParameterizedTest(name = "{0}")
+ @ValueSource(strings = {"id", "targetDirectory"})
+ void testInstallValidatesArgumentsBeforeTheRemoteSetting(String argument,
+ @TempDir Path dir)
+ throws IOException {
+ final DictionaryCatalog catalog = DictionaryCatalog.load(
+ new ByteArrayInputStream(new byte[0]));
+ final String previous = System.getProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
+ System.clearProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
+ try {
+ final Executable install = switch (argument) {
+ case "id" -> () -> catalog.install(null, dir);
+ case "targetDirectory" -> () -> catalog.install("demo", null);
+ default -> throw new IllegalArgumentException("unknown argument: " + argument);
+ };
+ final IllegalArgumentException thrown =
+ Assertions.assertThrows(IllegalArgumentException.class, install);
+ Assertions.assertEquals(argument + " must not be null", thrown.getMessage());
+ } finally {
+ restore(previous);
+ }
+ }
+
/**
- * Verifies that a catalog download without the remote-download property fails with
- * the property name in the message.
+ * Verifies that a catalog install without the remote-download property fails with
+ * the property name in the message, before anything is fetched or created.
*
* @param dir A scratch directory managed by the test framework.
* @throws Exception Thrown if the fixture catalog cannot be prepared.
*/
@Test
- void testDownloadRequiresRemoteProperty(@TempDir Path dir) throws Exception {
+ void testInstallRequiresRemoteProperty(@TempDir Path dir) throws Exception {
final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8);
final DictionaryCatalog loaded = demoCatalog(dir, payload);
+ final Path target = dir.resolve("out");
- final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY);
- System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY);
+ final String previous =
+ System.getProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
+ System.clearProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
try {
final IOException e = Assertions.assertThrows(IOException.class,
- () -> loaded.download("demo", dir.resolve("out.bin")));
- Assertions.assertTrue(e.getMessage().contains(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY));
+ () -> loaded.install("demo", target));
+ Assertions.assertTrue(
+ e.getMessage().contains(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY));
+ Assertions.assertTrue(Files.notExists(target));
} finally {
restore(previous);
}
}
/**
- * Verifies that an enabled catalog download fetches the entry and writes the
- * digest-verified bytes to the target.
+ * Verifies that an enabled catalog install fetches the entry and stores its
+ * digest-verified bytes under the source name in the target directory.
*
* @param dir A scratch directory managed by the test framework.
* @throws Exception Thrown if the fixture catalog cannot be prepared or fetched.
*/
@Test
- void testDownloadWithRemotePropertyEnabled(@TempDir Path dir) throws Exception {
+ void testInstallStoresTheEntryUnderItsSourceName(@TempDir Path dir) throws Exception {
final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8);
final DictionaryCatalog loaded = demoCatalog(dir, payload);
- final Path target = dir.resolve("out.bin");
+ final Path target = dir.resolve("out");
- final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY);
- System.setProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY, "true");
+ final String previous =
+ System.getProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
+ System.setProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY, "true");
try {
- loaded.download("demo", target);
- Assertions.assertArrayEquals(payload, Files.readAllBytes(target));
+ loaded.install("demo", target);
+ Assertions.assertArrayEquals(payload,
+ Files.readAllBytes(target.resolve("dict.bin")));
} finally {
restore(previous);
}
@@ -146,9 +255,9 @@ private static DictionaryCatalog demoCatalog(Path dir, byte[] payload)
*/
private static void restore(String previous) {
if (previous == null) {
- System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY);
+ System.clearProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY);
} else {
- System.setProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY, previous);
+ System.setProperty(DictionaryCatalog.REMOTE_DOWNLOAD_PROPERTY, previous);
}
}
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java
index af85b09a8e..20c9e08cc6 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java
@@ -22,22 +22,41 @@
import java.util.HexFormat;
/**
- * Computes SHA-512 digests for test fixtures.
+ * Computes digests for test fixtures.
*/
public final class DigestTestUtil {
private DigestTestUtil() {
}
+ /**
+ * {@return the SHA-256 digest of {@code bytes} as 64 lowercase hex digits}
+ *
+ * @param bytes The content to digest. Must not be {@code null}.
+ */
+ public static String sha256(byte[] bytes) {
+ return digest("SHA-256", bytes);
+ }
+
/**
* {@return the SHA-512 digest of {@code bytes} as 128 lowercase hex digits}
*
* @param bytes The content to digest. Must not be {@code null}.
*/
public static String sha512(byte[] bytes) {
+ return digest("SHA-512", bytes);
+ }
+
+ /**
+ * {@return the digest of {@code bytes} as lowercase hex digits}
+ *
+ * @param algorithm The digest algorithm name.
+ * @param bytes The content to digest. Must not be {@code null}.
+ */
+ private static String digest(String algorithm, byte[] bytes) {
try {
return HexFormat.of().formatHex(
- MessageDigest.getInstance("SHA-512").digest(bytes));
+ MessageDigest.getInstance(algorithm).digest(bytes));
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException(e);
}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java
deleted file mode 100644
index 493bb20503..0000000000
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package opennlp.tools.util;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.ValueSource;
-
-/**
- * Pins {@link DownloadUtil#download(java.net.URI, Path, String)} against local file URIs
- * so digest verification and the size ceiling are covered without a network.
- */
-public class DownloadUtilFileTest {
-
- /** The fixture bytes the download tests serve and digest. */
- private static final byte[] PAYLOAD = "dictionary-bytes".getBytes(StandardCharsets.UTF_8);
-
- /**
- * Verifies that a download whose bytes match the expected digest lands in the
- * target file.
- *
- * @param dir A scratch directory managed by the test framework.
- * @throws IOException Thrown if the fixture cannot be written or fetched.
- */
- @Test
- void testDownloadAcceptsMatchingDigest(@TempDir Path dir) throws IOException {
- final Path source = dir.resolve("source.bin");
- Files.write(source, PAYLOAD);
- final Path target = dir.resolve("target.bin");
-
- DownloadUtil.download(source.toUri(), target, DigestTestUtil.sha512(PAYLOAD));
-
- Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target));
- }
-
- /**
- * Verifies that a digest mismatch fails the download and leaves no target file
- * behind.
- *
- * @param dir A scratch directory managed by the test framework.
- * @throws IOException Thrown if the fixture cannot be written.
- */
- @Test
- void testDownloadRejectsMismatchedDigest(@TempDir Path dir) throws IOException {
- final Path source = dir.resolve("source.bin");
- Files.write(source, PAYLOAD);
- final Path target = dir.resolve("target.bin");
- final String wrong = DigestTestUtil.sha512("other".getBytes(StandardCharsets.UTF_8));
-
- final IOException e = Assertions.assertThrows(IOException.class,
- () -> DownloadUtil.download(source.toUri(), target, wrong));
- Assertions.assertTrue(e.getMessage().contains("SHA512 checksum validation failed"));
- Assertions.assertTrue(Files.notExists(target));
- }
-
- /** Verifies that a {@code null} digest is rejected with the documented exception. */
- @Test
- void testDownloadRequiresSha512() {
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> DownloadUtil.download(Path.of("x").toUri(), Path.of("y"), null));
- }
-
- /**
- * Verifies that a digest shorter than 128 hex digits is rejected before anything is
- * fetched.
- *
- * @param dir A scratch directory managed by the test framework.
- * @throws IOException Thrown if the fixture cannot be written.
- */
- @Test
- void testDownloadRejectsMalformedSha512(@TempDir Path dir) throws IOException {
- final Path source = dir.resolve("source.bin");
- Files.write(source, PAYLOAD);
-
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> DownloadUtil.download(source.toUri(), dir.resolve("target.bin"), "abc123"));
- }
-
- /**
- * Verifies that a source larger than the byte ceiling fails the download and leaves
- * no target file behind.
- *
- * @param dir A scratch directory managed by the test framework.
- * @throws IOException Thrown if the fixture cannot be written.
- */
- @Test
- void testDownloadRejectsOversizedSource(@TempDir Path dir) throws IOException {
- final Path source = dir.resolve("source.bin");
- Files.write(source, PAYLOAD);
- final Path target = dir.resolve("target.bin");
-
- final IOException e = Assertions.assertThrows(IOException.class,
- () -> DownloadUtil.download(source.toUri(), target,
- DigestTestUtil.sha512(PAYLOAD), PAYLOAD.length - 1));
- Assertions.assertTrue(e.getMessage().contains("exceeds safe limit"));
- Assertions.assertTrue(Files.notExists(target));
- }
-
- /**
- * Pins the inclusive byte ceiling: a source of exactly the ceiling's size still
- * downloads.
- *
- * @param dir A scratch directory managed by the test framework.
- * @throws IOException Thrown if the fixture cannot be written or fetched.
- */
- @Test
- void testDownloadCeilingIsInclusive(@TempDir Path dir) throws IOException {
- final Path source = dir.resolve("source.bin");
- Files.write(source, PAYLOAD);
- final Path target = dir.resolve("target.bin");
-
- DownloadUtil.download(source.toUri(), target,
- DigestTestUtil.sha512(PAYLOAD), PAYLOAD.length);
-
- Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target));
- }
-
- /** Verifies that a positive property value overrides the fallback limit. */
- @Test
- void testConfiguredLimitOverridesFromProperty() {
- final String property = "opennlp.test.limit.override";
- System.setProperty(property, "1024");
- try {
- Assertions.assertEquals(1024L, DownloadUtil.configuredLimit(property, 7L));
- } finally {
- System.clearProperty(property);
- }
- }
-
- /** Verifies that an unset property falls back to the given default. */
- @Test
- void testConfiguredLimitFallsBackWhenAbsent() {
- Assertions.assertEquals(7L,
- DownloadUtil.configuredLimit("opennlp.test.limit.absent", 7L));
- }
-
- /** Verifies that blank, non-numeric, and non-positive values fall back. */
- @ParameterizedTest(name = "value \"{0}\" falls back")
- @ValueSource(strings = {"", " ", "abc", "-1", "0"})
- void testConfiguredLimitRejectsInvalidValues(String invalid) {
- final String property = "opennlp.test.limit.invalid";
- System.setProperty(property, invalid);
- try {
- Assertions.assertEquals(7L, DownloadUtil.configuredLimit(property, 7L));
- } finally {
- System.clearProperty(property);
- }
- }
-
- /** Pins the default download ceiling of 64 MiB when no override property is set. */
- @Test
- void testDefaultBudgetsWithoutOverrides() {
- Assertions.assertEquals(64L * 1024 * 1024, DownloadUtil.MAX_DOWNLOAD_BYTES);
- }
-}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/InstallerTestSupport.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/InstallerTestSupport.java
new file mode 100644
index 0000000000..a341bc9c20
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/InstallerTestSupport.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Stream;
+
+import opennlp.tools.util.archive.TarArchives;
+
+/**
+ * Shared fixtures for the {@code ResourceInstaller} test classes: tar archive building,
+ * gzip compression, digest computation, and installed-file listing.
+ */
+final class InstallerTestSupport {
+
+ static final int BLOCK = TarArchives.BLOCK;
+ static final int TERMINATOR_SIZE = TarArchives.TERMINATOR_SIZE;
+
+ /** One kibibyte, a convenient small ceiling for limit tests. */
+ static final long KIBIBYTE = 1024;
+
+ /** One mebibyte, a convenient generous ceiling for tests that do not exercise it. */
+ static final long MEBIBYTE = 1024 * KIBIBYTE;
+
+ private InstallerTestSupport() {
+ }
+
+ /**
+ * Writes one regular-file tar entry into the given buffer.
+ *
+ * @param tar The buffer receiving the entry bytes. Must not be {@code null}.
+ * @param name The entry name; at most 100 bytes when encoded as UTF-8.
+ * @param content The entry content. Must not be {@code null}.
+ * @throws IOException Thrown if writing to the buffer fails.
+ * @throws IllegalArgumentException Thrown if the name exceeds the tar name field.
+ */
+ static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content)
+ throws IOException {
+ TarArchives.entry(tar, name, content);
+ }
+
+ /**
+ * Builds a gzip-compressed tar archive from name and content pairs.
+ *
+ * @param entries Pairs of entry name and UTF-8 text content. Must not be {@code null}.
+ * @return The archive bytes. Never {@code null}.
+ * @throws IOException Thrown if assembling the archive fails.
+ */
+ static byte[] tarGz(String[][] entries) throws IOException {
+ return TarArchives.gzippedTar(entries);
+ }
+
+ /**
+ * Compresses the given bytes with gzip.
+ *
+ * @param content The bytes to compress. Must not be {@code null}.
+ * @return The gzip-compressed bytes. Never {@code null}.
+ * @throws IOException Thrown if compressing fails.
+ */
+ static byte[] gzip(byte[] content) throws IOException {
+ return TarArchives.gzip(content);
+ }
+
+ /**
+ * Computes the SHA-256 of the given bytes as a lowercase hex string.
+ *
+ * @param content The bytes to digest. Must not be {@code null}.
+ * @return The 64-character lowercase hex digest. Never {@code null}.
+ */
+ static String sha256(byte[] content) {
+ return DigestTestUtil.sha256(content);
+ }
+
+ /**
+ * Computes the SHA-512 of the given bytes as a lowercase hex string.
+ *
+ * @param content The bytes to digest. Must not be {@code null}.
+ * @return The 128-character lowercase hex digest. Never {@code null}.
+ */
+ static String sha512(byte[] content) {
+ return DigestTestUtil.sha512(content);
+ }
+
+ /**
+ * Lists every regular file below the given directory as relative paths with forward
+ * slashes, sorted lexicographically, so tests can assert the exact installed file
+ * set.
+ *
+ * @param root The directory to walk. Must not be {@code null}.
+ * @return The sorted relative paths. Never {@code null}.
+ * @throws IOException Thrown if walking the directory fails.
+ */
+ static List installedFiles(Path root) throws IOException {
+ try (Stream walk = Files.walk(root)) {
+ return walk.filter(Files::isRegularFile)
+ .map(file -> root.relativize(file).toString().replace('\\', '/'))
+ .sorted()
+ .toList();
+ }
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerHttpTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerHttpTest.java
new file mode 100644
index 0000000000..7e27bb528b
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerHttpTest.java
@@ -0,0 +1,635 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.function.Executable;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import static opennlp.tools.util.InstallerTestSupport.KIBIBYTE;
+import static opennlp.tools.util.InstallerTestSupport.MEBIBYTE;
+import static opennlp.tools.util.InstallerTestSupport.installedFiles;
+import static opennlp.tools.util.InstallerTestSupport.sha256;
+import static opennlp.tools.util.InstallerTestSupport.tarGz;
+
+/**
+ * Exercises {@link ResourceInstaller} against a local scripted HTTP server: happy
+ * downloads, redirect handling and its policy, error statuses, stalled responses
+ * against the read timeout, and download limits against incorrect or oversized bodies.
+ */
+public class ResourceInstallerHttpTest {
+
+ private static final Duration GENEROUS = Duration.ofSeconds(10);
+
+ /** The limits these tests never exercise, kept at their defaults. */
+ private static final long DEFAULT_ENTRIES =
+ ResourceInstaller.Limits.DEFAULT.maxEntries();
+ private static final long DEFAULT_RATIO =
+ ResourceInstaller.Limits.DEFAULT.maxExpansionRatio();
+
+ /** Long enough that a stalled route outlives any timeout a test configures. */
+ private static final Duration STALL = Duration.ofSeconds(30);
+
+ /**
+ * A well-formed digest for fetches that fail before verification runs; its value is
+ * never compared.
+ */
+ private static final String UNREACHED_CHECKSUM = "0".repeat(64);
+
+ private StubServer server;
+
+ @BeforeEach
+ void startServer() throws IOException {
+ server = new StubServer();
+ }
+
+ @AfterEach
+ void stopServer() throws IOException {
+ server.close();
+ }
+
+ /**
+ * Builds installation limits with the given read timeout and otherwise generous
+ * values, so timeout tests state only the value they exercise.
+ *
+ * @param readTimeout The read timeout to apply.
+ * @return The limits. Never {@code null}.
+ */
+ private static ResourceInstaller.Limits withReadTimeout(Duration readTimeout) {
+ return new ResourceInstaller.Limits(GENEROUS, readTimeout, 5, MEBIBYTE, MEBIBYTE,
+ DEFAULT_ENTRIES, DEFAULT_RATIO);
+ }
+
+ /**
+ * Builds installation limits with the given redirect allowance and otherwise
+ * generous values.
+ *
+ * @param maxRedirects The number of redirects to follow.
+ * @return The limits. Never {@code null}.
+ */
+ private static ResourceInstaller.Limits withMaxRedirects(int maxRedirects) {
+ return new ResourceInstaller.Limits(GENEROUS, GENEROUS, maxRedirects,
+ MEBIBYTE, MEBIBYTE, DEFAULT_ENTRIES, DEFAULT_RATIO);
+ }
+
+ /**
+ * Builds installation limits with the given download limit and otherwise generous
+ * values.
+ *
+ * @param maxDownloadBytes The download limit in bytes.
+ * @return The limits. Never {@code null}.
+ */
+ private static ResourceInstaller.Limits withDownloadLimit(long maxDownloadBytes) {
+ return new ResourceInstaller.Limits(GENEROUS, GENEROUS, 5, maxDownloadBytes,
+ MEBIBYTE, DEFAULT_ENTRIES, DEFAULT_RATIO);
+ }
+
+ /**
+ * Asserts that the given call is rejected as an argument error demanding a checksum
+ * for the remote source.
+ *
+ * @param source The remote source the message must name.
+ * @param call The call under test.
+ */
+ private static void assertChecksumRequired(URI source, Executable call) {
+ final IllegalArgumentException thrown =
+ Assertions.assertThrows(IllegalArgumentException.class, call);
+ Assertions.assertEquals(
+ "checksum must be given for an http or https source: " + source,
+ thrown.getMessage());
+ }
+
+ @Test
+ void testHttpSourceWithoutChecksumIsRejectedBeforeFetching(@TempDir Path target)
+ throws Exception {
+ final AtomicBoolean fetched = new AtomicBoolean();
+ server.route("/corpus.tar.gz", out -> {
+ fetched.set(true);
+ StubServer.ok(out, tarGz(new String[][] {{"corpus/data.txt", "unverified"}}));
+ });
+
+ final URI source = server.uri("/corpus.tar.gz");
+ Assertions.assertAll(
+ () -> assertChecksumRequired(source,
+ () -> ResourceInstaller.install(source, target)),
+ () -> assertChecksumRequired(source,
+ () -> ResourceInstaller.install(source, target, null)),
+ () -> assertChecksumRequired(source,
+ () -> ResourceInstaller.install(source, target, null, withMaxRedirects(5))));
+ Assertions.assertFalse(fetched.get());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testHttpsSourceWithoutChecksumIsRejectedBeforeCreatingTheTarget(
+ @TempDir Path parent) {
+ final Path target = parent.resolve("not-created-yet");
+ final URI source = URI.create("https://example.invalid/corpus.tar.gz");
+
+ assertChecksumRequired(source, () -> ResourceInstaller.install(source, target));
+ Assertions.assertTrue(Files.notExists(target));
+ }
+
+ @Test
+ void testHttpDownloadInstallsArchive(@TempDir Path target) throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "over http"}});
+ server.route("/corpus.tar.gz", out -> StubServer.ok(out, archive));
+
+ ResourceInstaller.install(server.uri("/corpus.tar.gz"), target, sha256(archive));
+
+ Assertions.assertEquals("over http",
+ Files.readString(target.resolve("corpus/data.txt")));
+ }
+
+ @Test
+ void testInvalidSourceNameIsRejectedBeforeFetching(@TempDir Path parent)
+ throws Exception {
+ final AtomicBoolean fetched = new AtomicBoolean();
+ final byte[] content = "dictionary".getBytes(StandardCharsets.UTF_8);
+ server.route("/bad%00name.dat", out -> {
+ fetched.set(true);
+ StubServer.ok(out, content);
+ });
+ final Path target = parent.resolve("not-created-yet");
+
+ final IllegalArgumentException thrown = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> ResourceInstaller.install(server.uri("/bad%00name.dat"), target,
+ sha256(content)));
+
+ Assertions.assertEquals("name must be a file name", thrown.getMessage());
+ Assertions.assertFalse(fetched.get());
+ Assertions.assertTrue(Files.notExists(target));
+ }
+
+ @Test
+ void testAbsoluteRedirectIsFollowed(@TempDir Path target) throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "moved"}});
+ server.route("/old.tar.gz", out -> StubServer.redirect(out,
+ server.uri("/new.tar.gz").toString()));
+ server.route("/new.tar.gz", out -> StubServer.ok(out, archive));
+
+ ResourceInstaller.install(server.uri("/old.tar.gz"), target, sha256(archive));
+
+ Assertions.assertEquals("moved",
+ Files.readString(target.resolve("corpus/data.txt")));
+ }
+
+ @Test
+ void testRelativeRedirectIsResolvedAgainstTheSource(@TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "relative"}});
+ server.route("/mirror/old.tar.gz",
+ out -> StubServer.redirect(out, "new.tar.gz"));
+ server.route("/mirror/new.tar.gz", out -> StubServer.ok(out, archive));
+
+ ResourceInstaller.install(server.uri("/mirror/old.tar.gz"), target, sha256(archive));
+
+ Assertions.assertEquals("relative",
+ Files.readString(target.resolve("corpus/data.txt")));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @ValueSource(strings = {"301 Moved Permanently", "302 Found", "303 See Other",
+ "307 Temporary Redirect", "308 Permanent Redirect"})
+ void testEveryRedirectStatusIsFollowed(String status, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "followed"}});
+ server.route("/old.tar.gz", out -> StubServer.redirect(out, status, "/new.tar.gz"));
+ server.route("/new.tar.gz", out -> StubServer.ok(out, archive));
+
+ ResourceInstaller.install(server.uri("/old.tar.gz"), target, sha256(archive));
+
+ Assertions.assertEquals("followed",
+ Files.readString(target.resolve("corpus/data.txt")));
+ }
+
+ @Test
+ void testZeroRedirectAllowanceRejectsTheFirstRedirect(@TempDir Path target)
+ throws Exception {
+ server.route("/once", out -> StubServer.redirect(out,
+ server.uri("/anywhere.tar.gz").toString()));
+
+ final URI source = server.uri("/once");
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(source, target, UNREACHED_CHECKSUM, withMaxRedirects(0)));
+ Assertions.assertEquals("more than 0 redirects: " + source, thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testMalformedRedirectLocationFails() {
+ final URI from = URI.create("http://example.invalid/archive.tar.gz");
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.resolveRedirect(from, "http://mirror.invalid/bad path"));
+ Assertions.assertEquals("redirect from " + from
+ + " contains a malformed Location: http://mirror.invalid/bad path",
+ thrown.getMessage());
+ }
+
+ @Test
+ void testRedirectChainBeyondLimitFails(@TempDir Path target) throws Exception {
+ server.route("/loop", out -> StubServer.redirect(out, "/loop"));
+
+ final URI source = server.uri("/loop");
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(source, target, UNREACHED_CHECKSUM, withMaxRedirects(3)));
+ Assertions.assertEquals("more than 3 redirects: " + source, thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testRedirectWithoutLocationFails(@TempDir Path target) throws Exception {
+ server.route("/broken", out -> StubServer.head(out, "302 Found",
+ "Content-Length: 4", "", "gone"));
+
+ final URI source = server.uri("/broken");
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(source, target, UNREACHED_CHECKSUM));
+ Assertions.assertEquals("redirect from " + source + " contains no Location header",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testRedirectToNonHttpSchemeFails(@TempDir Path target) throws Exception {
+ server.route("/non-http", out -> StubServer.redirect(out, "file:///etc/passwd"));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(server.uri("/non-http"), target,
+ UNREACHED_CHECKSUM));
+ Assertions.assertEquals(
+ "redirect target is not an http or https location: file:///etc/passwd",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testHttpsToHttpDowngradeIsRejected() {
+ final URI https = URI.create("https://example.invalid/archive.tar.gz");
+
+ // Cover the rejected downgrade and both accepted upgrade and same-scheme cases.
+ Assertions.assertAll(
+ () -> {
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.resolveRedirect(https,
+ "http://example.invalid/archive.tar.gz"));
+ Assertions.assertEquals(
+ "redirect downgrades https to http: http://example.invalid/archive.tar.gz",
+ thrown.getMessage());
+ },
+ () -> Assertions.assertDoesNotThrow(() -> ResourceInstaller.resolveRedirect(https,
+ "https://mirror.invalid/archive.tar.gz")),
+ () -> Assertions.assertDoesNotThrow(() -> ResourceInstaller.resolveRedirect(
+ URI.create("http://example.invalid/archive.tar.gz"),
+ "https://mirror.invalid/archive.tar.gz")));
+ }
+
+ @Test
+ void testHttpErrorStatusFails(@TempDir Path target) throws Exception {
+ server.route("/missing.tar.gz", out -> StubServer.head(out, "404 Not Found",
+ "Content-Length: 0", ""));
+
+ final URI source = server.uri("/missing.tar.gz");
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(source, target, UNREACHED_CHECKSUM));
+ Assertions.assertEquals("download failed with HTTP status 404: " + source,
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testStalledResponseHitsReadTimeout(@TempDir Path target) {
+ server.route("/stall", out -> StubServer.sleep(STALL));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(server.uri("/stall"), target,
+ UNREACHED_CHECKSUM, withReadTimeout(Duration.ofMillis(250))));
+ Assertions.assertInstanceOf(SocketTimeoutException.class, thrown);
+ }
+
+ @Test
+ void testStalledBodyHitsReadTimeout(@TempDir Path target) throws Exception {
+ server.route("/drip", out -> {
+ StubServer.head(out, "200 OK", "Content-Length: 100000", "");
+ out.write("just a few bytes".getBytes(StandardCharsets.UTF_8));
+ out.flush();
+ StubServer.sleep(STALL);
+ });
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(server.uri("/drip"), target,
+ UNREACHED_CHECKSUM, withReadTimeout(Duration.ofMillis(250))));
+ Assertions.assertInstanceOf(SocketTimeoutException.class, thrown);
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testDeclaredContentLengthBeyondLimitFailsFast(@TempDir Path target)
+ throws Exception {
+ server.route("/liar.tar.gz", out -> StubServer.head(out, "200 OK",
+ "Content-Length: 10000000", ""));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(server.uri("/liar.tar.gz"), target,
+ UNREACHED_CHECKSUM, withDownloadLimit(KIBIBYTE)));
+ Assertions.assertEquals(
+ "declared content length 10000000 exceeds the download limit of 1024 bytes",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testStreamedBodyBeyondLimitAborts(@TempDir Path target) throws Exception {
+ server.route("/endless", out -> {
+ StubServer.head(out, "200 OK", "");
+ out.write(new byte[64 * 1024]);
+ out.flush();
+ });
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(server.uri("/endless"), target,
+ UNREACHED_CHECKSUM, withDownloadLimit(KIBIBYTE)));
+ Assertions.assertEquals("download exceeds the limit of 1024 bytes",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ @Timeout(value = 15, threadMode = Timeout.ThreadMode.SEPARATE_THREAD)
+ void testSubMillisecondReadTimeoutStillTimesOut(@TempDir Path target) {
+ server.route("/stall", out -> StubServer.sleep(STALL));
+ final ResourceInstaller.Limits limits = ResourceInstaller.Limits.builder()
+ .readTimeout(Duration.ofNanos(1))
+ .build();
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(server.uri("/stall"), target,
+ UNREACHED_CHECKSUM, limits));
+
+ Assertions.assertInstanceOf(SocketTimeoutException.class, thrown);
+ }
+
+
+ @Test
+ void testTimeoutBeyondTheMillisecondRangeIsCapped(@TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"payload/data.txt", "content"}});
+ server.route("/payload.tar.gz", out -> StubServer.ok(out, archive));
+ final Duration beyondMillis = Duration.ofSeconds(Long.MAX_VALUE / 1000 + 1);
+ final ResourceInstaller.Limits limits = ResourceInstaller.Limits.builder()
+ .connectTimeout(beyondMillis)
+ .readTimeout(beyondMillis)
+ .build();
+
+ ResourceInstaller.install(server.uri("/payload.tar.gz"), target, sha256(archive),
+ limits);
+
+ Assertions.assertEquals("content",
+ Files.readString(target.resolve("payload/data.txt")));
+ }
+
+ /**
+ * A scripted HTTP server on a loopback socket. Each registered route writes a raw
+ * response for redirect, timeout, malformed-response, and size-limit tests.
+ */
+ private static final class StubServer implements AutoCloseable {
+
+ /** Writes a raw HTTP response to the connected client. */
+ @FunctionalInterface
+ interface Responder {
+
+ /**
+ * Writes the raw response bytes for one request.
+ *
+ * @param out The response stream.
+ * @throws IOException Thrown if writing fails.
+ */
+ void respond(OutputStream out) throws IOException;
+ }
+
+ private final ServerSocket socket;
+ private final Map routes = new ConcurrentHashMap<>();
+ private final List connections = new ArrayList<>();
+
+ StubServer() throws IOException {
+ socket = new ServerSocket(0, 50, InetAddress.getLoopbackAddress());
+ final Thread acceptor = new Thread(this::acceptLoop, "stub-http-acceptor");
+ acceptor.setDaemon(true);
+ acceptor.start();
+ }
+
+ /**
+ * Registers the responder serving the given absolute request path.
+ *
+ * @param path The absolute request path, starting with {@code /}.
+ * @param responder The script producing the raw response.
+ */
+ void route(String path, Responder responder) {
+ routes.put(path, responder);
+ }
+
+ /**
+ * Builds the http URI of the given absolute path on this server.
+ *
+ * @param path The absolute request path, starting with {@code /}.
+ * @return The URI. Never {@code null}.
+ */
+ URI uri(String path) {
+ return URI.create("http://127.0.0.1:" + socket.getLocalPort() + path);
+ }
+
+ /**
+ * Writes the HTTP/1.0 status line followed by the given lines, each terminated by
+ * CRLF. The caller supplies the header lines, then an empty string for the blank
+ * line separating head from body, then optional body text.
+ *
+ * @param out The response stream.
+ * @param status The status line content after the protocol, such as {@code 200 OK}.
+ * @param lines Header lines, then an empty string separator, then optional body
+ * text, each written with a trailing CRLF.
+ * @throws IOException Thrown if writing fails.
+ */
+ static void head(OutputStream out, String status, String... lines)
+ throws IOException {
+ final StringBuilder response = new StringBuilder("HTTP/1.0 ").append(status)
+ .append("\r\n");
+ for (final String line : lines) {
+ response.append(line).append("\r\n");
+ }
+ out.write(response.toString().getBytes(StandardCharsets.US_ASCII));
+ out.flush();
+ }
+
+ /**
+ * Writes a complete 200 response carrying the given body with its exact length.
+ *
+ * @param out The response stream.
+ * @param body The response body bytes.
+ * @throws IOException Thrown if writing fails.
+ */
+ static void ok(OutputStream out, byte[] body) throws IOException {
+ head(out, "200 OK", "Content-Length: " + body.length, "");
+ out.write(body);
+ out.flush();
+ }
+
+ /**
+ * Writes a 302 redirect to the given location.
+ *
+ * @param out The response stream.
+ * @param location The Location header value, absolute or relative.
+ * @throws IOException Thrown if writing fails.
+ */
+ static void redirect(OutputStream out, String location) throws IOException {
+ redirect(out, "302 Found", location);
+ }
+
+ /**
+ * Writes a redirect with the given status to the given location.
+ *
+ * @param out The response stream.
+ * @param status The status line content after the protocol, such as
+ * {@code 301 Moved Permanently}.
+ * @param location The Location header value, absolute or relative.
+ * @throws IOException Thrown if writing fails.
+ */
+ static void redirect(OutputStream out, String status, String location)
+ throws IOException {
+ head(out, status, "Location: " + location, "Content-Length: 0", "");
+ }
+
+ /**
+ * Blocks the handler thread, simulating a stalled server. Interruption during
+ * teardown ends the sleep early.
+ *
+ * @param duration How long to stall.
+ */
+ static void sleep(Duration duration) {
+ try {
+ Thread.sleep(duration.toMillis());
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private void acceptLoop() {
+ while (!socket.isClosed()) {
+ final Socket connection;
+ try {
+ connection = socket.accept();
+ } catch (IOException e) {
+ return;
+ }
+ synchronized (connections) {
+ connections.add(connection);
+ }
+ final Thread handler = new Thread(() -> handle(connection),
+ "stub-http-handler");
+ handler.setDaemon(true);
+ handler.start();
+ }
+ }
+
+ /**
+ * Reads one request, dispatches it to the registered responder, and closes the
+ * connection. Write failures from clients that abort mid-transfer are expected
+ * and ignored.
+ *
+ * @param connection The accepted client connection.
+ */
+ private void handle(Socket connection) {
+ try (connection) {
+ final String path = readRequestPath(connection.getInputStream());
+ final Responder responder = routes.get(path);
+ if (responder == null) {
+ head(connection.getOutputStream(), "404 Not Found", "Content-Length: 0", "");
+ return;
+ }
+ responder.respond(connection.getOutputStream());
+ } catch (IOException e) {
+ // The client hung up or the server is shutting down; both are test-normal.
+ }
+ }
+
+ /**
+ * Reads the request head up to its terminating blank line and returns the path
+ * from the request line.
+ *
+ * @param in The request stream.
+ * @return The request path. Never {@code null}.
+ * @throws IOException Thrown if the request head is malformed or truncated.
+ */
+ private static String readRequestPath(InputStream in) throws IOException {
+ final ByteArrayOutputStream headBytes = new ByteArrayOutputStream();
+ int last4 = 0;
+ int b;
+ while ((b = in.read()) >= 0) {
+ headBytes.write(b);
+ last4 = (last4 << 8) | b;
+ if (last4 == 0x0D0A0D0A) {
+ break;
+ }
+ }
+ final String head = headBytes.toString(StandardCharsets.US_ASCII);
+ final int firstLineEnd = head.indexOf("\r\n");
+ final String requestLine = firstLineEnd < 0 ? head : head.substring(0, firstLineEnd);
+ final String[] parts = requestLine.split(" ");
+ if (parts.length < 2) {
+ throw new IOException("malformed request line: " + requestLine);
+ }
+ return parts[1];
+ }
+
+ @Override
+ public void close() throws IOException {
+ socket.close();
+ synchronized (connections) {
+ for (final Socket connection : connections) {
+ connection.close();
+ }
+ }
+ }
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerTest.java
new file mode 100644
index 0000000000..f336234142
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerTest.java
@@ -0,0 +1,1412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.NoSuchAlgorithmException;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Stream;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.api.function.Executable;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.util.archive.TarArchives;
+
+import static opennlp.tools.util.InstallerTestSupport.BLOCK;
+import static opennlp.tools.util.InstallerTestSupport.KIBIBYTE;
+import static opennlp.tools.util.InstallerTestSupport.MEBIBYTE;
+import static opennlp.tools.util.InstallerTestSupport.TERMINATOR_SIZE;
+import static opennlp.tools.util.InstallerTestSupport.gzip;
+import static opennlp.tools.util.InstallerTestSupport.installedFiles;
+import static opennlp.tools.util.InstallerTestSupport.sha256;
+import static opennlp.tools.util.InstallerTestSupport.sha512;
+import static opennlp.tools.util.InstallerTestSupport.tarEntry;
+import static opennlp.tools.util.InstallerTestSupport.tarGz;
+
+public class ResourceInstallerTest {
+
+ private static final String CHECKSUM_ARGUMENT_ERROR =
+ "checksum must be 64 (SHA-256) or 128 (SHA-512) hex characters; pass null to skip";
+ private static final String ESCAPE_ERROR =
+ "archive entry escapes the target directory: ";
+ private static final String EXPANSION_LIMIT_ERROR =
+ "expanded content exceeds the limit of " + KIBIBYTE + " bytes";
+ private static final String ENTRY_LIMIT_ERROR =
+ "archive entry count exceeds the limit of 2 entries";
+ private static final String COLLISION_ERROR = "target already contains: ";
+ private static final String DUPLICATE_ENTRY_ERROR =
+ "archive contains duplicate file entry: ";
+ private static final String RATIO_ERROR =
+ "content expands beyond 100 times its compressed size";
+ private static final String ZIP_MISMATCH_ERROR =
+ "zip local headers and central directory list different files";
+
+ /** The property name used by the parser tests; never read by the installer. */
+ private static final String TEST_LIMIT_PROPERTY = "opennlp.test.limit";
+
+ /**
+ * Installs the given file under the default limits and asserts that it fails with the
+ * expected message, leaving the target directory without a single installed file.
+ *
+ * @param file The source file to install.
+ * @param target The target directory, which must have been empty before the attempt.
+ * @param message The exact failure message expected.
+ * @throws IOException Thrown if listing the target directory fails.
+ */
+ private static void assertInstallFails(Path file, Path target, String message)
+ throws IOException {
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target));
+ Assertions.assertEquals(message, thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Installs the given file under the given limits and asserts that it fails with the
+ * expected message, leaving the target directory without a single installed file.
+ *
+ * @param file The source file to install.
+ * @param target The target directory, which must have been empty before the attempt.
+ * @param limits The limits to install under.
+ * @param message The exact failure message expected.
+ * @throws IOException Thrown if listing the target directory fails.
+ */
+ private static void assertInstallFails(Path file, Path target,
+ ResourceInstaller.Limits limits, String message) throws IOException {
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target, null, limits));
+ Assertions.assertEquals(message, thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Computes the SHA-256 of the given bytes as an uppercase hex string, so tests can
+ * prove that checksum comparison does not depend on the hex letter case.
+ *
+ * @param content The bytes to digest. Must not be {@code null}.
+ * @return The 64-character uppercase hex digest. Never {@code null}.
+ * @throws NoSuchAlgorithmException Thrown if the digest algorithm is unavailable.
+ */
+ private static String sha256UpperCase(byte[] content) throws NoSuchAlgorithmException {
+ return sha256(content).toUpperCase(Locale.ROOT);
+ }
+
+ /**
+ * Builds installation limits with generous timeouts and redirect allowance but the
+ * given size limits, so limit tests state only the values they exercise.
+ *
+ * @param maxDownloadBytes The download limit in bytes.
+ * @param maxExpandedBytes The expansion limit in bytes.
+ * @return The limits. Never {@code null}.
+ */
+ private static ResourceInstaller.Limits limits(long maxDownloadBytes,
+ long maxExpandedBytes) {
+ return new ResourceInstaller.Limits(Duration.ofSeconds(10), Duration.ofSeconds(10),
+ 5, maxDownloadBytes, maxExpandedBytes,
+ ResourceInstaller.Limits.DEFAULT.maxEntries(),
+ ResourceInstaller.Limits.DEFAULT.maxExpansionRatio());
+ }
+
+ @Test
+ void testInstallEndToEndUsageExample(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {
+ {"corpus/README", "A tiny example corpus.\n"},
+ {"corpus/tokens.txt", "the\ncat\n"},
+ {"corpus/pos/tags.tsv", "the\tDET\ncat\tNOUN\n"}});
+ final Path file = source.resolve("corpus.tar.gz");
+ Files.write(file, archive);
+
+ final Path result = ResourceInstaller.install(file.toUri(), target, sha256(archive));
+
+ Assertions.assertEquals(target, result);
+ Assertions.assertEquals(
+ List.of("corpus/README", "corpus/pos/tags.tsv", "corpus/tokens.txt"),
+ installedFiles(target));
+ Assertions.assertEquals("A tiny example corpus.\n",
+ Files.readString(target.resolve("corpus/README")));
+ Assertions.assertEquals("the\ncat\n",
+ Files.readString(target.resolve("corpus/tokens.txt")));
+ Assertions.assertEquals("the\tDET\ncat\tNOUN\n",
+ Files.readString(target.resolve("corpus/pos/tags.tsv")));
+ }
+
+ @Test
+ void testTarGzUnpacksWithStructure(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {
+ {"corpus-1.0/train.conllu", "# sent_id = 1\n"},
+ {"corpus-1.0/sub/readme.txt", "hello"}});
+ final Path file = source.resolve("corpus.tgz");
+ Files.write(file, archive);
+
+ ResourceInstaller.install(file.toUri(), target, sha256(archive));
+
+ Assertions.assertEquals("# sent_id = 1\n",
+ Files.readString(target.resolve("corpus-1.0/train.conllu")));
+ Assertions.assertEquals("hello",
+ Files.readString(target.resolve("corpus-1.0/sub/readme.txt")));
+ }
+
+ @Test
+ void testChecksumMismatchFailsBeforeUnpacking(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"a/file.txt", "content"}});
+ final Path file = source.resolve("archive.tar.gz");
+ Files.write(file, archive);
+
+ final String wrong = "0".repeat(64);
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target, wrong));
+ Assertions.assertEquals("checksum mismatch: expected " + wrong
+ + " but downloaded " + sha256(archive), thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testChecksumComparisonIgnoresHexLetterCase(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"data/entry.txt", "payload"}});
+ final Path file = source.resolve("cased.tar.gz");
+ Files.write(file, archive);
+ // The uppercase digest must differ textually from the lowercase one, otherwise
+ // this test would not exercise the case handling at all.
+ Assertions.assertNotEquals(sha256(archive), sha256UpperCase(archive));
+
+ ResourceInstaller.install(file.toUri(), target, sha256UpperCase(archive));
+
+ Assertions.assertEquals("payload",
+ Files.readString(target.resolve("data/entry.txt")));
+ }
+
+ @Test
+ void testChecksumIgnoresUnicodeWhitespace(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "verified"}});
+ final Path file = source.resolve("unicode-space.tar.gz");
+ Files.write(file, archive);
+ final String emSpace = Character.toString(0x2003);
+
+ ResourceInstaller.install(file.toUri(), target,
+ emSpace + sha256(archive) + emSpace);
+
+ Assertions.assertEquals("verified",
+ Files.readString(target.resolve("corpus/data.txt")));
+ }
+
+ @Test
+ void testEscapingEntriesAreRejected(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"../escape.txt", "bad"}});
+ final Path file = source.resolve("evil.tar.gz");
+ Files.write(file, archive);
+
+ assertInstallFails(file, target, ESCAPE_ERROR + "../escape.txt");
+ Assertions.assertTrue(Files.notExists(target.getParent().resolve("escape.txt")));
+ }
+
+ @Test
+ void testAbsoluteTarEntryIsRejected(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {
+ {"/absolute-escape-attempt/evil.txt", "bad"}});
+ final Path file = source.resolve("absolute.tar.gz");
+ Files.write(file, archive);
+
+ assertInstallFails(file, target,
+ ESCAPE_ERROR + "/absolute-escape-attempt/evil.txt");
+ Assertions.assertTrue(Files.notExists(Path.of("/absolute-escape-attempt")));
+ }
+
+ @Test
+ void testEscapingTarDirectoryEntryIsRejected(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ TarArchives.entry(tar, "../outside/", new byte[0], TarArchives.TYPE_DIRECTORY);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final Path file = source.resolve("directory-escape.tar.gz");
+ Files.write(file, gzip(tar.toByteArray()));
+
+ assertInstallFails(file, target, ESCAPE_ERROR + "../outside/");
+ }
+
+ @Test
+ void testZipEntryWithTraversalIsRejected(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("../zip-escape.txt"));
+ zip.write("bad".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ final Path file = source.resolve("evil.zip");
+ Files.write(file, out.toByteArray());
+
+ assertInstallFails(file, target, ESCAPE_ERROR + "../zip-escape.txt");
+ Assertions.assertTrue(Files.notExists(target.getParent().resolve("zip-escape.txt")));
+ }
+
+ @Test
+ void testEscapingZipDirectoryEntryIsRejected(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("../outside/"));
+ zip.closeEntry();
+ }
+ final Path file = source.resolve("directory-escape.zip");
+ Files.write(file, out.toByteArray());
+
+ assertInstallFails(file, target, ESCAPE_ERROR + "../outside/");
+ }
+
+ @Test
+ void testZipUnpacks(@TempDir Path source, @TempDir Path target) throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("lexicon/words.txt"));
+ zip.write("cat 100\n".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ final Path file = source.resolve("lexicon.zip");
+ Files.write(file, out.toByteArray());
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ Assertions.assertEquals("cat 100\n",
+ Files.readString(target.resolve("lexicon/words.txt")));
+ }
+
+ /**
+ * Checks that ZIP validation works when the target uses a non-default file system.
+ *
+ * @param source The directory containing the source ZIP file.
+ * @param scratch The directory containing the target file system.
+ * @throws IOException Thrown if the fixture or installation cannot be read or written.
+ */
+ @Test
+ void testZipUnpacksIntoANonDefaultFileSystem(@TempDir Path source,
+ @TempDir Path scratch) throws IOException {
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(bytes)) {
+ zip.putNextEntry(new ZipEntry("payload/data.txt"));
+ zip.write("content".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ final Path archive = source.resolve("payload.zip");
+ Files.write(archive, bytes.toByteArray());
+ final URI targetUri = URI.create("jar:" + scratch.resolve("target.zip").toUri());
+
+ try (FileSystem fileSystem =
+ FileSystems.newFileSystem(targetUri, Map.of("create", "true"))) {
+ final Path target = fileSystem.getPath("/installed");
+
+ ResourceInstaller.install(archive.toUri(), target);
+
+ Assertions.assertEquals("content",
+ Files.readString(target.resolve("payload/data.txt")));
+ }
+ }
+
+ @Test
+ void testModelBinIsStoredPacked(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("manifest.properties"));
+ zip.write("OpenNLP-Version: 0.0.0\n".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ final Path file = source.resolve("en-ner-person.bin");
+ Files.write(file, out.toByteArray());
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ Assertions.assertEquals(List.of("en-ner-person.bin"), installedFiles(target));
+ Assertions.assertArrayEquals(out.toByteArray(),
+ Files.readAllBytes(target.resolve("en-ner-person.bin")));
+ }
+
+ @Test
+ void testPlainGzipDecompressesToTheSourceName(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (GZIPOutputStream gzip = new GZIPOutputStream(out)) {
+ gzip.write("word\tlemma\n".getBytes(StandardCharsets.UTF_8));
+ }
+ final Path file = source.resolve("lexicon.tsv.gz");
+ Files.write(file, out.toByteArray());
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ Assertions.assertEquals("word\tlemma\n",
+ Files.readString(target.resolve("lexicon.tsv")));
+ }
+
+ @Test
+ void testPlainGzipWithoutABaseNameUsesResource(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final Path file = source.resolve(".gz");
+ Files.write(file, gzip("word\tlemma\n".getBytes(StandardCharsets.UTF_8)));
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ Assertions.assertEquals("word\tlemma\n",
+ Files.readString(target.resolve("resource")));
+ }
+
+ @Test
+ void testPlainFilesAreStoredUnderTheirSourceName(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final Path file = source.resolve("frequencies.txt");
+ Files.writeString(file, "cat 100");
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ Assertions.assertEquals("cat 100", Files.readString(target.resolve("frequencies.txt")));
+ }
+
+ /**
+ * Checks each required installation parameter.
+ *
+ * @param argument The invalid method parameter.
+ * @param target A scratch directory managed by the test framework.
+ */
+ @ParameterizedTest(name = "{0}")
+ @ValueSource(strings = {"source", "targetDirectory", "limits"})
+ void testInvalidArguments(String argument, @TempDir Path target) {
+ final Executable call = switch (argument) {
+ case "source" -> () -> ResourceInstaller.install(null, target);
+ case "targetDirectory" -> () -> ResourceInstaller.install(target.toUri(), null);
+ case "limits" ->
+ () -> ResourceInstaller.install(target.toUri(), target, null, null);
+ default -> throw new IllegalArgumentException("unknown argument: " + argument);
+ };
+
+ assertArgumentError(argument + " must not be null", call);
+ }
+
+ /**
+ * Supplies checksum arguments that are neither a valid SHA-256 nor a valid SHA-512
+ * digest.
+ *
+ * @return One case per rejected digest. Never {@code null}.
+ */
+ static Stream rejectedChecksums() {
+ return Stream.of(
+ Arguments.of("blank", " "),
+ Arguments.of("too short", "abc123"),
+ Arguments.of("right length, non-hex characters", "g".repeat(64)),
+ Arguments.of("between the two supported lengths", "a".repeat(96)),
+ Arguments.of("longer than SHA-512", "a".repeat(129)));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("rejectedChecksums")
+ void testInvalidChecksumIsRejected(String label, String checksum, @TempDir Path target) {
+ assertArgumentError(CHECKSUM_ARGUMENT_ERROR,
+ () -> ResourceInstaller.install(target.toUri(), target, checksum));
+ }
+
+ /**
+ * Asserts that the given call fails as an argument error carrying the exact message.
+ *
+ * @param message The exact failure message expected.
+ * @param call The call under test.
+ */
+ private static void assertArgumentError(String message, Executable call) {
+ final IllegalArgumentException thrown =
+ Assertions.assertThrows(IllegalArgumentException.class, call);
+ Assertions.assertEquals(message, thrown.getMessage());
+ }
+
+ @Test
+ void testSha512ChecksumVerifies(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"data/entry.txt", "payload"}});
+ final Path file = source.resolve("checked.tar.gz");
+ Files.write(file, archive);
+
+ ResourceInstaller.install(file.toUri(), target, sha512(archive));
+
+ Assertions.assertEquals("payload",
+ Files.readString(target.resolve("data/entry.txt")));
+ }
+
+ @Test
+ void testSha512ChecksumMismatchFailsBeforeUnpacking(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"a/file.txt", "content"}});
+ final Path file = source.resolve("archive.tar.gz");
+ Files.write(file, archive);
+
+ final String wrong = "0".repeat(128);
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target, wrong));
+ Assertions.assertEquals("checksum mismatch: expected " + wrong
+ + " but downloaded " + sha512(archive), thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testFailedTarUnpackLeavesTargetEmpty(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {
+ {"good.txt", "fine"},
+ {"../escape.txt", "bad"}});
+ final Path file = source.resolve("partial.tar.gz");
+ Files.write(file, archive);
+
+ assertInstallFails(file, target, ESCAPE_ERROR + "../escape.txt");
+ }
+
+ @Test
+ void testFailedZipUnpackLeavesTargetEmpty(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("good.txt"));
+ zip.write("fine".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ zip.putNextEntry(new ZipEntry("../zip-escape.txt"));
+ zip.write("bad".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ final Path file = source.resolve("partial.zip");
+ Files.write(file, out.toByteArray());
+
+ assertInstallFails(file, target, ESCAPE_ERROR + "../zip-escape.txt");
+ }
+
+ @Test
+ void testDuplicateTarFileEntryIsRejected(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tarEntry(tar, "corpus/data.txt", "first".getBytes(StandardCharsets.UTF_8));
+ tarEntry(tar, "corpus/data.txt", "second".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final Path file = source.resolve("duplicate.tar.gz");
+ Files.write(file, gzip(tar.toByteArray()));
+
+ assertInstallFails(file, target, DUPLICATE_ENTRY_ERROR + "corpus/data.txt");
+ }
+
+ @Test
+ void testEquivalentZipFileEntriesAreRejected(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("corpus/data.txt"));
+ zip.write("first".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ zip.putNextEntry(new ZipEntry("corpus/./data.txt"));
+ zip.write("second".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ final Path file = source.resolve("duplicate.zip");
+ Files.write(file, out.toByteArray());
+
+ assertInstallFails(file, target, DUPLICATE_ENTRY_ERROR + "corpus/./data.txt");
+ }
+
+ @Test
+ void testPlainFileBeginningWithPkIsStored(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] content = "PK plain dictionary".getBytes(StandardCharsets.UTF_8);
+ final Path file = source.resolve("dictionary.dat");
+ Files.write(file, content);
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ Assertions.assertArrayEquals(content,
+ Files.readAllBytes(target.resolve("dictionary.dat")));
+ }
+
+ @Test
+ void testTruncatedZipHeaderIsRejected(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final Path file = source.resolve("truncated.zip");
+ Files.write(file, new byte[] {'P', 'K', 3, 4});
+
+ assertInstallFails(file, target, "malformed zip archive");
+ }
+
+ @Test
+ void testZipWithoutCentralDirectoryIsRejected(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("corpus/data.txt"));
+ zip.write("content".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ final byte[] complete = out.toByteArray();
+ int centralDirectory = -1;
+ for (int i = 0; i <= complete.length - 4; i++) {
+ if (complete[i] == 'P' && complete[i + 1] == 'K'
+ && complete[i + 2] == 1 && complete[i + 3] == 2) {
+ centralDirectory = i;
+ break;
+ }
+ }
+ Assertions.assertTrue(centralDirectory > 0);
+ final Path file = source.resolve("missing-central-directory.zip");
+ Files.write(file, Arrays.copyOf(complete, centralDirectory));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target));
+ Assertions.assertEquals("malformed zip archive", thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testTruncatedEmptyZipHeaderIsRejected(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final Path file = source.resolve("truncated-empty.zip");
+ Files.write(file, new byte[] {'P', 'K', 5, 6});
+
+ assertInstallFails(file, target, "malformed zip archive");
+ }
+
+ @Test
+ void testValidEmptyZipInstallsNothing(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] endHeader = new byte[22];
+ endHeader[0] = 'P';
+ endHeader[1] = 'K';
+ endHeader[2] = 5;
+ endHeader[3] = 6;
+ final Path file = source.resolve("empty.zip");
+ Files.write(file, endHeader);
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testValidEmptyTarGzInstallsNothing(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final Path file = source.resolve("empty.tar.gz");
+ Files.write(file, gzip(new byte[TERMINATOR_SIZE]));
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testInvalidZipEntryPathLeavesNoStagingFiles(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("good.txt"));
+ zip.write("good".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ zip.putNextEntry(new ZipEntry("bad\0name.txt"));
+ zip.write("bad".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ final Path file = source.resolve("invalid-path.zip");
+ Files.write(file, out.toByteArray());
+
+ assertInstallFails(file, target,
+ "archive entry has an invalid path: bad\0name.txt");
+ }
+
+ @Test
+ void testTruncatedTarLeavesTargetEmpty(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tarEntry(tar, "first.txt", "complete".getBytes(StandardCharsets.UTF_8));
+ tarEntry(tar, "second.txt", "never finished".getBytes(StandardCharsets.UTF_8));
+ final byte[] whole = tar.toByteArray();
+ // Cut inside the second entry's header: first entry occupies two 512-byte blocks.
+ final byte[] truncated = Arrays.copyOf(whole, 2 * BLOCK + 100);
+ final Path file = source.resolve("truncated.tar.gz");
+ Files.write(file, gzip(truncated));
+
+ Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target));
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testFailedInstallKeepsPreexistingTargetContent(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ Files.writeString(target.resolve("existing.txt"), "keep");
+ final byte[] archive = tarGz(new String[][] {
+ {"good.txt", "fine"},
+ {"../escape.txt", "bad"}});
+ final Path file = source.resolve("partial.tar.gz");
+ Files.write(file, archive);
+
+ Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target));
+ Assertions.assertEquals(List.of("existing.txt"), installedFiles(target));
+ Assertions.assertEquals("keep", Files.readString(target.resolve("existing.txt")));
+ }
+
+ @Test
+ void testInstallationLeavesNoStagingResidue(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "content"}});
+ final Path file = source.resolve("clean.tar.gz");
+ Files.write(file, archive);
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ try (Stream walk = Files.walk(target)) {
+ final List hidden = walk
+ .filter(path -> !path.equals(target))
+ .map(path -> path.getFileName().toString())
+ .filter(fileName -> fileName.startsWith("."))
+ .toList();
+ Assertions.assertEquals(List.of(), hidden);
+ }
+ Assertions.assertEquals(List.of("corpus/data.txt"), installedFiles(target));
+ }
+
+ /**
+ * Supplies invalid {@code Limits} constructions with the argument error each one
+ * must raise.
+ *
+ * @return One case per invalid argument. Never {@code null}.
+ */
+ static Stream invalidLimits() {
+ final Duration valid = Duration.ofSeconds(10);
+ return Stream.of(
+ Arguments.of("null connectTimeout", (Executable)
+ () -> new ResourceInstaller.Limits(null, valid, 5, 1024, 1024, 10, 100),
+ "connectTimeout must not be null"),
+ Arguments.of("zero connectTimeout", (Executable)
+ () -> new ResourceInstaller.Limits(Duration.ZERO, valid, 5, 1024, 1024, 10, 100),
+ "connectTimeout must be positive"),
+ Arguments.of("null readTimeout", (Executable)
+ () -> new ResourceInstaller.Limits(valid, null, 5, 1024, 1024, 10, 100),
+ "readTimeout must not be null"),
+ Arguments.of("negative readTimeout", (Executable)
+ () -> new ResourceInstaller.Limits(valid, Duration.ofSeconds(-1),
+ 5, 1024, 1024, 10, 100),
+ "readTimeout must be positive"),
+ Arguments.of("negative maxRedirects", (Executable)
+ () -> new ResourceInstaller.Limits(valid, valid, -1, 1024, 1024, 10, 100),
+ "maxRedirects must not be negative"),
+ Arguments.of("zero maxDownloadBytes", (Executable)
+ () -> new ResourceInstaller.Limits(valid, valid, 5, 0, 1024, 10, 100),
+ "maxDownloadBytes must be positive"),
+ Arguments.of("zero maxExpandedBytes", (Executable)
+ () -> new ResourceInstaller.Limits(valid, valid, 5, 1024, 0, 10, 100),
+ "maxExpandedBytes must be positive"),
+ Arguments.of("zero maxEntries", (Executable)
+ () -> new ResourceInstaller.Limits(valid, valid, 5, 1024, 1024, 0, 100),
+ "maxEntries must be positive"),
+ Arguments.of("zero maxExpansionRatio", (Executable)
+ () -> new ResourceInstaller.Limits(valid, valid, 5, 1024, 1024, 10, 0),
+ "maxExpansionRatio must be positive"),
+ Arguments.of("builder zero connectTimeout", (Executable) () ->
+ ResourceInstaller.Limits.builder().connectTimeout(Duration.ZERO).build(),
+ "connectTimeout must be positive"),
+ Arguments.of("builder null readTimeout", (Executable) () ->
+ ResourceInstaller.Limits.builder().readTimeout(null).build(),
+ "readTimeout must not be null"),
+ Arguments.of("builder negative maxRedirects", (Executable) () ->
+ ResourceInstaller.Limits.builder().maxRedirects(-1).build(),
+ "maxRedirects must not be negative"),
+ Arguments.of("builder zero maxDownloadBytes", (Executable) () ->
+ ResourceInstaller.Limits.builder().maxDownloadBytes(0).build(),
+ "maxDownloadBytes must be positive"),
+ Arguments.of("builder negative maxExpandedBytes", (Executable) () ->
+ ResourceInstaller.Limits.builder().maxExpandedBytes(-1).build(),
+ "maxExpandedBytes must be positive"),
+ Arguments.of("builder zero maxEntries", (Executable) () ->
+ ResourceInstaller.Limits.builder().maxEntries(0).build(),
+ "maxEntries must be positive"));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("invalidLimits")
+ void testLimitsRejectInvalidValues(String label, Executable construction,
+ String message) {
+ final IllegalArgumentException thrown =
+ Assertions.assertThrows(IllegalArgumentException.class, construction);
+ Assertions.assertEquals(message, thrown.getMessage());
+ }
+
+ @Test
+ void testDownloadLimitRejectsOversizedSource(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final Path file = source.resolve("large.txt");
+ Files.write(file, new byte[8192]);
+
+ assertInstallFails(file, target, limits(KIBIBYTE, MEBIBYTE),
+ "download exceeds the limit of " + KIBIBYTE + " bytes");
+ }
+
+ @Test
+ void testTarExpansionLimitRejectsArchive(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tarEntry(tar, "bomb/zeros.bin", new byte[64 * 1024]);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final Path file = source.resolve("bomb.tar.gz");
+ Files.write(file, gzip(tar.toByteArray()));
+
+ assertInstallFails(file, target, limits(MEBIBYTE, KIBIBYTE),
+ EXPANSION_LIMIT_ERROR);
+ }
+
+ @Test
+ void testTarExpansionLimitCountsBytesAfterTheTerminator(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tarEntry(tar, "corpus/data.txt", "content".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ tar.write(new byte[64 * 1024]);
+ final Path file = source.resolve("trailing-data.tar.gz");
+ Files.write(file, gzip(tar.toByteArray()));
+
+ assertInstallFails(file, target, limits(MEBIBYTE, 16 * KIBIBYTE),
+ "expanded content exceeds the limit of " + 16 * KIBIBYTE + " bytes");
+ }
+
+ @Test
+ void testTarGzipTrailerIsVerified(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tarEntry(tar, "corpus/data.txt", "content".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ tar.write(new byte[64 * 1024]);
+ final byte[] archive = gzip(tar.toByteArray());
+ archive[archive.length - 8] ^= 1;
+ final Path file = source.resolve("bad-trailer.tar.gz");
+ Files.write(file, archive);
+
+ Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target));
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testTarMetadataCountsTowardExpansionLimit(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ TarArchives.entry(tar, "pax_global_header",
+ TarArchives.paxRecord("comment", "a".repeat(64 * 1024)), 'g');
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final Path file = source.resolve("metadata-bomb.tar.gz");
+ Files.write(file, gzip(tar.toByteArray()));
+
+ assertInstallFails(file, target, limits(MEBIBYTE, KIBIBYTE),
+ EXPANSION_LIMIT_ERROR);
+ }
+
+ @Test
+ void testZipExpansionLimitRejectsArchive(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("bomb/zeros.bin"));
+ zip.write(new byte[64 * 1024]);
+ zip.closeEntry();
+ }
+ final Path file = source.resolve("bomb.zip");
+ Files.write(file, out.toByteArray());
+
+ assertInstallFails(file, target, limits(MEBIBYTE, KIBIBYTE),
+ EXPANSION_LIMIT_ERROR);
+ }
+
+ @Test
+ void testZipDirectoryContentCountsTowardExpansionLimit(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("payload/"));
+ zip.write(new byte[64 * 1024]);
+ zip.closeEntry();
+ }
+ final Path file = source.resolve("directory-content.zip");
+ Files.write(file, out.toByteArray());
+
+ assertInstallFails(file, target, limits(MEBIBYTE, KIBIBYTE),
+ EXPANSION_LIMIT_ERROR);
+ }
+
+ @Test
+ void testPlainGzipExpansionLimitRejectsFile(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final Path file = source.resolve("zeros.bin.gz");
+ Files.write(file, gzip(new byte[64 * 1024]));
+
+ assertInstallFails(file, target, limits(MEBIBYTE, KIBIBYTE),
+ EXPANSION_LIMIT_ERROR);
+ }
+
+ @Test
+ void testInstallWithinCustomLimitsSucceeds(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "small"}});
+ final Path file = source.resolve("small.tar.gz");
+ Files.write(file, archive);
+
+ ResourceInstaller.install(file.toUri(), target, sha256(archive),
+ limits(MEBIBYTE, MEBIBYTE));
+
+ Assertions.assertEquals("small", Files.readString(target.resolve("corpus/data.txt")));
+ }
+
+ @Test
+ void testDefaultLimitsArePinned() {
+ final ResourceInstaller.Limits defaults = ResourceInstaller.Limits.DEFAULT;
+ Assertions.assertEquals(Duration.ofSeconds(20), defaults.connectTimeout());
+ Assertions.assertEquals(Duration.ofSeconds(60), defaults.readTimeout());
+ Assertions.assertEquals(5, defaults.maxRedirects());
+ Assertions.assertEquals(1L << 30, defaults.maxDownloadBytes());
+ Assertions.assertEquals(4L << 30, defaults.maxExpandedBytes());
+ Assertions.assertEquals(100_000L, defaults.maxEntries());
+ Assertions.assertEquals(100L, defaults.maxExpansionRatio());
+ }
+
+ @Test
+ void testLimitPropertyNames() {
+ Assertions.assertEquals("opennlp.download.max.bytes",
+ ResourceInstaller.Limits.MAX_DOWNLOAD_BYTES_PROPERTY);
+ Assertions.assertEquals("opennlp.install.max.total.bytes",
+ ResourceInstaller.Limits.MAX_EXPANDED_BYTES_PROPERTY);
+ Assertions.assertEquals("opennlp.install.max.entries",
+ ResourceInstaller.Limits.MAX_ENTRIES_PROPERTY);
+ Assertions.assertEquals("opennlp.install.max.expansion.ratio",
+ ResourceInstaller.Limits.MAX_EXPANSION_RATIO_PROPERTY);
+ }
+
+ @Test
+ void testLimitPropertyOverrideIsRead() {
+ System.setProperty(TEST_LIMIT_PROPERTY, " 123 ");
+ try {
+ Assertions.assertEquals(123L,
+ ResourceInstaller.Limits.longProperty(TEST_LIMIT_PROPERTY, 7L));
+ } finally {
+ System.clearProperty(TEST_LIMIT_PROPERTY);
+ }
+ }
+
+ /**
+ * Supplies property values that must fall back to the built-in default: absent,
+ * not a number, zero, negative, and empty.
+ *
+ * @return One case per unusable value. Never {@code null}.
+ */
+ static Stream unusableLimitProperties() {
+ return Stream.of(
+ Arguments.of("absent", null),
+ Arguments.of("not a number", "abc"),
+ Arguments.of("zero", "0"),
+ Arguments.of("negative", "-5"),
+ Arguments.of("empty", ""));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("unusableLimitProperties")
+ void testLimitPropertyFallsBackOnUnusableValues(String label, String value) {
+ if (value == null) {
+ System.clearProperty(TEST_LIMIT_PROPERTY);
+ } else {
+ System.setProperty(TEST_LIMIT_PROPERTY, value);
+ }
+ try {
+ Assertions.assertEquals(7L,
+ ResourceInstaller.Limits.longProperty(TEST_LIMIT_PROPERTY, 7L));
+ } finally {
+ System.clearProperty(TEST_LIMIT_PROPERTY);
+ }
+ }
+
+ /**
+ * Builds installation limits with the given entry limit and otherwise default
+ * values, so entry-count tests state only the value they exercise.
+ *
+ * @param maxEntries The entry limit.
+ * @return The limits. Never {@code null}.
+ */
+ private static ResourceInstaller.Limits entryLimit(long maxEntries) {
+ return ResourceInstaller.Limits.builder().maxEntries(maxEntries).build();
+ }
+
+ @Test
+ void testTarEntryCountLimitRejectsArchive(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {
+ {"corpus/one.txt", "1"},
+ {"corpus/two.txt", "2"},
+ {"corpus/three.txt", "3"}});
+ final Path file = source.resolve("many.tar.gz");
+ Files.write(file, archive);
+
+ assertInstallFails(file, target, entryLimit(2),
+ ENTRY_LIMIT_ERROR);
+ }
+
+ /**
+ * Checks that tar extension headers count toward the archive entry limit.
+ *
+ * @param source A scratch directory for the source archive.
+ * @param target A scratch installation directory.
+ * @throws Exception Thrown if the fixture cannot be created or installed.
+ */
+ @Test
+ void testTarEntryCountLimitIncludesMetadata(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ TarArchives.entry(tar, "PaxHeaders/one",
+ TarArchives.paxRecord("comment", "first"), 'x');
+ TarArchives.entry(tar, "PaxHeaders/b",
+ TarArchives.paxRecord("comment", "right"), 'x');
+ tarEntry(tar, "payload/data.txt", "content".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final Path file = source.resolve("metadata-entries.tar.gz");
+ Files.write(file, gzip(tar.toByteArray()));
+
+ assertInstallFails(file, target, entryLimit(2), ENTRY_LIMIT_ERROR);
+ }
+
+ @Test
+ void testZipEntryCountLimitRejectsArchive(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ for (int i = 0; i < 3; i++) {
+ zip.putNextEntry(new ZipEntry("corpus/entry-" + i + ".txt"));
+ zip.write("x".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ }
+ final Path file = source.resolve("many.zip");
+ Files.write(file, out.toByteArray());
+
+ assertInstallFails(file, target, entryLimit(2),
+ ENTRY_LIMIT_ERROR);
+ }
+
+ @Test
+ void testEntryCountExactlyAtLimitSucceeds(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {
+ {"corpus/one.txt", "1"},
+ {"corpus/two.txt", "2"}});
+ final Path file = source.resolve("exact.tar.gz");
+ Files.write(file, archive);
+
+ ResourceInstaller.install(file.toUri(), target, sha256(archive), entryLimit(2));
+
+ Assertions.assertEquals(List.of("corpus/one.txt", "corpus/two.txt"),
+ installedFiles(target));
+ }
+
+ @Test
+ void testSha512ChecksumComparisonIgnoresHexLetterCase(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"data/entry.txt", "payload"}});
+ final Path file = source.resolve("cased512.tar.gz");
+ Files.write(file, archive);
+ final String upperCase = sha512(archive).toUpperCase(Locale.ROOT);
+ Assertions.assertNotEquals(sha512(archive), upperCase);
+
+ ResourceInstaller.install(file.toUri(), target, upperCase);
+
+ Assertions.assertEquals("payload",
+ Files.readString(target.resolve("data/entry.txt")));
+ }
+
+ @Test
+ void testDownloadExactlyAtLimitSucceeds(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final Path file = source.resolve("exact.dat");
+ Files.write(file, new byte[1024]);
+
+ ResourceInstaller.install(file.toUri(), target, null, limits(KIBIBYTE, MEBIBYTE));
+
+ Assertions.assertEquals(List.of("exact.dat"), installedFiles(target));
+ Assertions.assertEquals(1024, Files.size(target.resolve("exact.dat")));
+ }
+
+ @Test
+ void testExpansionExactlyAtLimitSucceeds(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tarEntry(tar, "corpus/exact.bin", new byte[1024]);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final Path file = source.resolve("exact.tar.gz");
+ Files.write(file, gzip(tar.toByteArray()));
+
+ ResourceInstaller.install(file.toUri(), target, null,
+ limits(MEBIBYTE, tar.size()));
+
+ Assertions.assertEquals(List.of("corpus/exact.bin"), installedFiles(target));
+ Assertions.assertEquals(1024, Files.size(target.resolve("corpus/exact.bin")));
+ }
+
+ @Test
+ void testCumulativeExpansionAcrossEntriesHitsLimit(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tarEntry(tar, "corpus/first.bin", new byte[768]);
+ tarEntry(tar, "corpus/second.bin", new byte[768]);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final Path file = source.resolve("cumulative.tar.gz");
+ Files.write(file, gzip(tar.toByteArray()));
+
+ assertInstallFails(file, target, limits(MEBIBYTE, KIBIBYTE),
+ EXPANSION_LIMIT_ERROR);
+ }
+
+ @Test
+ void testReinstallOverAnExistingFileIsRejected(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] first = tarGz(new String[][] {{"corpus/data.txt", "version one"}});
+ final byte[] second = tarGz(new String[][] {{"corpus/data.txt", "version two"}});
+ final Path firstFile = source.resolve("first.tar.gz");
+ final Path secondFile = source.resolve("second.tar.gz");
+ Files.write(firstFile, first);
+ Files.write(secondFile, second);
+ ResourceInstaller.install(firstFile.toUri(), target, sha256(first));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(secondFile.toUri(), target, sha256(second)));
+
+ Assertions.assertEquals(
+ COLLISION_ERROR + target.resolve("corpus/data.txt"),
+ thrown.getMessage());
+ Assertions.assertEquals("version one",
+ Files.readString(target.resolve("corpus/data.txt")));
+
+ Files.delete(target.resolve("corpus/data.txt"));
+ ResourceInstaller.install(secondFile.toUri(), target, sha256(second));
+ Assertions.assertEquals("version two",
+ Files.readString(target.resolve("corpus/data.txt")));
+ }
+
+ @Test
+ void testCollidingInstallPromotesNothing(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ Files.createDirectories(target.resolve("corpus"));
+ Files.writeString(target.resolve("corpus/data.txt"), "keep");
+ final byte[] archive = tarGz(new String[][] {
+ {"corpus/fresh.txt", "new"},
+ {"corpus/data.txt", "replacement"}});
+ final Path file = source.resolve("colliding.tar.gz");
+ Files.write(file, archive);
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target, sha256(archive)));
+
+ Assertions.assertEquals(
+ COLLISION_ERROR + target.resolve("corpus/data.txt"),
+ thrown.getMessage());
+ Assertions.assertEquals("keep", Files.readString(target.resolve("corpus/data.txt")));
+ Assertions.assertTrue(Files.notExists(target.resolve("corpus/fresh.txt")));
+ Assertions.assertEquals(List.of("corpus/data.txt"), installedFiles(target));
+ }
+
+ @Test
+ @DisabledOnOs(OS.WINDOWS)
+ void testPromotionRejectsToFollowASymlinkedDirectory(@TempDir Path source,
+ @TempDir Path target, @TempDir Path outside) throws Exception {
+ final Path link = target.resolve("link");
+ Files.createSymbolicLink(link, outside);
+ final byte[] archive = tarGz(new String[][] {{"link/planted.txt", "escaped"}});
+ final Path file = source.resolve("symlink.tar.gz");
+ Files.write(file, archive);
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target));
+
+ Assertions.assertEquals("installation path crosses a symbolic link: " + link,
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(outside));
+ Assertions.assertTrue(Files.notExists(outside.resolve("planted.txt")));
+ }
+
+ @Test
+ void testNestedDirectoryThatIsNotASymlinkStillInstalls(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ Files.createDirectory(target.resolve("link"));
+ final byte[] archive = tarGz(new String[][] {{"link/planted.txt", "fine"}});
+ final Path file = source.resolve("nested.tar.gz");
+ Files.write(file, archive);
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ Assertions.assertEquals("fine",
+ Files.readString(target.resolve("link/planted.txt")));
+ }
+
+ /**
+ * Supplies source schemes the installer rejects because they are outside the bounded
+ * HTTP and local-file paths.
+ *
+ * @return One case per rejected scheme. Never {@code null}.
+ */
+ static Stream rejectedSourceSchemes() {
+ return Stream.of(
+ Arguments.of("ftp", URI.create("ftp://example.invalid/corpus.tar.gz")),
+ Arguments.of("jar", URI.create("jar:file:/tmp/a.jar!/corpus.tar.gz")),
+ Arguments.of("mailto", URI.create("mailto:someone@example.invalid")),
+ Arguments.of("no scheme at all", URI.create("corpus.tar.gz")));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("rejectedSourceSchemes")
+ void testUnsupportedSourceSchemeIsRejected(String label, URI source,
+ @TempDir Path target) throws IOException {
+ assertArgumentError("source scheme must be http, https, or file, but was: " + source,
+ () -> ResourceInstaller.install(source, target));
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ @Test
+ void testUnsupportedSourceSchemeIsRejectedBeforeCreatingTheTarget(@TempDir Path parent) {
+ final Path target = parent.resolve("not-created-yet");
+
+ assertArgumentError(
+ "source scheme must be http, https, or file, but was: ftp://example.invalid/c.gz",
+ () -> ResourceInstaller.install(URI.create("ftp://example.invalid/c.gz"), target));
+ Assertions.assertTrue(Files.notExists(target));
+ }
+
+
+
+
+ @Test
+ void testBuilderWithoutOverridesEqualsTheDefaults() {
+ Assertions.assertEquals(ResourceInstaller.Limits.DEFAULT,
+ ResourceInstaller.Limits.builder().build());
+ }
+
+ @Test
+ void testBuilderOverridesOnlyWhatIsSet() {
+ final ResourceInstaller.Limits limits = ResourceInstaller.Limits.builder()
+ .maxDownloadBytes(MEBIBYTE)
+ .maxExpandedBytes(2 * MEBIBYTE)
+ .build();
+
+ Assertions.assertEquals(MEBIBYTE, limits.maxDownloadBytes());
+ Assertions.assertEquals(2 * MEBIBYTE, limits.maxExpandedBytes());
+ Assertions.assertEquals(ResourceInstaller.Limits.DEFAULT.connectTimeout(),
+ limits.connectTimeout());
+ Assertions.assertEquals(ResourceInstaller.Limits.DEFAULT.readTimeout(),
+ limits.readTimeout());
+ Assertions.assertEquals(ResourceInstaller.Limits.DEFAULT.maxRedirects(),
+ limits.maxRedirects());
+ Assertions.assertEquals(ResourceInstaller.Limits.DEFAULT.maxEntries(),
+ limits.maxEntries());
+ }
+
+ @Test
+ void testDownloadFileIsCreatedInTheTargetDirectory(@TempDir Path target)
+ throws Exception {
+ final Path downloaded = ResourceInstaller.createDownloadFile(target);
+ try {
+ Assertions.assertEquals(target, downloaded.getParent());
+ Assertions.assertTrue(
+ downloaded.getFileName().toString().startsWith(".opennlp-download"),
+ downloaded.getFileName().toString());
+ } finally {
+ Files.deleteIfExists(downloaded);
+ }
+ }
+
+ static Stream expansionBombs() throws IOException {
+ // Four mebibytes of repeated text compress to a few kibibytes in either format: far
+ // beyond the ratio ceiling, far below the absolute expansion limit. Repeated text
+ // rather than zeros, because a gzip stream of zeros reads as an empty tar archive.
+ final byte[] repetitive =
+ "opennlp ".repeat(512 * 1024).getBytes(StandardCharsets.UTF_8);
+ return Stream.of(
+ Arguments.of("corpus.txt.gz", gzip(repetitive)),
+ Arguments.of("corpus.zip", zipOf("corpus.txt", repetitive)));
+ }
+
+ @ParameterizedTest
+ @MethodSource("expansionBombs")
+ void testExpansionRatioIsBounded(String name, byte[] content, @TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final Path file = source.resolve(name);
+ Files.write(file, content);
+
+ assertInstallFails(file, target, RATIO_ERROR);
+ }
+
+ @ParameterizedTest
+ @MethodSource("expansionBombs")
+ void testRaisingTheExpansionRatioAcceptsWhatTheDefaultRejects(String name,
+ byte[] content, @TempDir Path source, @TempDir Path target) throws Exception {
+ final Path file = source.resolve(name);
+ Files.write(file, content);
+
+ // Raising the byte limit alone cannot lift the ratio: the tighter budget wins.
+ final ResourceInstaller.Limits raised = ResourceInstaller.Limits.builder()
+ .maxExpansionRatio(2000)
+ .build();
+ ResourceInstaller.install(file.toUri(), target, null, raised);
+
+ Assertions.assertEquals(4 << 20, Files.size(target.resolve("corpus.txt")));
+ }
+
+ @Test
+ void testMoveIntoPlaceDoesNotReplaceAnExistingFile(@TempDir Path directory)
+ throws Exception {
+ final Path staged = directory.resolve("staged.txt");
+ final Path destination = directory.resolve("installed.txt");
+ Files.writeString(staged, "new");
+ Files.writeString(destination, "old");
+
+ Assertions.assertThrows(FileAlreadyExistsException.class,
+ () -> ResourceInstaller.moveIntoPlace(staged, destination));
+
+ Assertions.assertEquals("old", Files.readString(destination));
+ Assertions.assertEquals("new", Files.readString(staged));
+ }
+
+ @Test
+ void testModelSuffixIsMatchedIgnoringCase(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("manifest.properties"));
+ zip.write("OpenNLP-Version: 0.0.0\n".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ final Path file = source.resolve("en-ner-person.BIN");
+ Files.write(file, out.toByteArray());
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ Assertions.assertEquals(List.of("en-ner-person.BIN"), installedFiles(target));
+ }
+
+ @Test
+ void testGzipSuffixIsStrippedIgnoringCase(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final Path file = source.resolve("lexicon.tsv.GZ");
+ Files.write(file, gzip("word\tlemma\n".getBytes(StandardCharsets.UTF_8)));
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ Assertions.assertEquals(List.of("lexicon.tsv"), installedFiles(target));
+ }
+
+ @Test
+ void testStaleWorkFilesOfAKilledInstallAreRemoved(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final Path staleStaging = Files.createDirectory(target.resolve(".opennlp-stagingOLD"));
+ Files.writeString(staleStaging.resolve("partial.txt"), "half");
+ Files.writeString(target.resolve(".opennlp-downloadOLD.part"), "half");
+ final Path file = source.resolve("corpus.tar.gz");
+ Files.write(file, tarGz(new String[][] {{"corpus/data.txt", "content"}}));
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ try (Stream entries = Files.list(target)) {
+ Assertions.assertEquals(List.of("corpus"),
+ entries.map(path -> path.getFileName().toString()).sorted().toList());
+ }
+ }
+
+ @Test
+ void testFailedInstallRemovesTheTargetDirectoryItCreated(@TempDir Path source,
+ @TempDir Path parent) throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "content"}});
+ final Path file = source.resolve("corpus.tar.gz");
+ Files.write(file, archive);
+ final Path target = parent.resolve("fresh");
+
+ Assertions.assertThrows(IOException.class, () -> ResourceInstaller.install(
+ file.toUri(), target, sha256("other".getBytes(StandardCharsets.UTF_8))));
+
+ Assertions.assertTrue(Files.notExists(target));
+ }
+
+ @Test
+ void testZipLocalHeadersMustMatchTheCentralDirectory(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ // The same content under two names, so the local file sections are the same size and
+ // one archive's central directory fits the other's local headers.
+ final byte[] first = zipOf("a.txt", "content");
+ final byte[] second = zipOf("b.txt", "content");
+ final int centralFirst = centralDirectoryStart(first);
+ final int centralSecond = centralDirectoryStart(second);
+ final byte[] hybrid = new byte[centralFirst + second.length - centralSecond];
+ System.arraycopy(first, 0, hybrid, 0, centralFirst);
+ System.arraycopy(second, centralSecond, hybrid, centralFirst,
+ second.length - centralSecond);
+ final Path file = source.resolve("hybrid.zip");
+ Files.write(file, hybrid);
+
+ assertInstallFails(file, target, ZIP_MISMATCH_ERROR);
+ }
+
+ /** Builds a zip archive holding one text entry. */
+ private static byte[] zipOf(String name, String content) throws IOException {
+ return zipOf(name, content.getBytes(StandardCharsets.UTF_8));
+ }
+
+ /** Builds a zip archive holding one entry with the given bytes. */
+ private static byte[] zipOf(String name, byte[] content) throws IOException {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry(name));
+ zip.write(content);
+ zip.closeEntry();
+ }
+ return out.toByteArray();
+ }
+
+ /** Finds the first central directory header signature in a zip archive. */
+ private static int centralDirectoryStart(byte[] zip) {
+ for (int i = 0; i + 3 < zip.length; i++) {
+ if (zip[i] == 'P' && zip[i + 1] == 'K' && zip[i + 2] == 1 && zip[i + 3] == 2) {
+ return i;
+ }
+ }
+ throw new AssertionError("no central directory");
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/archive/TarArchives.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/archive/TarArchives.java
new file mode 100644
index 0000000000..6646cd1732
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/archive/TarArchives.java
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util.archive;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * Builds tar archives byte by byte, so tests can assemble well-formed, boundary, and
+ * malformed archives without an extra library. Each header has the ustar
+ * magic and a valid checksum, as a real archive does; {@link #reseal(byte[])} restores
+ * the checksum after a test has corrupted some other field on purpose.
+ */
+public final class TarArchives {
+
+ /** The tar block size; headers, contents, and terminators are multiples of it. */
+ public static final int BLOCK = 512;
+
+ /** The size of the two all-zero blocks that terminate a tar archive. */
+ public static final int TERMINATOR_SIZE = 2 * BLOCK;
+
+ /** The type flag of a regular file. */
+ public static final char TYPE_REGULAR_FILE = '0';
+
+ /** The type flag an old-style archive leaves NUL for a regular file. */
+ public static final char TYPE_REGULAR_FILE_CLASSIC = '\0';
+
+ /** The type flag of a directory. */
+ public static final char TYPE_DIRECTORY = '5';
+
+ /** The offset of the 155-byte ustar name prefix field. */
+ public static final int PREFIX_OFFSET = 345;
+
+ private static final int NAME_LENGTH = 100;
+ private static final int PREFIX_LENGTH = 155;
+ private static final int MODE_OFFSET = 100;
+ private static final int SIZE_OFFSET = 124;
+ private static final int SIZE_LENGTH = 12;
+ private static final int BASE_256_MARKER = 0x80;
+ private static final int BASE_256_NEGATIVE = 0x40;
+ private static final int CHECKSUM_OFFSET = 148;
+ private static final int CHECKSUM_LENGTH = 8;
+ private static final int TYPE_OFFSET = 156;
+ private static final int MAGIC_OFFSET = 257;
+ private static final int BLANK = ' ';
+ private static final String SIZE_FORMAT = "%011o";
+ private static final String CHECKSUM_FORMAT = "%06o";
+ private static final String MODE = "0000644 ";
+
+ /** The ustar magic and version fields: {@code "ustar"}, NUL, then {@code "00"}. */
+ private static final byte[] USTAR_MAGIC = {'u', 's', 't', 'a', 'r', 0, '0', '0'};
+
+ /** The GNU magic and version fields: {@code "ustar"}, two blanks, then NUL. */
+ private static final byte[] GNU_MAGIC = {'u', 's', 't', 'a', 'r', ' ', ' ', 0};
+
+ private TarArchives() {
+ }
+
+ /**
+ * Builds one 512-byte tar header block with the given name, declared content size,
+ * and type flag. The mode field is populated with a non-zero octal value so that a
+ * name filling the whole 100-byte name field is followed by non-zero bytes, which
+ * makes name-boundary tests meaningful.
+ *
+ * @param name The entry name; at most 100 bytes when encoded as UTF-8.
+ * @param size The content size to declare in the octal size field; must not be
+ * negative.
+ * @param typeFlag The tar type flag, for example {@link #TYPE_REGULAR_FILE}.
+ * @return The header block. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if the name does not fit the name field or
+ * the size is negative.
+ */
+ public static byte[] header(String name, long size, char typeFlag) {
+ return header(name, "", size, typeFlag, true);
+ }
+
+ /**
+ * Builds one 512-byte classic v7 header block, which carries no ustar magic and no
+ * name prefix field. Real archives in this format still exist, and a reader that keys
+ * off the magic rather than the checksum will not see them.
+ *
+ * @param name The entry name; at most 100 bytes when encoded as UTF-8.
+ * @param size The content size to declare in the octal size field; must not be
+ * negative.
+ * @param typeFlag The tar type flag, for example {@link #TYPE_REGULAR_FILE}.
+ * @return The header block. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if the name does not fit the name field or
+ * the size is negative.
+ */
+ public static byte[] classicHeader(String name, long size, char typeFlag) {
+ return header(name, "", size, typeFlag, false);
+ }
+
+ /**
+ * Builds one 512-byte GNU header block. GNU writes {@code "ustar"} followed by two
+ * blanks and a NUL where ustar writes {@code "ustar"}, a NUL, and the version, and it
+ * uses the offset ustar gives to the name prefix for {@code atime} instead.
+ *
+ * @param name The entry name; at most 100 bytes when encoded as UTF-8.
+ * @param size The content size to declare in the octal size field; must not be
+ * negative.
+ * @param typeFlag The tar type flag, for example {@link #TYPE_REGULAR_FILE}.
+ * @param atime The octal {@code atime} to write at {@link #PREFIX_OFFSET}, as a GNU
+ * incremental archive does, or empty to leave it NUL.
+ * @return The header block. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if the name does not fit the name field or
+ * the size is negative.
+ */
+ public static byte[] gnuHeader(String name, long size, char typeFlag, String atime) {
+ final byte[] block = header(name, "", size, typeFlag, false);
+ System.arraycopy(GNU_MAGIC, 0, block, MAGIC_OFFSET, GNU_MAGIC.length);
+ write(block, PREFIX_OFFSET, atime);
+ return reseal(block);
+ }
+
+ /**
+ * Builds one 512-byte header block whose size field uses the base-256 encoding, which
+ * GNU writes when a length does not fit the eleven octal digits the field holds.
+ *
+ * The leading bit marks the encoding, the next bit carries the sign, and the
+ * remaining bits of that byte followed by every later byte form a big-endian two's
+ * complement number.
+ *
+ * @param name The entry name; at most 100 bytes when encoded as UTF-8.
+ * @param size The content size to encode. May be negative, which no real writer emits
+ * for a size, so that the reader's rejection of it can be exercised.
+ * @param typeFlag The tar type flag, for example {@link #TYPE_REGULAR_FILE}.
+ * @return The header block. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if the name does not fit the name field.
+ */
+ public static byte[] base256Header(String name, long size, char typeFlag) {
+ final byte[] block = header(name, 0, typeFlag);
+ long remaining = size;
+ for (int i = SIZE_OFFSET + SIZE_LENGTH - 1; i >= SIZE_OFFSET; i--) {
+ block[i] = (byte) (remaining & 0xFF);
+ remaining >>= Byte.SIZE;
+ }
+ block[SIZE_OFFSET] = (byte) (size < 0
+ ? block[SIZE_OFFSET] | BASE_256_MARKER | BASE_256_NEGATIVE
+ : block[SIZE_OFFSET] & ~BASE_256_NEGATIVE | BASE_256_MARKER);
+ return reseal(block);
+ }
+
+ /**
+ * Encodes one pax extended header record, {@code " =\n"}. The
+ * length includes all record bytes: digits, the blank, and the newline.
+ *
+ * @param keyword The pax keyword.
+ * @param value The keyword's value.
+ * @return The encoded record. Never {@code null}.
+ */
+ public static byte[] paxRecord(String keyword, String value) {
+ final byte[] body = (keyword + "=" + value + "\n").getBytes(StandardCharsets.UTF_8);
+ int digits = 1;
+ while (Integer.toString(body.length + 1 + digits).length() > digits) {
+ digits++;
+ }
+ final byte[] prefix = ((body.length + 1 + digits) + " ")
+ .getBytes(StandardCharsets.US_ASCII);
+ final byte[] record = new byte[prefix.length + body.length];
+ System.arraycopy(prefix, 0, record, 0, prefix.length);
+ System.arraycopy(body, 0, record, prefix.length, body.length);
+ return record;
+ }
+
+ /**
+ * Builds one 512-byte ustar header block whose entry name is split across the name
+ * prefix field and the name field, which is how a real archive stores a name longer
+ * than 100 bytes.
+ *
+ * @param name The name field content; at most 100 bytes when encoded as UTF-8.
+ * @param prefix The name prefix field content; at most 155 bytes when encoded as
+ * UTF-8. Empty for a header without a prefix.
+ * @param size The content size to declare in the octal size field; must not be
+ * negative.
+ * @param typeFlag The tar type flag, for example {@link #TYPE_REGULAR_FILE}.
+ * @return The header block. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if either name part does not fit its field
+ * or the size is negative.
+ */
+ public static byte[] header(String name, String prefix, long size, char typeFlag) {
+ return header(name, prefix, size, typeFlag, true);
+ }
+
+ /**
+ * Builds one 512-byte header block, in either the ustar or the classic v7 format.
+ *
+ * @param name The name field content; at most 100 bytes when encoded as UTF-8.
+ * @param prefix The name prefix field content; at most 155 bytes when encoded as
+ * UTF-8. Empty for a header without a prefix, and required to be empty
+ * for a classic header, which has no prefix field.
+ * @param size The content size to declare in the octal size field; must not be
+ * negative.
+ * @param typeFlag The tar type flag, for example {@link #TYPE_REGULAR_FILE}.
+ * @param ustar Whether to write the ustar magic and version fields.
+ * @return The header block. Never {@code null}.
+ * @throws IllegalArgumentException Thrown if either name part does not fit its field,
+ * the size is negative, or a classic header is asked for with a prefix.
+ */
+ private static byte[] header(String name, String prefix, long size, char typeFlag,
+ boolean ustar) {
+ final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8);
+ if (nameBytes.length > NAME_LENGTH) {
+ throw new IllegalArgumentException(
+ "entry name exceeds " + NAME_LENGTH + " bytes: " + name);
+ }
+ final byte[] prefixBytes = prefix.getBytes(StandardCharsets.UTF_8);
+ if (prefixBytes.length > PREFIX_LENGTH) {
+ throw new IllegalArgumentException(
+ "entry name prefix exceeds " + PREFIX_LENGTH + " bytes: " + prefix);
+ }
+ if (size < 0) {
+ throw new IllegalArgumentException("size must not be negative: " + size);
+ }
+ if (!ustar && prefixBytes.length > 0) {
+ throw new IllegalArgumentException("a classic header has no name prefix field");
+ }
+ final byte[] block = new byte[BLOCK];
+ System.arraycopy(nameBytes, 0, block, 0, nameBytes.length);
+ System.arraycopy(prefixBytes, 0, block, PREFIX_OFFSET, prefixBytes.length);
+ write(block, MODE_OFFSET, MODE);
+ write(block, SIZE_OFFSET, String.format(SIZE_FORMAT, size));
+ if (ustar) {
+ System.arraycopy(USTAR_MAGIC, 0, block, MAGIC_OFFSET, USTAR_MAGIC.length);
+ }
+ block[TYPE_OFFSET] = (byte) typeFlag;
+ return reseal(block);
+ }
+
+ /**
+ * Recomputes and writes the header checksum of the given block, so a test can corrupt
+ * a field on purpose and still hand the reader an otherwise well-formed header.
+ *
+ * @param block The header block to seal. Must be 512 bytes long.
+ * @return The same block, with its checksum field filled in. Never {@code null}.
+ */
+ public static byte[] reseal(byte[] block) {
+ for (int i = CHECKSUM_OFFSET; i < CHECKSUM_OFFSET + CHECKSUM_LENGTH; i++) {
+ block[i] = BLANK;
+ }
+ int sum = 0;
+ for (final byte b : block) {
+ sum += b & 0xFF;
+ }
+ write(block, CHECKSUM_OFFSET, String.format(CHECKSUM_FORMAT, sum));
+ block[CHECKSUM_OFFSET + CHECKSUM_LENGTH - 1] = BLANK;
+ return block;
+ }
+
+ /**
+ * Writes US-ASCII text into a header field.
+ *
+ * @param block The header block to write into.
+ * @param offset The field's offset in the block.
+ * @param text The text to write.
+ */
+ private static void write(byte[] block, int offset, String text) {
+ final byte[] bytes = text.getBytes(StandardCharsets.US_ASCII);
+ System.arraycopy(bytes, 0, block, offset, bytes.length);
+ }
+
+ /**
+ * Writes one complete tar entry into the given buffer: the header block declaring the
+ * content's actual length, the content itself, and zero padding up to the next
+ * 512-byte block boundary.
+ *
+ * @param tar The buffer receiving the entry bytes. Must not be {@code null}.
+ * @param name The entry name; at most 100 bytes when encoded as UTF-8.
+ * @param content The entry content. Must not be {@code null}.
+ * @param typeFlag The tar type flag for the header.
+ * @throws IOException Thrown if writing to the buffer fails.
+ * @throws IllegalArgumentException Thrown if the name does not fit the name field.
+ */
+ public static void entry(ByteArrayOutputStream tar, String name, byte[] content,
+ char typeFlag) throws IOException {
+ tar.write(header(name, content.length, typeFlag));
+ tar.write(content);
+ tar.write(new byte[(BLOCK - content.length % BLOCK) % BLOCK]);
+ }
+
+ /**
+ * Writes one complete regular-file tar entry into the given buffer.
+ *
+ * @param tar The buffer receiving the entry bytes. Must not be {@code null}.
+ * @param name The entry name; at most 100 bytes when encoded as UTF-8.
+ * @param content The entry content. Must not be {@code null}.
+ * @throws IOException Thrown if writing to the buffer fails.
+ * @throws IllegalArgumentException Thrown if the name does not fit the name field.
+ */
+ public static void entry(ByteArrayOutputStream tar, String name, byte[] content)
+ throws IOException {
+ entry(tar, name, content, TYPE_REGULAR_FILE);
+ }
+
+ /**
+ * Builds a gzip-compressed tar archive from name and content pairs, terminated by
+ * the two all-zero blocks that end a tar archive.
+ *
+ * @param entries The entries as {@code {name, content}} pairs of UTF-8 text. Must not
+ * be {@code null}.
+ * @return The compressed archive bytes. Never {@code null}.
+ * @throws IOException Thrown if writing to the in-memory streams fails.
+ */
+ public static byte[] gzippedTar(String[][] entries) throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ for (final String[] entry : entries) {
+ entry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8));
+ }
+ tar.write(new byte[TERMINATOR_SIZE]);
+ return gzip(tar.toByteArray());
+ }
+
+ /**
+ * Compresses raw tar bytes the way a {@code .tar.gz} distribution is shipped, so
+ * tests can wrap hand-built or deliberately truncated tar content.
+ *
+ * @param content The raw tar bytes. Must not be {@code null}.
+ * @return The gzip-compressed bytes. Never {@code null}.
+ * @throws IOException Thrown if writing to the in-memory stream fails.
+ */
+ public static byte[] gzip(byte[] content) throws IOException {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (GZIPOutputStream compressed = new GZIPOutputStream(out)) {
+ compressed.write(content);
+ }
+ return out.toByteArray();
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/archive/TarStreamTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/archive/TarStreamTest.java
new file mode 100644
index 0000000000..52b4bd96c1
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/archive/TarStreamTest.java
@@ -0,0 +1,783 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util.archive;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import static opennlp.tools.util.archive.TarArchives.BLOCK;
+import static opennlp.tools.util.archive.TarArchives.TERMINATOR_SIZE;
+import static opennlp.tools.util.archive.TarArchives.TYPE_DIRECTORY;
+import static opennlp.tools.util.archive.TarArchives.TYPE_REGULAR_FILE;
+import static opennlp.tools.util.archive.TarArchives.TYPE_REGULAR_FILE_CLASSIC;
+import static opennlp.tools.util.archive.TarArchives.entry;
+import static opennlp.tools.util.archive.TarArchives.header;
+
+/**
+ * Exercises {@link TarStream} against archives built byte by byte in the test, covering
+ * regular traversal as well as boundary and corruption cases.
+ */
+public class TarStreamTest {
+
+ private static final int NAME_LENGTH = 100;
+ private static final int SIZE_OFFSET = 124;
+ private static final int CHECKSUM_OFFSET = 148;
+
+ /** The largest length eleven octal digits can hold, one byte short of 8 GiB. */
+ private static final long LARGEST_OCTAL_SIZE = (1L << 33) - 1;
+
+ private static final String MALFORMED_RECORD = "malformed pax extended header record";
+ private static final String SPARSE_REJECTED = "sparse tar entries are not supported: "
+ + "the archived bytes describe file holes, not contiguous content";
+
+ /** The metadata records GNU tar and bsdtar write ahead of an ordinary entry. */
+ private static final String[][] METADATA = {
+ {"mtime", "1786557589.505896519"},
+ {"atime", "1786557589.496852337"},
+ {"ctime", "1786557589.505896519"},
+ {"uid", "1000"},
+ {"gname", "krickert"},
+ {"SCHILY.dev", "66306"},
+ {"hdrcharset", "BINARY"}};
+
+ /**
+ * Supplies blocks that must not be mistaken for a tar header, each with a description
+ * naming the reason.
+ *
+ * @return The rejection cases. Never {@code null}.
+ */
+ private static Stream nonHeaderContent() {
+ final byte[] filler = new byte[BLOCK];
+ Arrays.fill(filler, (byte) 'x');
+ return Stream.of(
+ Arguments.of("fewer bytes than one block",
+ "too short to be a tar header".getBytes(StandardCharsets.US_ASCII)),
+ Arguments.of("no ustar magic and a non-octal size field", filler),
+ Arguments.of("an all-zero block", new byte[BLOCK]));
+ }
+
+ @Test
+ void testEmptyStreamHasNoEntries() throws IOException {
+ final TarStream stream = new TarStream(new ByteArrayInputStream(new byte[0]));
+ Assertions.assertFalse(stream.next());
+ }
+
+ @Test
+ void testTerminatorOnlyArchiveHasNoEntries() throws IOException {
+ final TarStream stream =
+ new TarStream(new ByteArrayInputStream(new byte[TERMINATOR_SIZE]));
+ Assertions.assertFalse(stream.next());
+ }
+
+ /**
+ * Checks that repeated reads after the terminator remain at end of archive.
+ *
+ * @throws IOException Thrown if the fixture archive cannot be read.
+ */
+ @Test
+ void testNextRemainsAtEndAfterAnUnalignedEntry() throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "data.txt", "x".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertArrayEquals("x".getBytes(StandardCharsets.UTF_8),
+ stream.entryStream().readAllBytes());
+ Assertions.assertFalse(stream.next());
+ Assertions.assertFalse(stream.next());
+ }
+
+ @Test
+ void testReadsEntriesSizesTypesAndContent() throws IOException {
+ final byte[] blockSized = new byte[BLOCK];
+ for (int i = 0; i < blockSized.length; i++) {
+ blockSized[i] = (byte) (i % 251);
+ }
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "data/", new byte[0], TYPE_DIRECTORY);
+ entry(tar, "data/skip.bin", "0123456789".getBytes(StandardCharsets.US_ASCII),
+ TYPE_REGULAR_FILE);
+ entry(tar, "data/alpha.txt", "alpha\n".getBytes(StandardCharsets.UTF_8),
+ TYPE_REGULAR_FILE_CLASSIC);
+ entry(tar, "block.bin", blockSized, TYPE_REGULAR_FILE);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("data/", stream.name());
+ Assertions.assertEquals(0, stream.size());
+ Assertions.assertFalse(stream.isFile());
+
+ // The next call must skip the unread content and padding of this entry.
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("data/skip.bin", stream.name());
+ Assertions.assertEquals(10, stream.size());
+ Assertions.assertTrue(stream.isFile());
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("data/alpha.txt", stream.name());
+ Assertions.assertEquals(6, stream.size());
+ Assertions.assertTrue(stream.isFile());
+ Assertions.assertArrayEquals("alpha\n".getBytes(StandardCharsets.UTF_8),
+ stream.entryStream().readAllBytes());
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("block.bin", stream.name());
+ Assertions.assertEquals(BLOCK, stream.size());
+ Assertions.assertTrue(stream.isFile());
+ Assertions.assertArrayEquals(blockSized, stream.entryStream().readAllBytes());
+
+ Assertions.assertFalse(stream.next());
+ }
+
+ @Test
+ void testNameFillsFullHundredByteField() throws IOException {
+ final StringBuilder longName = new StringBuilder("d/");
+ while (longName.length() < NAME_LENGTH) {
+ longName.append('x');
+ }
+ final String name = longName.toString();
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, name, "n".getBytes(StandardCharsets.US_ASCII), TYPE_REGULAR_FILE);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals(NAME_LENGTH, stream.name().length());
+ Assertions.assertEquals(name, stream.name());
+ Assertions.assertArrayEquals("n".getBytes(StandardCharsets.US_ASCII),
+ stream.entryStream().readAllBytes());
+ Assertions.assertFalse(stream.next());
+ }
+
+ @Test
+ void testTruncatedHeaderReportsError() {
+ final byte[] partial =
+ Arrays.copyOf(header("cut.bin", 0, TYPE_REGULAR_FILE), BLOCK / 2);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(partial));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals("truncated tar header", thrown.getMessage());
+ }
+
+ @Test
+ void testTruncatedEntryContentReportsError() throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tar.write(header("data.bin", 10, TYPE_REGULAR_FILE));
+ tar.write("1234".getBytes(StandardCharsets.US_ASCII));
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> stream.entryStream().readAllBytes());
+ Assertions.assertEquals("truncated tar entry: data.bin", thrown.getMessage());
+ }
+
+ @Test
+ void testTruncatedArchiveWhenSkippingReportsError() throws IOException {
+ final TarStream stream = new TarStream(
+ new ByteArrayInputStream(header("gone.bin", 600, TYPE_REGULAR_FILE)));
+
+ Assertions.assertTrue(stream.next());
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals("truncated tar archive", thrown.getMessage());
+ }
+
+ @Test
+ void testMalformedSizeFieldReportsError() {
+ final byte[] block = header("bad.bin", 0, TYPE_REGULAR_FILE);
+ block[SIZE_OFFSET] = '9';
+ // Reseal, so the reader reaches the size field instead of stopping at the checksum.
+ TarArchives.reseal(block);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(block));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals("malformed tar size field in entry header",
+ thrown.getMessage());
+ }
+
+ @Test
+ void testSizeFieldWithEmbeddedPaddingIsRejected() {
+ final byte[] block = header("bad.bin", 0, TYPE_REGULAR_FILE);
+ Arrays.fill(block, SIZE_OFFSET, SIZE_OFFSET + 12, (byte) 0);
+ block[SIZE_OFFSET] = '1';
+ block[SIZE_OFFSET + 1] = ' ';
+ block[SIZE_OFFSET + 2] = '2';
+ TarArchives.reseal(block);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(block));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals("malformed tar size field in entry header",
+ thrown.getMessage());
+ }
+
+ @Test
+ void testStartsWithHeaderDetectsTarAndKeepsPosition() throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "data/alpha.txt", "alpha\n".getBytes(StandardCharsets.UTF_8),
+ TYPE_REGULAR_FILE);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final InputStream in =
+ new BufferedInputStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(TarStream.startsWithHeader(in));
+
+ final TarStream stream = new TarStream(in);
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("data/alpha.txt", stream.name());
+ Assertions.assertArrayEquals("alpha\n".getBytes(StandardCharsets.UTF_8),
+ stream.entryStream().readAllBytes());
+ }
+
+ @ParameterizedTest
+ @MethodSource("nonHeaderContent")
+ void testStartsWithHeaderRejectsNonTarContent(String description, byte[] content)
+ throws IOException {
+ Assertions.assertFalse(
+ TarStream.startsWithHeader(new ByteArrayInputStream(content)), description);
+ }
+
+ @Test
+ void testStartsWithHeaderRejectsUnusableStreams() {
+ final InputStream notMarkable = new InputStream() {
+ @Override
+ public int read() {
+ return -1;
+ }
+ };
+ // assertAll so a missing check on one argument does not hide the other.
+ Assertions.assertAll(
+ () -> Assertions.assertThrows(IllegalArgumentException.class,
+ () -> TarStream.startsWithHeader(null)),
+ () -> Assertions.assertThrows(IllegalArgumentException.class,
+ () -> TarStream.startsWithHeader(notMarkable)));
+ }
+
+ @Test
+ void testNullStreamIsRejected() {
+ Assertions.assertThrows(IllegalArgumentException.class, () -> new TarStream(null));
+ }
+
+ /**
+ * @param maxEntries The invalid entry limit.
+ */
+ @ParameterizedTest(name = "{0}")
+ @ValueSource(longs = {0, -1})
+ void testInvalidEntryLimitIsRejected(long maxEntries) {
+ final IllegalArgumentException thrown = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> new TarStream(InputStream.nullInputStream(), maxEntries));
+ Assertions.assertEquals("maxEntries must be positive", thrown.getMessage());
+ }
+
+ @Test
+ void testHeaderWithWrongChecksumIsRejected() {
+ final byte[] block = header("tampered.bin", 10, TYPE_REGULAR_FILE);
+ block[0] = 'X';
+ final TarStream stream = new TarStream(new ByteArrayInputStream(block));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals("malformed tar header checksum", thrown.getMessage());
+ }
+
+ @Test
+ void testChecksumWithEmbeddedPaddingIsRejected() {
+ final byte[] block = header("bad-checksum.bin", 0, TYPE_REGULAR_FILE);
+ System.arraycopy(block, CHECKSUM_OFFSET + 1, block, CHECKSUM_OFFSET + 2, 5);
+ block[CHECKSUM_OFFSET + 1] = ' ';
+ final TarStream stream = new TarStream(new ByteArrayInputStream(block));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals("malformed tar header checksum", thrown.getMessage());
+ }
+
+ @Test
+ void testStartsWithHeaderRejectsUstarMagicWithoutAChecksum() throws IOException {
+ final byte[] block = header("looks-real.bin", 0, TYPE_REGULAR_FILE);
+ Arrays.fill(block, CHECKSUM_OFFSET, CHECKSUM_OFFSET + 8, (byte) '0');
+
+ Assertions.assertFalse(TarStream.startsWithHeader(new ByteArrayInputStream(block)));
+ }
+
+ @Test
+ void testUstarPrefixIsJoinedToTheName() throws IOException {
+ final String prefix = "corpus-1.0/" + "d".repeat(120);
+ final String tail = "annotations/train.conllu";
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ final byte[] content = "# sent_id = 1\n".getBytes(StandardCharsets.UTF_8);
+ tar.write(TarArchives.header(tail, prefix, content.length, TYPE_REGULAR_FILE));
+ tar.write(content);
+ tar.write(new byte[BLOCK - content.length]);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals(prefix + "/" + tail, stream.name());
+ Assertions.assertTrue(stream.name().length() > 100);
+ Assertions.assertArrayEquals(content, stream.entryStream().readAllBytes());
+ }
+
+ @Test
+ void testEmptyUstarPrefixLeavesTheNameAlone() throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "plain.txt", "x".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("plain.txt", stream.name());
+ }
+
+ @Test
+ void testPaxGlobalHeaderWithOnlyACommentIsConsumed() throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "pax_global_header",
+ TarArchives.paxRecord("comment", "0123456789abcdef"), 'g');
+ entry(tar, "data.txt", "content".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("data.txt", stream.name());
+ Assertions.assertTrue(stream.isFile());
+ Assertions.assertArrayEquals("content".getBytes(StandardCharsets.UTF_8),
+ stream.entryStream().readAllBytes());
+ Assertions.assertFalse(stream.next());
+ }
+
+ /**
+ * Supplies pax global headers that must be rejected: the two keywords that would change
+ * every entry after them, the sparse family whose content cannot be unpacked, and
+ * payloads that are not pax records at all.
+ *
+ * @return One case per rejected global header. Never {@code null}.
+ */
+ private static Stream rejectedGlobalHeaders() {
+ return Stream.of(
+ Arguments.of("path redirects every following entry",
+ TarArchives.paxRecord("path", "../escape.txt"),
+ "pax global header contains path, which would change every entry after it"),
+ Arguments.of("size resizes every following entry",
+ TarArchives.paxRecord("size", "999999"),
+ "pax global header contains size, which would change every entry after it"),
+ Arguments.of("sparse map cannot be unpacked",
+ TarArchives.paxRecord("GNU.sparse.name", "holes.bin"),
+ SPARSE_REJECTED),
+ Arguments.of("no length prefix",
+ "comment=0\n".getBytes(StandardCharsets.UTF_8), MALFORMED_RECORD),
+ Arguments.of("length longer than the payload",
+ "99 comment=0\n".getBytes(StandardCharsets.UTF_8), MALFORMED_RECORD),
+ Arguments.of("length shorter than its own prefix",
+ "1 comment=0\n".getBytes(StandardCharsets.UTF_8), MALFORMED_RECORD),
+ Arguments.of("no keyword before the equals sign",
+ "12 =value\n".getBytes(StandardCharsets.UTF_8), MALFORMED_RECORD),
+ Arguments.of("no equals sign at all",
+ "12 comment0\n".getBytes(StandardCharsets.UTF_8), MALFORMED_RECORD));
+ }
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("rejectedGlobalHeaders")
+ void testRejectedPaxGlobalHeader(String description, byte[] payload, String message)
+ throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "pax_global_header", payload, 'g');
+ entry(tar, "data.txt", "content".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals(message, thrown.getMessage());
+ }
+
+ @Test
+ void testClassicHeaderWithoutUstarMagicIsRead() throws IOException {
+ final byte[] content = "classic\n".getBytes(StandardCharsets.UTF_8);
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tar.write(TarArchives.classicHeader("v7/data.txt", content.length,
+ TYPE_REGULAR_FILE));
+ tar.write(content);
+ tar.write(new byte[BLOCK - content.length]);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final byte[] archive = tar.toByteArray();
+
+ Assertions.assertTrue(TarStream.startsWithHeader(
+ new BufferedInputStream(new ByteArrayInputStream(archive))));
+ final TarStream stream = new TarStream(new ByteArrayInputStream(archive));
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("v7/data.txt", stream.name());
+ Assertions.assertTrue(stream.isFile());
+ Assertions.assertArrayEquals(content, stream.entryStream().readAllBytes());
+ Assertions.assertFalse(stream.next());
+ }
+
+ @Test
+ void testHeaderWithAnEmptyNameIsRejected() throws IOException {
+ final byte[] block = TarArchives.reseal(header("", 0, TYPE_REGULAR_FILE));
+ final TarStream stream = new TarStream(new ByteArrayInputStream(block));
+
+ Assertions.assertAll(
+ () -> {
+ final IOException thrown =
+ Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals("tar entry header contains an empty name",
+ thrown.getMessage());
+ },
+ () -> Assertions.assertFalse(
+ TarStream.startsWithHeader(new ByteArrayInputStream(block))));
+ }
+
+ @Test
+ void testHeaderNameWithMalformedUtf8IsRejected() {
+ final byte[] block = header("data.txt", 0, TYPE_REGULAR_FILE);
+ block[0] = (byte) 0xC3;
+ block[1] = 0;
+ TarArchives.reseal(block);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(block));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals("tar entry name is not valid UTF-8", thrown.getMessage());
+ }
+
+ @Test
+ void testEntryStreamRejectsInvalidReadRanges() throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "data.txt", "content".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+ Assertions.assertTrue(stream.next());
+ final InputStream content = stream.entryStream();
+ final byte[] buffer = new byte[8];
+
+ Assertions.assertAll(
+ () -> Assertions.assertThrows(NullPointerException.class,
+ () -> content.read(null, 0, 1)),
+ () -> Assertions.assertThrows(NullPointerException.class,
+ () -> content.read(null, 0, 0)),
+ () -> Assertions.assertThrows(IndexOutOfBoundsException.class,
+ () -> content.read(buffer, -1, 1)),
+ () -> Assertions.assertThrows(IndexOutOfBoundsException.class,
+ () -> content.read(buffer, 0, -1)),
+ () -> Assertions.assertThrows(IndexOutOfBoundsException.class,
+ () -> content.read(buffer, 0, buffer.length + 1)),
+ () -> Assertions.assertThrows(IndexOutOfBoundsException.class,
+ () -> content.read(buffer, buffer.length, 1)),
+ // Zero length at a valid offset is legal and must not be mistaken for an error.
+ () -> Assertions.assertEquals(0, content.read(buffer, buffer.length, 0)));
+ }
+
+ @Test
+ void testZeroLengthReadReturnsZero() throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "data.txt", "content".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+ Assertions.assertTrue(stream.next());
+ final InputStream content = stream.entryStream();
+ final byte[] buffer = new byte[8];
+
+ Assertions.assertEquals(0, content.read(buffer, 0, 0));
+ Assertions.assertArrayEquals("content".getBytes(StandardCharsets.UTF_8),
+ content.readAllBytes());
+ // Exhausted: a zero-length read still reports zero, not end of stream.
+ Assertions.assertEquals(0, content.read(buffer, 0, 0));
+ Assertions.assertEquals(-1, content.read(buffer, 0, buffer.length));
+ // Exhausted, and the range is still checked before the end-of-stream answer.
+ Assertions.assertThrows(IndexOutOfBoundsException.class,
+ () -> content.read(buffer, 0, buffer.length + 1));
+ }
+
+ /**
+ * Concatenates byte arrays, so a test can assemble one extension header payload from
+ * several records.
+ *
+ * @param parts The pieces to join. Must not be {@code null}.
+ * @return The concatenation. Never {@code null}.
+ */
+ private static byte[] concat(byte[]... parts) {
+ final ByteArrayOutputStream joined = new ByteArrayOutputStream();
+ for (final byte[] part : parts) {
+ joined.writeBytes(part);
+ }
+ return joined.toByteArray();
+ }
+
+ /**
+ * Encodes metadata records emitted by real archive writers, so tests cover their output.
+ *
+ * @return The encoded records. Never {@code null}.
+ */
+ private static byte[] metadataRecords() {
+ final ByteArrayOutputStream records = new ByteArrayOutputStream();
+ for (final String[] record : METADATA) {
+ records.writeBytes(TarArchives.paxRecord(record[0], record[1]));
+ }
+ return records.toByteArray();
+ }
+
+ @Test
+ void testPaxExtendedHeaderSuppliesTheEntryName() throws IOException {
+ final String path = "./corpus-1.0/" + "d".repeat(120) + "/annotations/train.conllu";
+ final byte[] content = "content\n".getBytes(StandardCharsets.UTF_8);
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "./PaxHeaders/train.conllu",
+ concat(TarArchives.paxRecord("path", path), metadataRecords()), 'x');
+ // GNU tar stores the truncated name in the entry header.
+ entry(tar, path.substring(0, 100), content);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals(path, stream.name());
+ Assertions.assertTrue(stream.isFile());
+ Assertions.assertArrayEquals(content, stream.entryStream().readAllBytes());
+ Assertions.assertFalse(stream.next());
+ }
+
+ @Test
+ void testPaxPathWithMalformedUtf8IsRejected() throws IOException {
+ final byte[] malformedPath = {
+ '1', '0', ' ', 'p', 'a', 't', 'h', '=', (byte) 0xC3, '\n'};
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "./PaxHeaders/data.txt", malformedPath, 'x');
+ entry(tar, "data.txt", "content".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals("pax record is not valid UTF-8", thrown.getMessage());
+ }
+
+ @Test
+ void testPaxMetadataOnlyHeaderLeavesTheEntryAlone() throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "./PaxHeaders/short.txt", metadataRecords(), 'x');
+ entry(tar, "./short.txt", "short\n".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("./short.txt", stream.name());
+ Assertions.assertEquals(6, stream.size());
+ Assertions.assertFalse(stream.next());
+ }
+
+ @Test
+ void testPaxExtendedHeaderAppliesOnlyToTheNextEntry() throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "./PaxHeaders/first",
+ TarArchives.paxRecord("path", "overridden/first.txt"), 'x');
+ entry(tar, "truncated-first", "one".getBytes(StandardCharsets.UTF_8));
+ entry(tar, "second.txt", "two".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("overridden/first.txt", stream.name());
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("second.txt", stream.name());
+ Assertions.assertFalse(stream.next());
+ }
+
+ @Test
+ void testPaxExtendedHeaderSuppliesTheEntrySize() throws IOException {
+ final byte[] content = "0123456789".getBytes(StandardCharsets.US_ASCII);
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "./PaxHeaders/data.bin", TarArchives.paxRecord("size", "10"), 'x');
+ tar.write(header("data.bin", 0, TYPE_REGULAR_FILE));
+ tar.write(content);
+ tar.write(new byte[BLOCK - content.length]);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals(10, stream.size());
+ Assertions.assertArrayEquals(content, stream.entryStream().readAllBytes());
+ }
+
+ @Test
+ void testGnuLongNameHeaderSuppliesTheEntryName() throws IOException {
+ final String path = "./corpus-1.0/" + "d".repeat(120) + "/annotations/train.conllu";
+ final byte[] content = "content\n".getBytes(StandardCharsets.UTF_8);
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "././@LongLink",
+ (path + "\0").getBytes(StandardCharsets.UTF_8), 'L');
+ entry(tar, path.substring(0, 100), content);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals(path, stream.name());
+ Assertions.assertArrayEquals(content, stream.entryStream().readAllBytes());
+ Assertions.assertFalse(stream.next());
+ }
+
+ @Test
+ void testGnuLongNameWithMalformedUtf8IsRejected() throws IOException {
+ final byte[] malformedName = {'b', 'a', 'd', (byte) 0xC3, 0};
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "././@LongLink", malformedName, 'L');
+ entry(tar, "data.txt", "content".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals("GNU long name is not valid UTF-8", thrown.getMessage());
+ }
+
+ @Test
+ void testGnuLongLinkHeaderIsConsumed() throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ entry(tar, "././@LongLink",
+ ("../" + "t".repeat(120) + "/target\0").getBytes(StandardCharsets.UTF_8), 'K');
+ entry(tar, "data.txt", "content".getBytes(StandardCharsets.UTF_8));
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("data.txt", stream.name());
+ Assertions.assertArrayEquals("content".getBytes(StandardCharsets.UTF_8),
+ stream.entryStream().readAllBytes());
+ }
+
+ @Test
+ void testGnuHeaderDoesNotReadItsAtimeAsANamePrefix() throws IOException {
+ final byte[] content = "short\n".getBytes(StandardCharsets.UTF_8);
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tar.write(TarArchives.gnuHeader("./short.txt", content.length, TYPE_REGULAR_FILE,
+ "15237132225"));
+ tar.write(content);
+ tar.write(new byte[BLOCK - content.length]);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("./short.txt", stream.name());
+ Assertions.assertArrayEquals(content, stream.entryStream().readAllBytes());
+ }
+
+ /**
+ * Supplies the two sparse entry declarations. Their archived bytes describe file holes,
+ * so copying those bytes would produce incorrect content.
+ *
+ * @return One case per sparse declaration. Never {@code null}.
+ */
+ private static Stream sparseEntries() {
+ return Stream.of(
+ Arguments.of("GNU sparse type flag", (Supplier) () -> {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tar.writeBytes(header("holes.bin", 0, 'S'));
+ return tar.toByteArray();
+ }),
+ Arguments.of("pax sparse records", (Supplier) () -> {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ try {
+ entry(tar, "./PaxHeaders/holes.bin",
+ concat(TarArchives.paxRecord("GNU.sparse.major", "1"),
+ TarArchives.paxRecord("GNU.sparse.name", "holes.bin")), 'x');
+ entry(tar, "holes.bin", "not the content".getBytes(StandardCharsets.UTF_8));
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ return tar.toByteArray();
+ }));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("sparseEntries")
+ void testSparseEntriesAreRejected(String description, Supplier archive) {
+ final TarStream stream = new TarStream(new ByteArrayInputStream(archive.get()));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals(SPARSE_REJECTED, thrown.getMessage());
+ }
+
+ @Test
+ void testBase256SizeFieldIsRead() throws IOException {
+ final byte[] content = "0123456789".getBytes(StandardCharsets.US_ASCII);
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tar.write(TarArchives.base256Header("big.bin", content.length, TYPE_REGULAR_FILE));
+ tar.write(content);
+ tar.write(new byte[BLOCK - content.length]);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(tar.toByteArray()));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals("big.bin", stream.name());
+ Assertions.assertEquals(content.length, stream.size());
+ Assertions.assertArrayEquals(content, stream.entryStream().readAllBytes());
+ }
+
+ @Test
+ void testBase256SizeFieldCarriesLengthsBeyondTheOctalRange() throws IOException {
+ final long beyondOctal = 8L * 1024 * 1024 * 1024;
+ Assertions.assertTrue(beyondOctal > LARGEST_OCTAL_SIZE);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(
+ TarArchives.base256Header("huge.bin", beyondOctal, TYPE_REGULAR_FILE)));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals(beyondOctal, stream.size());
+ }
+
+ @Test
+ void testBase256SizeFieldAcceptsTheLargestRepresentableLength() throws IOException {
+ final TarStream stream = new TarStream(new ByteArrayInputStream(
+ TarArchives.base256Header("max.bin", Long.MAX_VALUE, TYPE_REGULAR_FILE)));
+
+ Assertions.assertTrue(stream.next());
+ Assertions.assertEquals(Long.MAX_VALUE, stream.size());
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals("truncated tar archive", thrown.getMessage());
+ }
+
+ @Test
+ void testBase256SizeFieldBeyondTheLongRangeIsRejected() {
+ final byte[] block = TarArchives.base256Header("huge.bin", 0, TYPE_REGULAR_FILE);
+ // One bit above the 63 a long can hold, left of everything the encoder can write.
+ block[SIZE_OFFSET + 1] = 1;
+ TarArchives.reseal(block);
+ final TarStream stream = new TarStream(new ByteArrayInputStream(block));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals(
+ "tar size field exceeds the largest length this reader can represent",
+ thrown.getMessage());
+ }
+
+ @Test
+ void testNegativeBase256SizeFieldIsRejected() {
+ final TarStream stream = new TarStream(new ByteArrayInputStream(
+ TarArchives.base256Header("negative.bin", -1, TYPE_REGULAR_FILE)));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class, stream::next);
+ Assertions.assertEquals("tar size field is negative", thrown.getMessage());
+ }
+}
diff --git a/opennlp-docs/src/docbkx/model-loading.xml b/opennlp-docs/src/docbkx/model-loading.xml
index eb2356a80d..8a8d6cb394 100644
--- a/opennlp-docs/src/docbkx/model-loading.xml
+++ b/opennlp-docs/src/docbkx/model-loading.xml
@@ -164,4 +164,57 @@ for(ClassPathModelEntry entry : models) {
-
\ No newline at end of file
+
+
+ Installing third-party resources
+
+ ResourceInstaller copies a supplied http,
+ https, or file URI into a target directory. Http and https
+ sources require a SHA-256 or SHA-512 checksum. File sources may omit it.
+ Gzip-compressed tar and zip archives are unpacked, plain gzip files are decompressed,
+ and other content is stored under its source name. OpenNLP *.bin model
+ files remain packed.
+
+
+
+
+
+ Content is unpacked in a staging directory on the target filesystem. Archive paths
+ cannot escape that directory, duplicate file paths are rejected, and existing target
+ files are not replaced. A checksum, download, or unpacking error promotes no files.
+ Promotion also rejects symbolic links below the target. Concurrent changes to the
+ target directory are outside this guarantee.
+
+
+ Http and https requests use connection and read timeouts, follow up to five
+ redirects by default, and reject an https-to-http downgrade. Default limits are 1 GiB
+ downloaded, 4 GiB expanded, and 100000 archive entries; compressed content may
+ expand to at most 100 times its compressed size, with a 1 MiB floor for small
+ sources. These defaults can be set at
+ JVM startup with
+ opennlp.download.max.bytes, opennlp.install.max.total.bytes,
+ opennlp.install.max.entries, and
+ opennlp.install.max.expansion.ratio. Use
+ ResourceInstaller.Limits to set limits for one call.
+
+
+ The expansion ratio and the expansion limit are enforced independently and the
+ tighter one applies, so raising opennlp.install.max.total.bytes alone
+ does not admit an archive that exceeds the ratio. A resource that legitimately
+ compresses better than 100 to 1, such as a corpus of highly repetitive text, needs
+ opennlp.install.max.expansion.ratio raised as well.
+
+
+
+
+
+
diff --git a/opennlp-docs/src/docbkx/stemmer.xml b/opennlp-docs/src/docbkx/stemmer.xml
index d777842fd6..ddb5056411 100644
--- a/opennlp-docs/src/docbkx/stemmer.xml
+++ b/opennlp-docs/src/docbkx/stemmer.xml
@@ -78,7 +78,7 @@ new CachingStemmer(factory).stem("running"); // "run"]]>
.aff affix file and its .dic word list. OpenNLP
bundles no dictionary data; dictionaries are downloaded separately, and
each states its own license. The dictionary is immutable and safe to share;
- HunspellStemmerFactory hands out a fresh stemmer per call.
+ HunspellStemmerFactory creates a fresh stemmer per call.
HunspellManualExampleTest asserts the behavior shown here.
dev/README-hunspell-dictionaries.md.
An opt-in catalog download accepts an application-supplied
DictionaryCatalog, needs
- -Dopennlp.download.remote=true, and verifies SHA-512 digests.
+ -Dopennlp.download.remote=true, and fetches through the
+ digest-verified ResourceInstaller path. A file that already exists
+ in the target is not replaced. Remove old files before refreshing a dictionary.
Directives that would change stems when ignored
(ICONV, OCONV, COMPLEXPREFIXES,
COMPOUNDRULE, IGNORE, KEEPCASE)
diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml
index 32ee38e1f4..c666c9001b 100644
--- a/opennlp-docs/src/docbkx/tokenizer.xml
+++ b/opennlp-docs/src/docbkx/tokenizer.xml
@@ -545,19 +545,25 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> {
LatticeTokenizer scores paths over a MeCab-format dictionary and
emits the cheapest segmentation with spans in original text coordinates.
UnigramSegmenter does the same from a plain frequency lexicon.
- Install a trusted local dictionary archive with
- MecabDictionaryInstaller, load it as a
- MecabDictionary, and tokenize.
+ Install a dictionary archive with MecabDictionaryInstaller, load it
+ as a MecabDictionary, and tokenize. Remote archives require an
+ expected SHA-256 or SHA-512 digest; an application-supplied opt-in catalog
+ (installFromCatalog) also needs
+ -Dopennlp.download.remote=true.
matrix.def must list a cost for every declared cell; matrix
dimensions plus lexicon size are bounded by the shared
ResourceLimits.MAX_ENTRIES limit and the matrix cell count
by ResourceLimits.MAX_MATRIX_CELLS.
- Extraction limits per-entry size, total bytes, entry count, and gzip expansion.
- The byte limits default to 512 MiB per entry and 2 GiB total and can be raised
- at JVM startup via
- opennlp.install.max.entry.bytes and
- opennlp.install.max.total.bytes for larger dictionaries such as
- UniDic.
+ Fetching, verification, and unpacking use ResourceInstaller,
+ described in the model loading chapter. Default limits are a 1 GiB
+ download, 4 GiB unpacked, and 100000 archive entries. Set higher limits at
+ JVM startup via
+ opennlp.download.max.bytes,
+ opennlp.install.max.total.bytes,
+ opennlp.install.max.entries, and
+ opennlp.install.max.expansion.ratio for larger dictionaries such as
+ UniDic. Installed files are flattened to their base names. Existing target
+ files are not replaced and must be removed before a refresh.
Dictionary loading rejects empty lexicon surfaces, duplicate
matrix.def entries, duplicate char.def
categories, costs outside the signed 16-bit range, malformed text in the
@@ -570,13 +576,10 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> {
provides unknown-word settings. Grouping continues while successive
assignments overlap. Unknown words use their category's unk.def
template first, with DEFAULT used when no category template exists.
- Tar headers are checksum-validated before extraction. Files are staged on
- the target filesystem and published after the archive passes validation. The
- installer does not replace files already present in the target directory.
-
-LatticeUsageExampleTest asserts the install-load-tokenize and
+ lexicon flows shown here.
+
+