Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions java/src/main/java/ai/rapids/cudf/NativeDepUtil.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/
package ai.rapids.cudf;

import java.io.File;
import java.io.IOException;

/** Command-line utilities for inspecting and extracting packaged native dependencies. */
public final class NativeDepUtil {
private NativeDepUtil() {
}

/**
* Extract a packaged native dependency without loading it.
*
* @param args {@code extract <library-base-name> <destination>}
* @throws IOException if the native dependency cannot be extracted
*/
public static void main(String[] args) throws IOException {
String os = System.getProperty("os.name");
String arch = System.getProperty("os.arch");
File destination = execute(args, os, arch);
System.out.println(destination);
}

static File execute(String[] args, String os, String arch) throws IOException {
if (args.length != 3 || !"extract".equals(args[0])) {
throw new IllegalArgumentException(
"Usage: NativeDepUtil extract <library-base-name> <destination>");
}
return NativeDepsLoader.extractNativeDep(os, arch, args[1], new File(args[2]));
}
}
88 changes: 77 additions & 11 deletions java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
Expand Down Expand Up @@ -392,22 +397,12 @@ static File createFile(String os, String arch, String baseName) throws IOExcepti
}
return loc;
}
String path = arch + "/" + os + "/" + mappedName;
URL chunkManifestResource = loader.getResource(path + CHUNK_MANIFEST_SUFFIX);
URL resource = chunkManifestResource == null ? loader.getResource(path) : null;
if (chunkManifestResource == null && resource == null) {
throw new FileNotFoundException("Could not locate native dependency " + path);
}
long t0 = System.currentTimeMillis();
File loc = File.createTempFile(baseName, ".so");
loc.deleteOnExit();
boolean success = false;
try {
if (chunkManifestResource == null) {
extractConventionalResource(resource, loc);
} else {
extractChunkedResource(chunkManifestResource, mappedName, loc);
}
extractNativeResource(os, arch, baseName, loc);
success = true;
} finally {
if (!success && loc.exists() && !loc.delete()) {
Expand All @@ -422,6 +417,77 @@ static File createFile(String os, String arch, String baseName) throws IOExcepti
return loc;
}

/**
* Extract a native library resource without loading it. The library is searched for under
* {@code ${os.arch}/${os.name}/} in the class path using the class loader for this class.
* Both conventional resources and chunked resources are supported.
*
* <p>The destination is replaced only after the resource has been completely extracted and
* validated. If extraction fails, an existing destination is left unchanged.</p>
*
* @param depName the base name of the library; for example, use {@code "cudf"} for
* {@code libcudf.so}
* @param destination the file where the reconstructed library will be written
* @return the absolute destination file
* @throws IOException on any error locating or extracting the library
*/
public static File extractNativeDep(String depName, File destination) throws IOException {
String os = System.getProperty("os.name");
String arch = System.getProperty("os.arch");
return extractNativeDep(os, arch, depName, destination);
}

static File extractNativeDep(String os, String arch, String baseName, File destination)
throws IOException {
if (baseName == null || baseName.isEmpty()) {
throw new IllegalArgumentException("baseName must not be empty");
}
if (destination == null) {
throw new NullPointerException("destination");
}

Path destinationPath = destination.toPath().toAbsolutePath();
Path parent = destinationPath.getParent();
if (parent == null || !Files.isDirectory(parent)) {
throw new IOException("Native dependency destination directory does not exist: " + parent);
}

Path temporaryPath = Files.createTempFile(parent,
"." + destinationPath.getFileName(), ".tmp");
boolean moved = false;
try {
extractNativeResource(os, arch, baseName, temporaryPath.toFile());
try {
Files.move(temporaryPath, destinationPath,
StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (AtomicMoveNotSupportedException | FileAlreadyExistsException e) {
Files.move(temporaryPath, destinationPath, StandardCopyOption.REPLACE_EXISTING);
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
moved = true;
return destinationPath.toFile();
} finally {
if (!moved) {
Files.deleteIfExists(temporaryPath);
}
}
}

private static void extractNativeResource(
String os, String arch, String baseName, File destination) throws IOException {
String mappedName = System.mapLibraryName(baseName);
String path = arch + "/" + os + "/" + mappedName;
URL chunkManifestResource = loader.getResource(path + CHUNK_MANIFEST_SUFFIX);
URL resource = chunkManifestResource == null ? loader.getResource(path) : null;
if (chunkManifestResource == null && resource == null) {
throw new FileNotFoundException("Could not locate native dependency " + path);
}
if (chunkManifestResource == null) {
extractConventionalResource(resource, destination);
} else {
extractChunkedResource(chunkManifestResource, mappedName, destination);
}
}

private static void extractConventionalResource(URL resource, File loc) throws IOException {
try (InputStream in = resource.openStream();
OutputStream out = new FileOutputStream(loc)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,65 @@ void extractsChunkedResourceAndPrefersManifest() throws IOException {
}
}

@Test
void replacesExistingDestination() throws IOException {
String baseName = "destinationtest";
byte[] expected = "requested destination contents".getBytes(StandardCharsets.UTF_8);
writeChunkedResource(baseName, expected, 8, "1", null, null);
Path destinationDirectory = Files.createTempDirectory("native-dep-destination");
Path destination = destinationDirectory.resolve(System.mapLibraryName(baseName));
Files.write(destination, "existing contents".getBytes(StandardCharsets.UTF_8));
try {
File extracted = NativeDepsLoader.extractNativeDep(
TEST_OS, TEST_ARCH, baseName, destination.toFile());

assertEquals(destination.toAbsolutePath(), extracted.toPath());
assertArrayEquals(expected, Files.readAllBytes(destination));
} finally {
Files.deleteIfExists(destination);
Files.deleteIfExists(destinationDirectory);
}
}

@Test
void failedDestinationExtractionPreservesExistingFile() throws IOException {
String baseName = "preservedestinationtest";
byte[] expected = "chunk contents".getBytes(StandardCharsets.UTF_8);
writeChunkedResource(baseName, expected, 8, "1", 0L, null);
Path destinationDirectory = Files.createTempDirectory("native-dep-destination");
Path destination = destinationDirectory.resolve(System.mapLibraryName(baseName));
byte[] original = "existing contents".getBytes(StandardCharsets.UTF_8);
Files.write(destination, original);
try {
assertThrows(IOException.class, () -> NativeDepsLoader.extractNativeDep(
TEST_OS, TEST_ARCH, baseName, destination.toFile()));

assertArrayEquals(original, Files.readAllBytes(destination));
} finally {
Files.deleteIfExists(destination);
Files.deleteIfExists(destinationDirectory);
}
}

@Test
void nativeDepUtilExtractsResource() throws IOException {
String baseName = "nativeutiltest";
byte[] expected = "native util contents".getBytes(StandardCharsets.UTF_8);
writeChunkedResource(baseName, expected, 8, "1", null, null);
Path destinationDirectory = Files.createTempDirectory("native-dep-util");
Path destination = destinationDirectory.resolve(System.mapLibraryName(baseName));
try {
File extracted = NativeDepUtil.execute(
new String[]{"extract", baseName, destination.toString()}, TEST_OS, TEST_ARCH);

assertEquals(destination.toAbsolutePath(), extracted.toPath());
assertArrayEquals(expected, Files.readAllBytes(destination));
} finally {
Files.deleteIfExists(destination);
Files.deleteIfExists(destinationDirectory);
}
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

@Test
void extractsChunksConcurrently() throws Exception {
String mappedName = "libconcurrent.so";
Expand Down
2 changes: 0 additions & 2 deletions python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -3664,8 +3664,6 @@ def pytest_unconfigure(config):
"tests/tools/test_to_datetime.py::TestOrigin::test_julian": "AssertionError: Attributes of Series are different",
"tests/tools/test_to_datetime.py::TestOrigin::test_to_datetime_out_of_bounds_with_format_arg[%Y-%d-%m %H:%M:%S-None]": "TODO: Add a reason for failure",
"tests/tools/test_to_datetime.py::TestOrigin::test_to_datetime_out_of_bounds_with_format_arg[%Y-%m-%d %H:%M:%S-None]": "TODO: Add a reason for failure",
"tests/tools/test_to_datetime.py::TestTimeConversionFormats::test_to_datetime_format_weeks[False-2013020-%Y%U%w-2013-01-13]": "AssertionError: assert Timestamp('2013-01-19 00:00:00') == Timestamp('2013-01-13 00:00:00')",
"tests/tools/test_to_datetime.py::TestTimeConversionFormats::test_to_datetime_format_weeks[True-2013020-%Y%U%w-2013-01-13]": "AssertionError: assert Timestamp('2013-01-19 00:00:00') == Timestamp('2013-01-13 00:00:00')",
"tests/tools/test_to_datetime.py::TestToDatetime::test_mixed_offsets_with_native_datetime_utc_false_raises": "assert False",
"tests/tools/test_to_datetime.py::TestToDatetime::test_to_datetime_arrow[index-None-False]": "AssertionError: assert DatetimeIndex([1965-04-03 00:00:00, 1965-04-17 00:00:00, 1965-05-01 00:00:00,\n 1965-05-...",
"tests/tools/test_to_datetime.py::TestToDatetime::test_to_datetime_arrow[index-US/Central-False]": "AssertionError: assert Index([1965-04-03 00:00:00-06:00, 1965-04-17 00:00:00-06:00,\n 1965-05-01 00:00:00-05:00...",
Expand Down
Loading