diff --git a/java/src/main/java/ai/rapids/cudf/NativeDepUtil.java b/java/src/main/java/ai/rapids/cudf/NativeDepUtil.java new file mode 100644 index 000000000000..0fe5bde2880b --- /dev/null +++ b/java/src/main/java/ai/rapids/cudf/NativeDepUtil.java @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package ai.rapids.cudf; + +import java.io.File; +import java.io.IOException; + +/** Command-line utilities for inspecting and extracting packaged native dependencies. */ +public final class NativeDepUtil { + private NativeDepUtil() { + } + + /** + * Extract a packaged native dependency without loading it. + * + * @param args {@code extract } + * @throws IOException if the native dependency cannot be extracted + */ + public static void main(String[] args) throws IOException { + String os = System.getProperty("os.name"); + String arch = System.getProperty("os.arch"); + File destination = execute(args, os, arch); + System.out.println(destination); + } + + static File execute(String[] args, String os, String arch) throws IOException { + if (args.length != 3 || !"extract".equals(args[0])) { + throw new IllegalArgumentException( + "Usage: NativeDepUtil extract "); + } + return NativeDepsLoader.extractNativeDep(os, arch, args[1], new File(args[2])); + } +} diff --git a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java index 54fd1513d113..55681bc3851b 100755 --- a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java +++ b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java @@ -17,6 +17,11 @@ import java.net.URL; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -392,22 +397,12 @@ static File createFile(String os, String arch, String baseName) throws IOExcepti } return loc; } - String path = arch + "/" + os + "/" + mappedName; - URL chunkManifestResource = loader.getResource(path + CHUNK_MANIFEST_SUFFIX); - URL resource = chunkManifestResource == null ? loader.getResource(path) : null; - if (chunkManifestResource == null && resource == null) { - throw new FileNotFoundException("Could not locate native dependency " + path); - } long t0 = System.currentTimeMillis(); File loc = File.createTempFile(baseName, ".so"); loc.deleteOnExit(); boolean success = false; try { - if (chunkManifestResource == null) { - extractConventionalResource(resource, loc); - } else { - extractChunkedResource(chunkManifestResource, mappedName, loc); - } + extractNativeResource(os, arch, baseName, loc); success = true; } finally { if (!success && loc.exists() && !loc.delete()) { @@ -422,6 +417,77 @@ static File createFile(String os, String arch, String baseName) throws IOExcepti return loc; } + /** + * Extract a native library resource without loading it. The library is searched for under + * {@code ${os.arch}/${os.name}/} in the class path using the class loader for this class. + * Both conventional resources and chunked resources are supported. + * + *

The destination is replaced only after the resource has been completely extracted and + * validated. If extraction fails, an existing destination is left unchanged.

+ * + * @param depName the base name of the library; for example, use {@code "cudf"} for + * {@code libcudf.so} + * @param destination the file where the reconstructed library will be written + * @return the absolute destination file + * @throws IOException on any error locating or extracting the library + */ + public static File extractNativeDep(String depName, File destination) throws IOException { + String os = System.getProperty("os.name"); + String arch = System.getProperty("os.arch"); + return extractNativeDep(os, arch, depName, destination); + } + + static File extractNativeDep(String os, String arch, String baseName, File destination) + throws IOException { + if (baseName == null || baseName.isEmpty()) { + throw new IllegalArgumentException("baseName must not be empty"); + } + if (destination == null) { + throw new NullPointerException("destination"); + } + + Path destinationPath = destination.toPath().toAbsolutePath(); + Path parent = destinationPath.getParent(); + if (parent == null || !Files.isDirectory(parent)) { + throw new IOException("Native dependency destination directory does not exist: " + parent); + } + + Path temporaryPath = Files.createTempFile(parent, + "." + destinationPath.getFileName(), ".tmp"); + boolean moved = false; + try { + extractNativeResource(os, arch, baseName, temporaryPath.toFile()); + try { + Files.move(temporaryPath, destinationPath, + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException | FileAlreadyExistsException e) { + Files.move(temporaryPath, destinationPath, StandardCopyOption.REPLACE_EXISTING); + } + moved = true; + return destinationPath.toFile(); + } finally { + if (!moved) { + Files.deleteIfExists(temporaryPath); + } + } + } + + private static void extractNativeResource( + String os, String arch, String baseName, File destination) throws IOException { + String mappedName = System.mapLibraryName(baseName); + String path = arch + "/" + os + "/" + mappedName; + URL chunkManifestResource = loader.getResource(path + CHUNK_MANIFEST_SUFFIX); + URL resource = chunkManifestResource == null ? loader.getResource(path) : null; + if (chunkManifestResource == null && resource == null) { + throw new FileNotFoundException("Could not locate native dependency " + path); + } + if (chunkManifestResource == null) { + extractConventionalResource(resource, destination); + } else { + extractChunkedResource(chunkManifestResource, mappedName, destination); + } + } + private static void extractConventionalResource(URL resource, File loc) throws IOException { try (InputStream in = resource.openStream(); OutputStream out = new FileOutputStream(loc)) { diff --git a/java/src/test/java/ai/rapids/cudf/NativeDepsLoaderExtractionTest.java b/java/src/test/java/ai/rapids/cudf/NativeDepsLoaderExtractionTest.java index 6c65529afced..d22481a83b13 100644 --- a/java/src/test/java/ai/rapids/cudf/NativeDepsLoaderExtractionTest.java +++ b/java/src/test/java/ai/rapids/cudf/NativeDepsLoaderExtractionTest.java @@ -96,6 +96,65 @@ void extractsChunkedResourceAndPrefersManifest() throws IOException { } } + @Test + void replacesExistingDestination() throws IOException { + String baseName = "destinationtest"; + byte[] expected = "requested destination contents".getBytes(StandardCharsets.UTF_8); + writeChunkedResource(baseName, expected, 8, "1", null, null); + Path destinationDirectory = Files.createTempDirectory("native-dep-destination"); + Path destination = destinationDirectory.resolve(System.mapLibraryName(baseName)); + Files.write(destination, "existing contents".getBytes(StandardCharsets.UTF_8)); + try { + File extracted = NativeDepsLoader.extractNativeDep( + TEST_OS, TEST_ARCH, baseName, destination.toFile()); + + assertEquals(destination.toAbsolutePath(), extracted.toPath()); + assertArrayEquals(expected, Files.readAllBytes(destination)); + } finally { + Files.deleteIfExists(destination); + Files.deleteIfExists(destinationDirectory); + } + } + + @Test + void failedDestinationExtractionPreservesExistingFile() throws IOException { + String baseName = "preservedestinationtest"; + byte[] expected = "chunk contents".getBytes(StandardCharsets.UTF_8); + writeChunkedResource(baseName, expected, 8, "1", 0L, null); + Path destinationDirectory = Files.createTempDirectory("native-dep-destination"); + Path destination = destinationDirectory.resolve(System.mapLibraryName(baseName)); + byte[] original = "existing contents".getBytes(StandardCharsets.UTF_8); + Files.write(destination, original); + try { + assertThrows(IOException.class, () -> NativeDepsLoader.extractNativeDep( + TEST_OS, TEST_ARCH, baseName, destination.toFile())); + + assertArrayEquals(original, Files.readAllBytes(destination)); + } finally { + Files.deleteIfExists(destination); + Files.deleteIfExists(destinationDirectory); + } + } + + @Test + void nativeDepUtilExtractsResource() throws IOException { + String baseName = "nativeutiltest"; + byte[] expected = "native util contents".getBytes(StandardCharsets.UTF_8); + writeChunkedResource(baseName, expected, 8, "1", null, null); + Path destinationDirectory = Files.createTempDirectory("native-dep-util"); + Path destination = destinationDirectory.resolve(System.mapLibraryName(baseName)); + try { + File extracted = NativeDepUtil.execute( + new String[]{"extract", baseName, destination.toString()}, TEST_OS, TEST_ARCH); + + assertEquals(destination.toAbsolutePath(), extracted.toPath()); + assertArrayEquals(expected, Files.readAllBytes(destination)); + } finally { + Files.deleteIfExists(destination); + Files.deleteIfExists(destinationDirectory); + } + } + @Test void extractsChunksConcurrently() throws Exception { String mappedName = "libconcurrent.so"; diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 6ef0b3c6c088..411b5b9fb926 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -3664,8 +3664,6 @@ def pytest_unconfigure(config): "tests/tools/test_to_datetime.py::TestOrigin::test_julian": "AssertionError: Attributes of Series are different", "tests/tools/test_to_datetime.py::TestOrigin::test_to_datetime_out_of_bounds_with_format_arg[%Y-%d-%m %H:%M:%S-None]": "TODO: Add a reason for failure", "tests/tools/test_to_datetime.py::TestOrigin::test_to_datetime_out_of_bounds_with_format_arg[%Y-%m-%d %H:%M:%S-None]": "TODO: Add a reason for failure", - "tests/tools/test_to_datetime.py::TestTimeConversionFormats::test_to_datetime_format_weeks[False-2013020-%Y%U%w-2013-01-13]": "AssertionError: assert Timestamp('2013-01-19 00:00:00') == Timestamp('2013-01-13 00:00:00')", - "tests/tools/test_to_datetime.py::TestTimeConversionFormats::test_to_datetime_format_weeks[True-2013020-%Y%U%w-2013-01-13]": "AssertionError: assert Timestamp('2013-01-19 00:00:00') == Timestamp('2013-01-13 00:00:00')", "tests/tools/test_to_datetime.py::TestToDatetime::test_mixed_offsets_with_native_datetime_utc_false_raises": "assert False", "tests/tools/test_to_datetime.py::TestToDatetime::test_to_datetime_arrow[index-None-False]": "AssertionError: assert DatetimeIndex([1965-04-03 00:00:00, 1965-04-17 00:00:00, 1965-05-01 00:00:00,\n 1965-05-...", "tests/tools/test_to_datetime.py::TestToDatetime::test_to_datetime_arrow[index-US/Central-False]": "AssertionError: assert Index([1965-04-03 00:00:00-06:00, 1965-04-17 00:00:00-06:00,\n 1965-05-01 00:00:00-05:00...",