Add Google Cloud Storage reader support

CMakeLists.txt

@@ -101,6 +101,8 @@ cmake_dependent_option(BUILD_NVIMAGECODEC "Build with support for nvimagecodec l
                        "NOT BUILD_DALI_NODEPS" OFF)
 cmake_dependent_option(BUILD_AWSSDK "Build with support for AWS SKD library" ON
                        "NOT BUILD_DALI_NODEPS" OFF)
+cmake_dependent_option(BUILD_GCS "Build with support for Google Cloud Storage library" ON
+                       "NOT BUILD_DALI_NODEPS" OFF)
 set(NVIMGCODEC_DEFAULT_INSTALL_PATH "/opt/nvidia/nvimgcodec_cuda${CUDA_VERSION_MAJOR}" CACHE STRING
     "Path of the nvimagecodec installation")
 
@@ -344,6 +346,7 @@ propagate_option(BUILD_NVML)
 propagate_option(BUILD_CUFILE)
 propagate_option(BUILD_NVIMAGECODEC)
 propagate_option(BUILD_AWSSDK)
+propagate_option(BUILD_GCS)
 propagate_option(BUILD_FOR_CONDA)
 propagate_option(LINK_DRIVER)
 propagate_option(WITH_DYNAMIC_CUFFT)

cmake/Dependencies.common.cmake

@@ -452,3 +452,16 @@ if(BUILD_AWSSDK)
     message(STATUS "AWSSDK_LIBRARIES=${AWSSDK_LIBRARIES}")
   endif()
 endif()
+
+##################################################################
+# Google Cloud Storage (google-cloud-cpp)
+##################################################################
+if(BUILD_GCS)
+  find_package(google_cloud_cpp_storage CONFIG QUIET)
+  if (NOT google_cloud_cpp_storage_FOUND)
+    message(WARNING "google-cloud-cpp storage not found. Disabling Google Cloud Storage support.")
+    set(BUILD_GCS OFF)
+  else()
+    message(STATUS "google-cloud-cpp storage version=${google_cloud_cpp_storage_VERSION}")
+  endif()
+endif()

dali/CMakeLists.txt

@@ -94,6 +94,10 @@ if (BUILD_AWSSDK)
   target_link_libraries(dali PRIVATE ${AWSSDK_LIBRARIES})
 endif()
 
+if (BUILD_GCS)
+  target_link_libraries(dali PRIVATE google-cloud-cpp::storage)
+endif()
+
 # Build test suite
 ################################################
 if (BUILD_DALI_PIPELINE AND BUILD_TEST)

dali/operators/CMakeLists.txt

@@ -102,6 +102,12 @@ if (BUILD_CVCUDA)
   target_link_libraries(dali_operators PRIVATE cvcuda nvcv_types)
 endif(BUILD_CVCUDA)
 
+if (BUILD_GCS)
+  # GCSClientManager is header-only, so discover_files_gcs.cc needs the client library directly -
+  # it cannot rely on it being reachable through libdali.
+  target_link_libraries(dali_operators PRIVATE google-cloud-cpp::storage)
+endif()
+
 if (BUILD_NVIMAGECODEC)
   if (WITH_DYNAMIC_NVIMGCODEC)
     target_link_libraries(dali_operators PRIVATE dynlink_nvimgcodec)

dali/operators/reader/loader/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2017-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2017-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -67,5 +67,10 @@ if (BUILD_AWSSDK)
     "${CMAKE_CURRENT_SOURCE_DIR}/discover_files_s3.cc")
 endif()
 
+if (BUILD_GCS)
+  set(DALI_OPERATOR_SRCS ${DALI_OPERATOR_SRCS}
+    "${CMAKE_CURRENT_SOURCE_DIR}/discover_files_gcs.cc")
+endif()
+
 set(DALI_OPERATOR_SRCS ${DALI_OPERATOR_SRCS} PARENT_SCOPE)
 set(DALI_OPERATOR_TEST_SRCS ${DALI_OPERATOR_TEST_SRCS} PARENT_SCOPE)

dali/operators/reader/loader/discover_files.cc

@@ -1,4 +1,4 @@
-// Copyright (c) 2017-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright (c) 2017-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -33,6 +33,9 @@
 #if AWSSDK_ENABLED
 #include "dali/operators/reader/loader/discover_files_s3.h"
 #endif
+#if GCS_ENABLED
+#include "dali/operators/reader/loader/discover_files_gcs.h"
+#endif
 
 namespace dali {
 
@@ -122,6 +125,15 @@ std::vector<FileLabelEntry> discover_files(const std::string &file_root,
 #endif
   }
 
+  bool is_gcs = starts_with(file_root, "gs://");
+  if (is_gcs) {
+#if GCS_ENABLED
+    return gcs_discover_files(file_root, opts);
+#else
+    DALI_FAIL("This version of DALI was not built with Google Cloud Storage support.");
+#endif
+  }
+
   std::vector<std::string> subdirs;
   subdirs = list_subdirectories(file_root, opts.dir_filters, opts.case_sensitive_filter);
   std::vector<FileLabelEntry> entries;

dali/operators/reader/loader/discover_files_gcs.cc

@@ -0,0 +1,104 @@
+// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "dali/operators/reader/loader/discover_files_gcs.h"
+#include <fnmatch.h>
+#include <filesystem>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include "dali/operators/reader/loader/discover_files.h"
+#include "dali/util/gcs_client_manager.h"
+#include "dali/util/gcs_filesystem.h"
+
+namespace dali {
+
+// We are using std::filesystem to analyze URI relative paths, which wouldn't be OK in non-UNIX
+// based systems
+#ifndef __unix__
+#error This code works only in UNIX-based systems
+#endif
+
+std::vector<FileLabelEntry> gcs_discover_files(const std::string &file_root,
+                                               const FileDiscoveryOptions &opts) {
+  assert(starts_with(file_root, "gs://"));
+  auto gcs_object_location = gcs_filesystem::parse_uri(file_root);
+  std::filesystem::path parent_object_key(gcs_object_location.object);
+  auto count_elems = [](const std::filesystem::path &p) {
+    size_t k = 0;
+    for (auto &elem : p)
+      k++;
+    return k;
+  };
+  std::vector<FileLabelEntry> entries;
+  // in case that files are not visited in lexicographical order, we remember previously assigned
+  // labels
+  std::unordered_map<std::string, int> labels;
+  int next_label = 0;  // next free-label to be assigned
+  auto client = GCSClientManager::Instance().client();
+  gcs_filesystem::list_objects_f(
+      client, gcs_object_location, [&](const std::string &object_key, size_t object_size) {
+        auto p = std::filesystem::relative(object_key, parent_object_key);
+        auto path_elems = count_elems(p);
+        // We only look at one subdir level. Fewer than two components means either an object
+        // directly under the listed prefix, or the prefix's own directory marker, which relative()
+        // maps to "."; neither is a labelled file, and both must be rejected before dereferencing
+        // the second component below.
+        if (path_elems != 2)
+          return;
+        const auto& subdir = p.begin()->native();
+        const auto& fname = (++p.begin())->native();
+        // GCS directory markers are zero-byte objects whose name ends with '/'. A trailing
+        // separator becomes an empty final component, so "<prefix>/class/" arrives here as
+        // ("class", "") - a directory, not a file.
+        if (fname.empty())
+          return;
+        bool subdir_ok = opts.dir_filters.empty();
+        bool fname_ok = opts.file_filters.empty();
+        for (auto &filter : opts.dir_filters) {
+          if (fnmatch(filter.c_str(), subdir.c_str(),
+                      opts.case_sensitive_filter ? 0 : FNM_CASEFOLD) == 0) {
+            subdir_ok |= true;
+            break;
+          }
+        }
+
+        for (auto &filter : opts.file_filters) {
+          if (fnmatch(filter.c_str(), fname.c_str(),
+                      opts.case_sensitive_filter ? 0 : FNM_CASEFOLD) == 0) {
+            fname_ok |= true;
+            break;
+          }
+        }
+
+        if (!subdir_ok || !fname_ok)
+          return;
+
+        if (opts.label_from_subdir) {
+          int curr_label = -1;
+          auto it = labels.find(subdir);
+          if (it == labels.end()) {
+            curr_label = labels[subdir] = next_label++;
+          } else {
+            curr_label = it->second;
+          }
+          entries.push_back({p, curr_label, object_size});
+        } else {
+          entries.push_back({p, std::nullopt, object_size});
+        }
+      });
+  return entries;
+}
+
+}  // namespace dali

dali/operators/reader/loader/discover_files_gcs.h

@@ -0,0 +1,29 @@
+// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef DALI_OPERATORS_READER_LOADER_DISCOVER_FILES_GCS_H_
+#define DALI_OPERATORS_READER_LOADER_DISCOVER_FILES_GCS_H_
+
+#include <string>
+#include <vector>
+#include "dali/operators/reader/loader/discover_files.h"
+
+namespace dali {
+
+std::vector<FileLabelEntry> gcs_discover_files(const std::string &file_root,
+                                               const FileDiscoveryOptions &opts);
+
+}  // namespace dali
+
+#endif  // DALI_OPERATORS_READER_LOADER_DISCOVER_FILES_GCS_H_

dali/util/CMakeLists.txt

@@ -65,6 +65,12 @@ if (BUILD_AWSSDK)
   "${CMAKE_CURRENT_SOURCE_DIR}/s3_filesystem.cc")
 endif()
 
+if (BUILD_GCS)
+  set(DALI_SRCS ${DALI_SRCS}
+  "${CMAKE_CURRENT_SOURCE_DIR}/gcs_file.cc"
+  "${CMAKE_CURRENT_SOURCE_DIR}/gcs_filesystem.cc")
+endif()
+
 set(DALI_TEST_SRCS ${DALI_TEST_SRCS}
   "${CMAKE_CURRENT_SOURCE_DIR}/numpy_test.cc"
   "${CMAKE_CURRENT_SOURCE_DIR}/uri_test.cc")

dali/util/file.cc

@@ -1,4 +1,4 @@
-// Copyright (c) 2017-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright (c) 2017-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -25,6 +25,11 @@
 #include "dali/util/s3_file.h"
 #endif
 
+#if GCS_ENABLED
+#include "dali/util/gcs_client_manager.h"
+#include "dali/util/gcs_file.h"
+#endif
+
 namespace dali {
 
 std::unique_ptr<FileStream> FileStream::Open(const std::string& uri, FileStream::Options opts,
@@ -38,6 +43,16 @@ std::unique_ptr<FileStream> FileStream::Open(const std::string& uri, FileStream:
 #endif
   }
 
+  bool is_gcs = uri.rfind("gs://", 0) == 0;
+  if (is_gcs) {
+#if GCS_ENABLED
+    return std::make_unique<GCSFileStream>(GCSClientManager::Instance().client(), uri, size);
+#else
+    throw std::runtime_error(
+        "This version of DALI was not built with Google Cloud Storage support.");
+#endif
+  }
+
   std::string processed_uri;
   if (uri.find("file://") == 0) {
     processed_uri = uri.substr(std::string("file://").size());

dali/util/gcs_client_manager.h

@@ -0,0 +1,99 @@
+// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef DALI_UTIL_GCS_CLIENT_MANAGER_H_
+#define DALI_UTIL_GCS_CLIENT_MANAGER_H_
+
+#include <google/cloud/credentials.h>
+#include <google/cloud/options.h>
+#include <google/cloud/storage/client.h>
+#include <google/cloud/storage/options.h>
+#include <cstdio>
+#include <cstdlib>
+#include <string>
+#include "dali/core/common.h"
+#include "dali/core/error_handling.h"
+
+namespace dali {
+
+/**
+ * @brief Owns the process-wide configuration of the GCS client.
+ *
+ * Unlike `Aws::S3::S3Client`, `google::cloud::storage::Client` is not documented as safe for
+ * concurrent use of a *single* instance ("Two threads operating on the same instance of this
+ * class is not guaranteed to work"). Copies, on the other hand, share the underlying connection
+ * pool and are explicitly safe to use from different threads, and copying is about as expensive
+ * as copying a few shared pointers. Therefore `client()` hands out a copy and each caller
+ * (file stream, file discovery) keeps its own.
+ *
+ * There is no global init/shutdown to perform - the library initializes libcurl lazily.
+ */
+class GCSClientManager {
+ public:
+  static GCSClientManager& Instance() {
+    static GCSClientManager s_manager_;
+    return s_manager_;
+  }
+
+  /**
+   * @brief Returns a client sharing the connection pool with all other clients handed out here.
+   */
+  google::cloud::storage::Client client() const {
+    return client_;
+  }
+
+ private:
+  static bool EnvFlag(const char* name, bool default_value) {
+    auto* value = std::getenv(name);
+    if (!value)
+      return default_value;
+    return std::atoi(value) != 0;
+  }
+
+  static google::cloud::Options MakeOptions() {
+    namespace gcs = google::cloud::storage;
+    google::cloud::Options options;
+
+    // The library also honors CLOUD_STORAGE_EMULATOR_ENDPOINT on its own; this is the DALI-side
+    // counterpart of AWS_ENDPOINT_URL.
+    if (auto* endpoint_url = std::getenv("DALI_GCS_ENDPOINT_URL")) {
+      options.set<gcs::RestEndpointOption>(endpoint_url);
+    }
+
+    // By default the client uses Application Default Credentials. Reading a public bucket (or a
+    // local emulator) requires opting out of authentication explicitly.
+    if (EnvFlag("DALI_GCS_ANONYMOUS", false)) {
+      options.set<google::cloud::UnifiedCredentialsOption>(
+          google::cloud::MakeInsecureCredentials());
+    }
+
+    // DALI only ever issues ranged reads, and GCS reports checksums for whole objects only, so a
+    // per-read CRC32C over the payload cannot be validated end-to-end - it would just burn CPU in
+    // the data loading path. It can be turned back on for debugging.
+    if (!EnvFlag("DALI_GCS_VERIFY_CHECKSUMS", false)) {
+      options.set<gcs::DownloadChecksumValidationOption>(  // NOLINT(build/include_what_you_use)
+          gcs::ChecksumAlgorithm::kNone);
+    }
+
+    return options;
+  }
+
+  GCSClientManager() : client_(MakeOptions()) {}
+
+  google::cloud::storage::Client client_;
+};
+
+}  // namespace dali
+
+#endif  // DALI_UTIL_GCS_CLIENT_MANAGER_H_