sds: add support for polling file in SDS

api/envoy/config/core/v3/config_source.proto

@@ -157,22 +157,25 @@ message RateLimitSettings {
 
 // Local filesystem path configuration source.
 message PathConfigSource {
-  // Path on the filesystem to source and watch for configuration updates.
-  // When sourcing configuration for a :ref:`secret <envoy_v3_api_msg_extensions.transport_sockets.tls.v3.Secret>`,
-  // the certificate and key files are also watched for updates.
+  // Path on the filesystem from which to source configuration updates.
+  // When sourcing configuration for a
+  // :ref:`secret <envoy_v3_api_msg_extensions.transport_sockets.tls.v3.Secret>`, file-based secret
+  // data supported for SDS rotation is also monitored for updates.
   //
   // .. note::
   //
   //  The path to the source must exist at config load time.
   //
   // .. note::
   //
-  //   If ``watched_directory`` is *not* configured, Envoy will watch the file path for *moves*.
-  //   This is because in general only moves are atomic. The same method of swapping files as is
-  //   demonstrated in the :ref:`runtime documentation <config_runtime_symbolic_link_swap>` can be
-  //   used here also. If ``watched_directory`` is configured, no watch will be placed directly on
-  //   this path. Instead, the configured ``watched_directory`` will be used to trigger reloads of
-  //   this path. This is required in certain deployment scenarios. See below for more information.
+  //   If neither ``poll_interval`` nor ``watched_directory`` is configured, Envoy will watch the
+  //   file path for *moves*. This is because in general only moves are atomic. The same method of
+  //   swapping files as is demonstrated in the
+  //   :ref:`runtime documentation <config_runtime_symbolic_link_swap>` can be used here also.
+  //   One of ``poll_interval`` and ``watched_directory`` can also be configured; the configuration
+  //   is rejected if both are set. With ``poll_interval``, the same path is reloaded periodically.
+  //   With ``watched_directory``, no watch is placed directly on this path; events in the
+  //   configured directory trigger this path to be reloaded.
   string path = 1 [(validate.rules).string = {min_len: 1}];
 
   // If configured, this directory will be watched for *moves*. When an entry in this directory is
@@ -188,7 +191,24 @@ message PathConfigSource {
   //
   // The above configuration will ensure that Envoy watches the owning directory for moves which is
   // required due to how Kubernetes manages ConfigMap symbolic links during atomic updates.
+  //
+  // This field cannot be used together with ``poll_interval``.
   WatchedDirectory watched_directory = 2;
+
+  // If configured, the ``path`` will be polled at this interval instead of watched for filesystem
+  // events on either the file or ``watched_directory``. This is useful when the underlying
+  // filesystem does not reliably provide change notifications, or when a custom deployment model
+  // does not generate the move or modification events handled by watching the path or directory.
+  // The file is read on every poll, but an update is delivered only when its parsed contents
+  // change.
+  //
+  // When this configuration source provides a
+  // :ref:`Secret <envoy_v3_api_msg_extensions.transport_sockets.tls.v3.Secret>`, the same secret
+  // files that are watched in event-based mode are polled at this interval. The last successfully
+  // loaded configuration or secret remains active when a poll fails.
+  //
+  // This field cannot be used together with ``watched_directory`` and must be at least 1ms.
+  google.protobuf.Duration poll_interval = 3 [(validate.rules).duration = {gte {nanos: 1000000}}];
 }
 
 // Configuration for :ref:`listeners <config_listeners>`, :ref:`clusters

changelogs/current/new_features/sds__file-polling.rst

@@ -0,0 +1,6 @@
+Added :ref:`poll_interval
+<envoy_v3_api_field_config.core.v3.PathConfigSource.poll_interval>` to filesystem configuration
+sources. When used with SDS, Envoy polls both the SDS configuration and the same secret files that
+are watched in event-based mode, allowing rotation when filesystem notifications are unreliable or
+a custom deployment model does not generate the move or modification events handled by watching the
+path or directory.

docs/root/api-docs/xds_protocol.rst

@@ -74,8 +74,11 @@ Filesystem subscriptions
 
 The simplest approach to delivering dynamic configuration is to place it
 at a well known path specified in the :ref:`ConfigSource <envoy_v3_api_msg_config.core.v3.ConfigSource>`.
-Envoy will use ``inotify`` (``kqueue`` on macOS) to monitor the file for
-changes and parse the
+By default, Envoy will use ``inotify`` (``kqueue`` on macOS) to watch the path for move events.
+Alternatively, :ref:`poll_interval
+<envoy_v3_api_field_config.core.v3.PathConfigSource.poll_interval>` can be configured when
+filesystem notifications are unreliable or when a custom deployment model does not generate
+the move or modification events handled by watching the path or directory. Envoy parses the
 :ref:`DiscoveryResponse <envoy_v3_api_msg_service.discovery.v3.DiscoveryResponse>` proto in the file on update.
 Binary protobufs, JSON, YAML and proto text are supported formats for
 the

docs/root/configuration/security/secret.rst

@@ -185,6 +185,19 @@ In contrast, :ref:`sds_server_example` requires a restart to reload xDS certific
                 path_config_source:
                   path: /etc/envoy/validation_context_sds_secret.yaml
 
+When filesystem notifications are unreliable or a custom deployment model does not generate
+the move or modification events handled by watching the path or directory, configure :ref:`poll_interval
+<envoy_v3_api_field_config.core.v3.PathConfigSource.poll_interval>` on each
+``path_config_source``. Envoy will poll both the SDS configuration file and the same secret files
+that are watched in event-based mode:
+
+.. code-block:: yaml
+
+    sds_config:
+      path_config_source:
+        path: /etc/envoy/tls_certificate_sds_secret.yaml
+        poll_interval: 5s
+
 Paths to client certificate, including client's certificate chain and private key are given in SDS config file ``/etc/envoy/tls_certificate_sds_secret.yaml``:
 
 .. code-block:: yaml
@@ -415,4 +428,3 @@ the downstream filter chain, e.g. using the following filter configuration:
         text_format_source:
           inline_string: my_secret_name
       shared_with_upstream: ONCE
-

source/common/secret/sds_api.cc

@@ -30,9 +30,12 @@ SdsApi::SdsApi(envoy::config::core::v3::ConfigSource sds_config, absl::string_vi
           {Stats::TagStringView{Envoy::Config::TagNames::get().XDS_RESOURCE_NAME, sds_config_name}},
           absl::StrCat("sds.", sds_config_name, "."))),
       sds_api_stats_(generateStats(*scope_)), resource_type_helper_(validation_visitor, "name"),
-      sds_config_(std::move(sds_config)), sds_config_name_(sds_config_name),
-      clean_up_(std::move(destructor_cb)), subscription_factory_(subscription_factory),
-      time_source_(time_source),
+      sds_config_(std::move(sds_config)),
+      poll_interval_(sds_config_.has_path_config_source()
+                         ? PROTOBUF_GET_OPTIONAL_MS(sds_config_.path_config_source(), poll_interval)
+                         : std::nullopt),
+      sds_config_name_(sds_config_name), clean_up_(std::move(destructor_cb)),
+      subscription_factory_(subscription_factory), time_source_(time_source),
       secret_data_{sds_config_name_, "uninitialized", time_source_.systemTime()} {
   const auto resource_name = resource_type_helper_.getResourceName();
   // This has to happen here (rather than in initialize()) as it can throw exceptions.
@@ -41,6 +44,13 @@ SdsApi::SdsApi(envoy::config::core::v3::ConfigSource sds_config, absl::string_vi
                                 sds_config_, Grpc::Common::typeUrl(resource_name), *scope_, *this,
                                 resource_type_helper_.resourceDecoder(), {}),
                             Config::SubscriptionPtr);
+  if (poll_interval_.has_value()) {
+    poll_timer_ = dispatcher_.createTimer([this]() {
+      // Re-enable before running callbacks because a callback may destroy this SDS provider.
+      poll_timer_->enableTimer(*poll_interval_);
+      onFilesystemUpdate();
+    });
+  }
 }
 
 void SdsApi::resolveDataSource(const FileContentMap& files,
@@ -52,7 +62,7 @@ void SdsApi::resolveDataSource(const FileContentMap& files,
   }
 }
 
-void SdsApi::onWatchUpdate() {
+void SdsApi::onFilesystemUpdate() {
   // Filesystem reads and update callbacks can fail if the key material is missing or bad. We're not
   // under an onConfigUpdate() context, so we need to catch these cases explicitly here.
   TRY_ASSERT_MAIN_THREAD {
@@ -118,12 +128,18 @@ absl::Status SdsApi::onConfigUpdate(const std::vector<Config::DecodedResourceRef
     // tracking is available even if files don't exist yet.
     secret_data_.version_info_ = version_info;
 
-    // Set up per-file watchers before loadFiles() so that if loadFiles() fails, the watches
-    // are still setup for the next auto-recovery when files appear later.
-    // For watched_directory case, the callback is already set in setSecret().
-    if (getWatchedDirectory() == nullptr) {
-      // List DataSources that refer to files.
-      auto datasource_files = getDataSourceFilenames();
+    // Set up refresh triggers before loadFiles() so that if loadFiles() fails, the next poll or
+    // filesystem event can recover when files appear later.
+    auto datasource_files = getDataSourceFilenames();
+    if (poll_timer_ != nullptr) {
+      watcher_.reset();
+      if (datasource_files.empty()) {
+        poll_timer_->disableTimer();
+      } else {
+        poll_timer_->enableTimer(*poll_interval_);
+      }
+    } else if (getWatchedDirectory() == nullptr) {
+      // For watched_directory, the callback is already set in setSecret().
       if (!datasource_files.empty()) {
         // Create new watch, also destroys the old watch if any.
         watcher_ = dispatcher_.createFilesystemWatcher();
@@ -135,7 +151,7 @@ absl::Status SdsApi::onConfigUpdate(const std::vector<Config::DecodedResourceRef
           RETURN_IF_NOT_OK(watcher_->addWatch(absl::StrCat(result_or_error.value().directory_, "/"),
                                               Filesystem::Watcher::Events::MovedTo,
                                               [this](uint32_t) {
-                                                onWatchUpdate();
+                                                onFilesystemUpdate();
                                                 return absl::OkStatus();
                                               }));
         }
@@ -279,14 +295,14 @@ void TlsCertificateSdsApi::setSecret(
       std::make_unique<envoy::extensions::transport_sockets::tls::v3::TlsCertificate>(
           secret.tls_certificate());
   resolved_tls_certificate_secrets_ = nullptr;
-  if (secret.tls_certificate().has_watched_directory()) {
+  if (secret.tls_certificate().has_watched_directory() && !filesystemPollingEnabled()) {
     watched_directory_ = THROW_OR_RETURN_VALUE(
         Config::WatchedDirectory::create(secret.tls_certificate().watched_directory(), dispatcher_),
         std::unique_ptr<Config::WatchedDirectory>);
     // Set the callback immediately so that if subsequent operations fail, the watch is
     // still active and can trigger recovery when files appear later.
     watched_directory_->setCallback([this]() {
-      onWatchUpdate();
+      onFilesystemUpdate();
       return absl::OkStatus();
     });
   } else {
@@ -331,15 +347,15 @@ void CertificateValidationContextSdsApi::setSecret(
       std::make_unique<envoy::extensions::transport_sockets::tls::v3::CertificateValidationContext>(
           secret.validation_context());
   resolved_certificate_validation_context_secrets_ = nullptr;
-  if (secret.validation_context().has_watched_directory()) {
+  if (secret.validation_context().has_watched_directory() && !filesystemPollingEnabled()) {
     watched_directory_ =
         THROW_OR_RETURN_VALUE(Config::WatchedDirectory::create(
                                   secret.validation_context().watched_directory(), dispatcher_),
                               std::unique_ptr<Config::WatchedDirectory>);
     // Set the callback immediately so that if subsequent operations fail, the watch is
     // still active and can trigger recovery when files appear later.
     watched_directory_->setCallback([this]() {
-      onWatchUpdate();
+      onFilesystemUpdate();
       return absl::OkStatus();
     });
   } else {

source/common/secret/sds_api.h

@@ -1,6 +1,8 @@
 #pragma once
 
+#include <chrono>
 #include <functional>
+#include <optional>
 
 #include "envoy/api/api.h"
 #include "envoy/config/core/v3/config_source.pb.h"
@@ -92,8 +94,12 @@ class SdsApi : public Config::SubscriptionCallbacks {
   Event::Dispatcher& dispatcher_;
   Api::Api& api_;
 
-  // Invoked for filesystem watches on update. Protected so subclasses can set up the callback.
-  void onWatchUpdate();
+  // Invoked when a filesystem event fires or the polling timer expires. Protected so subclasses
+  // can set up filesystem event callbacks.
+  void onFilesystemUpdate();
+
+  // Whether filesystem-backed secret data is refreshed by polling rather than filesystem events.
+  bool filesystemPollingEnabled() const { return poll_timer_ != nullptr; }
 
   // Initializes the SDS API.
   void initialize(bool warm);
@@ -111,6 +117,8 @@ class SdsApi : public Config::SubscriptionCallbacks {
       resource_type_helper_;
 
   const envoy::config::core::v3::ConfigSource sds_config_;
+  const std::optional<std::chrono::milliseconds> poll_interval_;
+  Event::TimerPtr poll_timer_;
   Config::SubscriptionPtr subscription_;
   const std::string sds_config_name_;
 

source/extensions/config_subscription/filesystem/filesystem_subscription_impl.cc

@@ -23,9 +23,23 @@ FilesystemSubscriptionImpl::FilesystemSubscriptionImpl(
     const envoy::config::core::v3::PathConfigSource& path_config_source,
     SubscriptionCallbacks& callbacks, OpaqueResourceDecoderSharedPtr resource_decoder,
     SubscriptionStats stats, ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api)
-    : path_(path_config_source.path()), callbacks_(callbacks), resource_decoder_(resource_decoder),
-      stats_(stats), api_(api), validation_visitor_(validation_visitor) {
-  if (!path_config_source.has_watched_directory()) {
+    : path_(path_config_source.path()),
+      poll_interval_(PROTOBUF_GET_OPTIONAL_MS(path_config_source, poll_interval)),
+      callbacks_(callbacks), resource_decoder_(resource_decoder), stats_(stats), api_(api),
+      validation_visitor_(validation_visitor) {
+  if (poll_interval_.has_value() && path_config_source.has_watched_directory()) {
+    throw EnvoyException("PathConfigSource poll_interval and watched_directory cannot both be set");
+  }
+
+  if (poll_interval_.has_value()) {
+    poll_timer_ = dispatcher.createTimer([this]() {
+      // Re-enable before refreshing because an update callback may destroy the subscription.
+      poll_timer_->enableTimer(*poll_interval_);
+      if (started_) {
+        refresh();
+      }
+    });
+  } else if (!path_config_source.has_watched_directory()) {
     file_watcher_ = dispatcher.createFilesystemWatcher();
     THROW_IF_NOT_OK(
         file_watcher_->addWatch(path_, Filesystem::Watcher::Events::MovedTo, [this](uint32_t) {
@@ -50,6 +64,9 @@ FilesystemSubscriptionImpl::FilesystemSubscriptionImpl(
 // Config::Subscription
 void FilesystemSubscriptionImpl::start(const absl::flat_hash_set<std::string>&) {
   started_ = true;
+  if (poll_timer_ != nullptr) {
+    poll_timer_->enableTimer(*poll_interval_);
+  }
   // Attempt to read in case there is a file there already.
   refresh();
 }
@@ -72,11 +89,19 @@ std::string FilesystemSubscriptionImpl::refreshInternal(ProtobufTypes::MessagePt
   auto& message = *owned_message;
   THROW_IF_NOT_OK(MessageUtil::loadFromFile(path_, message, validation_visitor_, api_));
   *config_update = std::move(owned_message);
+  std::optional<uint64_t> new_hash;
+  if (poll_timer_ != nullptr) {
+    new_hash = MessageUtil::hash(message);
+    if (new_hash == config_hash_) {
+      return message.version_info();
+    }
+  }
   const auto decoded_resources =
       THROW_OR_RETURN_VALUE(DecodedResourcesWrapper::create(*resource_decoder_, message.resources(),
                                                             message.version_info()),
                             std::unique_ptr<DecodedResourcesWrapper>);
   THROW_IF_NOT_OK(callbacks_.onConfigUpdate(decoded_resources->refvec_, message.version_info()));
+  config_hash_ = new_hash;
   return message.version_info();
 }
 

source/extensions/config_subscription/filesystem/filesystem_subscription_impl.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include <chrono>
+#include <optional>
+
 #include "envoy/api/api.h"
 #include "envoy/config/core/v3/config_source.pb.h"
 #include "envoy/config/subscription.h"
@@ -47,8 +50,11 @@ class FilesystemSubscriptionImpl : public Config::Subscription,
 
   bool started_{};
   const std::string path_;
+  const std::optional<std::chrono::milliseconds> poll_interval_;
+  Event::TimerPtr poll_timer_;
   std::unique_ptr<Filesystem::Watcher> file_watcher_;
   WatchedDirectoryPtr directory_watcher_;
+  std::optional<uint64_t> config_hash_;
   SubscriptionCallbacks& callbacks_;
   OpaqueResourceDecoderSharedPtr resource_decoder_;
   SubscriptionStats stats_;