config: support hostnames for additional shred destinations

src/app/fdctl/config/default.toml

@@ -1471,12 +1471,15 @@ dynamic_port_range = "8900-9000"
         # example to run an unstaked RPC or for archiving.  Each new, valid
         # shred that the validator receives will be forwarded to the
         # addresses specified in additional_shred_destinations_retransmit.
-        # Each shred that the validor produced when it is leader will be
+        # Each shred that the validator produced when it is leader will be
         # sent to the addresses specified in
-        # additional_shred_destinations_leader.  Destinations must be in the
-        # form "ip:port", for example "1.2.3.4:5566".  Shreds will
-        # be sent to these destinations first, prior to sending to other
-        # validators.
+        # additional_shred_destinations_leader.  Destinations must be in
+        # the form "host:port", e.g. "1.2.3.4:5566" or
+        # "shred-receiver.example.com:5566".  Hosts may be IPv4 addresses
+        # or DNS/NSS names.  Names resolve once at startup to the first
+        # IPv4 result; failure prevents startup.  Restart for DNS changes.
+        # IPv6 is unsupported.  Shreds are sent to these destinations
+        # first, before other validators.
         additional_shred_destinations_retransmit = []
         additional_shred_destinations_leader = []
 

src/app/fdctl/topology.c

@@ -11,24 +11,6 @@
 
 extern fd_topo_obj_callbacks_t * CALLBACKS[];
 
-static void
-parse_ip_port( const char * name, const char * ip_port, fd_topo_ip_port_t *parsed_ip_port) {
-  char buf[ sizeof( "255.255.255.255:65536" ) ];
-  memcpy( buf, ip_port, sizeof( buf ) );
-  char *ip_end = strchr( buf, ':' );
-  if( FD_UNLIKELY( !ip_end ) )
-    FD_LOG_ERR(( "[%s] must in the form ip:port", name ));
-  *ip_end = '\0';
-
-  if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( buf, &( parsed_ip_port->ip ) ) ) ) {
-    FD_LOG_ERR(( "could not parse IP %s in [%s]", buf, name ));
-  }
-
-  parsed_ip_port->port = fd_cstr_to_ushort( ip_end+1 );
-  if( FD_UNLIKELY( !parsed_ip_port->port ) )
-    FD_LOG_ERR(( "could not parse port %s in [%s]", ip_end+1, name ));
-}
-
 void
 fd_topo_configure_tile( fd_topo_tile_t * tile,
                         fd_config_t *    config );
@@ -525,18 +507,6 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
     tile->shred.shred_listen_port             = config->tiles.shred.shred_listen_port;
     tile->shred.max_shreds_per_block          = config->limits.max_shreds_per_block;
     tile->shred.bench_max_shreds_per_block    = config->development.bench.max_shreds_per_block;
-    for( ulong i=0UL; i<config->tiles.shred.additional_shred_destinations_retransmit_cnt; i++ ) {
-      parse_ip_port( "tiles.shred.additional_shred_destinations_retransmit",
-                      config->tiles.shred.additional_shred_destinations_retransmit[ i ],
-                      &tile->shred.adtl_dests_retransmit[ i ] );
-    }
-    tile->shred.adtl_dests_retransmit_cnt = config->tiles.shred.additional_shred_destinations_retransmit_cnt;
-    for( ulong i=0UL; i<config->tiles.shred.additional_shred_destinations_leader_cnt; i++ ) {
-      parse_ip_port( "tiles.shred.additional_shred_destinations_leader",
-                      config->tiles.shred.additional_shred_destinations_leader[ i ],
-                      &tile->shred.adtl_dests_leader[ i ] );
-    }
-    tile->shred.adtl_dests_leader_cnt = config->tiles.shred.additional_shred_destinations_leader_cnt;
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "store" ) ) ) {
     tile->store.disable_blockstore_from_slot = config->development.bench.disable_blockstore_from_slot;

src/app/firedancer-dev/commands/forktest/forktest.c

@@ -4,6 +4,7 @@ Constructed using a full topology which is pruned down. */
 #define _GNU_SOURCE
 #include "../../../firedancer/topology.h"
 #include "../../../shared/fd_action.h"
+#include "../../../shared/fd_config_private.h"
 #include "../../../shared/commands/configure/configure.h"
 #include "../../../shared/commands/run/run.h"
 #include "../../../shared/commands/watch/watch.h"
@@ -441,6 +442,8 @@ forktest_cmd_args( int *    pargc,
 static void
 forktest_fn( args_t *   args,
              config_t * config ) {
+  fd_config_apply_shred_destinations( config );
+
   args_t c_args = configure_args();
   configure_cmd_fn( &c_args, config );
 

src/app/firedancer/config/default.toml

@@ -1663,9 +1663,12 @@ telemetry = true
         # Each shred that the validator produced when it is leader will
         # be sent to the addresses specified for leader.
         #
-        # Destinations must be in the form "ip:port", for example
-        # "1.2.3.4:5566".  Shreds will be sent to these destinations
-        # first, prior to sending to other validators.
+        # Destinations must be in the form "host:port", e.g.
+        # "1.2.3.4:5566" or "shred-receiver.example.com:5566".  Hosts may
+        # be IPv4 addresses or DNS/NSS names.  Names resolve once at
+        # startup to the first IPv4 result; failure prevents startup.
+        # Restart for DNS changes.  IPv6 is unsupported.  Shreds are sent
+        # to these destinations first, before other validators.
         additional_shred_destinations_retransmit = []
         additional_shred_destinations_leader = []
 

src/app/firedancer/topology.c

@@ -89,24 +89,6 @@ wire_event_links( fd_topo_t * topo ) {
   }
 }
 
-static void
-parse_ip_port( const char * name, const char * ip_port, fd_topo_ip_port_t *parsed_ip_port) {
-  char buf[ sizeof( "255.255.255.255:65536" ) ];
-  memcpy( buf, ip_port, sizeof( buf ) );
-  char *ip_end = strchr( buf, ':' );
-  if( FD_UNLIKELY( !ip_end ) )
-    FD_LOG_ERR(( "[%s] must in the form ip:port", name ));
-  *ip_end = '\0';
-
-  if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( buf, &( parsed_ip_port->ip ) ) ) ) {
-    FD_LOG_ERR(( "could not parse IP %s in [%s]", buf, name ));
-  }
-
-  parsed_ip_port->port = fd_cstr_to_ushort( ip_end+1 );
-  if( FD_UNLIKELY( !parsed_ip_port->port ) )
-    FD_LOG_ERR(( "could not parse port %s in [%s]", ip_end+1, name ));
-}
-
 fd_topo_obj_t *
 setup_topo_banks( fd_topo_t *  topo,
                   char const * wksp_name,
@@ -1770,18 +1752,6 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
     tile->shred.shred_listen_port             = config->tiles.shred.shred_listen_port;
     tile->shred.max_shreds_per_block          = config->limits.max_shreds_per_block;
     tile->shred.bench_max_shreds_per_block    = config->development.bench.max_shreds_per_block;
-    for( ulong i=0UL; i<config->tiles.shred.additional_shred_destinations_retransmit_cnt; i++ ) {
-      parse_ip_port( "tiles.shred.additional_shred_destinations_retransmit",
-                      config->tiles.shred.additional_shred_destinations_retransmit[ i ],
-                      &tile->shred.adtl_dests_retransmit[ i ] );
-    }
-    tile->shred.adtl_dests_retransmit_cnt = config->tiles.shred.additional_shred_destinations_retransmit_cnt;
-    for( ulong i=0UL; i<config->tiles.shred.additional_shred_destinations_leader_cnt; i++ ) {
-      parse_ip_port( "tiles.shred.additional_shred_destinations_leader",
-                      config->tiles.shred.additional_shred_destinations_leader[ i ],
-                      &tile->shred.adtl_dests_leader[ i ] );
-    }
-    tile->shred.adtl_dests_leader_cnt = config->tiles.shred.additional_shred_destinations_leader_cnt;
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "sign" ) ) ) {
 

src/app/shared/commands/run/run.c

@@ -1,5 +1,6 @@
 #define _GNU_SOURCE
 #include "run.h"
+#include "../../fd_config_private.h"
 #include "../../../../flamenco/accdb/fd_accdb.h"
 #include "../../../../disco/store/fd_store.h"
 
@@ -1034,6 +1035,10 @@ void
 run_firedancer_init( config_t * config,
                      int        init_workspaces,
                      int        check_configure ) {
+  /* Resolve before serializing the topology or starting any tiles.  Admin
+     commands also construct topologies, but must not depend on live DNS. */
+  fd_config_apply_shred_destinations( config );
+
   struct stat st;
   int err = stat( config->paths.identity_key, &st );
   if( FD_UNLIKELY( -1==err && errno==ENOENT ) ) FD_LOG_ERR(( "[consensus.identity_path] key does not exist `%s`. You can generate an identity key at this path by running `%s keys new %s --config <toml>`", config->paths.identity_key, FD_BINARY_NAME, config->paths.identity_key ));

src/app/shared/commands/run/run1.c

@@ -1,5 +1,6 @@
 #define _GNU_SOURCE
 #include "run.h"
+#include "../../fd_config_private.h"
 
 #include "../../../../util/tile/fd_tile_private.h"
 
@@ -65,6 +66,12 @@ run1_cmd_fn( args_t *   args,
   if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile %s:%lu not found", args->run1.tile_name, args->run1.kind_id ));
   fd_topo_tile_t * tile = &config->topo.tiles[ tile_id ];
 
+  /* Internal --config-fd children retain their parent's action and
+     already have resolved destinations.  Standalone run1 does not. */
+  if( !strcmp( config->action, NAME ) && !strcmp( tile->name, "shred" ) ) {
+    fd_config_apply_shred_destinations( config );
+  }
+
   char thread_name[ FD_LOG_NAME_MAX ] = {0};
   FD_TEST( fd_cstr_printf_check( thread_name, FD_LOG_NAME_MAX-1UL, NULL, "%s:%lu", tile->name, tile->kind_id ) );
   fd_log_thread_set( thread_name );

src/app/shared/fd_config.c

@@ -6,6 +6,7 @@
 #include "../platform/fd_net_util.h"
 #include "../platform/fd_sys_util.h"
 #include "../../ballet/toml/fd_toml.h"
+#include "../../disco/topo/fd_dns_resolve.h"
 #include "../../disco/genesis/fd_genesis_cluster.h"
 #include "../../discof/genesis/fd_genesi_tile.h"
 #include "../../disco/net/fd_net_tile.h"
@@ -47,6 +48,60 @@ replace( char *       in,
   }
 }
 
+int
+fd_config_resolve_ip4_endpoint( char const *        endpoint,
+                                fd_topo_ip_port_t * out ) {
+  char         host[ FD_FQDN_BUF_MAX ];
+  char const * colon = strchr( endpoint, ':' );
+  if( FD_UNLIKELY( !colon || colon==endpoint || (ulong)(colon-endpoint)>=sizeof(host) ) ) {
+    FD_LOG_WARNING(( "invalid additional shred destination `%s`: expected host:port with a 1-255 byte host", endpoint ));
+    return 0;
+  }
+
+  char * end;
+  ulong  port = strtoul( colon+1, &end, 10 );
+  if( FD_UNLIKELY( colon[ 1 ]<'0' || colon[ 1 ]>'9' || *end || !port || port>USHORT_MAX ) ) {
+    FD_LOG_WARNING(( "invalid additional shred destination `%s`: port must be a decimal integer between 1 and 65535", endpoint ));
+    return 0;
+  }
+
+  fd_cstr_fini( fd_cstr_append_text( fd_cstr_init( host ), endpoint, (ulong)(colon-endpoint) ) );
+
+  out->port = (ushort)port;
+  return fd_dns_resolve_address( host, &out->ip );
+}
+
+void
+fd_config_apply_shred_destinations( fd_config_t * config ) {
+  fd_topo_tile_t * first = NULL;
+  for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
+    fd_topo_tile_t * tile = &config->topo.tiles[ i ];
+    if( FD_LIKELY( strcmp( tile->name, "shred" ) ) ) continue;
+
+    tile->shred.adtl_dests_retransmit_cnt = config->tiles.shred.additional_shred_destinations_retransmit_cnt;
+    tile->shred.adtl_dests_leader_cnt     = config->tiles.shred.additional_shred_destinations_leader_cnt;
+    if( first ) {
+      fd_memcpy( tile->shred.adtl_dests_retransmit, first->shred.adtl_dests_retransmit, sizeof(tile->shred.adtl_dests_retransmit) );
+      fd_memcpy( tile->shred.adtl_dests_leader,     first->shred.adtl_dests_leader,     sizeof(tile->shred.adtl_dests_leader) );
+      continue;
+    }
+
+    first = tile;
+    for( ulong j=0UL; j<tile->shred.adtl_dests_retransmit_cnt; j++ ) {
+      if( FD_UNLIKELY( !fd_config_resolve_ip4_endpoint( config->tiles.shred.additional_shred_destinations_retransmit[ j ],
+                                                      &tile->shred.adtl_dests_retransmit[ j ] ) ) ) {
+        FD_LOG_ERR(( "failed to resolve [tiles.shred.additional_shred_destinations_retransmit] entry %lu", j ));
+      }
+    }
+    for( ulong j=0UL; j<tile->shred.adtl_dests_leader_cnt; j++ ) {
+      if( FD_UNLIKELY( !fd_config_resolve_ip4_endpoint( config->tiles.shred.additional_shred_destinations_leader[ j ],
+                                                      &tile->shred.adtl_dests_leader[ j ] ) ) ) {
+        FD_LOG_ERR(( "failed to resolve [tiles.shred.additional_shred_destinations_leader] entry %lu", j ));
+      }
+    }
+  }
+}
+
 
 FD_FN_CONST static inline int
 parse_log_level( char const * level ) {

src/app/shared/fd_config.h

@@ -507,9 +507,9 @@ struct fd_config {
       uint   max_pending_shred_sets;
       ushort shred_listen_port;
       ulong  additional_shred_destinations_retransmit_cnt;
-      char   additional_shred_destinations_retransmit[ FD_TOPO_ADTL_DESTS_MAX ][ sizeof("255.255.255.255:65536") ];
+      char   additional_shred_destinations_retransmit[ FD_TOPO_ADTL_DESTS_MAX ][ FD_HOSTPORT_BUF_MAX ];
       ulong  additional_shred_destinations_leader_cnt;
-      char   additional_shred_destinations_leader[ FD_TOPO_ADTL_DESTS_MAX ][ sizeof("255.255.255.255:65536") ];
+      char   additional_shred_destinations_leader[ FD_TOPO_ADTL_DESTS_MAX ][ FD_HOSTPORT_BUF_MAX ];
       ulong  shred_cache_size_mib;
     } shred;
 

src/app/shared/fd_config_json.c

@@ -11,7 +11,7 @@
    or knowingly skipped) before the constant is bumped.  String keys of
    the user's own file are separately forced through the classification
    lists below. */
-FD_STATIC_ASSERT( sizeof(fd_config_t)==22974464UL, update_fd_config_to_json_for_the_layout_change );
+FD_STATIC_ASSERT( sizeof(fd_config_t)==22989824UL, update_fd_config_to_json_for_the_layout_change );
 
 #define REDACTED "[redacted]"
 

src/app/shared/fd_config_private.h

@@ -48,6 +48,23 @@ fd_config_fill( fd_config_t * config,
 void
 fd_config_validate( fd_config_t const * config );
 
+/* fd_config_resolve_ip4_endpoint parses an IPv4 host:port endpoint.
+   Numeric addresses are parsed directly; hostnames use the system
+   resolver's first IPv4 result.  Returns 1 on success, or logs a warning
+   and returns 0 on failure. */
+
+int
+fd_config_resolve_ip4_endpoint( char const *        endpoint,
+                                fd_topo_ip_port_t * out );
+
+/* fd_config_apply_shred_destinations resolves each additional shred
+   destination once and copies it into every shred tile.  Call with a
+   loaded config when starting tiles, before serializing or sandboxing.
+   Does nothing without shred tiles. */
+
+void
+fd_config_apply_shred_destinations( fd_config_t * config );
+
 FD_PROTOTYPES_END
 
 #endif /* HEADER_fd_src_app_shared_fd_config_private_h */