failover: initial MVP with tower file steaming

contrib/codegen/generate_filters.py

@@ -61,6 +61,7 @@
     "lseek":           (  "int", "long",   "int",   None,   None,    None  ),
     "madvise":         ( "long", "long",   "int",   None,   None,    None  ),
     "newfstatat":      (  "int", "long",  "long",  "int",   None,    None  ),
+    "openat":          (  "int", "long",   "int",  "int",   None,    None  ),
     "poll":            ( "long",  "int",   "int",   None,   None,    None  ),
     "ppoll":           ( "long",  "int",  "long", "long", "long",    None  ),
     "pread64":         (  "int", "long",  "long", "long",   None,    None  ),
@@ -80,6 +81,7 @@
     "setsockopt":      (  "int",  "int",   "int", "long",  "int",    None  ),
     "shutdown":        (  "int",  "int",    None,   None,   None,    None  ),
     "socket":          (  "int",  "int",   "int",   None,   None,    None  ),
+    "unlinkat":        (  "int", "long",   "int",   None,   None,    None  ),
     "wait4":           (  "int", "long",   "int", "long",   None,    None  ),
     "write":           (  "int", "long",  "long",   None,   None,    None  ),
     "writev":          (  "int", "long",  "long",   None,   None,    None  ),

src/app/firedancer-dev/main.h

@@ -205,6 +205,7 @@ extern action_t fd_action_mem;
 extern action_t fd_action_netconf;
 extern action_t fd_action_set_identity;
 extern action_t fd_action_get_identity;
+extern action_t fd_action_failover;
 extern action_t fd_action_ps;
 extern action_t fd_action_version;
 extern action_t fd_action_bench;
@@ -250,6 +251,7 @@ action_t * ACTIONS[] = {
   &fd_action_netconf,
   &fd_action_set_identity,
   &fd_action_get_identity,
+  &fd_action_failover,
   &fd_action_ps,
   &fd_action_help,
   &fd_action_metrics,

src/app/firedancer/Local.mk

@@ -25,6 +25,7 @@ $(call add-objs,commands/shred_version,fd_firedancer)
 $(call add-objs,commands/snapshot_create,fd_firedancer)
 $(call add-objs,commands/set_identity,fd_firedancer)
 $(call add-objs,commands/get_identity,fd_firedancer)
+$(call add-objs,commands/failover,fd_firedancer)
 $(call add-objs,commands/adminctl_client,fd_firedancer)
 $(call add-objs,commands/monitor_gossip/monitor_gossip commands/monitor_gossip/gossip_diag,fd_firedancer)
 

src/app/firedancer/commands/failover.c

@@ -0,0 +1,145 @@
+#define _GNU_SOURCE
+#include "adminctl_client.h"
+#include "../../shared/fd_config.h"
+#include "../../shared/fd_action.h"
+#include "../../../discof/failover/fd_failover_proto.h"
+
+#include <unistd.h>
+
+static char const * const SESSION_NAMES[] = {
+  "listening", "dialing", "hello", "paired", "rejected", "backoff"
+};
+static char const * const READINESS_NAMES[] = {
+  "ready", "link down", "peer status stale", "role or term conflict",
+  "active is unhealthy", "standby is unhealthy", "standby is not caught up"
+};
+
+#define SESSION_NAME_CNT   ( sizeof(SESSION_NAMES  )/sizeof(SESSION_NAMES  [ 0 ]) )
+#define READINESS_NAME_CNT ( sizeof(READINESS_NAMES)/sizeof(READINESS_NAMES[ 0 ]) )
+
+FD_STATIC_ASSERT( SESSION_NAME_CNT  ==FD_FAILOVER_SESSION_CNT,   session_names   );
+FD_STATIC_ASSERT( READINESS_NAME_CNT==FD_FAILOVER_READINESS_CNT, readiness_names );
+
+static void
+failover_cmd_args( int *    pargc,
+                   char *** pargv,
+                   args_t * args ) {
+  char const * name = fd_env_strip_cmdline_cstr( pargc, pargv, "--name", NULL, NULL );
+  if( FD_UNLIKELY( name ) ) fd_cstr_ncpy( args->failover.name, name, sizeof(args->failover.name) );
+
+  if( FD_UNLIKELY( !( *pargc ) ) ) FD_LOG_ERR(( "missing subcommand, supported: status" ));
+  if( FD_UNLIKELY( strcmp( **pargv, "status" ) ) ) FD_LOG_ERR(( "unknown subcommand `%s`, supported: status", **pargv ));
+  ( *pargc )--;
+  ( *pargv )++;
+}
+
+static void
+print_slot( char const * label,
+            ulong        slot ) {
+  if( FD_UNLIKELY( slot==FD_FAILOVER_SLOT_NULL ) ) FD_LOG_STDOUT(( "%-22s unknown\n", label ));
+  else FD_LOG_STDOUT(( "%-22s %lu\n", label, slot ));
+}
+
+static char const *
+role_name( uchar role ) {
+  if( FD_LIKELY( role==FD_FAILOVER_ROLE_STANDBY ) ) return "standby";
+  if( FD_LIKELY( role==FD_FAILOVER_ROLE_ACTIVE  ) ) return "active";
+  return "unknown";
+}
+
+static void
+failover_cmd_fn( args_t *   args,
+                 config_t * config ) {
+  fd_adminctl_t * adminctl = adminctl_client_attach( config, args->failover.name );
+
+  void * payload     = NULL;
+  ulong  payload_max = 0UL;
+  ulong  slot_idx    = fd_adminctl_reserve( adminctl, &payload, &payload_max );
+  if( FD_UNLIKELY( slot_idx==ULONG_MAX ) ) FD_LOG_ERR(( "all admin command slots are busy" ));
+  if( FD_UNLIKELY( sizeof(fd_adminctl_failover_status_req_t)>payload_max ) ) FD_LOG_ERR(( "adminctl failover-status payload too large" ));
+
+  fd_adminctl_failover_status_req_t * req = (fd_adminctl_failover_status_req_t *)payload;
+  req->version = FD_ADMINCTL_FAILOVER_STATUS_PAYLOAD_VERSION;
+
+  fd_adminctl_publish( adminctl, slot_idx, FD_ADMINCTL_CMD_FAILOVER_STATUS, sizeof(fd_adminctl_failover_status_req_t) );
+
+  fd_adminctl_failover_status_resp_t resp = { 0 };
+  ulong resp_sz = 0UL;
+  ulong result  = fd_adminctl_wait_response( adminctl, slot_idx, &resp, sizeof(resp), &resp_sz );
+  switch( result ) {
+    case FD_ADMINCTL_RESULT_SUCCESS: {
+      if( FD_UNLIKELY( resp_sz!=sizeof(resp) || resp.version!=FD_ADMINCTL_FAILOVER_STATUS_PAYLOAD_VERSION ) ) {
+        FD_LOG_ERR(( "running validator returned an incompatible failover-status response" ));
+      }
+
+      if( FD_UNLIKELY( !resp.enabled ) ) {
+        FD_LOG_STDOUT(( "%-22s %s\n", "failover:", "disabled" ));
+        break;
+      }
+
+      FD_LOG_STDOUT(( "%-22s %s\n", "failover:", "enabled" ));
+      FD_LOG_STDOUT(( "%-22s %s at term %lu\n", "role:", role_name( resp.role ), resp.term ));
+      FD_LOG_STDOUT(( "%-22s %s\n", "link:", resp.link_state<SESSION_NAME_CNT ? SESSION_NAMES[ resp.link_state ] : "unknown" ));
+      if( FD_LIKELY( resp.peer_status_valid ) ) {
+        FD_LOG_STDOUT(( "%-22s %s at term %lu\n", "peer:", role_name( resp.peer_role ), resp.peer_term ));
+        if( FD_UNLIKELY( resp.peer_status_age_nanos==ULONG_MAX ) ) FD_LOG_STDOUT(( "%-22s unknown\n", "peer status age:" ));
+        else                                                       FD_LOG_STDOUT(( "%-22s %.3f ms\n", "peer status age:", (double)resp.peer_status_age_nanos/1e6 ));
+        print_slot( "peer replay slot:",    resp.peer_replay_slot    );
+        print_slot( "peer last vote slot:", resp.peer_last_vote_slot );
+      }
+      FD_LOG_STDOUT(( "%-22s %s\n", "handoff ready:", resp.ready ? "yes" : "no" ));
+      if( FD_UNLIKELY( !resp.ready ) ) {
+        char const * reason = resp.readiness_reason<READINESS_NAME_CNT
+          ? READINESS_NAMES[ resp.readiness_reason ]
+          : "unknown";
+        FD_LOG_STDOUT(( "%-22s %s\n", "reason:", reason ));
+      }
+      FD_LOG_STDOUT(( "%-22s 0x%08x (flags 0x%02x)\n", "status:", resp.status, (uint)resp.flags ));
+      if( FD_LIKELY( resp.peer_status_valid ) ) {
+        FD_LOG_STDOUT(( "%-22s 0x%08x (flags 0x%02x)\n", "peer status:", resp.peer_status, (uint)resp.peer_flags ));
+      }
+      print_slot( "replay slot:",    resp.replay_slot    );
+      print_slot( "root slot:",      resp.root_slot      );
+      print_slot( "last vote slot:", resp.last_vote_slot );
+      if( FD_UNLIKELY( resp.replication_lag_slots==FD_FAILOVER_SLOT_NULL ) ) FD_LOG_STDOUT(( "%-22s unknown\n", "replication lag:" ));
+      else FD_LOG_STDOUT(( "%-22s %lu slots\n", "replication lag:", resp.replication_lag_slots ));
+      if( FD_UNLIKELY( !resp.rtt_nanos ) ) FD_LOG_STDOUT(( "%-22s unmeasured\n", "round trip:" ));
+      else                                 FD_LOG_STDOUT(( "%-22s %.3f ms\n", "round trip:", (double)resp.rtt_nanos/1e6 ));
+      FD_LOG_STDOUT(( "%-22s %lu sent, %lu received, %lu authentication failures\n",
+                      "frames:", resp.frames_sent, resp.frames_received, resp.mac_failures ));
+      FD_LOG_STDOUT(( "%-22s %lu protocol failures, %lu HELLO rejections, %lu connections\n",
+                      "channel:", resp.wire_failures, resp.hello_rejections, resp.connections ));
+      break;
+    }
+    case FD_ADMINCTL_RESULT_UNKNOWN_COMMAND:
+    case FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH:
+    case FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH:
+      FD_LOG_ERR(( "failover status is incompatible with the running validator" ));
+    default:
+      FD_LOG_ERR(( "unexpected failover-status result %lu", result ));
+  }
+}
+
+static void
+failover_args_help( fd_action_help_t * help ) {
+  fd_action_help_arg( help, "--name", "<name>", "Name of the validator instance to attach to, if more than one is\n"
+                                                "running on this host" );
+}
+
+action_t fd_action_failover = {
+  .name           = "failover",
+  .args           = failover_cmd_args,
+  .fn             = failover_cmd_fn,
+  .require_config = 0,
+  .perm           = NULL,
+  .is_diagnostic  = 1,
+  .description    = "Inspect the failover pair channel of the running validator",
+  .detail         = "`failover status` prints the pair channel's role, link state, peer view,\n"
+                    "replication lag, measured round trip time, and frame counters.\n"
+                    "\n"
+                    "This command does not start a validator; it attaches to one that is already\n"
+                    "running.  With no arguments it discovers the running validator automatically.\n"
+                    "If multiple validators are running, pass --name to select one.\n",
+  .usage          = "failover status [--name <name>]",
+  .args_help      = failover_args_help,
+};

src/app/firedancer/config/default.toml

@@ -77,7 +77,8 @@ telemetry = true
     #  +-- snapshots
     #      +-- snapshot-368327930-E5ZsnTCNX7WToyrP4ZQYJj7pPii6Ssa7UnNcV7HZ3iAf.tar.zst
     #      +-- incremental-snapshot-368327930-368400391-8ybYYVq7Ecui9TLj13SvzXjTX7JiGP65f4KrtNRCPajW.tar.zst
-    #  +-- tower-1_9-8RDPvMTMSDASGQBvehBFkiFVi2UdLTSZQYuFsGh5JwWU.bin.new
+    #  +-- tower
+    #  |   +-- tower-1_9-8RDPvMTMSDASGQBvehBFkiFVi2UdLTSZQYuFsGh5JwWU.bin
     #  +-- vote-account.json
     #
     # Two substitutions will be performed on this string.  If "{user}"
@@ -1854,6 +1855,46 @@ telemetry = true
         # loading.
         delay_startup = true
 
+# Failover pairs two machines that manage one staked identity.
+[failover]
+    # When false, failover is completely disengaged.
+    enabled = false
+
+    # Exactly one member of the pair must set this to true and dial the other.
+    dial_peer = false
+
+    # Local listener address used when dial_peer is false.
+    bind_address = "0.0.0.0"
+    bind_port = 9700
+
+    # Address and port used when dial_peer is true.  It is not advised to
+    # use a public network for the pair link.
+    peer_address = ""
+    peer_port = 9700
+
+    # Pair credentials and the two local identity keypairs.
+    pair_secret_path = ""
+    junk_identity_path = ""
+    staked_identity_path = ""
+
+    # Permit a request initiated by the standby to trigger a handoff.
+    accept_peer_requests = false
+
+    # Handoff and channel parameter thresholds.
+    status_interval_millis = 800
+    min_slots_to_leader = 150
+    deadline_slots = 64
+    catchup_gap_slots = 8
+    replication_lag_slots = 8
+    peer_silence_intervals = 5
+    retry_backoff_min_millis = 800
+    retry_backoff_max_millis = 12800
+
+    # Write an Agave-compatible tower file under `paths.base/tower` on
+    # every locally produced vote.  This option is not supported with
+    # Alpenglow.
+    tower_file = false
+
 # These options can be useful for development, but should not be used
 # when connecting to a live cluster, as they may cause the validator to
 # be unstable or have degraded performance or security.  The program

src/app/firedancer/main.c

@@ -188,6 +188,7 @@ extern action_t fd_action_add_authorized_voter;
 extern action_t fd_action_remove_all_authorized_voters;
 extern action_t fd_action_set_identity;
 extern action_t fd_action_get_identity;
+extern action_t fd_action_failover;
 extern action_t fd_action_ps;
 extern action_t fd_action_monitor_gossip;
 extern action_t fd_action_snapshot_create;
@@ -211,6 +212,7 @@ action_t * ACTIONS[] = {
   &fd_action_monitor_gossip,
   &fd_action_set_identity,
   &fd_action_get_identity,
+  &fd_action_failover,
   &fd_action_ps,
   &fd_action_snapshot_create,
   NULL,